feat(bench): add scoreable persistent SWE arm

bench/src/benchmarks/swe-bench.test.mts

@@ -48,9 +48,12 @@ test('SWE evaluation command preserves the requested instance image', () => {
     runId: 'r364',
     instanceId: taskId,
     cacheLevel: 'instance',
+    namespace: 'none',
   })
   const cacheIndex = argv.indexOf('--cache_level')
+  const namespaceIndex = argv.indexOf('--namespace')
   assert.equal(argv[cacheIndex + 1], 'instance')
+  assert.equal(argv[namespaceIndex + 1], 'none')
   assert.throws(
     () => createSweBenchAdapter({ cacheLevel: 'invalid' as 'instance' }),
     /invalid cacheLevel/,

bench/src/benchmarks/swe-bench.ts

@@ -81,6 +81,14 @@ export interface SweBenchAdapterOptions {
 }
 
 const SWE_CACHE_LEVELS = new Set<SweBenchCacheLevel>(['none', 'base', 'env', 'instance'])
+
+function scorerNamespace(): 'swebench' | 'none' {
+  const namespace = process.env.SWEBENCH_NAMESPACE ?? 'swebench'
+  if (namespace !== 'swebench' && namespace !== 'none') {
+    throw new Error(`SWEBENCH_NAMESPACE must be swebench|none, got "${namespace}"`)
+  }
+  return namespace
+}
 const TEST_FILE_EXCLUDES = [
   "':(exclude,glob)**/tests/**'",
   "':(exclude,glob)**/test/**'",
@@ -157,6 +165,7 @@ export function sweEvaluationArgv(args: {
   readonly runId: string
   readonly instanceId: string
   readonly cacheLevel: SweBenchCacheLevel
+  readonly namespace?: 'swebench' | 'none'
 }): string[] {
   return [
     '-m', 'swebench.harness.run_evaluation',
@@ -165,6 +174,7 @@ export function sweEvaluationArgv(args: {
     '--run_id', args.runId,
     '--instance_ids', args.instanceId,
     '--max_workers', '1',
+    '--namespace', args.namespace ?? scorerNamespace(),
     '--cache_level', args.cacheLevel,
   ]
 }
@@ -311,6 +321,7 @@ print(json.dumps(out))
           runId,
           instanceId: task.id,
           cacheLevel,
+          namespace: scorerNamespace(),
         }),
         async parseReport(dir) {
           // Report file: agent-runtime-bench.<run_id>.json

bench/src/swe-bench-env.test.ts

@@ -1,9 +1,144 @@
 import assert from 'node:assert/strict'
-import { mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'
+import { execFileSync } from 'node:child_process'
+import { mkdirSync, mkdtempSync, readlinkSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'
 import { tmpdir } from 'node:os'
-import { join } from 'node:path'
+import { isAbsolute, join } from 'node:path'
 import { after, describe, it } from 'node:test'
-import { isInsideJail, isTestPath, jailPath } from './swe-bench-env'
+import {
+  copyPristineGitCheckout,
+  firstAvailableSweImageCandidate,
+  isInsideJail,
+  isTestPath,
+  jailPath,
+  parseSweImageIdentity,
+  parseSweImageCandidates,
+  SWE_RUN_TOOL_CONFIG,
+  SWE_SEED_PROMPT,
+  SWE_SEED_PROMPT_WITH_RUN,
+} from './swe-bench-env'
+import { absoluteSweTempDir } from './swe-temp'
+
+const makeRelativeSymlinkRepo = (): { root: string; source: string; destination: string; links: string[] } => {
+  const root = mkdtempSync(join(tmpdir(), 'swe-cache-copy-'))
+  const source = join(root, 'source')
+  const destination = join(root, 'destination')
+  const targetDir = join(source, 'docs/_theme/djangodocs/static')
+  const linkDir = join(source, 'docs/_theme/djangodocs-epub/static')
+  const hooksDir = join(root, 'empty-hooks')
+  const links = ['docicons-note.png', 'docicons-philosophy.png', 'docicons-behindscenes.png', 'docicons-warning.png']
+  mkdirSync(targetDir, { recursive: true })
+  mkdirSync(linkDir, { recursive: true })
+  mkdirSync(hooksDir)
+  mkdirSync(destination)
+  for (const name of links) {
+    writeFileSync(join(targetDir, name), `${name}\n`)
+    symlinkSync(`../../djangodocs/static/${name}`, join(linkDir, name))
+  }
+  execFileSync('git', ['-C', source, 'init', '--quiet'])
+  execFileSync('git', ['-C', source, 'add', '.'])
+  execFileSync('git', [
+    '-C',
+    source,
+    '-c',
+    'user.name=SWE Fixture',
+    '-c',
+    'user.email=swe-fixture@example.invalid',
+    '-c',
+    `core.hooksPath=${hooksDir}`,
+    '-c',
+    'commit.gpgsign=false',
+    'commit',
+    '--quiet',
+    '-m',
+    'fixture',
+  ])
+  return { root, source, destination, links }
+}
+
+describe('SWE worker prompts', () => {
+  it('keeps the run-capable prompt as the exact baseline prompt plus run guidance', () => {
+    assert.equal(SWE_SEED_PROMPT_WITH_RUN.startsWith(`${SWE_SEED_PROMPT} `), true)
+    assert.match(SWE_SEED_PROMPT_WITH_RUN, /You ALSO have a run tool/)
+  })
+
+  it('exports the exact run-tool settings used by the worker surface', () => {
+    assert.ok(SWE_RUN_TOOL_CONFIG.timeoutS > 0)
+    assert.ok(SWE_RUN_TOOL_CONFIG.outputLimit > 0)
+  })
+
+  it('captures immutable Docker image identity independently of a mutable tag', () => {
+    assert.deepEqual(
+      parseSweImageIdentity(
+        JSON.stringify([{ Id: 'sha256:image', RepoDigests: ['repo@sha256:z', 'repo@sha256:a'] }]),
+      ),
+      { id: 'sha256:image', repoDigests: ['repo@sha256:a', 'repo@sha256:z'] },
+    )
+    assert.throws(() => parseSweImageIdentity(JSON.stringify([{}])), /no image ID/)
+  })
+
+  it('preserves the local fallback namespace when only the unnamespaced image is cached', () => {
+    const candidates = parseSweImageCandidates(
+      JSON.stringify([
+        { tag: 'swebench/sweb.eval.example:latest', namespace: 'swebench' },
+        { tag: 'sweb.eval.example:latest', namespace: 'none' },
+      ]),
+    )
+    const localIdentity = { id: 'sha256:local', repoDigests: [] }
+    assert.deepEqual(
+      firstAvailableSweImageCandidate(candidates, new Map([['sweb.eval.example:latest', localIdentity]])),
+      { candidate: { tag: 'sweb.eval.example:latest', namespace: 'none' }, identity: localIdentity },
+    )
+  })
+})
+
+describe('SWE temporary directory', () => {
+  it('stays absolute when model temperature is configured through TEMPERATURE', () => {
+    const priorTemp = process.env.TEMP
+    const priorTemperature = process.env.TEMPERATURE
+    try {
+      delete process.env.TEMP
+      process.env.TEMPERATURE = '0.8'
+      assert.equal(isAbsolute(absoluteSweTempDir()), true)
+
+      process.env.TEMP = '0.8'
+      assert.throws(() => absoluteSweTempDir(), /must be absolute.*TEMPERATURE/)
+    } finally {
+      if (priorTemp === undefined) delete process.env.TEMP
+      else process.env.TEMP = priorTemp
+      if (priorTemperature === undefined) delete process.env.TEMPERATURE
+      else process.env.TEMPERATURE = priorTemperature
+    }
+  })
+})
+
+describe('SWE clone cache copy', () => {
+  it('preserves Django-style relative symlinks and produces a clean worktree', async () => {
+    const fixture = makeRelativeSymlinkRepo()
+    try {
+      await copyPristineGitCheckout(fixture.source, fixture.destination)
+      for (const name of fixture.links) {
+        const copiedLink = join(fixture.destination, 'docs/_theme/djangodocs-epub/static', name)
+        assert.equal(readlinkSync(copiedLink), `../../djangodocs/static/${name}`)
+      }
+      assert.equal(
+        execFileSync('git', ['-C', fixture.destination, 'status', '--porcelain'], { encoding: 'utf8' }),
+        '',
+      )
+    } finally {
+      rmSync(fixture.root, { recursive: true, force: true })
+    }
+  })
+
+  it('fails closed when a cached copy is not pristine', async () => {
+    const fixture = makeRelativeSymlinkRepo()
+    try {
+      writeFileSync(join(fixture.source, 'docs/_theme/djangodocs/static/docicons-note.png'), 'dirty\n')
+      await assert.rejects(copyPristineGitCheckout(fixture.source, fixture.destination), /not pristine/)
+    } finally {
+      rmSync(fixture.root, { recursive: true, force: true })
+    }
+  })
+})
 
 describe('isTestPath', () => {
   it('flags test directories and test-named python files', () => {