diff --git a/bench/src/benchmarks/swe-bench.test.mts b/bench/src/benchmarks/swe-bench.test.mts index 703807cd..3486d81a 100644 --- a/bench/src/benchmarks/swe-bench.test.mts +++ b/bench/src/benchmarks/swe-bench.test.mts @@ -48,9 +48,12 @@ test('SWE evaluation command preserves the requested instance image', () => { runId: 'r364', instanceId: taskId, cacheLevel: 'instance', + namespace: 'none', }) const cacheIndex = argv.indexOf('--cache_level') + const namespaceIndex = argv.indexOf('--namespace') assert.equal(argv[cacheIndex + 1], 'instance') + assert.equal(argv[namespaceIndex + 1], 'none') assert.throws( () => createSweBenchAdapter({ cacheLevel: 'invalid' as 'instance' }), /invalid cacheLevel/, diff --git a/bench/src/benchmarks/swe-bench.ts b/bench/src/benchmarks/swe-bench.ts index 60a77580..71fc9b83 100644 --- a/bench/src/benchmarks/swe-bench.ts +++ b/bench/src/benchmarks/swe-bench.ts @@ -81,6 +81,14 @@ export interface SweBenchAdapterOptions { } const SWE_CACHE_LEVELS = new Set(['none', 'base', 'env', 'instance']) + +function scorerNamespace(): 'swebench' | 'none' { + const namespace = process.env.SWEBENCH_NAMESPACE ?? 'swebench' + if (namespace !== 'swebench' && namespace !== 'none') { + throw new Error(`SWEBENCH_NAMESPACE must be swebench|none, got "${namespace}"`) + } + return namespace +} const TEST_FILE_EXCLUDES = [ "':(exclude,glob)**/tests/**'", "':(exclude,glob)**/test/**'", @@ -157,6 +165,7 @@ export function sweEvaluationArgv(args: { readonly runId: string readonly instanceId: string readonly cacheLevel: SweBenchCacheLevel + readonly namespace?: 'swebench' | 'none' }): string[] { return [ '-m', 'swebench.harness.run_evaluation', @@ -165,6 +174,7 @@ export function sweEvaluationArgv(args: { '--run_id', args.runId, '--instance_ids', args.instanceId, '--max_workers', '1', + '--namespace', args.namespace ?? scorerNamespace(), '--cache_level', args.cacheLevel, ] } @@ -311,6 +321,7 @@ print(json.dumps(out)) runId, instanceId: task.id, cacheLevel, + namespace: scorerNamespace(), }), async parseReport(dir) { // Report file: agent-runtime-bench..json diff --git a/bench/src/swe-bench-env.test.ts b/bench/src/swe-bench-env.test.ts index a2a93d78..b95d8bf5 100644 --- a/bench/src/swe-bench-env.test.ts +++ b/bench/src/swe-bench-env.test.ts @@ -1,9 +1,144 @@ import assert from 'node:assert/strict' -import { mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs' +import { execFileSync } from 'node:child_process' +import { mkdirSync, mkdtempSync, readlinkSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' -import { join } from 'node:path' +import { isAbsolute, join } from 'node:path' import { after, describe, it } from 'node:test' -import { isInsideJail, isTestPath, jailPath } from './swe-bench-env' +import { + copyPristineGitCheckout, + firstAvailableSweImageCandidate, + isInsideJail, + isTestPath, + jailPath, + parseSweImageIdentity, + parseSweImageCandidates, + SWE_RUN_TOOL_CONFIG, + SWE_SEED_PROMPT, + SWE_SEED_PROMPT_WITH_RUN, +} from './swe-bench-env' +import { absoluteSweTempDir } from './swe-temp' + +const makeRelativeSymlinkRepo = (): { root: string; source: string; destination: string; links: string[] } => { + const root = mkdtempSync(join(tmpdir(), 'swe-cache-copy-')) + const source = join(root, 'source') + const destination = join(root, 'destination') + const targetDir = join(source, 'docs/_theme/djangodocs/static') + const linkDir = join(source, 'docs/_theme/djangodocs-epub/static') + const hooksDir = join(root, 'empty-hooks') + const links = ['docicons-note.png', 'docicons-philosophy.png', 'docicons-behindscenes.png', 'docicons-warning.png'] + mkdirSync(targetDir, { recursive: true }) + mkdirSync(linkDir, { recursive: true }) + mkdirSync(hooksDir) + mkdirSync(destination) + for (const name of links) { + writeFileSync(join(targetDir, name), `${name}\n`) + symlinkSync(`../../djangodocs/static/${name}`, join(linkDir, name)) + } + execFileSync('git', ['-C', source, 'init', '--quiet']) + execFileSync('git', ['-C', source, 'add', '.']) + execFileSync('git', [ + '-C', + source, + '-c', + 'user.name=SWE Fixture', + '-c', + 'user.email=swe-fixture@example.invalid', + '-c', + `core.hooksPath=${hooksDir}`, + '-c', + 'commit.gpgsign=false', + 'commit', + '--quiet', + '-m', + 'fixture', + ]) + return { root, source, destination, links } +} + +describe('SWE worker prompts', () => { + it('keeps the run-capable prompt as the exact baseline prompt plus run guidance', () => { + assert.equal(SWE_SEED_PROMPT_WITH_RUN.startsWith(`${SWE_SEED_PROMPT} `), true) + assert.match(SWE_SEED_PROMPT_WITH_RUN, /You ALSO have a run tool/) + }) + + it('exports the exact run-tool settings used by the worker surface', () => { + assert.ok(SWE_RUN_TOOL_CONFIG.timeoutS > 0) + assert.ok(SWE_RUN_TOOL_CONFIG.outputLimit > 0) + }) + + it('captures immutable Docker image identity independently of a mutable tag', () => { + assert.deepEqual( + parseSweImageIdentity( + JSON.stringify([{ Id: 'sha256:image', RepoDigests: ['repo@sha256:z', 'repo@sha256:a'] }]), + ), + { id: 'sha256:image', repoDigests: ['repo@sha256:a', 'repo@sha256:z'] }, + ) + assert.throws(() => parseSweImageIdentity(JSON.stringify([{}])), /no image ID/) + }) + + it('preserves the local fallback namespace when only the unnamespaced image is cached', () => { + const candidates = parseSweImageCandidates( + JSON.stringify([ + { tag: 'swebench/sweb.eval.example:latest', namespace: 'swebench' }, + { tag: 'sweb.eval.example:latest', namespace: 'none' }, + ]), + ) + const localIdentity = { id: 'sha256:local', repoDigests: [] } + assert.deepEqual( + firstAvailableSweImageCandidate(candidates, new Map([['sweb.eval.example:latest', localIdentity]])), + { candidate: { tag: 'sweb.eval.example:latest', namespace: 'none' }, identity: localIdentity }, + ) + }) +}) + +describe('SWE temporary directory', () => { + it('stays absolute when model temperature is configured through TEMPERATURE', () => { + const priorTemp = process.env.TEMP + const priorTemperature = process.env.TEMPERATURE + try { + delete process.env.TEMP + process.env.TEMPERATURE = '0.8' + assert.equal(isAbsolute(absoluteSweTempDir()), true) + + process.env.TEMP = '0.8' + assert.throws(() => absoluteSweTempDir(), /must be absolute.*TEMPERATURE/) + } finally { + if (priorTemp === undefined) delete process.env.TEMP + else process.env.TEMP = priorTemp + if (priorTemperature === undefined) delete process.env.TEMPERATURE + else process.env.TEMPERATURE = priorTemperature + } + }) +}) + +describe('SWE clone cache copy', () => { + it('preserves Django-style relative symlinks and produces a clean worktree', async () => { + const fixture = makeRelativeSymlinkRepo() + try { + await copyPristineGitCheckout(fixture.source, fixture.destination) + for (const name of fixture.links) { + const copiedLink = join(fixture.destination, 'docs/_theme/djangodocs-epub/static', name) + assert.equal(readlinkSync(copiedLink), `../../djangodocs/static/${name}`) + } + assert.equal( + execFileSync('git', ['-C', fixture.destination, 'status', '--porcelain'], { encoding: 'utf8' }), + '', + ) + } finally { + rmSync(fixture.root, { recursive: true, force: true }) + } + }) + + it('fails closed when a cached copy is not pristine', async () => { + const fixture = makeRelativeSymlinkRepo() + try { + writeFileSync(join(fixture.source, 'docs/_theme/djangodocs/static/docicons-note.png'), 'dirty\n') + await assert.rejects(copyPristineGitCheckout(fixture.source, fixture.destination), /not pristine/) + } finally { + rmSync(fixture.root, { recursive: true, force: true }) + } + }) +}) describe('isTestPath', () => { it('flags test directories and test-named python files', () => { diff --git a/bench/src/swe-bench-env.ts b/bench/src/swe-bench-env.ts index e82b2c1a..7c0ac562 100644 --- a/bench/src/swe-bench-env.ts +++ b/bench/src/swe-bench-env.ts @@ -13,17 +13,132 @@ * memorization. Always report this; never claim a "clean" frontier number from this arena alone. */ import { execFile } from 'node:child_process' -import { existsSync, lstatSync, mkdtempSync, readdirSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs' -import { tmpdir } from 'node:os' +import { cpSync, existsSync, lstatSync, mkdtempSync, readdirSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs' import { join, sep } from 'node:path' import { promisify } from 'node:util' import type { AgenticSurface, AgenticTask, AgenticTool, ArtifactHandle, SurfaceScore } from '@tangle-network/agent-runtime/loops' -import { createSweBenchAdapter } from './benchmarks/swe-bench' +import { runVenvPython } from './benchmarks/_harness' +import { createSweBenchAdapter, type SweBenchAdapterOptions } from './benchmarks/swe-bench' import type { BenchTask } from './benchmarks/types' +import { absoluteSweTempDir } from './swe-temp' const exec = promisify(execFile) export const isTestPath = (p: string) => /(^|\/)(tests?)\//.test(p) || /test_.*\.py$|_test\.py$|conftest\.py$/.test(p) +/** Copy a cached git checkout without rewriting repository-relative symlinks, then prove that the + * copy is byte-for-byte clean from git's perspective before a worker can observe it. */ +export async function copyPristineGitCheckout(sourceDir: string, destinationDir: string): Promise { + cpSync(sourceDir, destinationDir, { recursive: true, verbatimSymlinks: true }) + let status: string + try { + const result = await exec('git', ['-C', destinationDir, 'status', '--porcelain'], { + timeout: 60_000, + maxBuffer: 20_000_000, + }) + status = result.stdout + } catch (error) { + throw new Error(`could not verify cached checkout copy: ${(error as Error).message}`) + } + if (status.length > 0) { + throw new Error(`cached checkout copy is not pristine:\n${status.slice(0, 4_000)}`) + } +} + +/** + * The read/edit-only SWE agent system prompt — the ESTABLISHED baseline surface (glm-5.2 raw = 7/12, + * glm-4.6 = 3/12). Exported as the single source of truth so `tasks()` here and the improve() seed in + * swe-improve.mts stay byte-identical (the baseline denominator depends on this — no drift). Do NOT + * edit this constant to add run-tool guidance; that lives in SWE_SEED_PROMPT_WITH_RUN so the read/edit + * baseline arm is reproducible unchanged. + */ +export const SWE_SEED_PROMPT = + 'You are a senior engineer fixing a real bug in the checked-out repository. Work PERSISTENTLY and do not ' + + 'stop early: use list_files + read_file to explore BROADLY (read many candidate files — the bug is rarely in ' + + 'the first file you open), trace the issue to its root cause, then fix it with edit_file. You MUST make at ' + + 'least one edit_file call — never finish with prose alone or without attempting a fix. Make a MINIMAL surgical ' + + 'change (a few lines, like a real PR), source only (test files are rejected). If an edit_file fails (old_string ' + + 'not unique/found), read the file again and retry with exact text. Keep going until you have made your best fix.' + +/** + * The WITH-TOOLS arm: the baseline prompt PLUS the run-tool workflow (write a failing repro, edit, re-run + * until it passes). Only used when the environment is built with `enableRun`. Kept as a separate constant + * so enabling the run tool never mutates the read/edit-only baseline. Also reconciles the now-active + * confusion from loadTasks' userPrompt (which says the repo is "at /work, cd /work" — the container path + * from the sandbox path; here the run tool's cwd is the repo root and there is nothing to cd into). + */ +// NOTE (WAF): the router sits behind a Cloudflare WAF whose RCE ruleset 403s any request body that +// contains BACKTICK-wrapped command-like text (backticks are shell command-substitution syntax). The +// SAME commands without backticks pass. So this prompt and the run tool description deliberately write +// example commands WITHOUT backticks. Verified: backtick form -> 403, plain form -> 200. +export const SWE_SEED_PROMPT_WITH_RUN = + `${SWE_SEED_PROMPT} ` + + 'You ALSO have a run tool: execute a shell command in the repo checkout (cwd is already the repo root — do ' + + 'NOT cd, and ignore any instruction that says the repo is at /work). Use run to VERIFY your fix, not to ' + + 'stall. DISCIPLINE: read a few files to locate the bug, then MAKE YOUR EDIT with edit_file EARLY — do not ' + + 'spend many turns running commands before your first edit. After you edit, use run to check the fix: a ' + + 'one-line python -c inline check, or python -m pytest on the nearest existing test file (use -k to select ' + + 'the case, plus -rA and -p no:cacheprovider). If it still fails, read the output, refine the edit, and ' + + 're-run — iterate EDIT then run until it passes, and run the nearest existing tests to catch regressions. ' + + 'The network is DISABLED and the hidden grading tests are NOT present, so verify with local, network-free ' + + 'checks. You MUST make at least one edit_file — a turn budget spent running with no edit is a failure.' + +/** Hard wall-clock cap (seconds) for a single `run` command, enforced BOTH in-container (`timeout`) and + * on the host (execFile timeout + SIGKILL). Env-overridable for slow suites; default fail-closed at 120s. */ +const RUN_TIMEOUT_S = Number(process.env.SWE_RUN_TIMEOUT ?? 120) +/** Combined stdout+stderr budget returned to the model: head + tail so the failure summary (which pytest + * prints at the tail) survives truncation. */ +const RUN_OUTPUT_LIMIT = Number(process.env.SWE_RUN_OUTPUT_LIMIT ?? 10_000) +/** Exact run-tool settings stamped into structural-experiment receipts. */ +export const SWE_RUN_TOOL_CONFIG = Object.freeze({ + timeoutS: RUN_TIMEOUT_S, + outputLimit: RUN_OUTPUT_LIMIT, +}) +/** Monotonic suffix so concurrent `run` calls get distinct container names (for reap-on-timeout). */ +let runNameCounter = 0 + +/** Truncate a long string keeping a head and a (larger) tail, with a marker between — the tail carries the + * test-failure summary, so it gets the bigger share. Pure. */ +function truncateHeadTail(s: string, limit: number): string { + if (s.length <= limit) return s + const head = Math.floor(limit * 0.35) + const tail = limit - head + return `${s.slice(0, head)}\n...[truncated ${s.length - limit} chars]...\n${s.slice(s.length - tail)}` +} + +/** Inline venv script: given the instance metadata row (argv[1] JSON), print the candidate Docker image + * tags swebench itself would use — the remote/namespaced tag (what a `swebench` pull caches, e.g. + * `swebench/sweb.eval.x86_64.psf_1776_requests-1142:latest`) FIRST, then the local-build tag + * (`sweb.eval.x86_64.psf__requests-1142:latest`). Delegated to `make_test_spec` so the naming stays + * correct across swebench versions instead of being hand-built. No network (it reads a dict, not the HF split). */ +const IMAGE_KEY_SCRIPT = ` +import json, sys +from swebench.harness.test_spec.test_spec import make_test_spec +row = json.loads(sys.argv[1]) +candidates = [] +for namespace in ("swebench", None): + try: + candidates.append({ + "tag": make_test_spec(row, namespace=namespace).instance_image_key, + "namespace": "swebench" if namespace == "swebench" else "none", + }) + except Exception: + pass +print(json.dumps(candidates)) +` + +const SWEBENCH_VERSION_SCRIPT = ` +import importlib.metadata +print(importlib.metadata.version("swebench")) +` + +/** Installed official scorer package stamped into experiment execution receipts. */ +export async function resolveSweBenchScorerVersion(): Promise { + const output = await runVenvPython(SWEBENCH_VERSION_SCRIPT, [], 60_000) + const version = output.trim().split('\n').filter(Boolean).pop() ?? '' + if (!version) throw new Error('could not resolve installed swebench scorer version') + return version +} + /** * Cheap string pre-filter for an agent-supplied repo-relative path, applied before the path is * joined to a workspace root: rejects absolute paths and any `..` segment, strips a leading `./`. @@ -43,24 +158,177 @@ export const jailPath = (_root: string, p: string): string | null => { */ export const isInsideJail = (jailRoot: string, real: string): boolean => real === jailRoot || real.startsWith(jailRoot + sep) +export interface SweImageIdentity { + id: string + repoDigests: string[] +} + +export interface SweImageCandidate { + tag: string + namespace: 'swebench' | 'none' +} + +export type SweImageResolution = + | { ok: true; tag: string; namespace: SweImageCandidate['namespace']; identity: SweImageIdentity } + | { ok: false; reason: string } + +export function parseSweImageCandidates(stdout: string): SweImageCandidate[] { + const parsed = JSON.parse(stdout) as Array<{ tag?: unknown; namespace?: unknown }> + return parsed.map((candidate, index) => { + if ( + typeof candidate.tag !== 'string' || + (candidate.namespace !== 'swebench' && candidate.namespace !== 'none') + ) { + throw new Error(`invalid SWE image candidate at index ${index}`) + } + return { tag: candidate.tag, namespace: candidate.namespace } + }) +} + +export function firstAvailableSweImageCandidate( + candidates: readonly SweImageCandidate[], + identities: ReadonlyMap, +): { candidate: SweImageCandidate; identity: SweImageIdentity } | null { + for (const candidate of candidates) { + const identity = identities.get(candidate.tag) + if (identity) return { candidate, identity } + } + return null +} + +/** Parse the immutable image identity returned by `docker image inspect`. */ +export function parseSweImageIdentity(stdout: string): SweImageIdentity { + const rows = JSON.parse(stdout) as Array<{ Id?: unknown; RepoDigests?: unknown }> + const row = rows[0] + const id = typeof row?.Id === 'string' ? row.Id : '' + if (!id) throw new Error('docker image inspect returned no image ID') + const repoDigests = Array.isArray(row?.RepoDigests) + ? row.RepoDigests.filter((value): value is string => typeof value === 'string').sort() + : [] + return { id, repoDigests } +} + interface Ws { dir: string task: BenchTask + /** Memoized `run`-tool image resolution: the local Docker tag to exec in, or a fail-closed reason. */ + image?: SweImageResolution +} + +/** Resolve the locally-present Docker image for an instance's METADATA row (no workspace needed). + * Asks swebench for the candidate tags, then picks the first that `docker image inspect` finds locally. + * Fail-closed: docker down / no cached image / resolver error → `{ ok:false }`. Exported so script-side + * calibrators (swe-repro-calibrate) hard-assert image presence through the SAME resolution the `run` + * tool uses instead of hand-building tags. */ +export async function resolveImageForMetadata( + metadata: Record, +): Promise { + let candidates: SweImageCandidate[] + try { + const out = await runVenvPython(IMAGE_KEY_SCRIPT, [JSON.stringify(metadata)], 60_000) + const lastLine = out.trim().split('\n').filter(Boolean).pop() ?? '[]' + candidates = parseSweImageCandidates(lastLine) + } catch (e) { + return { ok: false, reason: `image-key resolution failed: ${(e as Error).message.slice(0, 160)}` } + } + if (!candidates.length) return { ok: false, reason: 'swebench produced no image key for this instance' } + const identities = new Map() + for (const { tag } of candidates) { + try { + const inspected = await exec('docker', ['image', 'inspect', tag], { timeout: 20_000 }) + identities.set(tag, parseSweImageIdentity(inspected.stdout)) + } catch (e) { + const m = (e as { stderr?: string }).stderr ?? (e as Error).message ?? '' + if (/Cannot connect to the Docker daemon|Is the docker daemon running/i.test(m)) { + return { ok: false, reason: 'docker daemon unavailable' } + } + // otherwise: this tag is just not cached locally — try the next candidate + } + } + const selected = firstAvailableSweImageCandidate(candidates, identities) + if (selected) { + return { + ok: true, + tag: selected.candidate.tag, + namespace: selected.candidate.namespace, + identity: selected.identity, + } + } + return { ok: false, reason: `no cached image (tried: ${candidates.map(({ tag }) => tag).join(', ')})` } +} + +/** Per-workspace memoization of `resolveImageForMetadata` for the `run` tool (degrades to an + * `ERROR:` string so the agent falls back to read/edit-only instead of crashing). */ +async function resolveInstanceImage(ws: Ws, expected?: SweImageIdentity): Promise { + if (ws.image) { + if (ws.image.ok && expected && ws.image.identity.id !== expected.id) { + return { ok: false, reason: `image identity changed (${expected.id} -> ${ws.image.identity.id})` } + } + return ws.image + } + const resolved = await resolveImageForMetadata(ws.task.metadata ?? {}) + if (resolved.ok && expected && resolved.identity.id !== expected.id) { + return (ws.image = { + ok: false, + reason: `image identity changed (${expected.id} -> ${resolved.identity.id})`, + }) + } + return (ws.image = resolved) } /** Build the SWE-bench Environment + a DISJOINT-slice task supplier over the Verified split. The * supplier keys tasks by dataset offset so `runStrategyEvolution`'s train [0,trainN) and holdout * [trainN+off,…) never overlap. Verified is loaded once; instances carry their repo/base_commit. */ -export async function createSweBenchEnvironment(poolN = 80): Promise<{ +export async function createSweBenchEnvironment( + poolN = 80, + opts: { + ids?: readonly string[] + enableRun?: boolean + cloneCache?: boolean + expectedImageIdentities?: ReadonlyMap + adapterOptions?: SweBenchAdapterOptions + } = {}, +): Promise<{ environment: AgenticSurface tasks: (offset: number, n: number) => Promise adapter: ReturnType }> { - const adapter = createSweBenchAdapter() - const pool = await adapter.loadTasks({ limit: poolN, split: 'test' }) + const adapter = createSweBenchAdapter(opts.adapterOptions) + // WITH-TOOLS arm: expose the jailed `run` tool + use the run-aware seed prompt. Default OFF keeps + // the established read/edit-only baseline byte-identical. + const enableRun = opts.enableRun ?? false + const pool = opts.ids?.length + ? await adapter.loadTasks({ ids: [...opts.ids], split: 'test' }) + : await adapter.loadTasks({ limit: poolN, split: 'test' }) const byId = new Map(pool.map((t) => [t.id, t])) // Each environment owns its workspace registry so concurrent environments don't share state. const workspaces = new Map() + // Opt-in per-instance clone cache: clone each instance from GitHub once, then copy the pristine + // checkout for later sessions. This removes a mid-run network dependency without sharing edits. + const cloneCache = opts.cloneCache ?? false + const pristine = new Map>() + const clonedAt = async (md: Record, dir: string): Promise => { + await exec('git', ['clone', '--filter=blob:none', '--no-checkout', '--quiet', `https://github.com/${md.repo}.git`, dir], { timeout: 420_000 }) + await exec('git', ['-C', dir, 'checkout', '--quiet', md.base_commit], { timeout: 300_000 }) + } + const pristineClone = (id: string, md: Record): Promise => { + let pending = pristine.get(id) + if (!pending) { + pending = (async () => { + const dir = mkdtempSync(join(absoluteSweTempDir(), 'swe-cache-')) + try { + await clonedAt(md, dir) + return dir + } catch (error) { + rmSync(dir, { recursive: true, force: true }) + pristine.delete(id) + throw error + } + })() + pristine.set(id, pending) + } + return pending + } const environment: AgenticSurface = { name: 'swe-bench-verified', @@ -68,10 +336,10 @@ export async function createSweBenchEnvironment(poolN = 80): Promise<{ const bt = byId.get(task.id) if (!bt) throw new Error(`swe-bench-env: unknown task ${task.id}`) const md = bt.metadata as Record - const dir = mkdtempSync(join(tmpdir(), 'swe-')) + const dir = mkdtempSync(join(absoluteSweTempDir(), 'swe-')) try { - await exec('git', ['clone', '--filter=blob:none', '--no-checkout', '--quiet', `https://github.com/${md.repo}.git`, dir], { timeout: 420_000 }) - await exec('git', ['-C', dir, 'checkout', '--quiet', md.base_commit], { timeout: 300_000 }) + if (cloneCache) await copyPristineGitCheckout(await pristineClone(task.id, md), dir) + else await clonedAt(md, dir) const handle: ArtifactHandle = { id: dir, surface: 'swe-bench-verified' } workspaces.set(dir, { dir, task: bt }) return handle @@ -81,11 +349,28 @@ export async function createSweBenchEnvironment(poolN = 80): Promise<{ } }, async tools() { - return [ + const tools: AgenticTool[] = [ { type: 'function', function: { name: 'list_files', description: 'List source files under a repo subdirectory (recursive, bounded). "" = repo root.', parameters: { type: 'object', properties: { dir: { type: 'string' } }, required: ['dir'] } } }, { type: 'function', function: { name: 'read_file', description: 'Read a repo file by path.', parameters: { type: 'object', properties: { path: { type: 'string' } }, required: ['path'] } } }, { type: 'function', function: { name: 'edit_file', description: 'Surgical fix: replace the EXACT old_string (must occur once — copy whitespace precisely) with new_string in a SOURCE file. Minimal changes, never whole-file rewrites. Test files are rejected.', parameters: { type: 'object', properties: { path: { type: 'string' }, old_string: { type: 'string' }, new_string: { type: 'string' } }, required: ['path', 'old_string', 'new_string'] } } }, - ] satisfies AgenticTool[] + ] + if (enableRun) { + tools.push({ + type: 'function', + function: { + name: 'run', + description: + 'Run a shell command in the repo checkout to REPRODUCE the bug and VERIFY your fix. cwd is the ' + + 'repo root (do NOT cd). Use a one-line python -c inline check for a quick offline reproduction, ' + + 'or python -m pytest on an existing test file (add -k to select a case, plus -rA and ' + + '-p no:cacheprovider) to run tests near your change. Returns the exit code then the combined ' + + 'stdout+stderr. NETWORK IS DISABLED and hidden grading tests are absent. A non-zero exit is a ' + + 'normal failing-test signal, not a tool error.', + parameters: { type: 'object', properties: { cmd: { type: 'string' } }, required: ['cmd'] }, + }, + }) + } + return tools }, async call(handle, name, args) { const ws = workspaces.get(handle.id) @@ -175,6 +460,57 @@ export async function createSweBenchEnvironment(poolN = 80): Promise<{ writeFileSync(real, content.replace(oldStr, newStr)) return `edited ${p}: replaced 1 occurrence` } + if (enableRun && name === 'run') { + const cmd = String(args.cmd ?? '').trim() + if (!cmd) return 'ERROR: run requires a non-empty cmd' + const img = await resolveInstanceImage(ws, opts.expectedImageIdentities?.get(ws.task.id)) + if (!img.ok) return `ERROR: run unavailable (${img.reason}) — continue with read_file/edit_file only` + const T = RUN_TIMEOUT_S + const containerName = `swe-run-${process.pid}-${Date.now()}-${runNameCounter++}` + // The whole toolchain is interpreted ONLY by the container's bash — the host never sees a shell + // (execFile + args array). The agent cmd rides in as $SWE_CMD (an env var, no host-quoting hazard). + // `{ …; } 2>&1` merges stderr into stdout preserving order; the group's exit = the last command's. + // conda.sh only DEFINES `conda` (unlike bin/activate, which would consume our positional args). + const script = + '{ source /opt/miniconda3/etc/profile.d/conda.sh && conda activate testbed && cd /testbed && ' + + 'timeout -s KILL "$SWE_T"s bash -c "$SWE_CMD"; } 2>&1' + // READ-ONLY mount ⇒ the container physically cannot mutate the graded checkout (`run` is purely + // observational; the only writer stays edit_file). `--network none` + `--rm` + the dual timeout. + const dockerArgs = [ + 'run', '--rm', '--name', containerName, '--network', 'none', + '-v', `${ws.dir}:/testbed:ro`, '-w', '/testbed', + '-e', 'PYTHONDONTWRITEBYTECODE=1', '-e', `SWE_T=${T}`, '-e', `SWE_CMD=${cmd}`, + img.identity.id, 'bash', '-lc', script, + ] + let code = 0 + let out = '' + try { + const r = await exec('docker', dockerArgs, { timeout: (T + 20) * 1000, killSignal: 'SIGKILL', maxBuffer: 20_000_000 }) + out = r.stdout + } catch (e) { + const err = e as { code?: number; killed?: boolean; stdout?: string; message?: string } + if (typeof err.code === 'number') { + code = err.code + out = err.stdout ?? '' + } else { + // Host-level kill (execFile timeout) or docker failed to spawn: reap any orphaned container. + exec('docker', ['rm', '-f', containerName], { timeout: 20_000 }).catch(() => {}) + if (err.killed) { + code = 124 + out = `${err.stdout ?? ''}\n[host timeout: docker run exceeded ${T + 20}s and was killed]` + } else { + return `ERROR: run failed to execute (${String(err.message ?? e).slice(0, 200)})` + } + } + } + // Docker infra failures (container couldn't start / daemon vanished) are tool-misuse, not a test + // result → keep the ERROR: prefix so they don't pollute the failing-test signal. + if (code === 125 || /Cannot connect to the Docker daemon/i.test(out)) { + return `ERROR: run unavailable (docker: ${truncateHeadTail(out, 400)})` + } + const note = code === 124 || code === 137 ? `\n[command hit the ${T}s time/kill limit]` : '' + return `exit=${code}${note}\n${truncateHeadTail(out, RUN_OUTPUT_LIMIT)}` + } return `ERROR: unknown tool ${name}` }, async score(_task, handle): Promise { @@ -208,13 +544,7 @@ export async function createSweBenchEnvironment(poolN = 80): Promise<{ if (slice.length < n) throw new Error(`swe-bench-env: pool exhausted at offset ${offset} (need ${n}, have ${slice.length}; raise poolN)`) return slice.map((bt) => ({ id: bt.id, - systemPrompt: - 'You are a senior engineer fixing a real bug in the checked-out repository. Work PERSISTENTLY and do not ' + - 'stop early: use list_files + read_file to explore BROADLY (read many candidate files — the bug is rarely in ' + - 'the first file you open), trace the issue to its root cause, then fix it with edit_file. You MUST make at ' + - 'least one edit_file call — never finish with prose alone or without attempting a fix. Make a MINIMAL surgical ' + - 'change (a few lines, like a real PR), source only (test files are rejected). If an edit_file fails (old_string ' + - 'not unique/found), read the file again and retry with exact text. Keep going until you have made your best fix.', + systemPrompt: enableRun ? SWE_SEED_PROMPT_WITH_RUN : SWE_SEED_PROMPT, userPrompt: bt.prompt, meta: { instanceId: bt.id }, })) diff --git a/bench/src/swe-jail.ts b/bench/src/swe-jail.ts new file mode 100644 index 00000000..4d3c6bf9 --- /dev/null +++ b/bench/src/swe-jail.ts @@ -0,0 +1,293 @@ +/** + * Shared SWE execution-jail + zai-client primitives — extracted from swe-repro-calibrate.mts + * (the Stage-0 calibrator, commit 5102c43b) so Stage 1/2 drivers (swe-structural.mts) run on the + * SAME canary-verified jail instead of forking one. Behavior is byte-identical to the calibrator's + * in-file originals; the calibrator now imports these. + * + * The jail is the byte-identical invocation pattern of swe-bench-env's `run` tool: the instance's + * cached swebench Docker image, conda testbed, cwd=/testbed, --network none, dual timeout. + */ +import { execFile } from 'node:child_process' +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { promisify } from 'node:util' +import { absoluteSweTempDir } from './swe-temp' + +const exec = promisify(execFile) + +export const tail = (s: string, n: number): string => (s.length > n ? `…${s.slice(s.length - n)}` : s) + +// ---------- zai chat client (plain fetch; patient 429 ladder) ---------- + +export interface ZaiCfg { + base: string + key: string + timeoutMs: number + /** Optional hard wall-clock stop (epoch ms). No HTTP attempt starts — and no retry-ladder sleep + * runs — past this instant, so a per-instance deadline reaches INTO the 429 ladder instead of + * letting a doomed retry sleep for another 240s after the instance was already written off. */ + deadlineAt?: number +} + +export interface ZaiRaw { + /** The parsed /chat/completions JSON, verbatim. */ + json: Record + /** HTTP attempts spent (retries included). */ + attempts: number +} + +/** + * One OpenAI-shape chat completion against the zai coding endpoint, with the calibrator's retry + * discipline: zai 429s arrive in sustained bursts (measured: an overnight code-1305 storm zeroed + * 6/23 instances even on a 30s ladder), so rate-limit retries climb a long ladder (60s → 120s → + * 240s cap) while transient errors retry on a 2s base. An "empty" completion — no content AND no + * tool_calls — is the glm reasoning path starving `content` when reasoning eats max_tokens, and is + * retried too (a tool_calls turn with empty content is a NORMAL tool-loop turn, not starvation). + */ +export async function zaiChatRaw(cfg: ZaiCfg, body: Record): Promise { + let lastErr = '' + let delayBase = 2_000 + for (let attempt = 1; attempt <= 7; attempt += 1) { + if (attempt > 1) { + const delay = Math.min(delayBase * 2 ** (attempt - 2), 240_000) + if (cfg.deadlineAt !== undefined && Date.now() + delay >= cfg.deadlineAt) { + throw new Error(`completion abandoned at deadline (attempt ${attempt}): ${lastErr}`) + } + await new Promise((r) => setTimeout(r, delay)) + } + if (cfg.deadlineAt !== undefined && Date.now() >= cfg.deadlineAt) { + throw new Error(`completion abandoned at deadline (attempt ${attempt}): ${lastErr || 'no attempt made'}`) + } + const perCallTimeout = + cfg.deadlineAt !== undefined ? Math.max(1, Math.min(cfg.timeoutMs, cfg.deadlineAt - Date.now())) : cfg.timeoutMs + const ctl = new AbortController() + const timer = setTimeout(() => ctl.abort(), perCallTimeout) + try { + const res = await fetch(`${cfg.base}/chat/completions`, { + method: 'POST', + headers: { Authorization: `Bearer ${cfg.key}`, 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + signal: ctl.signal, + }) + if (!res.ok) { + lastErr = `HTTP ${res.status}: ${(await res.text()).slice(0, 200)}` + delayBase = res.status === 429 ? 60_000 : 2_000 + continue + } + const json = (await res.json()) as Record + const msg = ((json.choices as Array<{ message?: Record }> | undefined)?.[0]?.message ?? {}) as { + content?: string + tool_calls?: unknown[] + } + const hasToolCalls = Array.isArray(msg.tool_calls) && msg.tool_calls.length > 0 + if (!hasToolCalls && String(msg.content ?? '').trim() === '') { + lastErr = 'empty content' + continue + } + return { json, attempts: attempt } + } catch (e) { + lastErr = e instanceof Error ? e.message : String(e) + } finally { + clearTimeout(timer) + } + } + throw new Error(`completion failed after retries: ${lastErr}`) +} + +// ---------- judge-separation guard ---------- + +/** Refuse any outbound request whose SYSTEM/USER messages (the strings the harness authors) carry + * hidden-judge content marks (gold patch / test patch lines). Assistant and tool messages are + * exempt by construction: they are the model's own text and reads of the visible base tree — a + * model that independently authors the gold line is a success, not a leak. Returns the number of + * messages checked; throws on contact. */ +export function assertNoHiddenLeak(marks: readonly string[], msgs: ReadonlyArray<{ role?: string; content?: unknown }>): number { + let checked = 0 + for (const m of msgs) { + if (m.role !== 'system' && m.role !== 'user') continue + const c = typeof m.content === 'string' ? m.content : JSON.stringify(m.content ?? '') + for (const mark of marks) { + if (mark && c.includes(mark)) { + throw new Error('REFUSED: hidden judge content (gold/test patch) leaked into a model message') + } + } + checked += 1 + } + return checked +} + +// ---------- jailed python execution ---------- + +export interface JailRun { + code: number + out: string + timedOut: boolean + infraError?: string +} + +/** Echoed by the in-container shell after a successful `git apply` of the ride-along patch — + * unambiguously separates "patch failed to apply" (no sentinel, git's error in the output) from + * "patched code still fails" (sentinel present, the script's own nonzero exit). */ +export const APPLY_SENTINEL = '__PATCH_APPLIED__' + +let runSeq = 0 + +/** `treeDir === null` ⇒ run against the image's OWN built /testbed (the `image` substrate); a string + * ⇒ mount that host tree :ro over /testbed (the `mount` substrate — the run tool's exact pattern). + * `applyPatch` (image-substrate patch runs) applies a patch to the container's writable layer before + * running; --rm discards it, so every run still starts from the pristine image. */ +export async function runPyInJail( + imageTag: string, + treeDir: string | null, + pyScript: string, + applyPatch?: string, + opts: { timeoutS?: number } = {}, +): Promise { + const scriptDir = mkdtempSync(join(absoluteSweTempDir(), 'swe-repro-')) + writeFileSync(join(scriptDir, 'repro.py'), pyScript) + if (applyPatch) writeFileSync(join(scriptDir, 'ride.patch'), applyPatch.endsWith('\n') ? applyPatch : `${applyPatch}\n`) + const T = opts.timeoutS ?? 120 + const containerName = `swe-repro-${process.pid}-${Date.now()}-${runSeq++}` + // Identical shape to swe-bench-env's run tool; the command is the fixed `python /repro/repro.py`, + // so no SWE_CMD env ride-along is needed. The script rides in on a SECOND :ro mount. + // PYTHONPATH=/testbed is LOAD-BEARING: `python /repro/repro.py` puts /repro (the script's dir), not + // the cwd, at sys.path[0], so without it `import ` resolves to the image's baked-in install — + // the UNPATCHED code — and the patched tree is never exercised (verified on psf__requests-1142: + // post-gold run kept failing until PYTHONPATH pinned the mounted tree). The run tool never hits this + // because `python -c`/`python -m` put the cwd on sys.path. + const shell = + '{ source /opt/miniconda3/etc/profile.d/conda.sh && conda activate testbed && cd /testbed && ' + + (applyPatch ? `git apply --whitespace=nowarn /repro/ride.patch && echo ${APPLY_SENTINEL} && ` : '') + + 'timeout -s KILL "$SWE_T"s env PYTHONPATH=/testbed python /repro/repro.py; } 2>&1' + const dockerArgs = [ + 'run', '--rm', '--name', containerName, '--network', 'none', + ...(treeDir ? ['-v', `${treeDir}:/testbed:ro`] : []), + '-v', `${scriptDir}:/repro:ro`, '-w', '/testbed', + '-e', 'PYTHONDONTWRITEBYTECODE=1', '-e', `SWE_T=${T}`, + imageTag, 'bash', '-lc', shell, + ] + let code = 0 + let out = '' + // docker exit 125 (the daemon refused to start the container — transient resource pressure, + // a momentary daemon hiccup) is NOT a test result. Retry a few times with backoff before + // surfacing it as an infra error, so one bad daemon window can't nuke an instance (or, when it + // hits the canary of every instance in a fast error-loop, the whole stream). + for (let attempt = 0; ; attempt += 1) { + code = 0 + out = '' + try { + const r = await exec('docker', dockerArgs, { timeout: (T + 20) * 1000, killSignal: 'SIGKILL', maxBuffer: 20_000_000 }) + out = r.stdout + } catch (e) { + const err = e as { code?: number; killed?: boolean; stdout?: string; stderr?: string; message?: string } + if (typeof err.code === 'number') { + code = err.code + out = `${err.stdout ?? ''}${err.stderr ?? ''}` + } else if (err.killed) { + code = 124 + out = `${err.stdout ?? ''}\n[host timeout: docker run exceeded ${T + 20}s and was killed]` + } else { + rmSync(scriptDir, { recursive: true, force: true }) + return { code: -1, out: '', timedOut: false, infraError: `run failed to execute (${String(err.message ?? e).slice(0, 200)})` } + } + } + const transient = code === 125 || /Cannot connect to the Docker daemon|error creating overlay mount|no space left/i.test(out) + if (!transient || attempt >= 4) break + await exec('docker', ['rm', '-f', containerName], { timeout: 20_000 }).catch(() => {}) + await new Promise((r) => setTimeout(r, 2000 * 2 ** attempt)) + } + rmSync(scriptDir, { recursive: true, force: true }) + if (code === 125 || /Cannot connect to the Docker daemon/i.test(out)) { + return { code, out, timedOut: false, infraError: `docker: ${out.slice(0, 300)} (persisted through 5 attempts)` } + } + return { code, out, timedOut: code === 124 || code === 137 } +} + +// ---------- execution canary ---------- + +/** Import name of the repo's primary package, used by the per-instance EXECUTION CANARY. The canary + * decides whether this substrate can grade this instance AT ALL: with the gold patch applied to the + * tree under test, `import ` must succeed AND resolve INTO that tree (/testbed — not the image's + * site-packages install, which never receives the patch). A canary failure means any grade produced + * here measures the harness, not the model, so the instance is marked env-unresolvable before a + * single model call is spent. Never shown to the model. */ +export const IMPORT_NAME: Record = { + 'astropy/astropy': 'astropy', + 'django/django': 'django', + 'matplotlib/matplotlib': 'matplotlib', + 'mwaskom/seaborn': 'seaborn', + 'pallets/flask': 'flask', + 'pydata/xarray': 'xarray', + 'psf/requests': 'requests', + 'pylint-dev/pylint': 'pylint', + 'pytest-dev/pytest': 'pytest', + 'scikit-learn/scikit-learn': 'sklearn', + 'sphinx-doc/sphinx': 'sphinx', + 'sympy/sympy': 'sympy', +} + +/** The canary script: exit 0 iff `import ` succeeds AND resolves into /testbed. */ +export const importCanaryScript = (pkg: string): string => + `import sys\nimport ${pkg}\nf = getattr(${pkg}, '__file__', '') or ''\nprint(f)\n` + + "sys.exit(0 if f.startswith('/testbed') else 3)\n" + +// ---------- repro-authoring protocol (extracted from swe-repro-calibrate.mts, byte-identical +// behavior, so the stream driver authors fresh repros with the SAME calibrated protocol instead +// of forking one) ---------- + +/** The authoring system prompt, parameterized only on the jail's script timeout. */ +export const reproAuthorSystem = (reproTimeoutS: number): string => + 'You are an expert Python engineer writing a REPRODUCTION script for a reported bug in an open-source repository.\n\n' + + 'Contract for the script you produce:\n' + + '- A single self-contained Python file, executed as: python /repro/repro.py with cwd=/testbed, where /testbed is the ' + + "repository checkout. The project's own environment is active, so the repository package and its dependencies are importable.\n" + + '- Exit code 0 means the bug is FIXED. A nonzero exit (sys.exit(1), a failing assert, or an uncaught exception) means the ' + + 'bug is PRESENT. The script must exercise the EXACT behavior described in the issue.\n' + + '- The repository tree is READ-ONLY: never modify, create, or delete files inside it. If you genuinely need a scratch ' + + 'file, use the tempfile module (system tmp is writable).\n' + + `- No network access. Deterministic. Must finish well under ${reproTimeoutS} seconds.\n` + + "- Prefer plain Python with assert statements. Do not invoke the repository's test-suite runner and do not depend on " + + 'pytest fixtures; importing the repository package directly is the way.\n' + + '- Print one short line describing what was checked before exiting.\n\n' + + 'Be precise: the script must FAIL on the current buggy code and PASS once the underlying bug is properly fixed. Test the ' + + 'observable behavior the issue describes, not incidental implementation details that a legitimate fix might change.' + +/** All-READ-lines detector: models sometimes wrap their read requests in a python fence, which must + * NOT be mistaken for a script (observed live: a "script" of `READ: astropy/timeseries/core.py` + * reached the jail and crashed with NameError — measuring the parser, not the model). */ +export function asReadRequests(block: string): string[] | null { + const lines = block.split('\n').map((l) => l.trim()).filter(Boolean) + if (lines.length > 0 && lines.length <= 3 && lines.every((l) => /^READ:\s*\S+$/.test(l))) { + return lines.map((l) => l.replace(/^READ:\s*/, '')) + } + return null +} + +export function extractReproScript(text: string): string | null { + // Collapse a doubled fence OPENER (observed live: "```python\n```python\nimport os…" — the + // non-greedy fence match otherwise captures the empty span between the two openers and a + // complete script is thrown away as authoring-failed). + const cleaned = text.replace(/```(?:python|py)?[ \t]*\n(?=```(?:python|py)?[ \t]*\n)/g, '') + const fences = [...cleaned.matchAll(/```(?:python|py)?\s*\n([\s\S]*?)```/g)] + const last = fences.at(-1)?.[1]?.trim() + if (!last || asReadRequests(last)) return null + return last +} + +export function extractReadRequests(text: string): string[] { + return [...text.matchAll(/^READ:\s*(\S+)\s*$/gm)].map((m) => m[1]).slice(0, 3) +} + +// ---------- backbone enumeration ---------- + +/** The cached-instance backbone: every instance whose swebench eval image is cached locally. + * Tag → id mapping inverts make_test_spec's `__` → `_1776_` name mangling. */ +export async function cachedInstanceIds(): Promise { + const { stdout } = await exec('docker', ['images', '--format', '{{.Repository}}'], { timeout: 30_000 }) + const ids = new Set() + for (const line of stdout.split('\n')) { + const m = /(?:^|\/)sweb\.eval\.x86_64\.(.+)$/.exec(line.trim()) + if (m) ids.add(m[1].replace('_1776_', '__')) + } + return [...ids].sort() +} diff --git a/bench/src/swe-structural-judge-policy.test.ts b/bench/src/swe-structural-judge-policy.test.ts new file mode 100644 index 00000000..d41de225 --- /dev/null +++ b/bench/src/swe-structural-judge-policy.test.ts @@ -0,0 +1,117 @@ +import assert from 'node:assert/strict' +import { linkSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, relative } from 'node:path' +import { describe, it } from 'node:test' +import { createFingerprints } from './swe-structural-provenance' +import { + assertCompleteTaskSet, + assertDistinctArtifactPaths, + assertJudgeCompletionMatchesInput, + assertJudgeResumeFingerprints, + assertPairedExecutionFingerprint, + assertPairedFingerprints, + completeJudgeScore, +} from './swe-structural-judge-policy' + +const inputs = { + source: 'source', + config: { arm: 'independent-2' }, + commonConfig: { model: 'worker' }, + repro: 'repro', + prompt: 'prompt', + tools: ['run'], + task: 'task', +} + +describe('judge-only admission', () => { + it('requires a complete Phase-A task set before scoring', () => { + assert.doesNotThrow(() => assertCompleteTaskSet(['a', 'b'], ['a', 'b'], 'left')) + assert.throws(() => assertCompleteTaskSet(['a'], ['a', 'b'], 'left'), /incomplete Phase A/) + }) + + it('accepts arm-specific config hashes but rejects any shared-input mismatch', () => { + const left = createFingerprints(inputs) + const right = createFingerprints({ ...inputs, config: { arm: 'persistent-refine-2' } }) + assert.doesNotThrow(() => assertPairedFingerprints(left, right, 'task')) + const changedTools = createFingerprints({ ...inputs, config: { arm: 'persistent-refine-2' }, tools: ['edit'] }) + assert.throws(() => assertPairedFingerprints(left, changedTools, 'task'), /paired tools fingerprints/) + assert.doesNotThrow(() => assertPairedExecutionFingerprint('execution', 'execution', 'task')) + assert.throws(() => assertPairedExecutionFingerprint('left', 'right', 'task'), /paired execution/) + }) + + it('rejects a judge resume row tied to different Phase-A bytes', () => { + const expected = { + pairFingerprint: 'pair', + phaseFileFingerprint: 'phase-new', + inputFingerprint: 'input', + executionFingerprint: 'execution', + finalDiffHash: 'diff', + } + assert.doesNotThrow(() => assertJudgeResumeFingerprints(expected, expected, 'row')) + assert.throws(() => + assertJudgeResumeFingerprints({ ...expected, phaseFileFingerprint: 'phase-old' }, expected, 'row'), + /phaseFileFingerprint mismatch/, + ) + }) + + it('returns only completed scores and propagates scorer failures for retry', async () => { + assert.deepEqual(await completeJudgeScore('', async () => ({ resolved: true })), { + hiddenResolved: false, + judgeDetail: null, + judgeSkipped: 'empty-patch', + }) + await assert.rejects( + completeJudgeScore('diff --git a/a b/a', async () => { + throw new Error('transient scorer failure') + }), + /transient scorer failure/, + ) + assert.deepEqual(await completeJudgeScore('diff', async () => ({ resolved: true, detail: 'ok' })), { + hiddenResolved: true, + judgeDetail: 'ok', + judgeSkipped: null, + }) + assert.doesNotThrow(() => + assertJudgeCompletionMatchesInput( + { hiddenResolved: false, judgeDetail: null, judgeSkipped: 'empty-patch' }, + '', + 'row', + ), + ) + assert.throws(() => + assertJudgeCompletionMatchesInput( + { hiddenResolved: false, judgeDetail: null, judgeSkipped: 'empty-patch' }, + 'diff', + 'row', + ), + /non-empty patch cannot skip/, + ) + }) + + it('rejects relative, symlink, and hardlink aliases before scoring', () => { + const dir = mkdtempSync(join(tmpdir(), 'swe-judge-paths-')) + try { + const phase = join(dir, 'phase-a.jsonl') + const other = join(dir, 'phase-b.jsonl') + const symlink = join(dir, 'phase-link.jsonl') + const hardlink = join(dir, 'phase-hard.jsonl') + writeFileSync(phase, '{}\n') + writeFileSync(other, '{}\n') + symlinkSync(phase, symlink) + linkSync(phase, hardlink) + + assert.throws(() => + assertDistinctArtifactPaths({ phase, alias: relative(process.cwd(), phase), out: join(dir, 'out.jsonl') }), + /alias the same artifact file/, + ) + assert.throws(() => assertDistinctArtifactPaths({ phase, symlink, out: join(dir, 'out.jsonl') }), + /alias the same artifact file/, + ) + assert.throws(() => assertDistinctArtifactPaths({ phase, hardlink, other }), /alias the same artifact file/) + assert.doesNotThrow(() => assertDistinctArtifactPaths({ phase, other, out: join(dir, 'out.jsonl') })) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) +}) diff --git a/bench/src/swe-structural-judge-policy.ts b/bench/src/swe-structural-judge-policy.ts new file mode 100644 index 00000000..d4e2da18 --- /dev/null +++ b/bench/src/swe-structural-judge-policy.ts @@ -0,0 +1,133 @@ +import { realpathSync, statSync } from 'node:fs' +import { basename, dirname, join, resolve } from 'node:path' +import type { Fingerprints } from './swe-structural-provenance' + +export const PAIRED_FINGERPRINT_KEYS = [ + 'source', + 'commonConfig', + 'repro', + 'prompt', + 'tools', + 'task', +] as const satisfies ReadonlyArray + +export function assertCompleteTaskSet( + rowIds: Iterable, + taskIds: readonly string[], + context: string, +): void { + const rows = new Set(rowIds) + if (rows.size !== taskIds.length || taskIds.some((id) => !rows.has(id))) { + throw new Error(`${context}: incomplete Phase A (${rows.size}/${taskIds.length} required rows)`) + } +} + +export function assertPairedFingerprints( + left: Fingerprints, + right: Fingerprints, + context: string, +): void { + for (const key of PAIRED_FINGERPRINT_KEYS) { + if (left[key] !== right[key]) throw new Error(`${context}: paired ${key} fingerprints do not match`) + } +} + +export function assertPairedExecutionFingerprint(left: string, right: string, context: string): void { + if (!left || left !== right) throw new Error(`${context}: paired execution fingerprints do not match`) +} + +interface ArtifactPathIdentity { + canonicalPath: string + device: number | null + inode: number | null +} + +function artifactPathIdentity(path: string): ArtifactPathIdentity { + const absolute = resolve(path) + try { + const canonicalPath = realpathSync(absolute) + const stat = statSync(canonicalPath) + return { canonicalPath, device: stat.dev, inode: stat.ino } + } catch (error) { + const code = (error as NodeJS.ErrnoException).code + if (code !== 'ENOENT') throw error + const canonicalParent = realpathSync(dirname(absolute)) + return { canonicalPath: join(canonicalParent, basename(absolute)), device: null, inode: null } + } +} + +/** Reject relative/absolute, symlink, and hardlink aliases before any judge output is opened. */ +export function assertDistinctArtifactPaths(paths: Record): void { + const entries = Object.entries(paths).map(([name, path]) => ({ name, ...artifactPathIdentity(path) })) + for (let leftIndex = 0; leftIndex < entries.length; leftIndex += 1) { + for (let rightIndex = leftIndex + 1; rightIndex < entries.length; rightIndex += 1) { + const left = entries[leftIndex]! + const right = entries[rightIndex]! + const sameCanonicalPath = left.canonicalPath === right.canonicalPath + const sameExistingFile = + left.device !== null && + right.device !== null && + left.device === right.device && + left.inode === right.inode + if (sameCanonicalPath || sameExistingFile) { + throw new Error(`${left.name} and ${right.name} alias the same artifact file`) + } + } + } +} + +export interface CompletedJudgeScore { + hiddenResolved: boolean + judgeDetail: string | null + judgeSkipped: 'empty-patch' | null +} + +/** A scorer error rejects the promise, so callers cannot serialize an incomplete judge row. */ +export async function completeJudgeScore( + finalDiff: string, + judge: () => Promise<{ resolved?: boolean; detail?: unknown }>, +): Promise { + if (!finalDiff.trim()) { + return { hiddenResolved: false, judgeDetail: null, judgeSkipped: 'empty-patch' } + } + const score = await judge() + return { + hiddenResolved: score.resolved ?? false, + judgeDetail: String(score.detail ?? '').slice(0, 1_000), + judgeSkipped: null, + } +} + +export function assertJudgeCompletionMatchesInput( + row: CompletedJudgeScore, + finalDiff: string, + context: string, +): void { + if (!finalDiff.trim()) { + if (row.hiddenResolved !== false || row.judgeSkipped !== 'empty-patch') { + throw new Error(`${context}: empty patch has an invalid completed-judge receipt`) + } + return + } + if (row.judgeSkipped !== null) { + throw new Error(`${context}: non-empty patch cannot skip official scoring`) + } +} + +export interface JudgeResumeFingerprintReceipt { + pairFingerprint: string + phaseFileFingerprint: string + inputFingerprint: string + executionFingerprint: string + finalDiffHash: string +} + +export function assertJudgeResumeFingerprints( + actual: JudgeResumeFingerprintReceipt, + expected: JudgeResumeFingerprintReceipt, + context: string, +): void { + for (const key of Object.keys(expected) as Array) { + if (actual[key] !== expected[key]) throw new Error(`${context}: judge resume ${key} mismatch`) + } +} diff --git a/bench/src/swe-structural-policy.test.ts b/bench/src/swe-structural-policy.test.ts new file mode 100644 index 00000000..71fa5318 --- /dev/null +++ b/bench/src/swe-structural-policy.test.ts @@ -0,0 +1,124 @@ +import assert from 'node:assert/strict' +import { describe, it } from 'node:test' +import { assertNoHiddenLeak } from './swe-jail' +import { + assertExactCompletedWorkerSessions, + continuationDisposition, + continuationStateNotice, + preferLaterCandidate, + resolveExperimentArm, + resolveExperimentTemperature, + shouldAcceptContinuation, + shouldRunContinuation, +} from './swe-structural-policy' + +describe('structural experiment arms', () => { + it('derives exactly two independent sessions without repair', () => { + assert.deepEqual(resolveExperimentArm('independent-2'), { + arm: 'independent-2', + k: 2, + repairs: 0, + alwaysRunContinuation: false, + persistent: false, + workerSessions: 2, + }) + }) + + it('derives exactly one initial plus one persistent continuation session', () => { + const preset = resolveExperimentArm('persistent-refine-2') + assert.deepEqual(preset, { + arm: 'persistent-refine-2', + k: 1, + repairs: 1, + alwaysRunContinuation: true, + persistent: true, + workerSessions: 2, + }) + assert.equal(shouldRunContinuation({ round: 1, preset }), true) + assert.equal(shouldRunContinuation({ round: 2, preset }), false) + }) + + it('rejects every untyped arm instead of accepting free-form K/repair settings', () => { + assert.throws(() => resolveExperimentArm('system'), /independent-2\|persistent-refine-2/) + assert.throws(() => resolveExperimentArm(''), /independent-2\|persistent-refine-2/) + }) + + it('uses TEMPERATURE without repurposing Node TEMP as a model knob', () => { + assert.equal(resolveExperimentTemperature({}), 0.8) + assert.equal(resolveExperimentTemperature({ TEMPERATURE: '0.35' }), 0.35) + assert.equal(resolveExperimentTemperature({ TEMP: '/tmp', TEMPERATURE: '0.35' }), 0.35) + assert.throws( + () => resolveExperimentTemperature({ TEMP: '0.8' }), + /TEMP is the operating-system temporary-directory root.*TEMPERATURE/, + ) + assert.throws(() => resolveExperimentTemperature({ TEMPERATURE: 'hot' }), /finite number/) + }) +}) + +describe('shared later-on-visible-tie policy', () => { + it('prefers session two on a visible tie in both arm shapes', () => { + assert.equal(preferLaterCandidate(0, 0), true) + assert.equal(preferLaterCandidate(1, 1), true) + assert.equal(shouldAcceptContinuation(0, 0), true) + assert.equal(shouldAcceptContinuation(1, 1), true) + }) + + it('prefers a visible improvement and rejects a visible regression', () => { + assert.equal(preferLaterCandidate(1, 0), true) + assert.equal(preferLaterCandidate(0, 1), false) + assert.equal(shouldAcceptContinuation(1, 0), true) + assert.equal(shouldAcceptContinuation(0, 1), false) + }) + + it('never labels an identical second patch as a refinement', () => { + assert.deepEqual(continuationDisposition(true, false), { + finalFrom: 'session:2-identical', + stop: 'continuation-accepted-identical', + }) + assert.equal(continuationDisposition(true, true).finalFrom, 'session:2-refinement') + }) + + it('inherits parent state without echoing parent patch bytes into session two', () => { + const parent = 'diff --git a/a.py b/a.py\n+gold-like-added-line\n' + const notice = continuationStateNotice(parent) + assert.match(notice, /already applied to this checkout/) + assert.equal(notice.includes('gold-like-added-line'), false) + assert.doesNotThrow(() => assertNoHiddenLeak(['+gold-like-added-line'], [{ role: 'user', content: notice }])) + }) +}) + +describe('exact completed worker compute', () => { + const complete = { calls: 1, completions: 1, attemptError: null } + + it('admits exactly two completed model-backed sessions', () => { + assert.doesNotThrow(() => + assertExactCompletedWorkerSessions({ + started: 2, + completed: 2, + sessions: [complete, complete], + context: 'row', + }), + ) + }) + + it('rejects an attempted-but-failed or zero-call session', () => { + assert.throws(() => + assertExactCompletedWorkerSessions({ + started: 2, + completed: 1, + sessions: [complete, { calls: 0, completions: 0, attemptError: 'timeout' }], + context: 'row', + }), + /expected exactly 2 started and completed/, + ) + assert.throws(() => + assertExactCompletedWorkerSessions({ + started: 2, + completed: 2, + sessions: [complete, { calls: 0, completions: 1, attemptError: null }], + context: 'row', + }), + /session 2 did not complete successfully/, + ) + }) +}) diff --git a/bench/src/swe-structural-policy.ts b/bench/src/swe-structural-policy.ts new file mode 100644 index 00000000..ef1289fb --- /dev/null +++ b/bench/src/swe-structural-policy.ts @@ -0,0 +1,132 @@ +import { isAbsolute } from 'node:path' + +export type ExperimentArm = 'independent-2' | 'persistent-refine-2' + +export interface ExperimentArmPreset { + arm: ExperimentArm + k: 1 | 2 + repairs: 0 | 1 + alwaysRunContinuation: boolean + persistent: boolean + workerSessions: 2 +} + +export interface WorkerSessionReceipt { + calls: number + completions: number + attemptError: string | null +} + +const PRESETS: Record = { + 'independent-2': { + arm: 'independent-2', + k: 2, + repairs: 0, + alwaysRunContinuation: false, + persistent: false, + workerSessions: 2, + }, + 'persistent-refine-2': { + arm: 'persistent-refine-2', + k: 1, + repairs: 1, + alwaysRunContinuation: true, + persistent: true, + workerSessions: 2, + }, +} + +export function resolveExperimentArm(value: string): ExperimentArmPreset { + const preset = PRESETS[value as ExperimentArm] + if (!preset) { + throw new Error( + `EXPERIMENT_ARM must be independent-2|persistent-refine-2, got "${value}"`, + ) + } + return preset +} + +/** Keep model sampling configuration disjoint from Node's TEMP-controlled filesystem root. */ +export function resolveExperimentTemperature( + env: Readonly>, +): number { + if (env.TEMP?.trim() && !isAbsolute(env.TEMP)) { + throw new Error( + `TEMP is the operating-system temporary-directory root and must be absolute, got "${env.TEMP}"; ` + + 'use TEMPERATURE for model sampling', + ) + } + const value = Number(env.TEMPERATURE ?? 0.8) + if (!Number.isFinite(value)) { + throw new Error(`TEMPERATURE must be a finite number, got "${env.TEMPERATURE}"`) + } + return value +} + +/** Both arms use one selection rule: lower visible severity wins and a tie prefers session two. */ +export function preferLaterCandidate(currentSeverity: number, laterSeverity: number): boolean { + return laterSeverity <= currentSeverity +} + +/** The persistent arm always starts its one continuation, including after a visible pass. */ +export function shouldRunContinuation(input: { + round: number + preset: ExperimentArmPreset +}): boolean { + return input.preset.alwaysRunContinuation && input.preset.repairs === 1 && input.round === 1 +} + +/** A continuation replaces its parent under the same later-on-tie rule as independent selection. */ +export function shouldAcceptContinuation(currentSeverity: number, continuationSeverity: number): boolean { + return preferLaterCandidate(currentSeverity, continuationSeverity) +} + +export function continuationDisposition(accepted: boolean, changedFromParent: boolean): { + finalFrom: 'session:2-refinement' | 'session:2-identical' | 'session:1-visible-better' + stop: 'continuation-accepted-changed' | 'continuation-accepted-identical' | 'continuation-rejected-visible-regression' +} { + if (!accepted) { + return { finalFrom: 'session:1-visible-better', stop: 'continuation-rejected-visible-regression' } + } + return changedFromParent + ? { finalFrom: 'session:2-refinement', stop: 'continuation-accepted-changed' } + : { finalFrom: 'session:2-identical', stop: 'continuation-accepted-identical' } +} + +/** Describe inherited workspace state without echoing model-authored patch bytes back through the + * outbound prompt. The checkout already contains the parent state; repeating its diff can collide + * with hidden-patch leak marks when session one independently finds the correct line. */ +export function continuationStateNotice(parentDiff: string): string { + return parentDiff.trim() + ? '--- PREVIOUS FIX STATE ---\nSession one changes are already applied to this checkout. Inspect the current files directly.\n\n' + : '--- NO FIX APPLIED YET (session one produced no change) ---\n\n' +} + +/** Admit only two genuinely completed model-backed sessions, never two attempted dispatches. */ +export function assertExactCompletedWorkerSessions(input: { + started: number + completed: number + sessions: readonly WorkerSessionReceipt[] + context: string +}): void { + if (input.started !== 2 || input.completed !== 2 || input.sessions.length !== 2) { + throw new Error( + `${input.context}: expected exactly 2 started and completed worker sessions ` + + `(started=${input.started}, completed=${input.completed}, receipts=${input.sessions.length})`, + ) + } + input.sessions.forEach((session, index) => { + if ( + session.attemptError !== null || + !Number.isInteger(session.calls) || + session.calls < 1 || + !Number.isInteger(session.completions) || + session.completions < 1 + ) { + throw new Error( + `${input.context}: session ${index + 1} did not complete successfully ` + + `(error=${session.attemptError ?? 'none'}, calls=${session.calls}, completions=${session.completions})`, + ) + } + }) +} diff --git a/bench/src/swe-structural-provenance.test.ts b/bench/src/swe-structural-provenance.test.ts new file mode 100644 index 00000000..65611c08 --- /dev/null +++ b/bench/src/swe-structural-provenance.test.ts @@ -0,0 +1,93 @@ +import assert from 'node:assert/strict' +import { describe, it } from 'node:test' +import { + assertFingerprintsEqual, + createExecutionReceipt, + createFingerprints, + diffChanged, + diffFingerprint, + fingerprint, + runtimeImplementationFingerprint, +} from './swe-structural-provenance' + +const inputs = { + source: { files: ['runner', 'environment'] }, + config: { arm: 'independent-2', k: 2 }, + commonConfig: { model: 'worker' }, + repro: { script: 'raise SystemExit(1)' }, + prompt: 'prompt', + tools: [{ name: 'run' }], + task: { id: 'repo__repo-1' }, +} + +describe('structural provenance', () => { + it('hashes objects independent of key insertion order', () => { + assert.equal(fingerprint({ b: 2, a: 1 }), fingerprint({ a: 1, b: 2 })) + }) + + it('rejects a changed resume input by its named fingerprint', () => { + const expected = createFingerprints(inputs) + const actual = createFingerprints({ ...inputs, prompt: 'changed prompt' }) + assert.throws(() => assertFingerprintsEqual(actual, expected, 'resume row'), /prompt fingerprint mismatch/) + }) + + it('records identical and changed continuation patches exactly', () => { + const parent = 'diff --git a/a.py b/a.py\n+x\n' + assert.match(diffFingerprint(parent), /^sha256:[0-9a-f]{64}$/) + assert.equal(diffChanged(parent, parent), false) + assert.equal(diffChanged(parent, `${parent}+y\n`), true) + }) + + it('binds actual runtime callables, run settings, scorer version, and immutable image identity', () => { + const runtime = runtimeImplementationFingerprint({ + runAgentic: function runAgenticA() { return 'a' }, + refine: { name: 'refine', driver: function refineA() { return 'a' } }, + }) + const changedRuntime = runtimeImplementationFingerprint({ + runAgentic: function runAgenticB() { return 'b' }, + refine: { name: 'refine', driver: function refineA() { return 'a' } }, + }) + assert.notEqual(changedRuntime, runtime) + + const shared = { + runTool: { timeoutS: 120, outputLimit: 10_000 }, + runtimeImplementationFingerprint: runtime, + runtimeTreeFingerprint: fingerprint({ runtime: 'tree-a' }), + officialScorer: { + package: 'swebench' as const, + version: '4.0.0', + cacheLevel: 'instance' as const, + namespacePolicy: 'phase-a-image' as const, + }, + } + const receipt = createExecutionReceipt(shared, { + tag: 'sweb.eval.example:latest', + namespace: 'swebench', + identity: { id: 'sha256:image-a', repoDigests: ['sweb.eval.example@sha256:digest-a'] }, + }) + const movedTag = createExecutionReceipt(shared, { + tag: 'sweb.eval.example:latest', + namespace: 'swebench', + identity: { id: 'sha256:image-b', repoDigests: ['sweb.eval.example@sha256:digest-b'] }, + }) + const movedRuntimeTree = createExecutionReceipt( + { ...shared, runtimeTreeFingerprint: fingerprint({ runtime: 'tree-b' }) }, + { + tag: 'sweb.eval.example:latest', + namespace: 'swebench', + identity: { id: 'sha256:image-a', repoDigests: ['sweb.eval.example@sha256:digest-a'] }, + }, + ) + const movedScorer = createExecutionReceipt( + { ...shared, officialScorer: { ...shared.officialScorer, version: '4.0.1' } }, + { + tag: 'sweb.eval.example:latest', + namespace: 'swebench', + identity: { id: 'sha256:image-a', repoDigests: ['sweb.eval.example@sha256:digest-a'] }, + }, + ) + assert.notEqual(fingerprint(receipt), fingerprint(movedTag)) + assert.notEqual(fingerprint(receipt), fingerprint(movedRuntimeTree)) + assert.notEqual(fingerprint(receipt), fingerprint(movedScorer)) + }) +}) diff --git a/bench/src/swe-structural-provenance.ts b/bench/src/swe-structural-provenance.ts new file mode 100644 index 00000000..f69ef2a7 --- /dev/null +++ b/bench/src/swe-structural-provenance.ts @@ -0,0 +1,138 @@ +import { createHash } from 'node:crypto' + +export const FINGERPRINT_SCHEMA = 'swe-structural-v2' as const + +export interface Fingerprints { + schema: typeof FINGERPRINT_SCHEMA + source: string + config: string + commonConfig: string + repro: string + prompt: string + tools: string + task: string + composite: string +} + +export interface FingerprintInputs { + source: unknown + config: unknown + commonConfig: unknown + repro: unknown + prompt: unknown + tools: unknown + task: unknown +} + +export interface SharedExecutionReceipt { + runTool: { + timeoutS: number + outputLimit: number + } + runtimeImplementationFingerprint: string + runtimeTreeFingerprint: string + officialScorer: { + package: 'swebench' + version: string + cacheLevel: 'instance' + namespacePolicy: 'phase-a-image' + } +} + +export interface ExecutionReceipt extends SharedExecutionReceipt { + image: { + tag: string + namespace: 'swebench' | 'none' + id: string + repoDigests: string[] + } +} + +function canonicalize(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonicalize) + if (value && typeof value === 'object') { + return Object.fromEntries( + Object.entries(value as Record) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([key, child]) => [key, canonicalize(child)]), + ) + } + return value +} + +export function canonicalJson(value: unknown): string { + return JSON.stringify(canonicalize(value)) +} + +export function fingerprint(value: unknown): string { + return `sha256:${createHash('sha256').update(canonicalJson(value)).digest('hex')}` +} + +export function createFingerprints(inputs: FingerprintInputs): Fingerprints { + const parts = { + schema: FINGERPRINT_SCHEMA, + source: fingerprint(inputs.source), + config: fingerprint(inputs.config), + commonConfig: fingerprint(inputs.commonConfig), + repro: fingerprint(inputs.repro), + prompt: fingerprint(inputs.prompt), + tools: fingerprint(inputs.tools), + task: fingerprint(inputs.task), + } + return { ...parts, composite: fingerprint(parts) } +} + +export function assertFingerprintsEqual( + actual: Fingerprints | undefined, + expected: Fingerprints, + context: string, +): void { + if (!actual) throw new Error(`${context}: missing ${FINGERPRINT_SCHEMA} fingerprints`) + for (const key of Object.keys(expected) as Array) { + if (actual[key] !== expected[key]) { + throw new Error(`${context}: ${key} fingerprint mismatch (${actual[key]} != ${expected[key]})`) + } + } +} + +export function diffFingerprint(diff: string): string { + return fingerprint({ diff }) +} + +export function diffChanged(parent: string, candidate: string): boolean { + return diffFingerprint(parent) !== diffFingerprint(candidate) +} + +/** Hash the actual imported callables, not merely the source file expected to contain them. */ +export function runtimeImplementationFingerprint(input: { + runAgentic: unknown + refine: { name: string; driver: unknown } +}): string { + if (typeof input.runAgentic !== 'function' || typeof input.refine.driver !== 'function') { + throw new Error('runtime implementation receipt requires callable runAgentic and refine.driver') + } + return fingerprint({ + runAgentic: Function.prototype.toString.call(input.runAgentic), + refine: { + name: input.refine.name, + driver: Function.prototype.toString.call(input.refine.driver), + }, + }) +} + +export function createExecutionReceipt( + shared: SharedExecutionReceipt, + image: { tag: string; namespace: 'swebench' | 'none'; identity: { id: string; repoDigests: string[] } }, +): ExecutionReceipt { + return { + ...shared, + runTool: { ...shared.runTool }, + officialScorer: { ...shared.officialScorer }, + image: { + tag: image.tag, + namespace: image.namespace, + id: image.identity.id, + repoDigests: [...image.identity.repoDigests], + }, + } +} diff --git a/bench/src/swe-structural.mts b/bench/src/swe-structural.mts new file mode 100644 index 00000000..af02719c --- /dev/null +++ b/bench/src/swe-structural.mts @@ -0,0 +1,1260 @@ +/** + * Two-session SWE-bench experiment with generation and official scoring split into separate modes. + * + * MODE=generate requires EXPERIMENT_ARM=independent-2|persistent-refine-2 and writes only Phase A. + * The independent arm runs two fresh attempts. The persistent arm runs one attempt and one fresh + * continuation with attempt one's cumulative patch pre-applied. Both use the same run-capable tools, + * prompt, temperature, two worker sessions, and later-on-visible-tie selection rule. Per-row hashes + * bind source, config, reproduction, prompt, tools, task, parent patch, and final patch. + * + * MODE=judge-only requires INDEPENDENT_PHASE_A, PERSISTENT_PHASE_A, and a distinct OUT. It validates + * both complete Phase-A files and every paired fingerprint before the first serialized official score. + * + * Generate env: ZAI_API_KEY, REPRO_MANIFEST, IDS, OUT, MODEL, ZAI_BASE, MAX_TOKENS, TEMPERATURE, + * INNER_TURNS, CONC, REPRO_TIMEOUT, LLM_TIMEOUT_MS, SWE_RUN_TIMEOUT, SWE_RUN_OUTPUT_LIMIT, + * PRICE_IN, PRICE_OUT. Judge env: INDEPENDENT_PHASE_A, PERSISTENT_PHASE_A, OUT. + */ +import { execFile } from 'node:child_process' +import { appendFileSync, existsSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { join, relative } from 'node:path' +import { fileURLToPath } from 'node:url' +import { promisify } from 'node:util' +import type { AgenticSurface, AgenticTask, ArtifactHandle, SurfaceScore } from '@tangle-network/agent-runtime/loops' +import { refine, runAgentic } from '@tangle-network/agent-runtime/loops' +import type { BenchTask } from './benchmarks/types' +import { + createSweBenchEnvironment, + resolveImageForMetadata, + resolveSweBenchScorerVersion, + SWE_RUN_TOOL_CONFIG, + SWE_SEED_PROMPT_WITH_RUN, + type SweImageIdentity, +} from './swe-bench-env' +import { + APPLY_SENTINEL, + assertNoHiddenLeak, + cachedInstanceIds, + IMPORT_NAME, + importCanaryScript, + runPyInJail, + tail, + zaiChatRaw, +} from './swe-jail' +import { + type ExperimentArm, + type ExperimentArmPreset, + assertExactCompletedWorkerSessions, + continuationDisposition, + continuationStateNotice, + preferLaterCandidate, + resolveExperimentArm, + resolveExperimentTemperature, + shouldAcceptContinuation, + shouldRunContinuation, +} from './swe-structural-policy' +import { + assertFingerprintsEqual, + createExecutionReceipt, + createFingerprints, + diffChanged, + diffFingerprint, + fingerprint, + runtimeImplementationFingerprint, + type ExecutionReceipt, + type Fingerprints, + type SharedExecutionReceipt, +} from './swe-structural-provenance' +import { + assertCompleteTaskSet, + assertDistinctArtifactPaths, + assertJudgeCompletionMatchesInput, + assertJudgeResumeFingerprints, + assertPairedExecutionFingerprint, + assertPairedFingerprints, + completeJudgeScore, +} from './swe-structural-judge-policy' + +const exec = promisify(execFile) +const TEMPERATURE = resolveExperimentTemperature(process.env) + +function sourceTreeReceipt( + rootUrl: URL, + label: string, + include: (path: string) => boolean, +): Array<{ name: string; content: string }> { + const root = fileURLToPath(rootUrl) + const receipt: Array<{ name: string; content: string }> = [] + const visit = (dir: string): void => { + for (const entry of readdirSync(dir, { withFileTypes: true }).sort((left, right) => left.name.localeCompare(right.name))) { + const path = join(dir, entry.name) + if (entry.isDirectory()) visit(path) + else if (entry.isFile() && include(path)) { + receipt.push({ name: `${label}/${relative(root, path)}`, content: readFileSync(path, 'utf8') }) + } + } + } + visit(root) + return receipt +} + +// ---------- config ---------- + +type Mode = 'generate' | 'judge-only' +const MODE_INPUT = process.env.MODE ?? 'generate' +if (MODE_INPUT !== 'generate' && MODE_INPUT !== 'judge-only') { + throw new Error(`MODE must be generate|judge-only, got "${MODE_INPUT}"`) +} +const MODE: Mode = MODE_INPUT +const OFFICIAL_SCORER_CACHE_LEVEL = 'instance' as const +if ( + MODE === 'judge-only' && + process.env.SWEBENCH_CACHE_LEVEL !== undefined && + process.env.SWEBENCH_CACHE_LEVEL !== OFFICIAL_SCORER_CACHE_LEVEL +) { + throw new Error('MODE=judge-only requires SWEBENCH_CACHE_LEVEL=instance') +} +if (MODE === 'judge-only') process.env.SWEBENCH_CACHE_LEVEL = OFFICIAL_SCORER_CACHE_LEVEL +for (const legacy of ['ARM', 'ARM_NAME', 'K', 'REPAIRS', 'FORCE_TWO_SESSIONS', 'SOLO_TEMP', 'SKIP_JUDGE']) { + if (process.env[legacy] !== undefined) { + throw new Error(`${legacy} is not supported; use typed EXPERIMENT_ARM plus MODE=generate|judge-only`) + } +} +const ARM_PRESET: ExperimentArmPreset | null = MODE === 'generate' + ? resolveExperimentArm(process.env.EXPERIMENT_ARM ?? '') + : null +const ZAI_BASE = process.env.ZAI_BASE ?? 'https://api.z.ai/api/coding/paas/v4' +const ZAI_KEY = process.env.ZAI_API_KEY ?? '' +if (MODE === 'generate' && !ZAI_KEY) throw new Error('ZAI_API_KEY required for MODE=generate') +const MODEL = process.env.MODEL ?? 'glm-5.2' +// glm-5.2 is a reasoning model: hidden reasoning consumes max_tokens, so <8000 starves content. +const MAX_TOKENS = Number(process.env.MAX_TOKENS ?? 12_000) +const INNER_TURNS = Number(process.env.INNER_TURNS ?? 40) +const CONC = Math.max(1, Math.min(4, Number(process.env.CONC ?? 2))) +const REPRO_TIMEOUT_S = Number(process.env.REPRO_TIMEOUT ?? 120) +const LLM_TIMEOUT_MS = Number(process.env.LLM_TIMEOUT_MS ?? 480_000) +// Cost-table rates, USD per Mtok. ASSUMED (zai coding-plan tokens have no per-call list price); +// override with PRICE_IN/PRICE_OUT. The summary labels them as assumed. +const PRICE_IN = Number(process.env.PRICE_IN ?? 0.6) +const PRICE_OUT = Number(process.env.PRICE_OUT ?? 2.2) + +interface ReproManifestEntry { + script: string + source: string + stage0Class: string +} + +const MANIFEST: Record = (() => { + if (MODE !== 'generate') return {} + const p = process.env.REPRO_MANIFEST + if (!p) throw new Error('REPRO_MANIFEST required for MODE=generate') + return JSON.parse(readFileSync(p, 'utf8')) as Record +})() + +interface CommonConfigReceipt { + schema: 'swe-structural-v2' + model: string + zaiBase: string + maxTokens: number + temperature: number + innerTurns: number + concurrency: number + reproTimeoutS: number + llmTimeoutMs: number + sweRunTimeoutS: number + sweRunOutputLimit: number + runTool: true + seedPrompt: 'SWE_SEED_PROMPT_WITH_RUN' + taskIds: string[] +} + +interface ExperimentConfigReceipt extends CommonConfigReceipt { + arm: ExperimentArm + k: 1 | 2 + repairs: 0 | 1 + alwaysRunContinuation: boolean + persistent: boolean + workerSessions: 2 +} + +const SOURCE_RECEIPT = [ + ['swe-structural.mts', new URL('./swe-structural.mts', import.meta.url)], + ['swe-structural-policy.ts', new URL('./swe-structural-policy.ts', import.meta.url)], + ['swe-structural-provenance.ts', new URL('./swe-structural-provenance.ts', import.meta.url)], + ['swe-structural-judge-policy.ts', new URL('./swe-structural-judge-policy.ts', import.meta.url)], + ['swe-bench-env.ts', new URL('./swe-bench-env.ts', import.meta.url)], + ['swe-jail.ts', new URL('./swe-jail.ts', import.meta.url)], + ['swe-temp.ts', new URL('./swe-temp.ts', import.meta.url)], + ['benchmarks/swe-bench.ts', new URL('./benchmarks/swe-bench.ts', import.meta.url)], + ['benchmarks/_harness.ts', new URL('./benchmarks/_harness.ts', import.meta.url)], + ['runtime/strategy.ts', new URL('../../src/runtime/strategy.ts', import.meta.url)], +].map(([name, url]) => ({ name: String(name), content: readFileSync(url as URL, 'utf8') })) + +const RUNTIME_IMPLEMENTATION_FINGERPRINT = runtimeImplementationFingerprint({ runAgentic, refine }) +const RUNTIME_TREE_FINGERPRINT = fingerprint([ + ...sourceTreeReceipt(new URL('../../src/', import.meta.url), 'agent-runtime/src', (path) => path.endsWith('.ts')), + ...sourceTreeReceipt( + new URL('../../node_modules/@tangle-network/agent-eval/dist/', import.meta.url), + 'agent-eval/dist', + (path) => path.endsWith('.js'), + ), + { + name: 'agent-runtime/package.json', + content: readFileSync(new URL('../../package.json', import.meta.url), 'utf8'), + }, + { + name: 'agent-runtime/pnpm-lock.yaml', + content: readFileSync(new URL('../../pnpm-lock.yaml', import.meta.url), 'utf8'), + }, + { + name: 'agent-eval/package.json', + content: readFileSync(new URL('../../node_modules/@tangle-network/agent-eval/package.json', import.meta.url), 'utf8'), + }, +]) + +function makeExperimentConfig(preset: ExperimentArmPreset, taskIds: string[]): ExperimentConfigReceipt { + return { + schema: 'swe-structural-v2', + arm: preset.arm, + k: preset.k, + repairs: preset.repairs, + alwaysRunContinuation: preset.alwaysRunContinuation, + persistent: preset.persistent, + workerSessions: preset.workerSessions, + model: MODEL, + zaiBase: ZAI_BASE, + maxTokens: MAX_TOKENS, + temperature: TEMPERATURE, + innerTurns: INNER_TURNS, + concurrency: CONC, + reproTimeoutS: REPRO_TIMEOUT_S, + llmTimeoutMs: LLM_TIMEOUT_MS, + sweRunTimeoutS: SWE_RUN_TOOL_CONFIG.timeoutS, + sweRunOutputLimit: SWE_RUN_TOOL_CONFIG.outputLimit, + runTool: true, + seedPrompt: 'SWE_SEED_PROMPT_WITH_RUN', + taskIds: [...taskIds], + } +} + +function commonConfig(config: ExperimentConfigReceipt): CommonConfigReceipt { + const { + arm: _arm, + k: _k, + repairs: _repairs, + alwaysRunContinuation: _alwaysRunContinuation, + persistent: _persistent, + workerSessions: _workerSessions, + ...common + } = config + return common +} + +function reproReceipt(entry: ReproManifestEntry | undefined): ReproManifestEntry | null { + return entry ? { script: entry.script, source: entry.source, stage0Class: entry.stage0Class } : null +} + +function expectedFingerprints( + bt: BenchTask, + config: ExperimentConfigReceipt, + tools: unknown, + repro: ReproManifestEntry | undefined, +): Fingerprints { + return createFingerprints({ + source: SOURCE_RECEIPT, + config, + commonConfig: commonConfig(config), + repro: reproReceipt(repro), + prompt: SWE_SEED_PROMPT_WITH_RUN, + tools, + task: { id: bt.id, prompt: bt.prompt, metadata: bt.metadata ?? null }, + }) +} + +// ---------- transport: zai direct, patient ladder, leak guard at the chokepoint ---------- + +interface Counter { + workerSessionsStarted: number + workerSessionsCompleted: number + calls: number + httpAttempts: number + tokensIn: number + tokensOut: number + guardedMsgs: number +} + +const newCounter = (): Counter => ({ + workerSessionsStarted: 0, + workerSessionsCompleted: 0, + calls: 0, + httpAttempts: 0, + tokensIn: 0, + tokensOut: 0, + guardedMsgs: 0, +}) + +/** Distinctive content marks for the leak guard: the first substantive ADDED line of the gold patch + * and of the hidden test patch. Never shown to any model; used only to refuse outbound messages. */ +function leakMarks(md: Record): string[] { + const marks: string[] = [] + for (const src of [md.patch, md.test_patch]) { + const m = String(src ?? '') + .split('\n') + .find((l) => l.startsWith('+') && !l.startsWith('+++') && l.trim().length > 12) + ?.slice(0, 80) + if (m) marks.push(m) + } + return marks +} + +/** Every model call in this driver flows through here (runAgentic's `complete` seam), so the judge + * separation is asserted once, at the single chokepoint, for every arm and every round. */ +const makeTransport = + (marks: string[], counter: Counter) => + async (body: Record): Promise => { + const msgs = (body.messages ?? []) as Array<{ role?: string; content?: unknown }> + counter.guardedMsgs += assertNoHiddenLeak(marks, msgs) + const { json, attempts } = await zaiChatRaw({ base: ZAI_BASE, key: ZAI_KEY, timeoutMs: LLM_TIMEOUT_MS }, body) + counter.calls += 1 + counter.httpAttempts += attempts + const u = (json as { usage?: { prompt_tokens?: number; completion_tokens?: number } }).usage + counter.tokensIn += u?.prompt_tokens ?? 0 + counter.tokensOut += u?.completion_tokens ?? 0 + return json + } + +// ---------- one emit-patch attempt (the swe-emit-patch protocol, with an optional pre-applied +// base diff for repair rounds) ---------- + +interface AttemptOut { + diff: string + completions: number + tokensIn: number + tokensOut: number + calls: number + wallMs: number + error?: string +} + +async function emitAttempt( + environment: AgenticSurface, + bt: BenchTask, + cfg: { + temperature: number + marks: string[] + instanceCounter: Counter + preApply?: string + promptAppendix?: string + }, +): Promise { + const t0 = Date.now() + const counter = newCounter() + // Capture the patch from inside score() (called during the refine loop, BEFORE the surface closes + // and rms the checkout). Keep the LATEST non-empty diff — the emit-patch pattern. + const capture = { patch: '' } + const proxy: AgenticSurface = { + ...environment, + async open(t: AgenticTask): Promise { + const h = await environment.open(t) + const pre = cfg.preApply + if (pre?.trim()) { + const f = join(h.id, '.swe-preapply.diff') + writeFileSync(f, pre.endsWith('\n') ? pre : `${pre}\n`) + try { + await exec('git', ['-C', h.id, 'apply', '--whitespace=nowarn', f], { timeout: 60_000 }) + } finally { + rmSync(f, { force: true }) + } + } + return h + }, + async score(_t: AgenticTask, handle: ArtifactHandle): Promise { + try { + const d = await exec('git', ['-C', handle.id, 'diff'], { maxBuffer: 40_000_000, timeout: 60_000 }) + if (d.stdout.trim()) capture.patch = d.stdout + } catch { + /* workspace gone or git error → keep whatever we already captured */ + } + return { passes: capture.patch.trim() ? 1 : 0, total: 1, errored: 0 } + }, + } + const task: AgenticTask = { + id: bt.id, + systemPrompt: SWE_SEED_PROMPT_WITH_RUN, + userPrompt: cfg.promptAppendix ? `${bt.prompt}\n\n${cfg.promptAppendix}` : bt.prompt, + meta: { instanceId: bt.id }, + } + let error: string | undefined + try { + cfg.instanceCounter.workerSessionsStarted += 1 + const r = await runAgentic({ + surface: proxy, + task, + strategy: refine, + routerBaseUrl: 'zai-direct', // unused: the `complete` transport short-circuits the router + routerKey: 'zai-direct', + model: MODEL, + maxTokens: MAX_TOKENS, + temperature: cfg.temperature, + innerTurns: INNER_TURNS, + budget: 1, + complete: makeTransport(cfg.marks, counter), + }) + if (counter.calls < 1) throw new Error('worker session completed without a successful model call') + cfg.instanceCounter.workerSessionsCompleted += 1 + cfg.instanceCounter.calls += counter.calls + cfg.instanceCounter.httpAttempts += counter.httpAttempts + cfg.instanceCounter.tokensIn += counter.tokensIn + cfg.instanceCounter.tokensOut += counter.tokensOut + cfg.instanceCounter.guardedMsgs += counter.guardedMsgs + return { + diff: capture.patch, + completions: r.completions, + tokensIn: counter.tokensIn, + tokensOut: counter.tokensOut, + calls: counter.calls, + wallMs: Date.now() - t0, + } + } catch (e) { + error = e instanceof Error ? e.message.slice(0, 300) : String(e).slice(0, 300) + cfg.instanceCounter.calls += counter.calls + cfg.instanceCounter.httpAttempts += counter.httpAttempts + cfg.instanceCounter.tokensIn += counter.tokensIn + cfg.instanceCounter.tokensOut += counter.tokensOut + cfg.instanceCounter.guardedMsgs += counter.guardedMsgs + return { + diff: capture.patch, + completions: 0, + tokensIn: counter.tokensIn, + tokensOut: counter.tokensOut, + calls: counter.calls, + wallMs: Date.now() - t0, + error, + } + } +} + +// Arm composition deliberately stays outside the built-in sample/refine strategies while each +// worker session still runs through runAgentic(refine, budget=1). The built-ins cannot reproduce +// this controlled comparison: sample exposes only aggregate scores and has no stable later-on-tie +// patch receipt; refine may stop after shot one, adds an analyst call, and carries conversation +// history. This layer supplies only the missing experiment policy: exactly two fresh worker +// sessions, optional parent-patch state, shared visible selection, and exact per-session receipts. + +// ---------- in-image candidate scoring ---------- + +/** Severity ordering for selection (lower is better): 0 repro-pass, 1 repro-fail, 2 timeout, + * 3 apply-fail, 4 empty. Both experiment arms deterministically prefer session two on a tie. */ +interface CandScore { + applyOk: boolean | null + exit: number | null + timedOut: boolean + severity: number + out: string +} + +const EMPTY_SCORE: CandScore = { applyOk: null, exit: null, timedOut: false, severity: 4, out: '' } +const UNSCORED: CandScore = { applyOk: null, exit: null, timedOut: false, severity: 1, out: '(no repro signal)' } + +async function scoreCandidate(imageTag: string, repro: string | null, diff: string): Promise { + if (!diff.trim()) return EMPTY_SCORE + if (!repro) return UNSCORED + const r = await runPyInJail(imageTag, null, repro, diff, { timeoutS: REPRO_TIMEOUT_S }) + if (r.infraError) throw new Error(r.infraError) + const applyOk = r.out.includes(APPLY_SENTINEL) + if (!applyOk) return { applyOk, exit: r.code, timedOut: r.timedOut, severity: 3, out: tail(r.out, 800) } + if (r.timedOut) return { applyOk, exit: r.code, timedOut: true, severity: 2, out: tail(r.out, 800) } + return { applyOk, exit: r.code, timedOut: false, severity: r.code === 0 ? 0 : 1, out: tail(r.out, 1_500) } +} + +// ---------- rows ---------- + +interface CandidateRow { + idx: number + diff: string + diffHash: string + diffBytes: number + completions: number + calls: number + tokensIn: number + tokensOut: number + wallMs: number + attemptError: string | null + applyOk: boolean | null + reproExit: number | null + reproTimedOut: boolean + severity: number + reproOutTail: string +} + +interface RepairRow { + round: number + baseFrom: string + baseSeverity: number + parentDiffHash: string + diff: string + finalDiffHash: string + changedFromParent: boolean + diffBytes: number + completions: number + calls: number + tokensIn: number + tokensOut: number + wallMs: number + attemptError: string | null + applyOk: boolean | null + reproExit: number | null + severity: number + accepted: boolean +} + +interface PhaseARow { + instanceId: string + arm: ExperimentArm + config: ExperimentConfigReceipt + fingerprints: Fingerprints + repo: string + model: string + image: string | null + execution: ExecutionReceipt | null + executionFingerprint: string + execMode: 'image' + temperature: number + innerTurns: number + k: number + maxRepairs: number + // canary + repro provenance (system arm) + canaryExit: number | null + canaryPass: boolean | null + reproSource: string + reproStage0Class: string | null + reproStatus: string + reproScript: string | null + reproPreExit: number | null + reproGoldExit: number | null + reproOutcomeFingerprint: string + // candidates + selection + continuation receipts + candidates: CandidateRow[] + selection: { mode: string; selectedIdx: number; movedOffFirst: boolean } | null + repairs: RepairRow[] + repairStop: string | null + finalFrom: string + parentDiffHash: string | null + finalDiff: string + finalDiffHash: string + changedFromParent: boolean | null + // cost + guard receipts + /** Number of fresh runAgentic worker invocations, including invocations that returned an error. */ + workerSessionsStarted: number + /** Number of runAgentic invocations that returned successfully after at least one model call. */ + workerSessions: number + llmCalls: number + httpAttempts: number + tokensIn: number + tokensOut: number + guardedMsgs: number + wallMs: number + error?: string +} + +// ---------- phase A: all arm decisions (no judge anywhere) ---------- + +type Env = Awaited> + +interface PhaseAContext { + preset: ExperimentArmPreset + config: ExperimentConfigReceipt + tools: unknown + sharedExecution: SharedExecutionReceipt + expectedImageIdentities: Map +} + +async function phaseA(env: Env, bt: BenchTask, ctx: PhaseAContext): Promise { + const t0 = Date.now() + const md = bt.metadata as Record + const counter = newCounter() + const row: PhaseARow = { + instanceId: bt.id, arm: ctx.preset.arm, config: ctx.config, + fingerprints: expectedFingerprints(bt, ctx.config, ctx.tools, MANIFEST[bt.id]), + repo: md.repo, model: MODEL, image: null, execution: null, executionFingerprint: '', execMode: 'image', + temperature: TEMPERATURE, innerTurns: INNER_TURNS, + k: ctx.preset.k, maxRepairs: ctx.preset.repairs, + canaryExit: null, canaryPass: null, reproSource: 'none', reproStage0Class: null, + reproStatus: 'none', reproScript: null, + reproPreExit: null, reproGoldExit: null, reproOutcomeFingerprint: '', + candidates: [], selection: null, repairs: [], + repairStop: null, finalFrom: 'none', parentDiffHash: null, finalDiff: '', + finalDiffHash: diffFingerprint(''), changedFromParent: null, + workerSessionsStarted: 0, workerSessions: 0, llmCalls: 0, httpAttempts: 0, + tokensIn: 0, tokensOut: 0, + guardedMsgs: 0, wallMs: 0, + } + const marks = leakMarks(md) + try { + const img = await resolveImageForMetadata(bt.metadata ?? {}) + if (!img.ok) throw new Error(`image missing: ${img.reason}`) + row.image = img.tag + row.execution = createExecutionReceipt(ctx.sharedExecution, img) + row.executionFingerprint = fingerprint(row.execution) + ctx.expectedImageIdentities.set(bt.id, img.identity) + + // 1. EXECUTION CANARY (image substrate, gold in-container — script-side only, zero model calls). + const pkg = IMPORT_NAME[md.repo] + if (!pkg) throw new Error(`no IMPORT_NAME for ${md.repo} — canary not expressible`) + const gold = String(md.patch ?? '') + if (!gold.trim()) throw new Error('gold patch missing from metadata') + const c = await runPyInJail(img.identity.id, null, importCanaryScript(pkg), gold, { timeoutS: REPRO_TIMEOUT_S }) + if (c.infraError) throw new Error(c.infraError) + row.canaryExit = c.code + row.canaryPass = c.code === 0 && c.out.includes(APPLY_SENTINEL) + if (!row.canaryPass) throw new Error(`canary failed (exit ${c.code}): this substrate cannot grade this instance`) + + // 2. Repro reuse + re-verification on THIS substrate (validity without gold, soundness with). + const manifest = MANIFEST[bt.id] + let repro: string | null = null + if (manifest) { + row.reproSource = manifest.source + row.reproStage0Class = manifest.stage0Class + row.reproScript = manifest.script + const pre = await runPyInJail(img.identity.id, null, manifest.script, undefined, { timeoutS: REPRO_TIMEOUT_S }) + if (pre.infraError) throw new Error(pre.infraError) + row.reproPreExit = pre.code + if (pre.timedOut) row.reproStatus = 'degraded-timeout' + else if (pre.code === 0) row.reproStatus = 'degraded-invalid' + else { + const post = await runPyInJail(img.identity.id, null, manifest.script, gold, { timeoutS: REPRO_TIMEOUT_S }) + if (post.infraError) throw new Error(post.infraError) + row.reproGoldExit = post.code + if (post.code === 0 && post.out.includes(APPLY_SENTINEL)) { + row.reproStatus = 'ok' + repro = manifest.script + } else { + row.reproStatus = 'degraded-unsound' + } + } + } + + // 3. k independent candidates (serial within the instance — CONC instances bound zai concurrency). + const diffs: string[] = [] + for (let i = 0; i < ctx.preset.k; i += 1) { + const a = await emitAttempt(env.environment, bt, { temperature: TEMPERATURE, marks, instanceCounter: counter }) + diffs.push(a.diff) + row.candidates.push({ + idx: i, diff: a.diff, diffHash: diffFingerprint(a.diff), diffBytes: a.diff.length, + completions: a.completions, calls: a.calls, + tokensIn: a.tokensIn, tokensOut: a.tokensOut, wallMs: a.wallMs, attemptError: a.error ?? null, + applyOk: null, reproExit: null, reproTimedOut: false, severity: -1, reproOutTail: '', + }) + } + + // 4. In-image scoring + argmax. + const scores: CandScore[] = [] + for (let i = 0; i < ctx.preset.k; i += 1) { + const s = await scoreCandidate(img.identity.id, repro, diffs[i] as string) + scores.push(s) + const cand = row.candidates[i] as CandidateRow + cand.applyOk = s.applyOk + cand.reproExit = s.exit + cand.reproTimedOut = s.timedOut + cand.severity = s.severity + cand.reproOutTail = s.out + } + let selectedIdx = 0 + for (let i = 1; i < ctx.preset.k; i += 1) { + if (preferLaterCandidate((scores[selectedIdx] as CandScore).severity, (scores[i] as CandScore).severity)) { + selectedIdx = i + } + } + const mode = repro ? 'visible-severity-later-tie' : 'no-repro-later-tie' + row.selection = { mode, selectedIdx, movedOffFirst: selectedIdx !== 0 } + + // 5. The persistent arm starts exactly one continuation from session one's cumulative patch. + // Both arms use the same later-on-visible-tie policy. + let best = { + diff: diffs[selectedIdx] as string, + score: scores[selectedIdx] as CandScore, + from: ctx.preset.persistent ? 'session:1' : `attempt:${selectedIdx + 1}`, + } + if (!ctx.preset.persistent) { + row.repairStop = 'independent-arm' + } else { + for (let round = 1; shouldRunContinuation({ round, preset: ctx.preset }); round += 1) { + const parentDiff = best.diff + const parentDiffHash = diffFingerprint(parentDiff) + row.parentDiffHash = parentDiffHash + const reproductionEvidence = repro + ? `--- REPRODUCTION SCRIPT (written from the issue; exit 0 = fixed) ---\n${tail(repro, 6_000)}\n\n` + + `--- REPRODUCTION OUTPUT on the current state (exit ${best.score.exit ?? 'n/a'}) ---\n${best.score.out}\n\n` + : '--- EXTERNAL REPRODUCTION ---\nNo external reproduction is available. Use the run tool to construct local, issue-specific checks.\n\n' + const repairInstruction = best.score.severity === 0 + ? 'The visible reproduction PASSES, but it is only a partial check. Re-read the full issue and audit ' + + 'the current patch for missed cases or regressions. Keep the visible check passing while correcting ' + + 'any incomplete source behavior you find with minimal edit_file changes. Do not modify tests.' + : 'The visible reproduction still fails (or no external reproduction is available). Diagnose why the ' + + 'current state does not resolve the full issue, then correct the SOURCE with minimal edit_file changes ' + + '(you may revise or revert parts of the previous fix — it is already in the files). Do not modify tests.' + const appendix = + continuationStateNotice(best.diff) + + reproductionEvidence + + '--- REPAIR INSTRUCTIONS ---\n' + + repairInstruction + const a = await emitAttempt(env.environment, bt, { + temperature: TEMPERATURE, marks, instanceCounter: counter, preApply: best.diff, promptAppendix: appendix, + }) + const ns = await scoreCandidate(img.identity.id, repro, a.diff) + const finalDiffHash = diffFingerprint(a.diff) + const changedFromParent = diffChanged(parentDiff, a.diff) + const accepted = shouldAcceptContinuation(best.score.severity, ns.severity) + const disposition = continuationDisposition(accepted, changedFromParent) + row.repairs.push({ + round, baseFrom: best.from, baseSeverity: best.score.severity, parentDiffHash, + diff: a.diff, finalDiffHash, changedFromParent, diffBytes: a.diff.length, + completions: a.completions, calls: a.calls, tokensIn: a.tokensIn, tokensOut: a.tokensOut, + wallMs: a.wallMs, attemptError: a.error ?? null, applyOk: ns.applyOk, reproExit: ns.exit, + severity: ns.severity, accepted, + }) + if (accepted) { + best = { + diff: a.diff, + score: ns, + from: disposition.finalFrom, + } + row.repairStop = disposition.stop + } else { + best.from = disposition.finalFrom + row.repairStop = disposition.stop + } + } + } + row.finalDiff = best.diff + row.finalDiffHash = diffFingerprint(best.diff) + row.finalFrom = best.from + row.changedFromParent = row.parentDiffHash === null + ? null + : row.finalDiffHash !== row.parentDiffHash + return row + } catch (e) { + row.error = e instanceof Error ? e.message.slice(0, 400) : String(e).slice(0, 400) + return row + } finally { + row.reproOutcomeFingerprint = fingerprint({ + canaryExit: row.canaryExit, + canaryPass: row.canaryPass, + image: row.execution?.image ?? null, + reproGoldExit: row.reproGoldExit, + reproPreExit: row.reproPreExit, + reproSource: row.reproSource, + reproStage0Class: row.reproStage0Class, + reproStatus: row.reproStatus, + }) + row.workerSessionsStarted = counter.workerSessionsStarted + row.workerSessions = counter.workerSessionsCompleted + row.llmCalls = counter.calls + row.httpAttempts = counter.httpAttempts + row.tokensIn = counter.tokensIn + row.tokensOut = counter.tokensOut + row.guardedMsgs = counter.guardedMsgs + row.wallMs = Date.now() - t0 + } +} + +// ---------- driver ---------- + +function loadPhaseRows(path: string, required = false): Map { + if (!existsSync(path)) { + if (required) throw new Error(`required Phase-A file does not exist: ${path}`) + return new Map() + } + const rows = new Map() + for (const [index, line] of readFileSync(path, 'utf8').split('\n').entries()) { + if (!line.trim()) continue + const row = JSON.parse(line) as PhaseARow + if (!row.instanceId) throw new Error(`${path}:${index + 1}: missing instanceId`) + if (rows.has(row.instanceId)) throw new Error(`${path}:${index + 1}: duplicate instanceId ${row.instanceId}`) + rows.set(row.instanceId, row) + } + if (required && rows.size === 0) throw new Error(`required Phase-A file is empty: ${path}`) + return rows +} + +function manifestFromRow(row: PhaseARow): ReproManifestEntry | undefined { + if (row.reproScript === null) { + if (row.reproSource !== 'none' || row.reproStage0Class !== null) { + throw new Error(`${row.instanceId}: incomplete null-repro receipt`) + } + return undefined + } + if (row.reproSource === 'none' || row.reproStage0Class === null) { + throw new Error(`${row.instanceId}: incomplete repro receipt`) + } + return { script: row.reproScript, source: row.reproSource, stage0Class: row.reproStage0Class } +} + +function assertPresetConfig(config: ExperimentConfigReceipt, arm: ExperimentArm, context: string): void { + const preset = resolveExperimentArm(arm) + if ( + config.schema !== 'swe-structural-v2' || + !config.model || + !config.zaiBase || + !Number.isFinite(config.maxTokens) || + config.maxTokens <= 0 || + !Number.isFinite(config.temperature) || + !Number.isInteger(config.innerTurns) || + config.innerTurns <= 0 || + !Number.isInteger(config.concurrency) || + config.concurrency < 1 || + !Number.isFinite(config.reproTimeoutS) || + config.reproTimeoutS <= 0 || + !Number.isFinite(config.llmTimeoutMs) || + config.llmTimeoutMs <= 0 || + config.runTool !== true || + config.seedPrompt !== 'SWE_SEED_PROMPT_WITH_RUN' || + !Number.isFinite(config.sweRunTimeoutS) || + config.sweRunTimeoutS <= 0 || + !Number.isFinite(config.sweRunOutputLimit) || + config.sweRunOutputLimit <= 0 || + !Array.isArray(config.taskIds) || + config.taskIds.length === 0 + ) { + throw new Error(`${context}: unsupported config schema or worker surface`) + } + for (const key of ['arm', 'k', 'repairs', 'alwaysRunContinuation', 'persistent', 'workerSessions'] as const) { + if (config[key] !== preset[key]) { + throw new Error(`${context}: config ${key}=${String(config[key])}, expected ${String(preset[key])}`) + } + } +} + +async function assertPhaseRow( + row: PhaseARow, + bt: BenchTask, + config: ExperimentConfigReceipt, + tools: unknown, + repro: ReproManifestEntry | undefined, + sharedExecution: SharedExecutionReceipt, + context: string, +): Promise { + if (row.instanceId !== bt.id) throw new Error(`${context}: instance mismatch ${row.instanceId} != ${bt.id}`) + if (row.arm !== config.arm) throw new Error(`${context}: arm mismatch ${row.arm} != ${config.arm}`) + if (!row.config) throw new Error(`${context}: missing config receipt`) + assertPresetConfig(row.config, row.arm, context) + if (fingerprint(row.config) !== fingerprint(config)) throw new Error(`${context}: config receipt mismatch`) + assertFingerprintsEqual(row.fingerprints, expectedFingerprints(bt, config, tools, repro), context) + if ( + row.model !== config.model || + row.temperature !== config.temperature || + row.innerTurns !== config.innerTurns || + row.k !== config.k || + row.maxRepairs !== config.repairs + ) { + throw new Error(`${context}: row execution fields do not match config receipt`) + } + if (row.repo !== String((bt.metadata as Record | undefined)?.repo ?? '')) { + throw new Error(`${context}: row repo does not match task metadata`) + } + if (row.error) throw new Error(`${context}: Phase A contains an error: ${row.error}`) + assertExactCompletedWorkerSessions({ + started: row.workerSessionsStarted, + completed: row.workerSessions, + sessions: [...row.candidates, ...row.repairs], + context, + }) + const currentImage = await resolveImageForMetadata(bt.metadata ?? {}) + if (!currentImage.ok) throw new Error(`${context}: image unavailable during receipt validation: ${currentImage.reason}`) + const expectedExecution = createExecutionReceipt(sharedExecution, currentImage) + const actualExecution = row.execution + if (!actualExecution || fingerprint(actualExecution) !== fingerprint(expectedExecution)) { + throw new Error(`${context}: execution receipt mismatch`) + } + if ( + row.config.sweRunTimeoutS !== actualExecution.runTool.timeoutS || + row.config.sweRunOutputLimit !== actualExecution.runTool.outputLimit + ) { + throw new Error(`${context}: config and execution run-tool settings do not match`) + } + if (row.executionFingerprint !== fingerprint(expectedExecution)) { + throw new Error(`${context}: execution fingerprint mismatch`) + } + if (row.image !== expectedExecution.image.tag) throw new Error(`${context}: image tag receipt mismatch`) + if (row.finalDiffHash !== diffFingerprint(row.finalDiff)) throw new Error(`${context}: final diff hash mismatch`) + const expectedReproOutcome = fingerprint({ + canaryExit: row.canaryExit, + canaryPass: row.canaryPass, + image: actualExecution.image, + reproGoldExit: row.reproGoldExit, + reproPreExit: row.reproPreExit, + reproSource: row.reproSource, + reproStage0Class: row.reproStage0Class, + reproStatus: row.reproStatus, + }) + if (row.reproOutcomeFingerprint !== expectedReproOutcome) { + throw new Error(`${context}: reproduction outcome fingerprint mismatch`) + } + for (const candidate of row.candidates) { + if (candidate.diffHash !== diffFingerprint(candidate.diff)) { + throw new Error(`${context}: candidate ${candidate.idx} diff hash mismatch`) + } + } + + const preset = resolveExperimentArm(row.arm) + if (row.candidates.length !== preset.k || row.repairs.length !== preset.repairs) { + throw new Error( + `${context}: arm shape mismatch (candidates=${row.candidates.length}, repairs=${row.repairs.length})`, + ) + } + let selectedIdx = 0 + for (let i = 1; i < row.candidates.length; i += 1) { + if (preferLaterCandidate(row.candidates[selectedIdx]!.severity, row.candidates[i]!.severity)) selectedIdx = i + } + const expectedMode = row.reproStatus === 'ok' ? 'visible-severity-later-tie' : 'no-repro-later-tie' + if ( + row.selection?.selectedIdx !== selectedIdx || + row.selection.mode !== expectedMode || + row.selection.movedOffFirst !== (selectedIdx !== 0) + ) { + throw new Error(`${context}: selectedIdx violates later-on-visible-tie policy`) + } + + if (!preset.persistent) { + if (row.parentDiffHash !== null || row.changedFromParent !== null) { + throw new Error(`${context}: independent arm must not claim a parent/refinement`) + } + if (row.repairStop !== 'independent-arm') throw new Error(`${context}: independent arm stop receipt mismatch`) + const selected = row.candidates[selectedIdx]! + if (row.finalDiff !== selected.diff || row.finalFrom !== `attempt:${selectedIdx + 1}`) { + throw new Error(`${context}: independent final patch does not match selected attempt`) + } + return + } + + const continuation = row.repairs[0]! + const parent = row.candidates[0]!.diff + if (continuation.baseFrom !== 'session:1' || continuation.baseSeverity !== row.candidates[0]!.severity) { + throw new Error(`${context}: continuation parent receipt mismatch`) + } + if (continuation.parentDiffHash !== diffFingerprint(parent)) throw new Error(`${context}: parent diff hash mismatch`) + if (continuation.finalDiffHash !== diffFingerprint(continuation.diff)) throw new Error(`${context}: continuation diff hash mismatch`) + if (continuation.changedFromParent !== diffChanged(parent, continuation.diff)) { + throw new Error(`${context}: continuation changedFromParent mismatch`) + } + const shouldAccept = shouldAcceptContinuation(continuation.baseSeverity, continuation.severity) + if (continuation.accepted !== shouldAccept) throw new Error(`${context}: continuation violates shared tie policy`) + const expectedFinal = continuation.accepted ? continuation.diff : parent + const disposition = continuationDisposition(continuation.accepted, continuation.changedFromParent) + if (row.finalDiff !== expectedFinal || row.finalFrom !== disposition.finalFrom || row.repairStop !== disposition.stop) { + throw new Error(`${context}: persistent final patch/provenance mismatch`) + } + if (row.parentDiffHash !== continuation.parentDiffHash) throw new Error(`${context}: row parent diff hash mismatch`) + if (row.changedFromParent !== (row.finalDiffHash !== row.parentDiffHash)) { + throw new Error(`${context}: row changedFromParent mismatch`) + } +} + +async function assertOfficialScoreImagePinned(row: PhaseARow, bt: BenchTask, context: string): Promise { + const expected = row.execution?.image + if (!expected) throw new Error(`${context}: missing immutable image receipt`) + const current = await resolveImageForMetadata(bt.metadata ?? {}) + if (!current.ok) throw new Error(`${context}: image unavailable: ${current.reason}`) + if ( + current.tag !== expected.tag || + current.namespace !== expected.namespace || + current.identity.id !== expected.id + ) { + throw new Error( + `${context}: official-score image changed ` + + `(${expected.namespace}:${expected.tag}@${expected.id} -> ` + + `${current.namespace}:${current.tag}@${current.identity.id})`, + ) + } +} + +async function loadTasksAndTools(ids: string[]): Promise<{ + env: Env + taskById: Map + tools: unknown + sharedExecution: SharedExecutionReceipt + expectedImageIdentities: Map +}> { + const expectedImageIdentities = new Map() + const env = await createSweBenchEnvironment(ids.length, { + ids, + cloneCache: true, + enableRun: true, + expectedImageIdentities, + adapterOptions: { cacheLevel: OFFICIAL_SCORER_CACHE_LEVEL }, + }) + await env.adapter.preflight?.() + const scorerVersion = await resolveSweBenchScorerVersion() + const taskById = new Map((await env.adapter.loadTasks({ ids, split: 'test' })).map((task) => [task.id, task])) + const missing = ids.filter((id) => !taskById.has(id)) + if (missing.length) throw new Error(`instances not found in SWE-bench_Verified: ${missing.join(', ')}`) + const first = taskById.get(ids[0]!)! + const fingerprintTask: AgenticTask = { + id: first.id, + systemPrompt: SWE_SEED_PROMPT_WITH_RUN, + userPrompt: first.prompt, + meta: { instanceId: first.id }, + } + const fingerprintHandle: ArtifactHandle = { id: 'fingerprint-only', surface: 'swe-bench-verified' } + return { + env, + taskById, + tools: await env.environment.tools(fingerprintTask, fingerprintHandle), + sharedExecution: { + runTool: { ...SWE_RUN_TOOL_CONFIG }, + runtimeImplementationFingerprint: RUNTIME_IMPLEMENTATION_FINGERPRINT, + runtimeTreeFingerprint: RUNTIME_TREE_FINGERPRINT, + officialScorer: { + package: 'swebench', + version: scorerVersion, + cacheLevel: OFFICIAL_SCORER_CACHE_LEVEL, + namespacePolicy: 'phase-a-image', + }, + }, + expectedImageIdentities, + } +} + +async function generateMain(): Promise { + const preset = ARM_PRESET as ExperimentArmPreset + const ids = process.env.IDS + ? process.env.IDS.split(',').map((value) => value.trim()).filter(Boolean) + : await cachedInstanceIds() + if (!ids.length) throw new Error('no cached sweb.eval images found and no IDS given') + if (new Set(ids).size !== ids.length) throw new Error('IDS contains duplicates') + const out = process.env.OUT ?? `swe-stage1-${preset.arm}.phaseA.jsonl` + const config = makeExperimentConfig(preset, ids) + assertPresetConfig(config, preset.arm, 'generate config') + const { env, taskById, tools, sharedExecution, expectedImageIdentities } = await loadTasksAndTools(ids) + + console.log(`═══ SWE-bench Phase A — ${preset.arm} ═══`) + console.log( + `sessions=2 k=${preset.k} repairs=${preset.repairs} persistent=${preset.persistent ? 1 : 0} ` + + `model=${MODEL} maxTokens=${MAX_TOKENS} innerTurns=${INNER_TURNS} temperature=${TEMPERATURE} runTool=1 judge=DISABLED`, + ) + console.log(`instances=${ids.length} out=${out}`) + + const done = loadPhaseRows(out) + for (const [id, row] of done) { + if (!ids.includes(id)) throw new Error(`${out}: resume row ${id} is outside current IDS`) + await assertPhaseRow(row, taskById.get(id)!, config, tools, MANIFEST[id], sharedExecution, `${out}:${id}`) + } + const todo = ids.filter((id) => !done.has(id)) + if (done.size) console.log(`resume accepted: ${done.size}/${ids.length} fingerprint-matched rows`) + + let next = 0 + const worker = async (): Promise => { + while (next < todo.length) { + const index = next++ + const id = todo[index]! + const row = await phaseA(env, taskById.get(id)!, { + preset, + config, + tools, + sharedExecution, + expectedImageIdentities, + }) + await assertPhaseRow( + row, + taskById.get(id)!, + config, + tools, + MANIFEST[id], + sharedExecution, + `${preset.arm}:${id}`, + ) + done.set(id, row) + appendFileSync(out, `${JSON.stringify(row)}\n`) + console.log( + `[${index + 1}/${todo.length}] ${id} sessions=${row.workerSessions} ` + + `severity=[${row.candidates.map((candidate) => candidate.severity).join(',')}] ` + + `continuation=[${row.repairs.map((repair) => repair.severity).join(',')}] ` + + `final=${row.finalFrom} changed=${row.changedFromParent ?? 'n/a'} calls=${row.llmCalls}`, + ) + } + } + await Promise.all(Array.from({ length: CONC }, () => worker())) + + const rows = ids.map((id) => done.get(id)!) + const totalSessions = rows.reduce((sum, row) => sum + row.workerSessions, 0) + const totalIn = rows.reduce((sum, row) => sum + row.tokensIn, 0) + const totalOut = rows.reduce((sum, row) => sum + row.tokensOut, 0) + const usd = (totalIn / 1e6) * PRICE_IN + (totalOut / 1e6) * PRICE_OUT + console.log( + `Phase A complete: rows=${rows.length}/${ids.length} sessions=${totalSessions}/${2 * ids.length} ` + + `tokens=${totalIn}/${totalOut} assumedCost=$${usd.toFixed(2)}; no official scores were called`, + ) +} + +interface JudgeRow { + schema: 'swe-structural-judge-v2' + instanceId: string + arm: ExperimentArm + pairFingerprint: string + phaseFileFingerprint: string + inputFingerprint: string + executionFingerprint: string + finalDiffHash: string + hiddenResolved: boolean + judgeDetail: string | null + judgeMs: number + judgeSkipped: 'empty-patch' | null +} + +function loadJudgeRows(path: string): Map { + if (!existsSync(path)) return new Map() + const rows = new Map() + for (const [index, line] of readFileSync(path, 'utf8').split('\n').entries()) { + if (!line.trim()) continue + const row = JSON.parse(line) as JudgeRow + if (row.schema !== 'swe-structural-judge-v2') { + throw new Error(`${path}:${index + 1}: unsupported judge row schema`) + } + if (typeof row.hiddenResolved !== 'boolean') { + throw new Error(`${path}:${index + 1}: incomplete judge row must be removed and retried`) + } + if (row.judgeSkipped !== null && row.judgeSkipped !== 'empty-patch') { + throw new Error(`${path}:${index + 1}: unsupported judge skip receipt ${row.judgeSkipped}`) + } + resolveExperimentArm(row.arm) + const key = `${row.arm}:${row.instanceId}` + if (rows.has(key)) throw new Error(`${path}:${index + 1}: duplicate judge row ${key}`) + rows.set(key, row) + } + return rows +} + +function requirePath(name: string): string { + const value = process.env[name]?.trim() + if (!value) throw new Error(`${name} required for MODE=judge-only`) + return value +} + +async function judgeOnlyMain(): Promise { + const independentPath = requirePath('INDEPENDENT_PHASE_A') + const persistentPath = requirePath('PERSISTENT_PHASE_A') + const out = requirePath('OUT') + assertDistinctArtifactPaths({ INDEPENDENT_PHASE_A: independentPath, PERSISTENT_PHASE_A: persistentPath, OUT: out }) + const independent = loadPhaseRows(independentPath, true) + const persistent = loadPhaseRows(persistentPath, true) + const independentConfig = [...independent.values()][0]!.config + const persistentConfig = [...persistent.values()][0]!.config + if (!independentConfig || !persistentConfig) throw new Error('Phase-A file is missing its config receipt') + assertPresetConfig(independentConfig, 'independent-2', independentPath) + assertPresetConfig(persistentConfig, 'persistent-refine-2', persistentPath) + if (fingerprint(commonConfig(independentConfig)) !== fingerprint(commonConfig(persistentConfig))) { + throw new Error('Phase-A files have different common configuration fingerprints') + } + const ids = independentConfig.taskIds + if (fingerprint(ids) !== fingerprint(persistentConfig.taskIds)) { + throw new Error('Phase-A files have different task-id fingerprints') + } + for (const [path, rows] of [[independentPath, independent], [persistentPath, persistent]] as const) { + assertCompleteTaskSet(rows.keys(), ids, path) + } + + // No official judge call occurs before every row in both files passes these checks. + const { env, taskById, tools, sharedExecution } = await loadTasksAndTools(ids) + for (const id of ids) { + const left = independent.get(id)! + const right = persistent.get(id)! + await assertPhaseRow( + left, + taskById.get(id)!, + independentConfig, + tools, + manifestFromRow(left), + sharedExecution, + `${independentPath}:${id}`, + ) + await assertPhaseRow( + right, + taskById.get(id)!, + persistentConfig, + tools, + manifestFromRow(right), + sharedExecution, + `${persistentPath}:${id}`, + ) + assertPairedFingerprints(left.fingerprints, right.fingerprints, id) + assertPairedExecutionFingerprint(left.executionFingerprint, right.executionFingerprint, id) + if (left.reproOutcomeFingerprint !== right.reproOutcomeFingerprint) { + throw new Error(`${id}: paired reproduction outcomes do not match`) + } + } + + const phaseFileFingerprints: Record = { + 'independent-2': fingerprint({ bytes: readFileSync(independentPath, 'utf8') }), + 'persistent-refine-2': fingerprint({ bytes: readFileSync(persistentPath, 'utf8') }), + } + const pairFingerprint = fingerprint({ + commonConfig: independent.get(ids[0]!)!.fingerprints.commonConfig, + ids, + phaseFiles: phaseFileFingerprints, + prompt: independent.get(ids[0]!)!.fingerprints.prompt, + source: independent.get(ids[0]!)!.fingerprints.source, + tools: independent.get(ids[0]!)!.fingerprints.tools, + executions: ids.map((id) => independent.get(id)!.executionFingerprint), + }) + const judged = loadJudgeRows(out) + const arms: Array<{ arm: ExperimentArm; path: string; rows: Map }> = [ + { arm: 'independent-2', path: independentPath, rows: independent }, + { arm: 'persistent-refine-2', path: persistentPath, rows: persistent }, + ] + for (const existing of judged.values()) { + const input = arms.find(({ arm }) => arm === existing.arm)?.rows.get(existing.instanceId) + if (!input) throw new Error(`${out}: judge resume row has no paired Phase-A input`) + assertJudgeCompletionMatchesInput(existing, input.finalDiff, `${out}:${existing.arm}:${existing.instanceId}`) + assertJudgeResumeFingerprints(existing, { + pairFingerprint, + phaseFileFingerprint: phaseFileFingerprints[existing.arm], + inputFingerprint: input.fingerprints.composite, + executionFingerprint: input.executionFingerprint, + finalDiffHash: input.finalDiffHash, + }, `${out}:${existing.arm}:${existing.instanceId}`) + } + + console.log(`all ${2 * ids.length} Phase-A rows matched; official scoring starts now (serialized)`) + for (const id of ids) { + for (const { arm, rows } of arms) { + const key = `${arm}:${id}` + if (judged.has(key)) continue + const input = rows.get(id)! + const started = Date.now() + // Fail without appending: a transient official-scorer error must be retried on resume. + const { hiddenResolved, judgeDetail, judgeSkipped } = await completeJudgeScore( + input.finalDiff, + async () => { + const task = taskById.get(id)! + process.env.SWEBENCH_NAMESPACE = input.execution!.image.namespace + await assertOfficialScoreImagePinned(input, task, `${key}:before`) + const score = await env.adapter.judge(task, input.finalDiff) + await assertOfficialScoreImagePinned(input, task, `${key}:after`) + return score + }, + ) + const row: JudgeRow = { + schema: 'swe-structural-judge-v2', + instanceId: id, + arm, + pairFingerprint, + phaseFileFingerprint: phaseFileFingerprints[arm], + inputFingerprint: input.fingerprints.composite, + executionFingerprint: input.executionFingerprint, + finalDiffHash: input.finalDiffHash, + hiddenResolved, + judgeDetail, + judgeMs: Date.now() - started, + judgeSkipped, + } + judged.set(key, row) + appendFileSync(out, `${JSON.stringify(row)}\n`) + console.log(`[judge] ${key} resolved=${hiddenResolved} skipped=${judgeSkipped ?? '-'}`) + } + } +} + +const main = MODE === 'generate' ? generateMain : judgeOnlyMain +main().catch((e) => { + console.error(e instanceof Error ? (e.stack ?? e.message) : String(e)) + process.exit(1) +}) diff --git a/bench/src/swe-temp.ts b/bench/src/swe-temp.ts new file mode 100644 index 00000000..f58c40ac --- /dev/null +++ b/bench/src/swe-temp.ts @@ -0,0 +1,14 @@ +import { tmpdir } from 'node:os' +import { isAbsolute } from 'node:path' + +/** Docker bind mounts and mkdtemp parents must never inherit a relative operating-system temp root. */ +export function absoluteSweTempDir(): string { + const dir = tmpdir() + if (!isAbsolute(dir)) { + throw new Error( + `SWE temporary directory must be absolute, got "${dir}"; ` + + 'TEMP configures the operating-system temp root, not model temperature. Use TEMPERATURE instead.', + ) + } + return dir +}