diff --git a/bench/package.json b/bench/package.json index ae0d3d52..e364629c 100644 --- a/bench/package.json +++ b/bench/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-bench", - "version": "0.2.2", + "version": "0.3.0", "type": "module", "description": "The unified benchmark suite for agent-runtime agents: 31 adapters (commit0, enterpriseops-gym, ragbench, crag, nomiracl, open-rag-bench, t2-ragbench, tau3-banking, bfcl, finresearchbench, …) behind one resolveAdapter registry, each with a real judge or fail-loud unsupported scorer. Score any profile/skill/prompt change against them. Map: bench/HARNESS.md.", "repository": { @@ -27,9 +27,9 @@ "verify:pier": "tsx scripts/verify-pier-pair.mts" }, "dependencies": { - "@tangle-network/agent-eval": "^0.114.0", + "@tangle-network/agent-eval": "^0.115.2", "@tangle-network/agent-interface": "^0.25.0", - "@tangle-network/agent-runtime": "^0.92.0", + "@tangle-network/agent-runtime": "0.94.6", "@tangle-network/sandbox": "^0.10.3" }, "devDependencies": { @@ -39,7 +39,7 @@ }, "pnpm": { "overrides": { - "@tangle-network/agent-eval": "0.114.0" + "@tangle-network/agent-eval": "0.115.2" } }, "files": [ diff --git a/bench/pnpm-lock.yaml b/bench/pnpm-lock.yaml index 2454aa2a..98a033fa 100644 --- a/bench/pnpm-lock.yaml +++ b/bench/pnpm-lock.yaml @@ -5,21 +5,21 @@ settings: excludeLinksFromLockfile: false overrides: - '@tangle-network/agent-eval': 0.114.0 + '@tangle-network/agent-eval': 0.115.2 importers: .: dependencies: '@tangle-network/agent-eval': - specifier: 0.114.0 - version: 0.114.0(typescript@6.0.3) + specifier: 0.115.2 + version: 0.115.2(typescript@6.0.3) '@tangle-network/agent-interface': specifier: ^0.25.0 version: 0.25.0 '@tangle-network/agent-runtime': - specifier: ^0.92.0 - version: 0.92.0(@tangle-network/agent-eval@0.114.0(typescript@6.0.3))(@tangle-network/agent-interface@0.25.0)(@tangle-network/sandbox@0.10.3(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)))(typescript@6.0.3) + specifier: 0.94.6 + version: 0.94.6(@tangle-network/agent-eval@0.115.2(typescript@6.0.3))(@tangle-network/agent-interface@0.25.0)(@tangle-network/sandbox@0.10.3(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)))(typescript@6.0.3) '@tangle-network/sandbox': specifier: ^0.10.3 version: 0.10.3(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)) @@ -263,8 +263,8 @@ packages: '@tangle-network/agent-core@0.4.9': resolution: {integrity: sha512-BFU4WV12Y6D28PX+o8LIVkIMD+cXwrL1uyV182l12Bn9zEu7VHt2oVMEEzbsN8L+X0xM9baEfMCHTFKFNJv7Aw==} - '@tangle-network/agent-eval@0.114.0': - resolution: {integrity: sha512-pIZCLkKsHeKDpLOpRXRhnc2HaREcHduHm4u3jw0nN73B1BWe6Jaqh2wlNHTPnPUA1qQl5sCl1K6XF1p9R7eTZw==} + '@tangle-network/agent-eval@0.115.2': + resolution: {integrity: sha512-ZQi/V7sXD9/6jOjZaD5P4z6a8LUnQWtWWXfa0WOSsjqTmvZfxlG0yl2YSkPDWU1zg7h1qG50NKWJg6nB4cctKw==} engines: {node: '>=20'} hasBin: true @@ -286,20 +286,20 @@ packages: '@tangle-network/agent-interface@0.25.0': resolution: {integrity: sha512-bMKjyjk8bk2651HleiTEUOTPgNE2bWxCEl9VBVUyub/UkP87Y09byXOxeDEsNGc4TIT19Y5Uo0b+k1kRWXkMVQ==} - '@tangle-network/agent-knowledge@1.11.2': - resolution: {integrity: sha512-sJLLYai35JIHou0YgcPg5I4WCz90vhIUmyF+DQ5GtMBDB+KT92y+rwvLQ4MojIZCTq3p/xv8C3SEhkuqCfzVsw==} + '@tangle-network/agent-knowledge@1.12.1': + resolution: {integrity: sha512-j5riyIvz5C+ZBELTtNVtmUbcKAMpgTHMNmDQ12MhOIPJv5Y7AeX2oCWuF6c8aPqg1hOrsLqgqXSr1zkV0ePsrA==} engines: {node: '>=20'} hasBin: true - '@tangle-network/agent-profile-materialize@0.3.1': - resolution: {integrity: sha512-yA/DaxmC+DsHdPl00iovR3tg80lQY1ukkHgGViLtzFYY7O46yZR9Ykhw/tMGTFYWFi5bzqjmr0NShSxdNmSzcQ==} + '@tangle-network/agent-profile-materialize@0.3.2': + resolution: {integrity: sha512-jCj1Hc/brPQ5eS7or9A+Klv0FAZcGtyFr7kx2cKfk+wCj0/4Di3k1pMjayrmRNgI/7Oc47gt6/t+qvdtZxBkAw==} - '@tangle-network/agent-runtime@0.92.0': - resolution: {integrity: sha512-R4G7JTgpQJ/Xgzf0+YQ50VWewBoAnfJnF3nwQPCE3VzvDJzQ/kBVcVDaRTEtteP/ygS30PQi90tnbBSy2x3wzg==} + '@tangle-network/agent-runtime@0.94.6': + resolution: {integrity: sha512-2hpS9Qjq2hM5oa8uLx/8BwL8ivQ5b2QYaMg3/hSHDxt3OO6bT1P34NjCQFOPDFyJgAnZhvyv1xJettZXdse1KQ==} engines: {node: '>=20'} hasBin: true peerDependencies: - '@tangle-network/agent-eval': 0.114.0 + '@tangle-network/agent-eval': 0.115.2 '@tangle-network/agent-interface': '>=0.25.0 <1.0.0' '@tangle-network/sandbox': '>=0.8.0 <1.0.0' playwright: ^1.40.0 @@ -372,6 +372,51 @@ packages: zod: optional: true + b4a@1.8.1: + resolution: {integrity: sha512-aiqre1Nr0B/6DgE2N5vwTc+2/oQZ4Wh1t4NznYY4E00y8LCt6NqdRv81so00oo27D8MVKTpUa/MwUUtBLXCoDw==} + peerDependencies: + react-native-b4a: '*' + peerDependenciesMeta: + react-native-b4a: + optional: true + + bare-events@2.9.1: + resolution: {integrity: sha512-Z0oHEHAFDZkffN8Qc39zNZjQlMDkPJRyyyZieU1VH7u8c5S+qHZ2S8ixdKIAxEjfHO7FJxXmJWgteOghVanIsg==} + peerDependencies: + bare-abort-controller: '*' + peerDependenciesMeta: + bare-abort-controller: + optional: true + + bare-fs@4.7.4: + resolution: {integrity: sha512-y1kC+ffIx/tPLdTE693uNjHfzTfr+ravR5tvWlMXe25nELbkqV400S71qHDwbkAQ1FVEZobB1NFRzFbCCcyBCQ==} + engines: {bare: '>=1.16.0'} + peerDependencies: + bare-buffer: '*' + peerDependenciesMeta: + bare-buffer: + optional: true + + bare-path@3.1.1: + resolution: {integrity: sha512-JprUlveX3QjApC1cTpsUOiscADftCGVWkzitbHsRqv84hzYwYHw2mbluddsq5TvI8mH/8Ov1f4BiMAdcB0oYnQ==} + + bare-stream@2.13.3: + resolution: {integrity: sha512-Kc+brLqvEqGkjyfiwJmImAOqLZL7OsoLKuavx+hJjgVV3nLTOjloJyPMFxjUPerGGHrNH0fLU06jjykMLWrERQ==} + peerDependencies: + bare-abort-controller: '*' + bare-buffer: '*' + bare-events: '*' + peerDependenciesMeta: + bare-abort-controller: + optional: true + bare-buffer: + optional: true + bare-events: + optional: true + + bare-url@2.4.5: + resolution: {integrity: sha512-K+y9xF1tN+CdPu4qWwr0QiK1Al07eFPGYK5M2pDXcmHdMdgC/tT/bpmMe1hrmRHaidKLkXrC+cRNYf3XVDUhSQ==} + commander@14.0.3: resolution: {integrity: sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==} engines: {node: '>=20'} @@ -387,6 +432,12 @@ packages: eventemitter3@5.0.1: resolution: {integrity: sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==} + events-universal@1.0.1: + resolution: {integrity: sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==} + + fast-fifo@1.3.2: + resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==} + fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} @@ -412,6 +463,18 @@ packages: typescript: optional: true + streamx@2.28.0: + resolution: {integrity: sha512-1Yowhzjf0ivGMrTIkY9hav5TxobO9qIVqUE41fiCGMGgc3CLlf4MY+9AHmZqBWgDTue0fY9zWjYFVyf6Diuobw==} + + tar-stream@3.2.0: + resolution: {integrity: sha512-ojzvCvVaNp6aOTFmG7jaRD0meowIAuPc3cMMhSgKiVWws1GyHbGd/xvnyuRKcKlMpt3qvxx6r0hreCNITP9hIg==} + + teex@1.0.1: + resolution: {integrity: sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg==} + + text-decoder@1.2.7: + resolution: {integrity: sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==} + tsx@4.22.4: resolution: {integrity: sha512-X8EX+XV4QR5xCsrgxaED954zTDfY8KqlDtskKEL0cHhyS/P8b4IFOvGDQpsC9Q1XnLq915wEfwwY/zzskCtmhg==} engines: {node: '>=18.0.0'} @@ -603,7 +666,7 @@ snapshots: '@tangle-network/agent-interface': 0.24.0 zod: 4.4.3 - '@tangle-network/agent-eval@0.114.0(typescript@6.0.3)': + '@tangle-network/agent-eval@0.115.2(typescript@6.0.3)': dependencies: '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) '@ax-llm/ax': 19.0.45(zod@4.4.3) @@ -645,9 +708,9 @@ snapshots: dependencies: zod: 4.4.3 - '@tangle-network/agent-knowledge@1.11.2(typescript@6.0.3)': + '@tangle-network/agent-knowledge@1.12.1(typescript@6.0.3)': dependencies: - '@tangle-network/agent-eval': 0.114.0(typescript@6.0.3) + '@tangle-network/agent-eval': 0.115.2(typescript@6.0.3) zod: 4.4.3 transitivePeerDependencies: - '@mastra/core' @@ -658,24 +721,28 @@ snapshots: - typescript - utf-8-validate - '@tangle-network/agent-profile-materialize@0.3.1': + '@tangle-network/agent-profile-materialize@0.3.2': dependencies: - '@tangle-network/agent-interface': 0.22.0 + '@tangle-network/agent-interface': 0.25.0 - '@tangle-network/agent-runtime@0.92.0(@tangle-network/agent-eval@0.114.0(typescript@6.0.3))(@tangle-network/agent-interface@0.25.0)(@tangle-network/sandbox@0.10.3(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)))(typescript@6.0.3)': + '@tangle-network/agent-runtime@0.94.6(@tangle-network/agent-eval@0.115.2(typescript@6.0.3))(@tangle-network/agent-interface@0.25.0)(@tangle-network/sandbox@0.10.3(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)))(typescript@6.0.3)': dependencies: - '@tangle-network/agent-eval': 0.114.0(typescript@6.0.3) + '@tangle-network/agent-eval': 0.115.2(typescript@6.0.3) '@tangle-network/agent-interface': 0.25.0 - '@tangle-network/agent-knowledge': 1.11.2(typescript@6.0.3) - '@tangle-network/agent-profile-materialize': 0.3.1 + '@tangle-network/agent-knowledge': 1.12.1(typescript@6.0.3) + '@tangle-network/agent-profile-materialize': 0.3.2 + tar-stream: 3.2.0 optionalDependencies: '@tangle-network/sandbox': 0.10.3(viem@2.52.0(typescript@6.0.3)(zod@4.4.3)) transitivePeerDependencies: - '@mastra/core' - '@modelcontextprotocol/sdk' - ai + - bare-abort-controller + - bare-buffer - bufferutil - openai + - react-native-b4a - typescript - utf-8-validate @@ -722,6 +789,37 @@ snapshots: typescript: 6.0.3 zod: 4.4.3 + b4a@1.8.1: {} + + bare-events@2.9.1: {} + + bare-fs@4.7.4: + dependencies: + bare-events: 2.9.1 + bare-path: 3.1.1 + bare-stream: 2.13.3(bare-events@2.9.1) + bare-url: 2.4.5 + fast-fifo: 1.3.2 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + + bare-path@3.1.1: {} + + bare-stream@2.13.3(bare-events@2.9.1): + dependencies: + b4a: 1.8.1 + streamx: 2.28.0 + teex: 1.0.1 + optionalDependencies: + bare-events: 2.9.1 + transitivePeerDependencies: + - react-native-b4a + + bare-url@2.4.5: + dependencies: + bare-path: 3.1.1 + commander@14.0.3: {} dayjs@1.11.21: {} @@ -757,6 +855,14 @@ snapshots: eventemitter3@5.0.1: {} + events-universal@1.0.1: + dependencies: + bare-events: 2.9.1 + transitivePeerDependencies: + - bare-abort-controller + + fast-fifo@1.3.2: {} + fsevents@2.3.3: optional: true @@ -785,6 +891,39 @@ snapshots: transitivePeerDependencies: - zod + streamx@2.28.0: + dependencies: + events-universal: 1.0.1 + fast-fifo: 1.3.2 + text-decoder: 1.2.7 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + + tar-stream@3.2.0: + dependencies: + b4a: 1.8.1 + bare-fs: 4.7.4 + fast-fifo: 1.3.2 + streamx: 2.28.0 + transitivePeerDependencies: + - bare-abort-controller + - bare-buffer + - react-native-b4a + + teex@1.0.1: + dependencies: + streamx: 2.28.0 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + + text-decoder@1.2.7: + dependencies: + b4a: 1.8.1 + transitivePeerDependencies: + - react-native-b4a + tsx@4.22.4: dependencies: esbuild: 0.28.0 diff --git a/bench/scripts/verify-pier-agent.mts b/bench/scripts/verify-pier-agent.mts index add31075..cd5ae119 100644 --- a/bench/scripts/verify-pier-agent.mts +++ b/bench/scripts/verify-pier-agent.mts @@ -23,6 +23,7 @@ import type { Sha256Digest, } from '@tangle-network/agent-interface' import { + createAgentCandidateWorkspacePort, disposePreparedAgentCandidateExecution, FileAgentCandidateExecutionClaimStore, prepareAgentCandidateExecution, @@ -36,7 +37,6 @@ import { import { executePreparedPierCandidate } from '../src/pier-agent' import { createPierResultGrader } from '../src/pier-result-grader' import { FilePierCandidateTrialController } from '../src/pier-trial-controller' -import { materializePierWorkspaceArchive } from '../src/pier-workspace-archive' const pinnedPierCommit = 'e69a20e4e0ac073ec71fde0274bab3d9f40bac87' const pinnedPierVersion = '0.3.0' @@ -441,6 +441,7 @@ ${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner= maxCostUsd: 0, }, } + const workspaces = createAgentCandidateWorkspacePort() const ports: AgentCandidateExecutionPorts = { artifacts: { read: async () => { @@ -449,9 +450,14 @@ ${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner= }, repositories: { resolve: async () => taskRoot }, workspaces: { - materialize: async ({ snapshot, archive, destination }) => { + materialize: async ({ role, snapshot, archive, destination }) => { if (destination === taskRoot || destination === candidateRoot) return - await materializePierWorkspaceArchive({ archive, expected: snapshot.material, destination }) + await workspaces.materialize({ + role, + snapshot, + archive, + destination, + }) }, }, containers: { resolve: async () => container }, diff --git a/bench/src/index.ts b/bench/src/index.ts index 84c23efc..703118fd 100644 --- a/bench/src/index.ts +++ b/bench/src/index.ts @@ -73,10 +73,3 @@ export { type PierDockerConnection, } from './pier-trial-controller' export { createPierResultGrader } from './pier-result-grader' -export { - createPierWorkspaceArchive, - materializePierWorkspaceArchive, - type PierWorkspaceArchiveFile, - type PierWorkspaceArchiveV1, - type PierWorkspaceFile, -} from './pier-workspace-archive' diff --git a/bench/src/pier-agent.ts b/bench/src/pier-agent.ts index 7319b2b3..efd48343 100644 --- a/bench/src/pier-agent.ts +++ b/bench/src/pier-agent.ts @@ -592,6 +592,10 @@ export async function executePreparedPierCandidate( baseCommit: repository.baseCommit, baseTree: repository.baseTree, patch: result.taskPatch, + artifactPersistence: { + executionId: request.executionId, + outputArtifacts: options.outputArtifacts, + }, }) return { stopped: true, taskOutcome } }, diff --git a/bench/src/pier-task-outcome.test.mts b/bench/src/pier-task-outcome.test.mts index 55980ac7..2c6bcef6 100644 --- a/bench/src/pier-task-outcome.test.mts +++ b/bench/src/pier-task-outcome.test.mts @@ -1,12 +1,18 @@ import assert from 'node:assert/strict' import { execFileSync } from 'node:child_process' +import { createHash } from 'node:crypto' import { chmodSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import test from 'node:test' +import { + type AgentCandidateOutputArtifactPort, + captureAgentCandidateWorkspaceFiles, + createAgentCandidateWorkspacePort, +} from '@tangle-network/agent-runtime/candidate-execution' + import { capturePierTaskOutcome } from './pier-task-outcome' -import { materializePierWorkspaceArchive } from './pier-workspace-archive' test('Pier task outcome reconstructs the exact tree and deterministic archive from a patch', async () => { const root = mkdtempSync(join(tmpdir(), 'pier-task-outcome-test-')) @@ -28,10 +34,16 @@ test('Pier task outcome reconstructs the exact tree and deterministic archive fr const captured = await capturePierTaskOutcome({ repositoryRoot: root, baseCommit, baseTree, patch }) assert.notEqual(captured.resultTree, baseTree) assert.deepEqual(Buffer.from(captured.gitDiff), patch) + const expected = await captureAgentCandidateWorkspaceFiles([ + { path: 'src/status.txt', mode: 0o644, bytes: Buffer.from('ready\n') }, + ]) + assert.deepEqual(captured.afterState, expected.snapshot.material) + assert.deepEqual(Buffer.from(captured.archive), Buffer.from(expected.archive)) const destination = join(root, 'materialized') - await materializePierWorkspaceArchive({ + await createAgentCandidateWorkspacePort().materialize({ + role: 'candidate', archive: captured.archive, - expected: captured.afterState, + snapshot: expected.snapshot, destination, }) assert.equal(readFileSync(join(destination, 'src/status.txt'), 'utf8'), 'ready\n') @@ -65,6 +77,57 @@ test('Pier task outcome preserves the signed base for an empty patch', async () } }) +test('Pier task outcome externalizes workspace evidence above the embedded limit', async () => { + const root = mkdtempSync(join(tmpdir(), 'pier-task-outcome-test-')) + try { + const payload = Buffer.alloc(1024 * 1024 + 1, 7) + writeFileSync(join(root, 'large.bin'), payload) + git(root, ['init', '-b', 'main']) + git(root, ['config', 'user.email', 'fixture@example.com']) + git(root, ['config', 'user.name', 'Fixture']) + git(root, ['add', '-A']) + git(root, ['commit', '-m', 'base']) + const baseCommit = git(root, ['rev-parse', 'HEAD']) + const baseTree = git(root, ['rev-parse', 'HEAD^{tree}']) + const stored = new Map() + const outputArtifacts: AgentCandidateOutputArtifactPort = { + put: async ({ bytes, purpose }) => { + const detached = Uint8Array.from(bytes) + const digest = sha256(detached) + stored.set(digest, detached) + return { + locator: { + kind: 's3', + bucket: 'pier-test-artifacts', + key: `${purpose}/${digest}`, + }, + sha256: digest, + byteLength: detached.byteLength, + } + }, + read: async (ref) => Uint8Array.from(stored.get(ref.sha256) ?? []), + } + + const captured = await capturePierTaskOutcome({ + repositoryRoot: root, + baseCommit, + baseTree, + patch: Buffer.alloc(0), + artifactPersistence: { executionId: 'pier-large-1', outputArtifacts }, + }) + + assert.ok(captured.archive.byteLength > 1024 * 1024) + assert.equal(captured.afterState.files[0]?.byteLength, payload.byteLength) + assert.deepEqual(Buffer.from(stored.get(sha256(captured.archive)) ?? []), captured.archive) + } finally { + rmSync(root, { recursive: true, force: true }) + } +}) + +function sha256(bytes: Uint8Array): `sha256:${string}` { + return `sha256:${createHash('sha256').update(bytes).digest('hex')}` +} + function git(root: string, args: string[]): string { return execFileSync('git', ['-C', root, ...args], { encoding: 'utf8', diff --git a/bench/src/pier-task-outcome.ts b/bench/src/pier-task-outcome.ts index ced6c50c..9ae8b4c1 100644 --- a/bench/src/pier-task-outcome.ts +++ b/bench/src/pier-task-outcome.ts @@ -4,9 +4,11 @@ import { mkdir, mkdtemp, realpath, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join, resolve } from 'node:path' -import type { AgentCandidateExecutorTaskOutcomeCapture } from '@tangle-network/agent-runtime' - -import { createPierWorkspaceArchive } from './pier-workspace-archive' +import { + type AgentCandidateExecutorTaskOutcomeCapture, + type AgentCandidateOutputArtifactPort, + captureAgentCandidateWorkspaceFiles, +} from '@tangle-network/agent-runtime/candidate-execution' interface GitResult { readonly stdout: Buffer @@ -20,6 +22,10 @@ export async function capturePierTaskOutcome(input: { readonly baseTree: string readonly patch: Uint8Array readonly signal?: AbortSignal + readonly artifactPersistence?: { + readonly executionId: string + readonly outputArtifacts: AgentCandidateOutputArtifactPort + } }): Promise { input.signal?.throwIfAborted() const repositoryRoot = resolve(input.repositoryRoot) @@ -115,10 +121,21 @@ export async function capturePierTaskOutcome(input: { } }), ) - const captured = createPierWorkspaceArchive(files) + // The executor outcome must still return exact bytes; the runtime verifies + // and persists its final task references after this capture completes. + const captured = await captureAgentCandidateWorkspaceFiles(files, { + ...(input.artifactPersistence + ? { + artifactPersistence: { + ...input.artifactPersistence, + ...(input.signal ? { signal: input.signal } : {}), + }, + } + : {}), + }) return { resultTree, - afterState: captured.manifest, + afterState: captured.snapshot.material, archive: captured.archive, gitDiff: patch, } diff --git a/bench/src/pier-workspace-archive.test.mts b/bench/src/pier-workspace-archive.test.mts deleted file mode 100644 index 7fdb694f..00000000 --- a/bench/src/pier-workspace-archive.test.mts +++ /dev/null @@ -1,58 +0,0 @@ -import assert from 'node:assert/strict' -import { chmod, lstat, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import test from 'node:test' - -import { - createPierWorkspaceArchive, - materializePierWorkspaceArchive, -} from './pier-workspace-archive' - -test('Pier workspace archive round-trips exact bytes, paths, and modes', async () => { - const encoded = createPierWorkspaceArchive([ - { path: 'src/run.sh', mode: 0o755, bytes: Buffer.from('#!/bin/sh\n') }, - { path: 'README.md', mode: 0o644, bytes: Buffer.from('fixture\n') }, - ]) - const root = await mkdtemp(join(tmpdir(), 'pier-archive-test-')) - const destination = join(root, 'workspace') - try { - await materializePierWorkspaceArchive({ ...encoded, expected: encoded.manifest, destination }) - assert.equal(await readFile(join(destination, 'README.md'), 'utf8'), 'fixture\n') - assert.equal((await lstat(join(destination, 'src/run.sh'))).mode & 0o777, 0o755) - } finally { - await rm(root, { recursive: true, force: true }) - } -}) - -test('Pier workspace archive rejects unsafe, noncanonical, and nonempty materialization', async () => { - assert.throws( - () => createPierWorkspaceArchive([{ path: '../escape', mode: 0o644, bytes: Buffer.alloc(0) }]), - /unsafe/, - ) - const encoded = createPierWorkspaceArchive([ - { path: 'source.ts', mode: 0o644, bytes: Buffer.from('export {}\n') }, - ]) - const root = await mkdtemp(join(tmpdir(), 'pier-archive-test-')) - const destination = join(root, 'workspace') - try { - await writeFile(join(root, 'tampered.json'), Buffer.from(encoded.archive)) - const noncanonical = Buffer.from(`${Buffer.from(encoded.archive).toString('utf8')}\n`) - await assert.rejects( - materializePierWorkspaceArchive({ - archive: noncanonical, - expected: encoded.manifest, - destination, - }), - /not canonical/, - ) - await materializePierWorkspaceArchive({ ...encoded, expected: encoded.manifest, destination }) - await chmod(join(destination, 'source.ts'), 0o644) - await assert.rejects( - materializePierWorkspaceArchive({ ...encoded, expected: encoded.manifest, destination }), - /must be empty/, - ) - } finally { - await rm(root, { recursive: true, force: true }) - } -}) diff --git a/bench/src/pier-workspace-archive.ts b/bench/src/pier-workspace-archive.ts deleted file mode 100644 index 8a6a7d8f..00000000 --- a/bench/src/pier-workspace-archive.ts +++ /dev/null @@ -1,182 +0,0 @@ -/** Deterministic regular-file archive used for runtime/Pier task outcomes. */ -import { createHash } from 'node:crypto' -import { chmod, lstat, mkdir, readFile, readdir, realpath, writeFile } from 'node:fs/promises' -import { dirname, isAbsolute, join, resolve } from 'node:path' - -import { canonicalJson } from '@tangle-network/agent-eval' -import type { AgentCandidateWorkspaceManifestMaterialV1 } from '@tangle-network/agent-interface' - -export interface PierWorkspaceArchiveFile { - readonly path: string - readonly mode: 0o644 | 0o755 - readonly content: string - readonly sha256: `sha256:${string}` - readonly byteLength: number -} - -export interface PierWorkspaceArchiveV1 { - readonly schemaVersion: 1 - readonly kind: 'pier-workspace-archive' - readonly files: readonly PierWorkspaceArchiveFile[] -} - -export interface PierWorkspaceFile { - readonly path: string - readonly mode: 0o644 | 0o755 - readonly bytes: Uint8Array -} - -const sha256 = (bytes: Uint8Array): `sha256:${string}` => - `sha256:${createHash('sha256').update(bytes).digest('hex')}` - -/** Encode sorted, detached file bytes and return the exact matching runtime manifest. */ -export function createPierWorkspaceArchive(files: readonly PierWorkspaceFile[]): { - readonly archive: Uint8Array - readonly manifest: AgentCandidateWorkspaceManifestMaterialV1 -} { - const seen = new Set() - const ordered = [...files] - .map((file) => { - const path = safeRelativePath(file.path) - if (seen.has(path)) throw new Error(`Pier workspace archive repeats path ${path}`) - seen.add(path) - if (file.mode !== 0o644 && file.mode !== 0o755) { - throw new Error(`Pier workspace archive has unsupported mode for ${path}`) - } - const bytes = Uint8Array.from(file.bytes) - return { - path, - mode: file.mode, - content: Buffer.from(bytes).toString('base64'), - sha256: sha256(bytes), - byteLength: bytes.byteLength, - } - }) - .sort((left, right) => left.path.localeCompare(right.path)) - const document: PierWorkspaceArchiveV1 = { - schemaVersion: 1, - kind: 'pier-workspace-archive', - files: ordered, - } - return { - archive: Buffer.from(canonicalJson(document), 'utf8'), - manifest: { - schemaVersion: 1, - kind: 'agent-candidate-workspace-manifest', - files: ordered.map(({ path, mode, sha256, byteLength }) => ({ - path, - mode, - sha256, - byteLength, - })), - }, - } -} - -/** Decode only the canonical archive format and materialize its exact declared file set. */ -export async function materializePierWorkspaceArchive(input: { - readonly archive: Uint8Array - readonly expected: AgentCandidateWorkspaceManifestMaterialV1 - readonly destination: string -}): Promise { - const parsed = parseArchive(input.archive) - const observed = createPierWorkspaceArchive( - parsed.files.map((file) => ({ - path: file.path, - mode: file.mode, - bytes: Buffer.from(file.content, 'base64'), - })), - ) - if (!Buffer.from(observed.archive).equals(Buffer.from(input.archive))) { - throw new Error('Pier workspace archive is not canonical') - } - if (canonicalJson(observed.manifest) !== canonicalJson(input.expected)) { - throw new Error('Pier workspace archive does not match the signed manifest') - } - - const destination = resolve(input.destination) - await mkdir(destination, { recursive: true, mode: 0o700 }) - const stats = await lstat(destination) - if (!stats.isDirectory() || stats.isSymbolicLink() || (await realpath(destination)) !== destination) { - throw new Error('Pier workspace archive destination must be a real directory') - } - if ((await readdir(destination)).length !== 0) { - throw new Error('Pier workspace archive destination must be empty') - } - for (const file of parsed.files) { - const bytes = Buffer.from(file.content, 'base64') - if (bytes.byteLength !== file.byteLength || sha256(bytes) !== file.sha256) { - throw new Error(`Pier workspace archive content differs at ${file.path}`) - } - const output = join(destination, file.path) - await mkdir(dirname(output), { recursive: true, mode: 0o700 }) - await writeFile(output, bytes, { flag: 'wx', mode: 0o600 }) - await chmod(output, file.mode) - const written = await readFile(output) - if (!written.equals(bytes)) throw new Error(`Pier workspace archive write drifted at ${file.path}`) - } -} - -function parseArchive(bytes: Uint8Array): PierWorkspaceArchiveV1 { - let value: unknown - try { - value = JSON.parse(Buffer.from(bytes).toString('utf8')) - } catch (error) { - throw new Error('Pier workspace archive is not UTF-8 JSON', { cause: error }) - } - if (!value || typeof value !== 'object' || Array.isArray(value)) { - throw new Error('Pier workspace archive must be an object') - } - const record = value as Record - if ( - record.schemaVersion !== 1 || - record.kind !== 'pier-workspace-archive' || - !Array.isArray(record.files) || - Object.keys(record).sort().join(',') !== 'files,kind,schemaVersion' - ) { - throw new Error('Pier workspace archive has an invalid envelope') - } - const files = record.files.map((value, index) => parseFile(value, index)) - return { schemaVersion: 1, kind: 'pier-workspace-archive', files } -} - -function parseFile(value: unknown, index: number): PierWorkspaceArchiveFile { - if (!value || typeof value !== 'object' || Array.isArray(value)) { - throw new Error(`Pier workspace archive file ${index} must be an object`) - } - const file = value as Record - if ( - Object.keys(file).sort().join(',') !== 'byteLength,content,mode,path,sha256' || - typeof file.path !== 'string' || - (file.mode !== 0o644 && file.mode !== 0o755) || - typeof file.content !== 'string' || - typeof file.sha256 !== 'string' || - !/^sha256:[a-f0-9]{64}$/.test(file.sha256) || - !Number.isSafeInteger(file.byteLength) || - (file.byteLength as number) < 0 - ) { - throw new Error(`Pier workspace archive file ${index} is invalid`) - } - return { - path: safeRelativePath(file.path), - mode: file.mode, - content: file.content, - sha256: file.sha256 as `sha256:${string}`, - byteLength: file.byteLength as number, - } -} - -function safeRelativePath(value: string): string { - const parts = value.split('/') - if ( - !value || - isAbsolute(value) || - value.includes('\\') || - value.includes('\0') || - parts.some((part) => !part || part === '.' || part === '..') || - parts[0]?.toLowerCase() === '.git' - ) { - throw new Error(`Pier workspace archive path is unsafe: ${value}`) - } - return value -}