Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions bench/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@tangle-network/agent-bench",
"version": "0.2.2",
"version": "0.3.0",
"type": "module",
"description": "The unified benchmark suite for agent-runtime agents: 31 adapters (commit0, enterpriseops-gym, ragbench, crag, nomiracl, open-rag-bench, t2-ragbench, tau3-banking, bfcl, finresearchbench, …) behind one resolveAdapter registry, each with a real judge or fail-loud unsupported scorer. Score any profile/skill/prompt change against them. Map: bench/HARNESS.md.",
"repository": {
Expand All @@ -27,9 +27,9 @@
"verify:pier": "tsx scripts/verify-pier-pair.mts"
},
"dependencies": {
"@tangle-network/agent-eval": "^0.114.0",
"@tangle-network/agent-eval": "^0.115.2",
"@tangle-network/agent-interface": "^0.25.0",
"@tangle-network/agent-runtime": "^0.92.0",
"@tangle-network/agent-runtime": "0.94.6",
"@tangle-network/sandbox": "^0.10.3"
},
"devDependencies": {
Expand All @@ -39,7 +39,7 @@
},
"pnpm": {
"overrides": {
"@tangle-network/agent-eval": "0.114.0"
"@tangle-network/agent-eval": "0.115.2"
}
},
"files": [
Expand Down
185 changes: 162 additions & 23 deletions bench/pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 9 additions & 3 deletions bench/scripts/verify-pier-agent.mts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import type {
Sha256Digest,
} from '@tangle-network/agent-interface'
import {
createAgentCandidateWorkspacePort,
disposePreparedAgentCandidateExecution,
FileAgentCandidateExecutionClaimStore,
prepareAgentCandidateExecution,
Expand All @@ -36,7 +37,6 @@ import {
import { executePreparedPierCandidate } from '../src/pier-agent'
import { createPierResultGrader } from '../src/pier-result-grader'
import { FilePierCandidateTrialController } from '../src/pier-trial-controller'
import { materializePierWorkspaceArchive } from '../src/pier-workspace-archive'

const pinnedPierCommit = 'e69a20e4e0ac073ec71fde0274bab3d9f40bac87'
const pinnedPierVersion = '0.3.0'
Expand Down Expand Up @@ -441,6 +441,7 @@ ${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner=
maxCostUsd: 0,
},
}
const workspaces = createAgentCandidateWorkspacePort()
const ports: AgentCandidateExecutionPorts = {
artifacts: {
read: async () => {
Expand All @@ -449,9 +450,14 @@ ${proofArm === 'success' ? "(task / 'src/status.txt').write_text('ready\\nowner=
},
repositories: { resolve: async () => taskRoot },
workspaces: {
materialize: async ({ snapshot, archive, destination }) => {
materialize: async ({ role, snapshot, archive, destination }) => {
if (destination === taskRoot || destination === candidateRoot) return
await materializePierWorkspaceArchive({ archive, expected: snapshot.material, destination })
await workspaces.materialize({
role,
snapshot,
archive,
destination,
})
},
},
containers: { resolve: async () => container },
Expand Down
7 changes: 0 additions & 7 deletions bench/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -73,10 +73,3 @@ export {
type PierDockerConnection,
} from './pier-trial-controller'
export { createPierResultGrader } from './pier-result-grader'
export {
createPierWorkspaceArchive,
materializePierWorkspaceArchive,
type PierWorkspaceArchiveFile,
type PierWorkspaceArchiveV1,
type PierWorkspaceFile,
} from './pier-workspace-archive'
4 changes: 4 additions & 0 deletions bench/src/pier-agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -592,6 +592,10 @@ export async function executePreparedPierCandidate(
baseCommit: repository.baseCommit,
baseTree: repository.baseTree,
patch: result.taskPatch,
artifactPersistence: {
executionId: request.executionId,
outputArtifacts: options.outputArtifacts,
},
})
return { stopped: true, taskOutcome }
},
Expand Down
Loading
Loading