From e5dcf41e6ecf9176fd7306680cee343cecdd50f5 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 10 Jul 2026 15:54:04 -0600 Subject: [PATCH] feat(analyst): type every agent improvement surface --- CHANGELOG.md | 11 ++ clients/python/pyproject.toml | 2 +- clients/python/src/agent_eval_rpc/__init__.py | 2 +- clients/python/uv.lock | 2 +- package.json | 4 +- pnpm-lock.yaml | 33 +--- src/analyst/finding-subject.test.ts | 58 ++++++ src/analyst/finding-subject.ts | 172 +++++++++++++++--- src/analyst/index.ts | 2 + src/analyst/kinds/failure-mode.ts | 7 +- src/analyst/kinds/improvement.ts | 10 +- src/analyst/kinds/knowledge-gap.ts | 18 +- src/analyst/kinds/knowledge-poisoning.ts | 7 +- src/analyst/policy-edit.ts | 56 ++++++ tests/policy-edit.test.ts | 15 ++ 15 files changed, 328 insertions(+), 71 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90e50842..32bc1c9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- --- +## [0.112.0] — 2026-07-10 — complete agent-surface findings + +### Added + +- Added typed finding subjects for skills, MCP servers and tools, hooks, subagents, workflows, rollout policy, generic agent-profile fields, and code paths. +- Routed every new subject into a typed policy edit so products do not need local string classifiers. + +### Changed + +- Updated the direct `@tangle-network/agent-interface` dependency from `^0.10.0` to `^0.22.0`. + ## [0.111.0] — 2026-07-09 — repository-clustered paired inference ### Added diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 687c6007..59d4b596 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agent-eval-rpc" -version = "0.111.0" +version = "0.112.0" description = "Python RPC client for @tangle-network/agent-eval — judge content against rubrics over HTTP or stdio RPC. Eval logic runs in the Node runtime; this package is a thin wire client." readme = "README.md" requires-python = ">=3.10" diff --git a/clients/python/src/agent_eval_rpc/__init__.py b/clients/python/src/agent_eval_rpc/__init__.py index 5d63aab6..03ed89b8 100644 --- a/clients/python/src/agent_eval_rpc/__init__.py +++ b/clients/python/src/agent_eval_rpc/__init__.py @@ -58,7 +58,7 @@ try: __version__ = version("agent-eval-rpc") except PackageNotFoundError: - __version__ = "0.111.0" + __version__ = "0.112.0" __all__ = [ "Client", diff --git a/clients/python/uv.lock b/clients/python/uv.lock index 5af1ec79..a3a87c32 100644 --- a/clients/python/uv.lock +++ b/clients/python/uv.lock @@ -4,7 +4,7 @@ requires-python = ">=3.10" [[package]] name = "agent-eval-rpc" -version = "0.111.0" +version = "0.112.0" source = { editable = "." } dependencies = [ { name = "httpx" }, diff --git a/package.json b/package.json index bb4041db..60123c4b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-eval", - "version": "0.111.0", + "version": "0.112.0", "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.", "homepage": "https://github.com/tangle-network/agent-eval#readme", "repository": { @@ -146,7 +146,7 @@ "@asteasolutions/zod-to-openapi": "^8.5.0", "@ax-llm/ax": "^19.0.25", "@hono/node-server": "^2.0.0", - "@tangle-network/agent-interface": "^0.10.0", + "@tangle-network/agent-interface": "^0.22.0", "@tangle-network/tcloud": "^0.4.14", "hono": "^4.12.16", "zod": "^4.3.6" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 54859d2d..8a911577 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -22,8 +22,8 @@ importers: specifier: ^2.0.0 version: 2.0.0(hono@4.12.18) '@tangle-network/agent-interface': - specifier: ^0.10.0 - version: 0.10.0 + specifier: ^0.22.0 + version: 0.22.0 '@tangle-network/tcloud': specifier: ^0.4.14 version: 0.4.14(typescript@5.9.3)(zod@4.3.6) @@ -100,28 +100,24 @@ packages: engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] - libc: [musl] '@biomejs/cli-linux-arm64@2.4.15': resolution: {integrity: sha512-owaAMZD/T4LrD0ELNCk0Km3qrRHuM0X6EAyVE1FSqGY0rbLoiDLrO4Us2tllm6cAeB2Ioa9C2C08NZPdr8+0Ug==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] - libc: [glibc] '@biomejs/cli-linux-x64-musl@2.4.15': resolution: {integrity: sha512-CNq/9W38SYSH023lfcQ4KKU8K0YX8T//FZUhcgtMMRABDojx5XsMV7jlweAvGSl389wJQB29Qo6Zb/a+jdvt+w==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] - libc: [musl] '@biomejs/cli-linux-x64@2.4.15': resolution: {integrity: sha512-0jj7THz12GbUOLmMibktK6DZjqz2zV64KFxyBtcFTKPiiOIY0a7vns1elpO1dERvxpsZ5ik0oFfz0oGwFde1+g==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] - libc: [glibc] '@biomejs/cli-win32-arm64@2.4.15': resolution: {integrity: sha512-ouhkYdlhp/1GghEJPdWwD/Vi3gQ1nFxuSpMolWsbq3Lsq3QUR4jl6UdhhscdCugKU5vOEuMiJhvKj66O0OCq+w==} @@ -368,79 +364,66 @@ packages: resolution: {integrity: sha512-L+34Qqil+v5uC0zEubW7uByo78WOCIrBvci69E7sFASRl0X7b/MB6Cqd1lky/CtcSVTydWa2WZwFuWexjS5o6g==} cpu: [arm] os: [linux] - libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.60.1': resolution: {integrity: sha512-n83O8rt4v34hgFzlkb1ycniJh7IR5RCIqt6mz1VRJD6pmhRi0CXdmfnLu9dIUS6buzh60IvACM842Ffb3xd6Gg==} cpu: [arm] os: [linux] - libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.60.1': resolution: {integrity: sha512-Nql7sTeAzhTAja3QXeAI48+/+GjBJ+QmAH13snn0AJSNL50JsDqotyudHyMbO2RbJkskbMbFJfIJKWA6R1LCJQ==} cpu: [arm64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.60.1': resolution: {integrity: sha512-+pUymDhd0ys9GcKZPPWlFiZ67sTWV5UU6zOJat02M1+PiuSGDziyRuI/pPue3hoUwm2uGfxdL+trT6Z9rxnlMA==} cpu: [arm64] os: [linux] - libc: [musl] '@rollup/rollup-linux-loong64-gnu@4.60.1': resolution: {integrity: sha512-VSvgvQeIcsEvY4bKDHEDWcpW4Yw7BtlKG1GUT4FzBUlEKQK0rWHYBqQt6Fm2taXS+1bXvJT6kICu5ZwqKCnvlQ==} cpu: [loong64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-loong64-musl@4.60.1': resolution: {integrity: sha512-4LqhUomJqwe641gsPp6xLfhqWMbQV04KtPp7/dIp0nzPxAkNY1AbwL5W0MQpcalLYk07vaW9Kp1PBhdpZYYcEw==} cpu: [loong64] os: [linux] - libc: [musl] '@rollup/rollup-linux-ppc64-gnu@4.60.1': resolution: {integrity: sha512-tLQQ9aPvkBxOc/EUT6j3pyeMD6Hb8QF2BTBnCQWP/uu1lhc9AIrIjKnLYMEroIz/JvtGYgI9dF3AxHZNaEH0rw==} cpu: [ppc64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-ppc64-musl@4.60.1': resolution: {integrity: sha512-RMxFhJwc9fSXP6PqmAz4cbv3kAyvD1etJFjTx4ONqFP9DkTkXsAMU4v3Vyc5BgzC+anz7nS/9tp4obsKfqkDHg==} cpu: [ppc64] os: [linux] - libc: [musl] '@rollup/rollup-linux-riscv64-gnu@4.60.1': resolution: {integrity: sha512-QKgFl+Yc1eEk6MmOBfRHYF6lTxiiiV3/z/BRrbSiW2I7AFTXoBFvdMEyglohPj//2mZS4hDOqeB0H1ACh3sBbg==} cpu: [riscv64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.60.1': resolution: {integrity: sha512-RAjXjP/8c6ZtzatZcA1RaQr6O1TRhzC+adn8YZDnChliZHviqIjmvFwHcxi4JKPSDAt6Uhf/7vqcBzQJy0PDJg==} cpu: [riscv64] os: [linux] - libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.60.1': resolution: {integrity: sha512-wcuocpaOlaL1COBYiA89O6yfjlp3RwKDeTIA0hM7OpmhR1Bjo9j31G1uQVpDlTvwxGn2nQs65fBFL5UFd76FcQ==} cpu: [s390x] os: [linux] - libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.60.1': resolution: {integrity: sha512-77PpsFQUCOiZR9+LQEFg9GClyfkNXj1MP6wRnzYs0EeWbPcHs02AXu4xuUbM1zhwn3wqaizle3AEYg5aeoohhg==} cpu: [x64] os: [linux] - libc: [glibc] '@rollup/rollup-linux-x64-musl@4.60.1': resolution: {integrity: sha512-5cIATbk5vynAjqqmyBjlciMJl1+R/CwX9oLk/EyiFXDWd95KpHdrOJT//rnUl4cUcskrd0jCCw3wpZnhIHdD9w==} cpu: [x64] os: [linux] - libc: [musl] '@rollup/rollup-openbsd-x64@4.60.1': resolution: {integrity: sha512-cl0w09WsCi17mcmWqqglez9Gk8isgeWvoUZ3WiJFYSR3zjBQc2J5/ihSjpl+VLjPqjQ/1hJRcqBfLjssREQILw==} @@ -493,15 +476,15 @@ packages: '@tangle-network/agent-core@0.3.8': resolution: {integrity: sha512-bZfVpdiFjXbcQwSxSQABdtXEmUuapWChCcXrHkP6fAnwqEy9hWEfeZLlMZUy+1XaOfpTaMLUOEAwYhXfN018wQ==} - '@tangle-network/agent-interface@0.10.0': - resolution: {integrity: sha512-oiREgihkeX/xcGEtFfi9AkAfU2VzuF7SSla2s0iliXPUXyHCIIx6jwzHiYdwb1ZGCfvC+T+0SWOIa6fN5u195g==} - '@tangle-network/agent-interface@0.13.0': resolution: {integrity: sha512-CeTPGRLoXqpt0h+BCyFgZPkfU1zyRpWmqfD+85i/uk+uvbqxkfI+JprfKVf3tBsQuCgJPSjPt5qjdW8n3h2BVg==} '@tangle-network/agent-interface@0.17.1': resolution: {integrity: sha512-B7dRJTo0HSUtgBCB1VMwkTFYkLUaRr/4BcRglrQuGhGUwOzKv1RYyMejOVh5M3a5AagY9N79f7GYbjcA3UmnIA==} + '@tangle-network/agent-interface@0.22.0': + resolution: {integrity: sha512-7fsJhNdvTmOB1X9E2owl06jzyrqaN+jMkOPVKbK7dvNqQvf627PowCtL/edhUqEEv+K0FWtkwG3R3xqjX7VNZg==} + '@tangle-network/sandbox@0.9.5': resolution: {integrity: sha512-yvX2OX6uISBVnMQ+v6Upkesa3u8yj6BHxsfcS6p8Vze+M4WBpyhkwA+onzFHuo9rti557ItZn8yDu4a/klljvQ==} peerDependencies: @@ -1386,15 +1369,15 @@ snapshots: '@tangle-network/agent-interface': 0.17.1 zod: 4.4.3 - '@tangle-network/agent-interface@0.10.0': + '@tangle-network/agent-interface@0.13.0': dependencies: zod: 4.4.3 - '@tangle-network/agent-interface@0.13.0': + '@tangle-network/agent-interface@0.17.1': dependencies: zod: 4.4.3 - '@tangle-network/agent-interface@0.17.1': + '@tangle-network/agent-interface@0.22.0': dependencies: zod: 4.4.3 diff --git a/src/analyst/finding-subject.test.ts b/src/analyst/finding-subject.test.ts index c77440cd..baaf4c16 100644 --- a/src/analyst/finding-subject.test.ts +++ b/src/analyst/finding-subject.test.ts @@ -1,11 +1,19 @@ import { describe, expect, it } from 'vitest' import { FINDING_SUBJECT_KINDS, + FINDING_SUBJECT_SYNTAX, type FindingSubject, + findingSubjectGrammarPromptFor, KIND_EXPECTED_SUBJECTS, parseFindingSubject, renderFindingSubject, } from './finding-subject' +import { + FAILURE_MODE_KIND_SPEC, + IMPROVEMENT_KIND_SPEC, + KNOWLEDGE_GAP_KIND_SPEC, + KNOWLEDGE_POISONING_KIND_SPEC, +} from './kinds' describe('parseFindingSubject — knowledge loci', () => { it('parses agent-knowledge:wiki:', () => { @@ -91,6 +99,30 @@ describe('parseFindingSubject — runtime surfaces', () => { }) }) + it.each([ + ['skill:linear-triage', { kind: 'skill', name: 'linear-triage' }], + ['mcp:linear', { kind: 'mcp', server: 'linear' }], + ['mcp:linear:get_issue', { kind: 'mcp', server: 'linear', tool: 'get_issue' }], + ['hook:post-tool-use', { kind: 'hook', name: 'post-tool-use' }], + ['subagent:reviewer', { kind: 'subagent', name: 'reviewer' }], + ['workflow:linear-ticket', { kind: 'workflow', name: 'linear-ticket' }], + ['rollout-policy:maxTurns', { kind: 'rollout-policy', field: 'maxTurns' }], + ['agent-profile:model', { kind: 'agent-profile', field: 'model' }], + ['code:src/dispatch.ts', { kind: 'code', path: 'src/dispatch.ts' }], + ])('parses %s', (raw, expected) => { + expect(parseFindingSubject(raw)).toEqual(expected) + }) + + it.each([ + 'skill:CodeReview', + 'mcp:Linear', + 'hook:', + 'subagent:', + 'workflow:', + ])('rejects malformed runtime subject %s', (raw) => { + expect(parseFindingSubject(raw)).toBeNull() + }) + it('parses rag::', () => { expect(parseFindingSubject('rag:irs-rulings:rev-rul-2024-12')).toEqual({ kind: 'rag', @@ -244,9 +276,18 @@ describe('renderFindingSubject', () => { { kind: 'knowledge.raw', sourceId: 'irs-pub-501-2024' }, { kind: 'knowledge.stale', slug: 'old-vat-rates' }, { kind: 'system-prompt', section: 'request-classification' }, + { kind: 'skill', name: 'linear-triage' }, { kind: 'tool-doc', tool: 'list_invoices' }, { kind: 'tool-doc', tool: 'list_invoices', aspect: 'examples' }, { kind: 'new-tool', name: 'diff_csv' }, + { kind: 'mcp', server: 'linear' }, + { kind: 'mcp', server: 'linear', tool: 'get_issue' }, + { kind: 'hook', name: 'post-tool-use' }, + { kind: 'subagent', name: 'reviewer' }, + { kind: 'workflow', name: 'linear-ticket' }, + { kind: 'rollout-policy', field: 'maxTurns' }, + { kind: 'agent-profile', field: 'model' }, + { kind: 'code', path: 'src/dispatch.ts' }, { kind: 'rag', corpus: 'irs-rulings', docId: 'rev-rul-2024-12' }, { kind: 'memory', key: 'last-customer-id' }, { kind: 'scaffolding', concern: 'retry-policy' }, @@ -293,4 +334,21 @@ describe('KIND_EXPECTED_SUBJECTS', () => { expect(improvement).not.toContain('websearch.outdated') expect(improvement).not.toContain('prior-run-summary') }) + + it.each([ + ['failure-mode', FAILURE_MODE_KIND_SPEC], + ['improvement', IMPROVEMENT_KIND_SPEC], + ['knowledge-gap', KNOWLEDGE_GAP_KIND_SPEC], + ['knowledge-poisoning', KNOWLEDGE_POISONING_KIND_SPEC], + ])('%s actor prompt embeds its complete allowed grammar', (kindId, spec) => { + const grammar = findingSubjectGrammarPromptFor(kindId) + expect(spec.actorDescription).toContain(grammar) + for (const subjectKind of KIND_EXPECTED_SUBJECTS[kindId] ?? []) { + expect(grammar).toContain(FINDING_SUBJECT_SYNTAX[subjectKind]) + } + }) + + it('rejects an unknown analyst kind instead of emitting an empty grammar', () => { + expect(() => findingSubjectGrammarPromptFor('unknown')).toThrow(/unknown analyst kind/) + }) }) diff --git a/src/analyst/finding-subject.ts b/src/analyst/finding-subject.ts index cc0c16ad..70399540 100644 --- a/src/analyst/finding-subject.ts +++ b/src/analyst/finding-subject.ts @@ -53,8 +53,16 @@ export type FindingSubject = // ── system-prompt / tool / new-tool / rag / memory / scaffolding / output-schema ── // routed to the ImprovementAdapter | { kind: 'system-prompt'; section: string } + | { kind: 'skill'; name: string } | { kind: 'tool-doc'; tool: string; aspect?: string } | { kind: 'new-tool'; name: string } + | { kind: 'mcp'; server: string; tool?: string } + | { kind: 'hook'; name: string } + | { kind: 'subagent'; name: string } + | { kind: 'workflow'; name: string } + | { kind: 'rollout-policy'; field: string } + | { kind: 'agent-profile'; field: string } + | { kind: 'code'; path: string } | { kind: 'rag'; corpus: string; docId: string } | { kind: 'memory'; key: string } | { kind: 'scaffolding'; concern: string } @@ -73,8 +81,16 @@ export const FINDING_SUBJECT_KINDS: ReadonlyArray = [ 'knowledge.raw', 'knowledge.stale', 'system-prompt', + 'skill', 'tool-doc', 'new-tool', + 'mcp', + 'hook', + 'subagent', + 'workflow', + 'rollout-policy', + 'agent-profile', + 'code', 'rag', 'memory', 'scaffolding', @@ -132,6 +148,10 @@ export function parseFindingSubject(raw: string | null | undefined): FindingSubj const sp = trimmed.match(/^system-prompt:(.+)$/) if (sp && sp[1]!.trim().length > 0) return { kind: 'system-prompt', section: sp[1]!.trim() } + // skill: + const skill = trimmed.match(/^skill:([a-z0-9][a-z0-9_.-]*)$/) + if (skill) return { kind: 'skill', name: skill[1]! } + // tool-doc:[:] const tdAspect = trimmed.match(/^tool-doc:([a-z0-9][a-z0-9_-]*):(.+)$/) if (tdAspect && tdAspect[2]!.trim().length > 0) { @@ -144,6 +164,34 @@ export function parseFindingSubject(raw: string | null | undefined): FindingSubj const nt = trimmed.match(/^new-tool:([a-z0-9][a-z0-9_-]*)$/) if (nt) return { kind: 'new-tool', name: nt[1]! } + // mcp:[:] + const mcp = trimmed.match(/^mcp:([a-z0-9][a-z0-9_.-]*)(?::([a-z0-9][a-z0-9_.-]*))?$/) + if (mcp) { + return { kind: 'mcp', server: mcp[1]!, ...(mcp[2] ? { tool: mcp[2] } : {}) } + } + + // hook / subagent / workflow: + const hook = trimmed.match(/^hook:([a-z0-9][a-z0-9_.-]*)$/) + if (hook) return { kind: 'hook', name: hook[1]! } + const subagent = trimmed.match(/^subagent:([a-z0-9][a-z0-9_.-]*)$/) + if (subagent) return { kind: 'subagent', name: subagent[1]! } + const workflow = trimmed.match(/^workflow:([a-z0-9][a-z0-9_.-]*)$/) + if (workflow) return { kind: 'workflow', name: workflow[1]! } + + // rollout-policy / agent-profile: + const rolloutPolicy = trimmed.match(/^rollout-policy:(.+)$/) + if (rolloutPolicy && rolloutPolicy[1]!.trim().length > 0) { + return { kind: 'rollout-policy', field: rolloutPolicy[1]!.trim() } + } + const agentProfile = trimmed.match(/^agent-profile:(.+)$/) + if (agentProfile && agentProfile[1]!.trim().length > 0) { + return { kind: 'agent-profile', field: agentProfile[1]!.trim() } + } + + // code: + const code = trimmed.match(/^code:(.+)$/) + if (code && code[1]!.trim().length > 0) return { kind: 'code', path: code[1]!.trim() } + // rag:: const rag = trimmed.match(/^rag:([a-z0-9][a-z0-9_-]*):(.+)$/) if (rag && rag[2]!.trim().length > 0) { @@ -200,10 +248,26 @@ export function renderFindingSubject(s: FindingSubject): string { return `agent-knowledge:stale:${s.slug}` case 'system-prompt': return `system-prompt:${s.section}` + case 'skill': + return `skill:${s.name}` case 'tool-doc': return s.aspect ? `tool-doc:${s.tool}:${s.aspect}` : `tool-doc:${s.tool}` case 'new-tool': return `new-tool:${s.name}` + case 'mcp': + return s.tool ? `mcp:${s.server}:${s.tool}` : `mcp:${s.server}` + case 'hook': + return `hook:${s.name}` + case 'subagent': + return `subagent:${s.name}` + case 'workflow': + return `workflow:${s.name}` + case 'rollout-policy': + return `rollout-policy:${s.field}` + case 'agent-profile': + return `agent-profile:${s.field}` + case 'code': + return `code:${s.path}` case 'rag': return `rag:${s.corpus}:${s.docId}` case 'memory': @@ -232,33 +296,65 @@ export function renderFindingSubject(s: FindingSubject): string { * this constant + the matching `expects` set, and the unit tests below * lock the table to the parser. */ -export const FINDING_SUBJECT_GRAMMAR_PROMPT = [ - 'Subjects MUST match this grammar — anything else is rejected at parse time and your work is wasted:', - '', - ' Knowledge loci (write to the agent-knowledge base):', - ' agent-knowledge:wiki:[#] create / update a wiki page', - ' agent-knowledge:claim: draft a claim / relation triple', - ' agent-knowledge:raw: lift a raw source into a curated page', - ' agent-knowledge:stale: mark a page superseded', - '', - ' Runtime mutable surfaces (write to prompts / tools / scaffolding):', - ' system-prompt:
add / replace a system-prompt section', - ' tool-doc:[:] rewrite a tool description', - ' new-tool: propose a new tool surface', - ' rag:: ingest / correct a RAG document', - ' memory: invalidate / set a memory entry', - ' scaffolding: change a precondition / retry / verifier', - ' output-schema: constrain the agent output shape', - '', - ' Stale signals (knowledge-poisoning only):', - ' websearch:outdated: stale web result', - ' prior-run-summary: stale prior-run summary', - '', - ' Cluster label (failure-mode only):', - ' short cluster id, e.g. "tool-call-loop"', - '', - 'Slugs / tool ids: [a-z0-9-]+ (lowercase kebab). Topics / keys / sections: free-form, trimmed.', -].join('\n') +export const FINDING_SUBJECT_SYNTAX: Readonly> = { + 'knowledge.wiki': 'agent-knowledge:wiki:[#]', + 'knowledge.claim': 'agent-knowledge:claim:', + 'knowledge.raw': 'agent-knowledge:raw:', + 'knowledge.stale': 'agent-knowledge:stale:', + 'system-prompt': 'system-prompt:
', + skill: 'skill:', + 'tool-doc': 'tool-doc:[:]', + 'new-tool': 'new-tool:', + mcp: 'mcp:[:]', + hook: 'hook:', + subagent: 'subagent:', + workflow: 'workflow:', + 'rollout-policy': 'rollout-policy:', + 'agent-profile': 'agent-profile:', + code: 'code:', + rag: 'rag::', + memory: 'memory:', + scaffolding: 'scaffolding:', + 'output-schema': 'output-schema:', + 'websearch.outdated': 'websearch:outdated:', + 'prior-run-summary': 'prior-run-summary:', + cluster: '', +} + +const FINDING_SUBJECT_PURPOSE: Readonly> = { + 'knowledge.wiki': 'create or update a wiki page', + 'knowledge.claim': 'draft a claim or relation', + 'knowledge.raw': 'curate a raw source', + 'knowledge.stale': 'mark a stale page', + 'system-prompt': 'revise a system-prompt section', + skill: 'create or revise a skill', + 'tool-doc': 'revise a tool contract', + 'new-tool': 'propose a new tool', + mcp: 'revise an MCP server or tool', + hook: 'revise a lifecycle hook', + subagent: 'revise a delegated agent', + workflow: 'revise an orchestration workflow', + 'rollout-policy': 'revise budget, sampling, or stop policy', + 'agent-profile': 'revise another AgentProfile field', + code: 'revise an implementation path', + rag: 'ingest or correct a RAG document', + memory: 'invalidate or set memory', + scaffolding: 'revise preconditions, retries, or verification', + 'output-schema': 'constrain the output shape', + 'websearch.outdated': 'identify a stale web result', + 'prior-run-summary': 'identify a stale prior-run summary', + cluster: 'name one failure cluster', +} + +function renderFindingSubjectGrammar(kinds: ReadonlyArray): string { + return [ + 'Subjects MUST match one of these forms — anything else is rejected at parse time:', + ...kinds.map((kind) => ` ${FINDING_SUBJECT_SYNTAX[kind]} — ${FINDING_SUBJECT_PURPOSE[kind]}`), + 'Runtime ids are lowercase [a-z0-9_.-]+. Topics, keys, paths, and sections are free-form and trimmed.', + ].join('\n') +} + +export const FINDING_SUBJECT_GRAMMAR_PROMPT = renderFindingSubjectGrammar(FINDING_SUBJECT_KINDS) // ── kind expects sets ───────────────────────────────────────────────── @@ -279,6 +375,10 @@ export const KIND_EXPECTED_SUBJECTS: Record/\` of the most representative span diff --git a/src/analyst/kinds/improvement.ts b/src/analyst/kinds/improvement.ts index 1a30cb43..3ee0b713 100644 --- a/src/analyst/kinds/improvement.ts +++ b/src/analyst/kinds/improvement.ts @@ -19,13 +19,18 @@ * picking the highest-leverage one to recommend. */ +import { findingSubjectGrammarPromptFor } from '../finding-subject' import type { TraceAnalystKindSpec } from '../kind-factory' import { buildTraceToolsForGroup } from '../tool-groups' +const subjectGrammar = findingSubjectGrammarPromptFor('improvement') + const ACTOR_PROMPT = `You are a recursive-self-improvement analyst. Your job is to propose **concrete, locus-named edits** the agent's runtime should adopt to fix the failure modes, knowledge gaps, and poisonings present in this dataset. Upstream analysts have already classified the problems. Your job is to convert each problem into a *change to make* and grade its expected leverage. Each finding is one proposed edit. +${subjectGrammar} + DISCOVERY → CANDIDATE-FIXES → COMPETE → CITE protocol: 1. \`traces.getDatasetOverview({})\` first. Note the agents, tools, and any system-prompt fingerprints (look for the prompt text echoed in early spans). @@ -37,13 +42,16 @@ DISCOVERY → CANDIDATE-FIXES → COMPETE → CITE protocol: - **Memory invalidation** — clear cached prior-run decisions that no longer apply - **Scaffolding** — add a precondition check, a retry policy, a turn budget, a verification step - **Output schema** — narrow the agent's output to forbid the failure shape + - **Skill / MCP / hook / subagent** — change the reusable profile component responsible for the behavior + - **Workflow / rollout policy** — change orchestration, budget, sampling, or stopping behavior + - **Code** — change an implementation path when profile edits cannot repair the behavior 3. **Compete candidate fixes via subagents.** For each failure cluster, spawn one \`llmQuery\` per candidate-fix axis you want to evaluate. Each subagent's job: simulate the fix on the cited traces and report (i) likely effect, (ii) side effects, (iii) implementation cost as small/medium/large. Pass the cluster's failing trace_ids and the candidate axis as context. 4. After subagents return, **pick the winning candidate per cluster** based on (effect / cost) and emit ONE finding. Discard the losing candidates — the output is the recommendation, not the candidate set. 5. **Cross-reference upstream findings.** If a finding cites a prior failure-mode or knowledge-gap finding, use \`evidence_uri = "finding://"\` (the registry supports this kind). This builds the dependency graph that lets the dashboard show "fix #X resolves failure modes A, B, C." For each winning recommendation, emit ONE finding with: - \`area\` = "improvement" -- \`subject\` = the locus to edit: \`system-prompt:
\`, \`tool-doc:\`, \`new-tool:\`, \`rag::\`, \`memory:\`, \`scaffolding:\`, \`output-schema:\` +- \`subject\` = one exact locus form from the subject grammar above - \`claim\` = one sentence stating the edit ("Add a precondition check to refuse tool X calls without arg Y") - \`severity\` = leverage rating: "critical" when fix resolves a critical failure mode; "high" when it resolves a high; "medium" when it's a quality-of-life win; "info" when it's a cleanup with no behavioral effect - \`evidence_uri\` = the failure-mode finding id this fix targets (\`finding://\`) when it exists; else the most representative span diff --git a/src/analyst/kinds/knowledge-gap.ts b/src/analyst/kinds/knowledge-gap.ts index 365d7862..c1a3dc02 100644 --- a/src/analyst/kinds/knowledge-gap.ts +++ b/src/analyst/kinds/knowledge-gap.ts @@ -24,13 +24,18 @@ * candidate gap-source layer; each subagent runs a focused detection. */ +import { findingSubjectGrammarPromptFor } from '../finding-subject' import type { TraceAnalystKindSpec } from '../kind-factory' import { buildTraceToolsForGroup } from '../tool-groups' +const subjectGrammar = findingSubjectGrammarPromptFor('knowledge-gap') + const ACTOR_PROMPT = `You are a knowledge-gap analyst for an OTLP trace dataset. Your job is to identify the **specific pieces of information the agent lacked, or that were stale**, that caused poor decisions. The agent under analysis maintains a curated knowledge base via \`@tangle-network/agent-knowledge\` — a wiki of \`KnowledgePage\`s with raw source anchors, claims, and relations. The primary expected store of agent-knowable facts IS that wiki. A "knowledge gap" is anything the agent had to discover or guess at run-time that the wiki should have held — or an outdated/contradictory fact the agent picked up from a non-wiki source. +${subjectGrammar} + DISCOVERY → ATTRIBUTE-TO-LAYER → CITE protocol: 1. \`traces.getDatasetOverview({})\` first. Note which agents, tools, and models appear. @@ -43,19 +48,10 @@ DISCOVERY → ATTRIBUTE-TO-LAYER → CITE protocol: - Agent quoting a tool's docs / system prompt incorrectly because the actual text was insufficient - Fabricated identifiers that don't appear in dataset \`sample_trace_ids\` Use \`traces.searchTrace\` with patterns like \`I (don.?t|do not) know\`, \`assumed\`, \`unclear\`, \`could you (clarify|tell me|provide)\`, \`not found\`, \`undefined\`, \`unknown\`, \`null\`, dates older than the analysis window, or the agent's specific clarification phrases. -3. For each gap, identify the **layer of the runtime that should have prevented it**. The locus is the value of \`subject\` on the finding. Use one of: - - \`agent-knowledge:wiki:\` — the wiki page that should exist but doesn't, or exists but lacks the claim - - \`agent-knowledge:wiki:#\` — wiki page exists but a specific section is missing - - \`agent-knowledge:claim:\` — a specific claim/relation triple that should be in the wiki - - \`agent-knowledge:raw:\` — raw source captured but never lifted into a curated page - - \`agent-knowledge:stale:\` — wiki page exists but contradicts ground-truth evidence in this trace (the wiki itself drifted) - - \`websearch:outdated:\` — agent relied on a web result that was stale; wiki should have superseded it - - \`tool-doc::\` — tool description missed a behavior aspect (return shape, failure modes, side effects) - - \`system-prompt:
\` — system prompt should have stated the rule directly - - \`memory:\` — prior-run memory should have surfaced an earlier decision +3. For each gap, identify the **layer of the runtime that should have prevented it** and use its exact locus from the subject grammar above. 4. For each gap you can defend with evidence, emit ONE finding with: - \`area\` = "knowledge-gap" - - \`subject\` = the locus string from the list above + - \`subject\` = one exact locus form from the subject grammar above - \`claim\` = a sentence naming the missing or stale knowledge ("wiki has no page on invoice line-item shape, agent had to re-derive it from raw spans") - \`severity\` = "high" when the gap caused a failure or a clarifying question; "medium" when it caused unnecessary turns; "low" when it caused minor inefficiency - \`evidence_uri\` = \`span:///\` of the moment the gap surfaced (the question, the self-correction, the retrieval miss, the stale web result) diff --git a/src/analyst/kinds/knowledge-poisoning.ts b/src/analyst/kinds/knowledge-poisoning.ts index dc38e0dd..6a2c5ee6 100644 --- a/src/analyst/kinds/knowledge-poisoning.ts +++ b/src/analyst/kinds/knowledge-poisoning.ts @@ -18,11 +18,16 @@ * itself is actually false in ground truth. */ +import { findingSubjectGrammarPromptFor } from '../finding-subject' import type { TraceAnalystKindSpec } from '../kind-factory' import { buildTraceToolsForGroup } from '../tool-groups' +const subjectGrammar = findingSubjectGrammarPromptFor('knowledge-poisoning') + const ACTOR_PROMPT = `You are a knowledge-poisoning analyst for an OTLP trace dataset. Your job is to identify cases where the agent **confidently used wrong information** — not where it lacked information (that's the knowledge-gap analyst). +${subjectGrammar} + DISCOVERY → DUAL-VERIFY → CITE protocol: 1. \`traces.getDatasetOverview({})\` first. Identify the agents, models, and tools. @@ -43,7 +48,7 @@ DISCOVERY → DUAL-VERIFY → CITE protocol: For each confirmed poisoning, emit ONE finding with: - \`area\` = "knowledge-poisoning" -- \`subject\` = the source of the false belief, one of: \`agent-knowledge:wiki:\` (wiki page contradicts current ground truth), \`agent-knowledge:claim:\` (a specific claim/relation went stale), \`agent-knowledge:raw:\` (the raw source is outdated and the wiki inherited the drift), \`websearch:outdated:\`, \`tool-doc:\`, \`system-prompt:
\`, \`memory:\`, \`prior-run-summary:\` +- \`subject\` = the source of the false belief using one exact form from the subject grammar above - \`claim\` = one sentence: "agent believed X (from source S); evidence in trace shows X is false" - \`severity\` = "critical" when poisoning caused a wrong user-visible action; "high" when caught internally but wasted significant work; "medium" for inefficiency only - \`evidence_uri\` = \`span:///\` of the action span (the moment the agent acted on the false belief) diff --git a/src/analyst/policy-edit.ts b/src/analyst/policy-edit.ts index f58e124b..50523383 100644 --- a/src/analyst/policy-edit.ts +++ b/src/analyst/policy-edit.ts @@ -30,6 +30,7 @@ export const POLICY_EDIT_TARGET_SURFACES = [ 'runtime-config', 'memory', 'agent-profile', + 'code', 'deployment', ] as const @@ -335,6 +336,11 @@ function routeParsedSubject(subject: FindingSubject): { axis: 'representation', target: { surface: 'prompt', path: `system-prompt:${subject.section}` }, } + case 'skill': + return { + axis: 'agent_profile', + target: { surface: 'agent-profile', path: `skill:${subject.name}` }, + } case 'tool-doc': return { axis: 'tool_contract', @@ -350,6 +356,44 @@ function routeParsedSubject(subject: FindingSubject): { axis: 'tool_contract', target: { surface: 'tool-contract', path: `new-tool:${subject.name}` }, } + case 'mcp': + return { + axis: 'tool_contract', + target: { + surface: 'agent-profile', + path: subject.tool ? `mcp:${subject.server}:${subject.tool}` : `mcp:${subject.server}`, + }, + } + case 'hook': + return { + axis: 'agent_profile', + target: { surface: 'agent-profile', path: `hook:${subject.name}` }, + } + case 'subagent': + return { + axis: 'routing', + target: { surface: 'agent-profile', path: `subagent:${subject.name}` }, + } + case 'workflow': + return { + axis: 'routing', + target: { surface: 'runtime-config', path: `workflow:${subject.name}` }, + } + case 'rollout-policy': + return { + axis: rolloutPolicyAxis(subject.field), + target: { surface: 'runtime-config', path: `rollout-policy:${subject.field}` }, + } + case 'agent-profile': + return { + axis: 'agent_profile', + target: { surface: 'agent-profile', path: `agent-profile:${subject.field}` }, + } + case 'code': + return { + axis: 'representation', + target: { surface: 'code', path: `code:${subject.path}` }, + } case 'rag': return { axis: 'memory', @@ -405,6 +449,18 @@ function routeParsedSubject(subject: FindingSubject): { } } +function rolloutPolicyAxis(field: string): PolicyEditAxis { + const normalized = field.toLowerCase() + if (/budget|max(?:imum)?[-_. ]?(?:turns?|tokens?|cost)|timeout|deadline/.test(normalized)) { + return 'budget' + } + if (/temperature|top[-_. ]?p|sampling|seed|shots?|parallel|concurrency/.test(normalized)) { + return 'sampling' + } + if (/output|schema|format/.test(normalized)) return 'output_contract' + return 'routing' +} + function resolveExpectedGain( finding: AnalystFinding, opts: FindingToPolicyEditOptions, diff --git a/tests/policy-edit.test.ts b/tests/policy-edit.test.ts index abff4286..2cafe665 100644 --- a/tests/policy-edit.test.ts +++ b/tests/policy-edit.test.ts @@ -121,6 +121,21 @@ describe('PolicyEdit contract', () => { expect(validatePolicyEdit(edit)).toBe(edit) }) + it.each([ + ['skill:linear-triage', 'agent_profile', 'agent-profile', 'skill:linear-triage'], + ['mcp:linear:get_issue', 'tool_contract', 'agent-profile', 'mcp:linear:get_issue'], + ['hook:post-tool-use', 'agent_profile', 'agent-profile', 'hook:post-tool-use'], + ['subagent:reviewer', 'routing', 'agent-profile', 'subagent:reviewer'], + ['workflow:linear-ticket', 'routing', 'runtime-config', 'workflow:linear-ticket'], + ['rollout-policy:maxTurns', 'budget', 'runtime-config', 'rollout-policy:maxTurns'], + ['rollout-policy:temperature', 'sampling', 'runtime-config', 'rollout-policy:temperature'], + ['agent-profile:model', 'agent_profile', 'agent-profile', 'agent-profile:model'], + ['code:src/dispatch.ts', 'representation', 'code', 'code:src/dispatch.ts'], + ])('routes %s to %s / %s / %s', (subject, axis, surface, path) => { + const edit = policyEditFromFinding(finding({ subject })) + expect(edit).toMatchObject({ axis, target: { surface, path } }) + }) + it('returns no edit when the finding lacks a typed expected gain', () => { const edit = policyEditFromFinding(finding({ metadata: undefined })) expect(edit).toBeNull()