From 41e62eeb99aa36cf3c4efe8ded69c19db7068c24 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 13 Jul 2026 01:14:26 -0600 Subject: [PATCH 1/3] fix(campaign): preserve proposer state and private profile fields --- CHANGELOG.md | 10 ++ src/campaign/proposers/composite.test.ts | 44 +++++ src/campaign/proposers/composite.ts | 24 ++- src/campaign/proposers/llm-policy-edit.ts | 82 +++++++++- .../campaign/llm-policy-edit-proposer.test.ts | 154 ++++++++++++++++++ 5 files changed, 310 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 762eb897..fccc8f1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- --- +## Unreleased + +### Added + +- `llmPolicyEditProposer({ projectAuthorSurface })` can remove credentials and unrelated private fields outside editable paths from the current surface sent to the model while applying validated edits to the complete original surface. + +### Fixed + +- `compositeProposer` restores each member's original labels when replaying history, so stateful members do not repeat candidates whose labels were decorated for provenance. + ## [0.116.0] — 2026-07-12 — evidence-linked AgentProfile optimization ### Added diff --git a/src/campaign/proposers/composite.test.ts b/src/campaign/proposers/composite.test.ts index 05bf4ad4..346d5af0 100644 --- a/src/campaign/proposers/composite.test.ts +++ b/src/campaign/proposers/composite.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest' import type { CodeSurface, ProposeContext, SurfaceProposer } from '../types' import { compositeProposer } from './composite' +import { parameterSweepProposer } from './fapo' function ctxOf(populationSize: number): ProposeContext { return { @@ -103,6 +104,49 @@ describe('compositeProposer (N proposers, one generation pool)', () => { expect((pool[0] as { surface: CodeSurface }).surface.worktreeRef).toBe(first.worktreeRef) }) + it('lets stateful members recognize their own prior labels', async () => { + const composite = compositeProposer({ + proposers: [ + parameterSweepProposer({ + candidates: [ + { label: 'low', rationale: 'try low', patch: { effort: 'low' } }, + { label: 'high', rationale: 'try high', patch: { effort: 'high' } }, + ], + }), + ], + }) + const first = await composite.propose({ + ...ctxOf(1), + currentSurface: '{"effort":"medium"}', + }) + const firstCandidate = first[0] as { surface: string; label: string; rationale: string } + const second = await composite.propose({ + ...ctxOf(2), + currentSurface: '{"effort":"medium"}', + history: [ + { + generationIndex: 0, + promoted: [], + candidates: [ + { + surfaceHash: 'first', + label: firstCandidate.label, + rationale: firstCandidate.rationale, + composite: 0.5, + ci95: [0.5, 0.5], + dimensions: {}, + scenarios: [], + }, + ], + }, + ], + }) + + expect(firstCandidate.label).toBe('parameter-sweep:low') + expect(second).toHaveLength(1) + expect((second[0] as { label: string }).label).toBe('parameter-sweep:high') + }) + it('isolates a failing member; throws only when ALL members fail', async () => { const oneDown = compositeProposer({ proposers: [stub('boom', [], { fail: true }), stub('ok', ['x1', 'x2'])], diff --git a/src/campaign/proposers/composite.ts b/src/campaign/proposers/composite.ts index 07c0df51..9eeb069a 100644 --- a/src/campaign/proposers/composite.ts +++ b/src/campaign/proposers/composite.ts @@ -91,7 +91,11 @@ export function compositeProposer( const share = shares[i] ?? 0 if (!member || share === 0 || ctx.signal.aborted) continue try { - const proposals = await member.propose({ ...ctx, populationSize: share }) + const proposals = await member.propose({ + ...ctx, + history: historyForMember(ctx.history, member.kind), + populationSize: share, + }) for (const proposal of proposals) { const isCandidate = typeof proposal === 'object' && proposal !== null && 'surface' in proposal @@ -133,7 +137,11 @@ export function compositeProposer( decide(args: { history: GenerationRecord[] }): { stop: boolean; reason?: string } { const votes = members .filter((m) => typeof m.decide === 'function') - .map((m) => (m.decide as NonNullable['decide']>)(args)) + .map((m) => + (m.decide as NonNullable['decide']>)({ + history: historyForMember(args.history, m.kind), + }), + ) if (votes.length === 0) return { stop: false } const allStop = votes.every((v) => v.stop) return allStop @@ -149,3 +157,15 @@ export function compositeProposer( }, } } + +function historyForMember(history: GenerationRecord[], memberKind: string): GenerationRecord[] { + const prefix = `${memberKind}:` + return history.map((generation) => ({ + ...generation, + candidates: generation.candidates.map((candidate) => + candidate.label?.startsWith(prefix) + ? { ...candidate, label: candidate.label.slice(prefix.length) } + : candidate, + ), + })) +} diff --git a/src/campaign/proposers/llm-policy-edit.ts b/src/campaign/proposers/llm-policy-edit.ts index 4a270fa8..2da54537 100644 --- a/src/campaign/proposers/llm-policy-edit.ts +++ b/src/campaign/proposers/llm-policy-edit.ts @@ -41,6 +41,10 @@ const JSON_POLICY_EDIT_TARGET_SURFACES = [ export type JsonPolicyEditTargetSurface = (typeof JSON_POLICY_EDIT_TARGET_SURFACES)[number] const NonEmptyStringSchema = z.string().trim().min(1) +const JsonObjectKeySchema = z + .string() + .min(1) + .refine((key) => key.trim() === key, 'JSON object keys must not have surrounding whitespace') const JsonValueSchema: z.ZodType = z.lazy(() => z.union([ @@ -49,7 +53,7 @@ const JsonValueSchema: z.ZodType = z.lazy(() => z.boolean(), z.null(), z.array(JsonValueSchema), - z.record(NonEmptyStringSchema, JsonValueSchema), + z.record(JsonObjectKeySchema, JsonValueSchema), ]), ) @@ -414,6 +418,13 @@ export interface LlmPolicyEditProposerOptions { maxAuthorContextChars?: number /** Optional one-to-one pseudonymizer applied to every author-visible evidence field. */ scenarioIdTransform?: (scenarioId: string) => string + /** + * Project the current JSON surface before it enters model context. Use this + * to remove credentials or unrelated private fields outside allowedJsonPaths. + * Every editable path must remain unchanged. Scored history is not projected. + * Authored edits are validated and applied to the original surface. + */ + projectAuthorSurface?: (surface: AgentProfileJson) => AgentProfileJson onAdmission?: (admission: PolicyEditAdmission) => void } @@ -493,6 +504,12 @@ export function llmPolicyEditProposer( { currentSurface, allowedJsonPaths, objectives, targetSurface: opts.targetSurface }, scenarioIds, ) + const authorSurface = projectAuthorSurface( + currentSurface, + allowedJsonPaths, + opts.projectAuthorSurface, + ) + assertSurfaceIsTaskAgnostic(authorSurface, scenarioIds) const measuredSources = measuredSourceMeasurements(ctx) const findings = citableFindings(ctx.findings, measuredSources, maxFindings) const findingByKey = new Map( @@ -505,7 +522,7 @@ export function llmPolicyEditProposer( objectives, candidateCount: limit, generation: ctx.generation, - currentSurface, + currentSurface: authorSurface, findings: findings.map((finding, index) => renderFinding(finding, `finding-${index + 1}`, scenarioIds, measuredSources), ), @@ -1247,6 +1264,67 @@ function parseJsonSurface(surface: MutableSurface): AgentProfileJson { return parsed as AgentProfileJson } +function projectAuthorSurface( + surface: AgentProfileJson, + allowedJsonPaths: readonly string[], + project: LlmPolicyEditProposerOptions['projectAuthorSurface'], +): AgentProfileJson { + if (!project) return surface + const projected = project(structuredClone(surface)) + const parsed = JsonValueSchema.safeParse(projected) + if ( + !parsed.success || + !parsed.data || + typeof parsed.data !== 'object' || + Array.isArray(parsed.data) + ) { + throw new Error('llmPolicyEditProposer: projectAuthorSurface must return a JSON object') + } + for (const path of allowedJsonPaths) { + if (!jsonValuesEqual(readJsonPath(surface, path), readJsonPath(parsed.data, path))) { + throw new Error( + `llmPolicyEditProposer: projectAuthorSurface must not change or hide allowed JSON path '${path}'`, + ) + } + } + return parsed.data +} + +function readJsonPath(root: AgentProfileJson, path: string): AgentProfileJson | undefined { + let cursor: AgentProfileJson | undefined = root + for (const part of path + .split('.') + .map((segment) => segment.trim()) + .filter(Boolean)) { + if (!cursor || typeof cursor !== 'object' || Array.isArray(cursor)) return undefined + cursor = cursor[part] + } + return cursor +} + +function jsonValuesEqual( + left: AgentProfileJson | undefined, + right: AgentProfileJson | undefined, +): boolean { + if (left === right) return true + if (left === undefined || right === undefined || left === null || right === null) return false + if (Array.isArray(left) || Array.isArray(right)) { + return ( + Array.isArray(left) && + Array.isArray(right) && + left.length === right.length && + left.every((value, index) => jsonValuesEqual(value, right[index])) + ) + } + if (typeof left !== 'object' || typeof right !== 'object') return false + const leftKeys = Object.keys(left) + const rightKeys = Object.keys(right) + return ( + leftKeys.length === rightKeys.length && + leftKeys.every((key) => Object.hasOwn(right, key) && jsonValuesEqual(left[key], right[key])) + ) +} + interface CitableFinding { finding: AnalystFinding sources: PolicyEditFindingSource[] diff --git a/tests/campaign/llm-policy-edit-proposer.test.ts b/tests/campaign/llm-policy-edit-proposer.test.ts index d5608e77..4cc79fd1 100644 --- a/tests/campaign/llm-policy-edit-proposer.test.ts +++ b/tests/campaign/llm-policy-edit-proposer.test.ts @@ -158,6 +158,7 @@ function proposer(input: { targetSurface?: 'agent-profile' | 'code' admissionMode?: 'evidence-only' | 'strict' admission?: PolicyEditAdmissionOptions + projectAuthorSurface?: (surface: Record) => Record }) { return llmPolicyEditProposer({ llm: { @@ -192,6 +193,9 @@ function proposer(input: { : { maxAuthorContextChars: input.maxAuthorContextChars }), ...(input.admissionMode === undefined ? {} : { admissionMode: input.admissionMode }), ...(input.admission === undefined ? {} : { admission: input.admission }), + ...(input.projectAuthorSurface === undefined + ? {} + : { projectAuthorSurface: input.projectAuthorSurface }), }) } @@ -273,6 +277,156 @@ describe('llmPolicyEditProposer', () => { expect(providerSchema).toContain('"mode":{"const":"remove"}') }) + it('authors from a private-field projection but applies edits to the complete surface', async () => { + const capture: CapturedRequest = {} + const out = await proposer({ + response: { edits: [authoredEdit('finding-1')] }, + capture, + projectAuthorSurface: (surface) => ({ prompt: surface.prompt }), + }).propose( + context({ + finding: finding(), + currentSurface: JSON.stringify({ + prompt: { systemPrompt: 'Base' }, + mcp: { + linear: { + transport: 'http', + url: 'https://mcp.example.test', + headers: { Authorization: 'Bearer private-token' }, + }, + }, + }), + }), + ) + + expect(capture.user?.currentSurface).toEqual({ prompt: { systemPrompt: 'Base' } }) + expect(JSON.stringify(capture.user)).not.toContain('private-token') + expect(JSON.parse(candidateSurface(out[0]!))).toEqual({ + prompt: { systemPrompt: 'Read repository instructions first.' }, + mcp: { + linear: { + transport: 'http', + url: 'https://mcp.example.test', + headers: { Authorization: 'Bearer private-token' }, + }, + }, + }) + }) + + it('rejects a non-object author projection before model dispatch', async () => { + let called = false + const configured = llmPolicyEditProposer({ + llm: { + apiKey: 'test-key', + baseUrl: 'https://router.test/v1', + fetch: (async () => { + called = true + throw new Error('must not dispatch') + }) as typeof fetch, + }, + model: 'test-model-snapshot', + target: 'canonical agent profile JSON', + targetSurface: 'agent-profile', + allowedJsonPaths: ['prompt.systemPrompt'], + objectives: OBJECTIVES, + projectAuthorSurface: () => [] as unknown as Record, + }) + + await expect(configured.propose(context({ finding: finding() }))).rejects.toThrow( + /projectAuthorSurface must return a JSON object/, + ) + expect(called).toBe(false) + }) + + it('rejects a projection that hides an editable subtree before model dispatch', async () => { + let called = false + const configured = llmPolicyEditProposer({ + llm: { + apiKey: 'test-key', + baseUrl: 'https://router.test/v1', + fetch: (async () => { + called = true + throw new Error('must not dispatch') + }) as typeof fetch, + }, + model: 'test-model-snapshot', + target: 'canonical agent profile JSON', + targetSurface: 'agent-profile', + allowedJsonPaths: ['mcp.linear'], + objectives: OBJECTIVES, + projectAuthorSurface: (surface) => ({ prompt: surface.prompt }), + }) + + await expect( + configured.propose( + context({ + finding: finding(), + currentSurface: JSON.stringify({ + prompt: { systemPrompt: 'Base' }, + mcp: { + linear: { + transport: 'http', + headers: { Authorization: 'Bearer private-token' }, + }, + }, + }), + }), + ), + ).rejects.toThrow(/must not change or hide allowed JSON path 'mcp\.linear'/) + expect(called).toBe(false) + }) + + it('does not let a mutating projection alter the executable surface', async () => { + const out = await proposer({ + response: { edits: [authoredEdit('finding-1')] }, + projectAuthorSurface: (surface) => { + delete surface.mcp + return { prompt: surface.prompt } + }, + }).propose( + context({ + finding: finding(), + currentSurface: JSON.stringify({ + prompt: { systemPrompt: 'Base' }, + mcp: { linear: { headers: { Authorization: 'Bearer private-token' } } }, + }), + }), + ) + + expect(JSON.parse(candidateSurface(out[0]!))).toMatchObject({ + mcp: { linear: { headers: { Authorization: 'Bearer private-token' } } }, + }) + }) + + it('rejects projected JSON keys that validation would otherwise rewrite', async () => { + const configured = proposer({ + response: { edits: [] }, + projectAuthorSurface: (surface) => ({ ...surface, ' mcp': {} }), + }) + + await expect(configured.propose(context({ finding: finding() }))).rejects.toThrow( + /projectAuthorSurface must return a JSON object/, + ) + }) + + it('does not let an author projection hide task identifiers in the executable surface', async () => { + const configured = proposer({ + response: { edits: [authoredEdit('finding-1')] }, + scenarioIdTransform: () => 'task-1', + projectAuthorSurface: () => ({}), + }) + + await expect( + configured.propose( + context({ + finding: finding(), + currentSurface: '{"prompt":{"systemPrompt":"Handle private-task"}}', + baselineOutcome: measuredOutcome('baseline', 0.4, 'private-task'), + }), + ), + ).rejects.toThrow(/raw scenario identifier/) + }) + it('shows the author measured baseline, incumbent, and exact parent deltas', async () => { const source = finding() const capture: CapturedRequest = {} From bafeb28cbaa8a5d4b8c1191719b1b54d0e669c13 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 22 Jul 2026 15:59:39 -0600 Subject: [PATCH 2/3] test(campaign): cover composite decision replay --- src/campaign/proposers/composite.test.ts | 34 ++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/campaign/proposers/composite.test.ts b/src/campaign/proposers/composite.test.ts index 346d5af0..b5cce05e 100644 --- a/src/campaign/proposers/composite.test.ts +++ b/src/campaign/proposers/composite.test.ts @@ -176,6 +176,40 @@ describe('compositeProposer (N proposers, one generation pool)', () => { expect(unanimous.decide?.({ history: [] })?.stop).toBe(true) }) + it('restores member labels before asking members whether to stop', () => { + let observedLabel: string | undefined + const member: SurfaceProposer = { + kind: 'stateful', + propose: async () => [], + decide: ({ history }) => { + observedLabel = history[0]?.candidates[0]?.label + return { stop: true } + }, + } + const composite = compositeProposer({ proposers: [member] }) + + composite.decide?.({ + history: [ + { + generationIndex: 0, + promoted: [], + candidates: [ + { + surfaceHash: 'first', + label: 'stateful:attempt-1', + composite: 0.5, + ci95: [0.5, 0.5], + dimensions: {}, + scenarios: [], + }, + ], + }, + ], + }) + + expect(observedLabel).toBe('attempt-1') + }) + it('fails loud on empty membership or bad weights', () => { expect(() => compositeProposer({ proposers: [] })).toThrow(/at least one/) expect(() => compositeProposer({ proposers: [stub('a', ['x'])], weights: [0] })).toThrow( From 9514ae6a545f2f5fb95cdcaac9267a34cccb8ed1 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 22 Jul 2026 16:36:57 -0600 Subject: [PATCH 3/3] fix(campaign): harden composite state and model redaction --- CHANGELOG.md | 2 +- src/agent-profile-cell.ts | 4 +- src/campaign/proposers/composite.test.ts | 20 ++++- src/campaign/proposers/composite.ts | 57 ++++++------- src/campaign/proposers/llm-policy-edit.ts | 79 ++++++++++------- src/index.ts | 1 + .../campaign/llm-policy-edit-proposer.test.ts | 85 ++++++++----------- 7 files changed, 134 insertions(+), 114 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99b4d3d3..4f2e022e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- ### Added -- `llmPolicyEditProposer({ projectAuthorSurface })` can remove credentials and unrelated private fields outside editable paths from the current surface sent to the model while applying validated edits to the complete original surface. +- `llmPolicyEditProposer({ redactCurrentSurfaceForModel })` can remove credentials and unrelated fields from the current surface sent to the model while applying validated edits to the complete original surface. - `CostLedger.listPending()` exposes immutable pending paid calls and distinguishes calls that are active, late after cancellation, or interrupted by a prior process so durable workflows can reconcile exact reservations before resuming. - `traceAnalystProposer()` accepts an opt-in `resolvePriorFindings` callback that forwards canonical prior findings into the existing analyst registry. diff --git a/src/agent-profile-cell.ts b/src/agent-profile-cell.ts index 1781d34a..cb1e1229 100644 --- a/src/agent-profile-cell.ts +++ b/src/agent-profile-cell.ts @@ -4,13 +4,15 @@ import { hashJson } from './pre-registration' export type AgentProfileCellSchemaVersion = 'agent-profile-cell/v1' +export type AgentProfileJsonObject = { [key: string]: AgentProfileJson } + export type AgentProfileJson = | string | number | boolean | null | AgentProfileJson[] - | { [key: string]: AgentProfileJson } + | AgentProfileJsonObject export type AgentProfileDimensionValue = string | number | boolean | null diff --git a/src/campaign/proposers/composite.test.ts b/src/campaign/proposers/composite.test.ts index b5cce05e..5554810f 100644 --- a/src/campaign/proposers/composite.test.ts +++ b/src/campaign/proposers/composite.test.ts @@ -210,10 +210,28 @@ describe('compositeProposer (N proposers, one generation pool)', () => { expect(observedLabel).toBe('attempt-1') }) + it('rejects member kinds that make history ownership ambiguous', () => { + expect(() => + compositeProposer({ proposers: [stub('same', ['a']), stub('same', ['b'])] }), + ).toThrow(/duplicate member kind 'same'/) + expect(() => compositeProposer({ proposers: [stub('a:b', ['a'])] })).toThrow( + /must not contain ':'/, + ) + expect(() => compositeProposer({ proposers: [stub(' spaced ', ['a'])] })).toThrow( + /trimmed and non-empty/, + ) + }) + it('fails loud on empty membership or bad weights', () => { expect(() => compositeProposer({ proposers: [] })).toThrow(/at least one/) expect(() => compositeProposer({ proposers: [stub('a', ['x'])], weights: [0] })).toThrow( - /positive/, + /finite and positive/, ) + expect(() => + compositeProposer({ proposers: [stub('a', ['x'])], weights: [Number.POSITIVE_INFINITY] }), + ).toThrow(/finite and positive/) + expect(() => + compositeProposer({ proposers: [stub('a', ['x'])], weights: [Number.NaN] }), + ).toThrow(/finite and positive/) }) }) diff --git a/src/campaign/proposers/composite.ts b/src/campaign/proposers/composite.ts index 9eeb069a..e627ef0e 100644 --- a/src/campaign/proposers/composite.ts +++ b/src/campaign/proposers/composite.ts @@ -1,34 +1,9 @@ /** - * `compositeProposer` — run N proposers TOGETHER on the same surface. - * - * The question this answers ("why can't we combine GEPA + skillOpt + ACE + a - * trace-analyst?"): nothing in the loop cares where candidates come from — the - * generation's population is one pool and the Pareto frontier / promotion logic - * evaluates every candidate identically. The only missing piece was a proposer - * that fans the population budget out across members and merges their proposals. - * This is that piece. - * - * Semantics: - * - Budget: each member is asked for a share of `populationSize` - * (near-equal split by default, or explicit `weights`). Members may return - * fewer; the pool is topped up round-robin from members that can offer more - * is NOT attempted — proposers are not obligated to be re-entrant. - * - Provenance: every candidate's `label` is prefixed with its member's kind - * (`gepa:...`, `skill-opt:...`) so generation records and the promotion - * provenance attribute each winner to the proposer family that made it — - * the cheap, honest version of proposer-level credit assignment. - * - Dedup: identical surfaces from different members collapse to the first. - * - Failure isolation: one member throwing does not sink the generation; its - * error is logged into the surviving candidates' generation via a warning - * and the pool proceeds (a generation with zero candidates from all members - * failing still throws — that is a real failure). - * - Early stop: the composite stops only when EVERY member with a `decide` - * votes stop (a member without `decide` never votes stop). - * - * This is deliberately NOT joint multi-surface mutation: every member mutates - * the SAME `MutableSurface`. Joint profile-patch surfaces (prompt+skills+tools - * in one candidate) require the composite-surface contract and measured - * component attribution — see the experiment-optimal research brief. + * Split one generation's candidate budget across independent proposers. + * Candidate labels retain the originating proposer kind, duplicate surfaces + * collapse to the first result, and one failed proposer does not discard the + * other results. The composite stops only when every member with `decide` + * votes to stop. */ import { surfaceContentHash } from '../surface-identity' @@ -57,9 +32,27 @@ export function compositeProposer( const members = opts.proposers if (members.length === 0) throw new Error('compositeProposer: at least one member proposer required') + const memberKinds = new Set() + for (const member of members) { + if (!member.kind || member.kind.trim() !== member.kind) { + throw new Error('compositeProposer: member kinds must be trimmed and non-empty') + } + if (member.kind.includes(':')) { + throw new Error(`compositeProposer: member kind '${member.kind}' must not contain ':'`) + } + if (memberKinds.has(member.kind)) { + throw new Error(`compositeProposer: duplicate member kind '${member.kind}'`) + } + memberKinds.add(member.kind) + } const weights = opts.weights ?? members.map(() => 1) - if (weights.length !== members.length || weights.some((w) => !(w > 0))) { - throw new Error('compositeProposer: weights must match proposers length and be positive') + if ( + weights.length !== members.length || + weights.some((weight) => !Number.isFinite(weight) || weight <= 0) + ) { + throw new Error( + 'compositeProposer: weights must match proposers length and be finite and positive', + ) } return { diff --git a/src/campaign/proposers/llm-policy-edit.ts b/src/campaign/proposers/llm-policy-edit.ts index 9c320c41..c3adcce5 100644 --- a/src/campaign/proposers/llm-policy-edit.ts +++ b/src/campaign/proposers/llm-policy-edit.ts @@ -1,5 +1,5 @@ import { z } from 'zod' -import type { AgentProfileJson } from '../../agent-profile-cell' +import type { AgentProfileJson, AgentProfileJsonObject } from '../../agent-profile-cell' import { makePolicyEdit, POLICY_EDIT_AXES, @@ -429,12 +429,12 @@ export interface LlmPolicyEditProposerOptions { /** Optional one-to-one pseudonymizer applied to every author-visible evidence field. */ scenarioIdTransform?: (scenarioId: string) => string /** - * Project the current JSON surface before it enters model context. Use this - * to remove credentials or unrelated private fields outside allowedJsonPaths. - * Every editable path must remain unchanged. Scored history is not projected. - * Authored edits are validated and applied to the original surface. + * Remove credentials or unrelated fields from the current surface before it + * is sent to the model. The callback receives a clone and must preserve every + * editable path unchanged. Validated edits apply to the complete original. + * This callback does not redact findings or scored history. */ - projectAuthorSurface?: (surface: AgentProfileJson) => AgentProfileJson + redactCurrentSurfaceForModel?: (surface: AgentProfileJsonObject) => AgentProfileJsonObject onAdmission?: (admission: PolicyEditAdmission) => void } @@ -515,12 +515,12 @@ export function llmPolicyEditProposer( { currentSurface, allowedJsonPaths, objectives, targetSurface: opts.targetSurface }, scenarioIds, ) - const authorSurface = projectAuthorSurface( + const modelSurface = redactCurrentSurfaceForModel( currentSurface, allowedJsonPaths, - opts.projectAuthorSurface, + opts.redactCurrentSurfaceForModel, ) - assertSurfaceIsTaskAgnostic(authorSurface, scenarioIds) + assertSurfaceIsTaskAgnostic(modelSurface, scenarioIds) const measuredSources = measuredSourceMeasurements(ctx) const findings = citableFindings(ctx.findings, measuredSources, maxFindings) const findingByKey = new Map( @@ -533,7 +533,7 @@ export function llmPolicyEditProposer( objectives, candidateCount: limit, generation: ctx.generation, - currentSurface: authorSurface, + currentSurface: modelSurface, findings: findings.map((finding, index) => renderFinding(finding, `finding-${index + 1}`, scenarioIds, measuredSources), ), @@ -1268,7 +1268,7 @@ function bindAuthoredEdit( return makePolicyEdit(init) } -function parseJsonSurface(surface: MutableSurface): AgentProfileJson { +function parseJsonSurface(surface: MutableSurface): AgentProfileJsonObject { if (typeof surface !== 'string') { throw new Error('llmPolicyEditProposer: currentSurface must be serialized JSON') } @@ -1281,36 +1281,57 @@ function parseJsonSurface(surface: MutableSurface): AgentProfileJson { if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { throw new Error('llmPolicyEditProposer: currentSurface JSON root must be an object') } - return parsed as AgentProfileJson + return parsed as AgentProfileJsonObject } -function projectAuthorSurface( - surface: AgentProfileJson, +function redactCurrentSurfaceForModel( + surface: AgentProfileJsonObject, allowedJsonPaths: readonly string[], - project: LlmPolicyEditProposerOptions['projectAuthorSurface'], -): AgentProfileJson { - if (!project) return surface - const projected = project(structuredClone(surface)) - const parsed = JsonValueSchema.safeParse(projected) - if ( - !parsed.success || - !parsed.data || - typeof parsed.data !== 'object' || - Array.isArray(parsed.data) - ) { - throw new Error('llmPolicyEditProposer: projectAuthorSurface must return a JSON object') + redact: LlmPolicyEditProposerOptions['redactCurrentSurfaceForModel'], +): AgentProfileJsonObject { + if (!redact) return surface + const redacted = redact(structuredClone(surface)) + const parsed = JsonValueSchema.safeParse(redacted) + if (!parsed.success) { + const detail = formatJsonValidationError(parsed.error) + throw new Error( + `llmPolicyEditProposer: redactCurrentSurfaceForModel returned invalid JSON (${detail})`, + ) + } + if (!parsed.data || typeof parsed.data !== 'object' || Array.isArray(parsed.data)) { + throw new Error('llmPolicyEditProposer: redactCurrentSurfaceForModel must return a JSON object') } for (const path of allowedJsonPaths) { if (!jsonValuesEqual(readJsonPath(surface, path), readJsonPath(parsed.data, path))) { throw new Error( - `llmPolicyEditProposer: projectAuthorSurface must not change or hide allowed JSON path '${path}'`, + `llmPolicyEditProposer: redactCurrentSurfaceForModel must not change or hide editable JSON path '${path}'`, ) } } - return parsed.data + return parsed.data as AgentProfileJsonObject +} + +function formatJsonValidationError(error: z.ZodError): string { + const messages = [...new Set(collectZodMessages(error.issues))] + const informative = messages.filter( + (message) => message !== 'Invalid input' && message !== 'Invalid key in record', + ) + const custom = informative.filter((message) => !message.startsWith('Invalid input: expected')) + return (custom.length > 0 ? custom : informative).join('; ') || 'invalid JSON value' +} + +function collectZodMessages(value: unknown): string[] { + if (Array.isArray(value)) return value.flatMap(collectZodMessages) + if (!value || typeof value !== 'object') return [] + const issue = value as Record + return [ + ...(typeof issue.message === 'string' ? [issue.message] : []), + ...collectZodMessages(issue.errors), + ...collectZodMessages(issue.issues), + ] } -function readJsonPath(root: AgentProfileJson, path: string): AgentProfileJson | undefined { +function readJsonPath(root: AgentProfileJsonObject, path: string): AgentProfileJson | undefined { let cursor: AgentProfileJson | undefined = root for (const part of path .split('.') diff --git a/src/index.ts b/src/index.ts index 2dbfd184..d33ed70b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -20,6 +20,7 @@ export type { AgentProfileDimensionValue, AgentProfileHarness, AgentProfileJson, + AgentProfileJsonObject, AgentProfileKind, AgentProfileSource, AgentProfileSourceInput, diff --git a/tests/campaign/llm-policy-edit-proposer.test.ts b/tests/campaign/llm-policy-edit-proposer.test.ts index 4cc79fd1..74a32497 100644 --- a/tests/campaign/llm-policy-edit-proposer.test.ts +++ b/tests/campaign/llm-policy-edit-proposer.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from 'vitest' +import type { AgentProfileJsonObject } from '../../src/agent-profile-cell' import { makePolicyEdit, makePolicyEditCandidateRecord, @@ -158,7 +159,7 @@ function proposer(input: { targetSurface?: 'agent-profile' | 'code' admissionMode?: 'evidence-only' | 'strict' admission?: PolicyEditAdmissionOptions - projectAuthorSurface?: (surface: Record) => Record + redactCurrentSurfaceForModel?: (surface: AgentProfileJsonObject) => AgentProfileJsonObject }) { return llmPolicyEditProposer({ llm: { @@ -193,9 +194,9 @@ function proposer(input: { : { maxAuthorContextChars: input.maxAuthorContextChars }), ...(input.admissionMode === undefined ? {} : { admissionMode: input.admissionMode }), ...(input.admission === undefined ? {} : { admission: input.admission }), - ...(input.projectAuthorSurface === undefined + ...(input.redactCurrentSurfaceForModel === undefined ? {} - : { projectAuthorSurface: input.projectAuthorSurface }), + : { redactCurrentSurfaceForModel: input.redactCurrentSurfaceForModel }), }) } @@ -277,12 +278,12 @@ describe('llmPolicyEditProposer', () => { expect(providerSchema).toContain('"mode":{"const":"remove"}') }) - it('authors from a private-field projection but applies edits to the complete surface', async () => { + it('redacts private fields from model input and applies edits to the complete surface', async () => { const capture: CapturedRequest = {} const out = await proposer({ response: { edits: [authoredEdit('finding-1')] }, capture, - projectAuthorSurface: (surface) => ({ prompt: surface.prompt }), + redactCurrentSurfaceForModel: (surface) => ({ prompt: surface.prompt }), }).propose( context({ finding: finding(), @@ -313,48 +314,27 @@ describe('llmPolicyEditProposer', () => { }) }) - it('rejects a non-object author projection before model dispatch', async () => { - let called = false - const configured = llmPolicyEditProposer({ - llm: { - apiKey: 'test-key', - baseUrl: 'https://router.test/v1', - fetch: (async () => { - called = true - throw new Error('must not dispatch') - }) as typeof fetch, - }, - model: 'test-model-snapshot', - target: 'canonical agent profile JSON', - targetSurface: 'agent-profile', - allowedJsonPaths: ['prompt.systemPrompt'], - objectives: OBJECTIVES, - projectAuthorSurface: () => [] as unknown as Record, + it('rejects a non-object redaction result before model dispatch', async () => { + const capture: CapturedRequest = {} + const configured = proposer({ + response: { edits: [] }, + capture, + redactCurrentSurfaceForModel: () => [] as unknown as AgentProfileJsonObject, }) await expect(configured.propose(context({ finding: finding() }))).rejects.toThrow( - /projectAuthorSurface must return a JSON object/, + /redactCurrentSurfaceForModel must return a JSON object/, ) - expect(called).toBe(false) + expect(capture.user).toBeUndefined() }) - it('rejects a projection that hides an editable subtree before model dispatch', async () => { - let called = false - const configured = llmPolicyEditProposer({ - llm: { - apiKey: 'test-key', - baseUrl: 'https://router.test/v1', - fetch: (async () => { - called = true - throw new Error('must not dispatch') - }) as typeof fetch, - }, - model: 'test-model-snapshot', - target: 'canonical agent profile JSON', - targetSurface: 'agent-profile', + it('rejects redaction that hides an editable subtree before model dispatch', async () => { + const capture: CapturedRequest = {} + const configured = proposer({ + response: { edits: [] }, + capture, allowedJsonPaths: ['mcp.linear'], - objectives: OBJECTIVES, - projectAuthorSurface: (surface) => ({ prompt: surface.prompt }), + redactCurrentSurfaceForModel: (surface) => ({ prompt: surface.prompt }), }) await expect( @@ -372,14 +352,14 @@ describe('llmPolicyEditProposer', () => { }), }), ), - ).rejects.toThrow(/must not change or hide allowed JSON path 'mcp\.linear'/) - expect(called).toBe(false) + ).rejects.toThrow(/must not change or hide editable JSON path 'mcp\.linear'/) + expect(capture.user).toBeUndefined() }) - it('does not let a mutating projection alter the executable surface', async () => { + it('does not let a mutating redaction callback alter the executable surface', async () => { const out = await proposer({ response: { edits: [authoredEdit('finding-1')] }, - projectAuthorSurface: (surface) => { + redactCurrentSurfaceForModel: (surface) => { delete surface.mcp return { prompt: surface.prompt } }, @@ -398,33 +378,38 @@ describe('llmPolicyEditProposer', () => { }) }) - it('rejects projected JSON keys that validation would otherwise rewrite', async () => { + it('rejects redacted JSON keys that validation would otherwise rewrite', async () => { const configured = proposer({ response: { edits: [] }, - projectAuthorSurface: (surface) => ({ ...surface, ' mcp': {} }), + redactCurrentSurfaceForModel: (surface) => ({ ...surface, ' mcp': {} }), }) await expect(configured.propose(context({ finding: finding() }))).rejects.toThrow( - /projectAuthorSurface must return a JSON object/, + /redactCurrentSurfaceForModel returned invalid JSON.*surrounding whitespace/, ) }) - it('does not let an author projection hide task identifiers in the executable surface', async () => { + it('rejects task identifiers introduced by model-surface redaction', async () => { + let redactionCalled = false const configured = proposer({ response: { edits: [authoredEdit('finding-1')] }, scenarioIdTransform: () => 'task-1', - projectAuthorSurface: () => ({}), + redactCurrentSurfaceForModel: (surface) => { + redactionCalled = true + return { ...surface, modelContext: { note: 'Handle private-task' } } + }, }) await expect( configured.propose( context({ finding: finding(), - currentSurface: '{"prompt":{"systemPrompt":"Handle private-task"}}', + currentSurface: '{"prompt":{"systemPrompt":"Base"}}', baselineOutcome: measuredOutcome('baseline', 0.4, 'private-task'), }), ), ).rejects.toThrow(/raw scenario identifier/) + expect(redactionCalled).toBe(true) }) it('shows the author measured baseline, incumbent, and exact parent deltas', async () => {