diff --git a/CHANGELOG.md b/CHANGELOG.md index b3f22c60..4f2e022e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- ### Added +- `llmPolicyEditProposer({ redactCurrentSurfaceForModel })` can remove credentials and unrelated fields from the current surface sent to the model while applying validated edits to the complete original surface. - `CostLedger.listPending()` exposes immutable pending paid calls and distinguishes calls that are active, late after cancellation, or interrupted by a prior process so durable workflows can reconcile exact reservations before resuming. - `traceAnalystProposer()` accepts an opt-in `resolvePriorFindings` callback that forwards canonical prior findings into the existing analyst registry. @@ -20,6 +21,7 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- ### Fixed +- `compositeProposer` restores each member's original labels when replaying history, so stateful members do not repeat candidates whose labels were decorated for provenance. - `InsightReport.interRater.kappa` now reports quadratic weighted kappa instead of Pearson correlation. Read `interRater.pearson` for the previous correlation measure; `icc` and `spearman` are now reported separately. - Compute contextual-bandit doubly robust estimates with separate logged-action and target-policy value terms, expose how many rows use DR versus IPS or the deprecated scalar path, and carry both values through belief-state records. diff --git a/src/agent-profile-cell.ts b/src/agent-profile-cell.ts index 1781d34a..cb1e1229 100644 --- a/src/agent-profile-cell.ts +++ b/src/agent-profile-cell.ts @@ -4,13 +4,15 @@ import { hashJson } from './pre-registration' export type AgentProfileCellSchemaVersion = 'agent-profile-cell/v1' +export type AgentProfileJsonObject = { [key: string]: AgentProfileJson } + export type AgentProfileJson = | string | number | boolean | null | AgentProfileJson[] - | { [key: string]: AgentProfileJson } + | AgentProfileJsonObject export type AgentProfileDimensionValue = string | number | boolean | null diff --git a/src/campaign/proposers/composite.test.ts b/src/campaign/proposers/composite.test.ts index 05bf4ad4..5554810f 100644 --- a/src/campaign/proposers/composite.test.ts +++ b/src/campaign/proposers/composite.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest' import type { CodeSurface, ProposeContext, SurfaceProposer } from '../types' import { compositeProposer } from './composite' +import { parameterSweepProposer } from './fapo' function ctxOf(populationSize: number): ProposeContext { return { @@ -103,6 +104,49 @@ describe('compositeProposer (N proposers, one generation pool)', () => { expect((pool[0] as { surface: CodeSurface }).surface.worktreeRef).toBe(first.worktreeRef) }) + it('lets stateful members recognize their own prior labels', async () => { + const composite = compositeProposer({ + proposers: [ + parameterSweepProposer({ + candidates: [ + { label: 'low', rationale: 'try low', patch: { effort: 'low' } }, + { label: 'high', rationale: 'try high', patch: { effort: 'high' } }, + ], + }), + ], + }) + const first = await composite.propose({ + ...ctxOf(1), + currentSurface: '{"effort":"medium"}', + }) + const firstCandidate = first[0] as { surface: string; label: string; rationale: string } + const second = await composite.propose({ + ...ctxOf(2), + currentSurface: '{"effort":"medium"}', + history: [ + { + generationIndex: 0, + promoted: [], + candidates: [ + { + surfaceHash: 'first', + label: firstCandidate.label, + rationale: firstCandidate.rationale, + composite: 0.5, + ci95: [0.5, 0.5], + dimensions: {}, + scenarios: [], + }, + ], + }, + ], + }) + + expect(firstCandidate.label).toBe('parameter-sweep:low') + expect(second).toHaveLength(1) + expect((second[0] as { label: string }).label).toBe('parameter-sweep:high') + }) + it('isolates a failing member; throws only when ALL members fail', async () => { const oneDown = compositeProposer({ proposers: [stub('boom', [], { fail: true }), stub('ok', ['x1', 'x2'])], @@ -132,10 +176,62 @@ describe('compositeProposer (N proposers, one generation pool)', () => { expect(unanimous.decide?.({ history: [] })?.stop).toBe(true) }) + it('restores member labels before asking members whether to stop', () => { + let observedLabel: string | undefined + const member: SurfaceProposer = { + kind: 'stateful', + propose: async () => [], + decide: ({ history }) => { + observedLabel = history[0]?.candidates[0]?.label + return { stop: true } + }, + } + const composite = compositeProposer({ proposers: [member] }) + + composite.decide?.({ + history: [ + { + generationIndex: 0, + promoted: [], + candidates: [ + { + surfaceHash: 'first', + label: 'stateful:attempt-1', + composite: 0.5, + ci95: [0.5, 0.5], + dimensions: {}, + scenarios: [], + }, + ], + }, + ], + }) + + expect(observedLabel).toBe('attempt-1') + }) + + it('rejects member kinds that make history ownership ambiguous', () => { + expect(() => + compositeProposer({ proposers: [stub('same', ['a']), stub('same', ['b'])] }), + ).toThrow(/duplicate member kind 'same'/) + expect(() => compositeProposer({ proposers: [stub('a:b', ['a'])] })).toThrow( + /must not contain ':'/, + ) + expect(() => compositeProposer({ proposers: [stub(' spaced ', ['a'])] })).toThrow( + /trimmed and non-empty/, + ) + }) + it('fails loud on empty membership or bad weights', () => { expect(() => compositeProposer({ proposers: [] })).toThrow(/at least one/) expect(() => compositeProposer({ proposers: [stub('a', ['x'])], weights: [0] })).toThrow( - /positive/, + /finite and positive/, ) + expect(() => + compositeProposer({ proposers: [stub('a', ['x'])], weights: [Number.POSITIVE_INFINITY] }), + ).toThrow(/finite and positive/) + expect(() => + compositeProposer({ proposers: [stub('a', ['x'])], weights: [Number.NaN] }), + ).toThrow(/finite and positive/) }) }) diff --git a/src/campaign/proposers/composite.ts b/src/campaign/proposers/composite.ts index 07c0df51..e627ef0e 100644 --- a/src/campaign/proposers/composite.ts +++ b/src/campaign/proposers/composite.ts @@ -1,34 +1,9 @@ /** - * `compositeProposer` — run N proposers TOGETHER on the same surface. - * - * The question this answers ("why can't we combine GEPA + skillOpt + ACE + a - * trace-analyst?"): nothing in the loop cares where candidates come from — the - * generation's population is one pool and the Pareto frontier / promotion logic - * evaluates every candidate identically. The only missing piece was a proposer - * that fans the population budget out across members and merges their proposals. - * This is that piece. - * - * Semantics: - * - Budget: each member is asked for a share of `populationSize` - * (near-equal split by default, or explicit `weights`). Members may return - * fewer; the pool is topped up round-robin from members that can offer more - * is NOT attempted — proposers are not obligated to be re-entrant. - * - Provenance: every candidate's `label` is prefixed with its member's kind - * (`gepa:...`, `skill-opt:...`) so generation records and the promotion - * provenance attribute each winner to the proposer family that made it — - * the cheap, honest version of proposer-level credit assignment. - * - Dedup: identical surfaces from different members collapse to the first. - * - Failure isolation: one member throwing does not sink the generation; its - * error is logged into the surviving candidates' generation via a warning - * and the pool proceeds (a generation with zero candidates from all members - * failing still throws — that is a real failure). - * - Early stop: the composite stops only when EVERY member with a `decide` - * votes stop (a member without `decide` never votes stop). - * - * This is deliberately NOT joint multi-surface mutation: every member mutates - * the SAME `MutableSurface`. Joint profile-patch surfaces (prompt+skills+tools - * in one candidate) require the composite-surface contract and measured - * component attribution — see the experiment-optimal research brief. + * Split one generation's candidate budget across independent proposers. + * Candidate labels retain the originating proposer kind, duplicate surfaces + * collapse to the first result, and one failed proposer does not discard the + * other results. The composite stops only when every member with `decide` + * votes to stop. */ import { surfaceContentHash } from '../surface-identity' @@ -57,9 +32,27 @@ export function compositeProposer( const members = opts.proposers if (members.length === 0) throw new Error('compositeProposer: at least one member proposer required') + const memberKinds = new Set() + for (const member of members) { + if (!member.kind || member.kind.trim() !== member.kind) { + throw new Error('compositeProposer: member kinds must be trimmed and non-empty') + } + if (member.kind.includes(':')) { + throw new Error(`compositeProposer: member kind '${member.kind}' must not contain ':'`) + } + if (memberKinds.has(member.kind)) { + throw new Error(`compositeProposer: duplicate member kind '${member.kind}'`) + } + memberKinds.add(member.kind) + } const weights = opts.weights ?? members.map(() => 1) - if (weights.length !== members.length || weights.some((w) => !(w > 0))) { - throw new Error('compositeProposer: weights must match proposers length and be positive') + if ( + weights.length !== members.length || + weights.some((weight) => !Number.isFinite(weight) || weight <= 0) + ) { + throw new Error( + 'compositeProposer: weights must match proposers length and be finite and positive', + ) } return { @@ -91,7 +84,11 @@ export function compositeProposer( const share = shares[i] ?? 0 if (!member || share === 0 || ctx.signal.aborted) continue try { - const proposals = await member.propose({ ...ctx, populationSize: share }) + const proposals = await member.propose({ + ...ctx, + history: historyForMember(ctx.history, member.kind), + populationSize: share, + }) for (const proposal of proposals) { const isCandidate = typeof proposal === 'object' && proposal !== null && 'surface' in proposal @@ -133,7 +130,11 @@ export function compositeProposer( decide(args: { history: GenerationRecord[] }): { stop: boolean; reason?: string } { const votes = members .filter((m) => typeof m.decide === 'function') - .map((m) => (m.decide as NonNullable['decide']>)(args)) + .map((m) => + (m.decide as NonNullable['decide']>)({ + history: historyForMember(args.history, m.kind), + }), + ) if (votes.length === 0) return { stop: false } const allStop = votes.every((v) => v.stop) return allStop @@ -149,3 +150,15 @@ export function compositeProposer( }, } } + +function historyForMember(history: GenerationRecord[], memberKind: string): GenerationRecord[] { + const prefix = `${memberKind}:` + return history.map((generation) => ({ + ...generation, + candidates: generation.candidates.map((candidate) => + candidate.label?.startsWith(prefix) + ? { ...candidate, label: candidate.label.slice(prefix.length) } + : candidate, + ), + })) +} diff --git a/src/campaign/proposers/llm-policy-edit.ts b/src/campaign/proposers/llm-policy-edit.ts index 7350939d..c3adcce5 100644 --- a/src/campaign/proposers/llm-policy-edit.ts +++ b/src/campaign/proposers/llm-policy-edit.ts @@ -1,5 +1,5 @@ import { z } from 'zod' -import type { AgentProfileJson } from '../../agent-profile-cell' +import type { AgentProfileJson, AgentProfileJsonObject } from '../../agent-profile-cell' import { makePolicyEdit, POLICY_EDIT_AXES, @@ -49,6 +49,10 @@ const JSON_POLICY_EDIT_TARGET_SURFACES = [ export type JsonPolicyEditTargetSurface = (typeof JSON_POLICY_EDIT_TARGET_SURFACES)[number] const NonEmptyStringSchema = z.string().trim().min(1) +const JsonObjectKeySchema = z + .string() + .min(1) + .refine((key) => key.trim() === key, 'JSON object keys must not have surrounding whitespace') const JsonValueSchema: z.ZodType = z.lazy(() => z.union([ @@ -57,7 +61,7 @@ const JsonValueSchema: z.ZodType = z.lazy(() => z.boolean(), z.null(), z.array(JsonValueSchema), - z.record(NonEmptyStringSchema, JsonValueSchema), + z.record(JsonObjectKeySchema, JsonValueSchema), ]), ) @@ -424,6 +428,13 @@ export interface LlmPolicyEditProposerOptions { maxAuthorContextChars?: number /** Optional one-to-one pseudonymizer applied to every author-visible evidence field. */ scenarioIdTransform?: (scenarioId: string) => string + /** + * Remove credentials or unrelated fields from the current surface before it + * is sent to the model. The callback receives a clone and must preserve every + * editable path unchanged. Validated edits apply to the complete original. + * This callback does not redact findings or scored history. + */ + redactCurrentSurfaceForModel?: (surface: AgentProfileJsonObject) => AgentProfileJsonObject onAdmission?: (admission: PolicyEditAdmission) => void } @@ -504,6 +515,12 @@ export function llmPolicyEditProposer( { currentSurface, allowedJsonPaths, objectives, targetSurface: opts.targetSurface }, scenarioIds, ) + const modelSurface = redactCurrentSurfaceForModel( + currentSurface, + allowedJsonPaths, + opts.redactCurrentSurfaceForModel, + ) + assertSurfaceIsTaskAgnostic(modelSurface, scenarioIds) const measuredSources = measuredSourceMeasurements(ctx) const findings = citableFindings(ctx.findings, measuredSources, maxFindings) const findingByKey = new Map( @@ -516,7 +533,7 @@ export function llmPolicyEditProposer( objectives, candidateCount: limit, generation: ctx.generation, - currentSurface, + currentSurface: modelSurface, findings: findings.map((finding, index) => renderFinding(finding, `finding-${index + 1}`, scenarioIds, measuredSources), ), @@ -1251,7 +1268,7 @@ function bindAuthoredEdit( return makePolicyEdit(init) } -function parseJsonSurface(surface: MutableSurface): AgentProfileJson { +function parseJsonSurface(surface: MutableSurface): AgentProfileJsonObject { if (typeof surface !== 'string') { throw new Error('llmPolicyEditProposer: currentSurface must be serialized JSON') } @@ -1264,7 +1281,89 @@ function parseJsonSurface(surface: MutableSurface): AgentProfileJson { if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { throw new Error('llmPolicyEditProposer: currentSurface JSON root must be an object') } - return parsed as AgentProfileJson + return parsed as AgentProfileJsonObject +} + +function redactCurrentSurfaceForModel( + surface: AgentProfileJsonObject, + allowedJsonPaths: readonly string[], + redact: LlmPolicyEditProposerOptions['redactCurrentSurfaceForModel'], +): AgentProfileJsonObject { + if (!redact) return surface + const redacted = redact(structuredClone(surface)) + const parsed = JsonValueSchema.safeParse(redacted) + if (!parsed.success) { + const detail = formatJsonValidationError(parsed.error) + throw new Error( + `llmPolicyEditProposer: redactCurrentSurfaceForModel returned invalid JSON (${detail})`, + ) + } + if (!parsed.data || typeof parsed.data !== 'object' || Array.isArray(parsed.data)) { + throw new Error('llmPolicyEditProposer: redactCurrentSurfaceForModel must return a JSON object') + } + for (const path of allowedJsonPaths) { + if (!jsonValuesEqual(readJsonPath(surface, path), readJsonPath(parsed.data, path))) { + throw new Error( + `llmPolicyEditProposer: redactCurrentSurfaceForModel must not change or hide editable JSON path '${path}'`, + ) + } + } + return parsed.data as AgentProfileJsonObject +} + +function formatJsonValidationError(error: z.ZodError): string { + const messages = [...new Set(collectZodMessages(error.issues))] + const informative = messages.filter( + (message) => message !== 'Invalid input' && message !== 'Invalid key in record', + ) + const custom = informative.filter((message) => !message.startsWith('Invalid input: expected')) + return (custom.length > 0 ? custom : informative).join('; ') || 'invalid JSON value' +} + +function collectZodMessages(value: unknown): string[] { + if (Array.isArray(value)) return value.flatMap(collectZodMessages) + if (!value || typeof value !== 'object') return [] + const issue = value as Record + return [ + ...(typeof issue.message === 'string' ? [issue.message] : []), + ...collectZodMessages(issue.errors), + ...collectZodMessages(issue.issues), + ] +} + +function readJsonPath(root: AgentProfileJsonObject, path: string): AgentProfileJson | undefined { + let cursor: AgentProfileJson | undefined = root + for (const part of path + .split('.') + .map((segment) => segment.trim()) + .filter(Boolean)) { + if (!cursor || typeof cursor !== 'object' || Array.isArray(cursor)) return undefined + cursor = cursor[part] + } + return cursor +} + +function jsonValuesEqual( + left: AgentProfileJson | undefined, + right: AgentProfileJson | undefined, +): boolean { + if (left === right) return true + if (left === undefined || right === undefined || left === null || right === null) return false + if (Array.isArray(left) || Array.isArray(right)) { + return ( + Array.isArray(left) && + Array.isArray(right) && + left.length === right.length && + left.every((value, index) => jsonValuesEqual(value, right[index])) + ) + } + if (typeof left !== 'object' || typeof right !== 'object') return false + const leftKeys = Object.keys(left) + const rightKeys = Object.keys(right) + return ( + leftKeys.length === rightKeys.length && + leftKeys.every((key) => Object.hasOwn(right, key) && jsonValuesEqual(left[key], right[key])) + ) } interface CitableFinding { diff --git a/src/index.ts b/src/index.ts index 2dbfd184..d33ed70b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -20,6 +20,7 @@ export type { AgentProfileDimensionValue, AgentProfileHarness, AgentProfileJson, + AgentProfileJsonObject, AgentProfileKind, AgentProfileSource, AgentProfileSourceInput, diff --git a/tests/campaign/llm-policy-edit-proposer.test.ts b/tests/campaign/llm-policy-edit-proposer.test.ts index d5608e77..74a32497 100644 --- a/tests/campaign/llm-policy-edit-proposer.test.ts +++ b/tests/campaign/llm-policy-edit-proposer.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from 'vitest' +import type { AgentProfileJsonObject } from '../../src/agent-profile-cell' import { makePolicyEdit, makePolicyEditCandidateRecord, @@ -158,6 +159,7 @@ function proposer(input: { targetSurface?: 'agent-profile' | 'code' admissionMode?: 'evidence-only' | 'strict' admission?: PolicyEditAdmissionOptions + redactCurrentSurfaceForModel?: (surface: AgentProfileJsonObject) => AgentProfileJsonObject }) { return llmPolicyEditProposer({ llm: { @@ -192,6 +194,9 @@ function proposer(input: { : { maxAuthorContextChars: input.maxAuthorContextChars }), ...(input.admissionMode === undefined ? {} : { admissionMode: input.admissionMode }), ...(input.admission === undefined ? {} : { admission: input.admission }), + ...(input.redactCurrentSurfaceForModel === undefined + ? {} + : { redactCurrentSurfaceForModel: input.redactCurrentSurfaceForModel }), }) } @@ -273,6 +278,140 @@ describe('llmPolicyEditProposer', () => { expect(providerSchema).toContain('"mode":{"const":"remove"}') }) + it('redacts private fields from model input and applies edits to the complete surface', async () => { + const capture: CapturedRequest = {} + const out = await proposer({ + response: { edits: [authoredEdit('finding-1')] }, + capture, + redactCurrentSurfaceForModel: (surface) => ({ prompt: surface.prompt }), + }).propose( + context({ + finding: finding(), + currentSurface: JSON.stringify({ + prompt: { systemPrompt: 'Base' }, + mcp: { + linear: { + transport: 'http', + url: 'https://mcp.example.test', + headers: { Authorization: 'Bearer private-token' }, + }, + }, + }), + }), + ) + + expect(capture.user?.currentSurface).toEqual({ prompt: { systemPrompt: 'Base' } }) + expect(JSON.stringify(capture.user)).not.toContain('private-token') + expect(JSON.parse(candidateSurface(out[0]!))).toEqual({ + prompt: { systemPrompt: 'Read repository instructions first.' }, + mcp: { + linear: { + transport: 'http', + url: 'https://mcp.example.test', + headers: { Authorization: 'Bearer private-token' }, + }, + }, + }) + }) + + it('rejects a non-object redaction result before model dispatch', async () => { + const capture: CapturedRequest = {} + const configured = proposer({ + response: { edits: [] }, + capture, + redactCurrentSurfaceForModel: () => [] as unknown as AgentProfileJsonObject, + }) + + await expect(configured.propose(context({ finding: finding() }))).rejects.toThrow( + /redactCurrentSurfaceForModel must return a JSON object/, + ) + expect(capture.user).toBeUndefined() + }) + + it('rejects redaction that hides an editable subtree before model dispatch', async () => { + const capture: CapturedRequest = {} + const configured = proposer({ + response: { edits: [] }, + capture, + allowedJsonPaths: ['mcp.linear'], + redactCurrentSurfaceForModel: (surface) => ({ prompt: surface.prompt }), + }) + + await expect( + configured.propose( + context({ + finding: finding(), + currentSurface: JSON.stringify({ + prompt: { systemPrompt: 'Base' }, + mcp: { + linear: { + transport: 'http', + headers: { Authorization: 'Bearer private-token' }, + }, + }, + }), + }), + ), + ).rejects.toThrow(/must not change or hide editable JSON path 'mcp\.linear'/) + expect(capture.user).toBeUndefined() + }) + + it('does not let a mutating redaction callback alter the executable surface', async () => { + const out = await proposer({ + response: { edits: [authoredEdit('finding-1')] }, + redactCurrentSurfaceForModel: (surface) => { + delete surface.mcp + return { prompt: surface.prompt } + }, + }).propose( + context({ + finding: finding(), + currentSurface: JSON.stringify({ + prompt: { systemPrompt: 'Base' }, + mcp: { linear: { headers: { Authorization: 'Bearer private-token' } } }, + }), + }), + ) + + expect(JSON.parse(candidateSurface(out[0]!))).toMatchObject({ + mcp: { linear: { headers: { Authorization: 'Bearer private-token' } } }, + }) + }) + + it('rejects redacted JSON keys that validation would otherwise rewrite', async () => { + const configured = proposer({ + response: { edits: [] }, + redactCurrentSurfaceForModel: (surface) => ({ ...surface, ' mcp': {} }), + }) + + await expect(configured.propose(context({ finding: finding() }))).rejects.toThrow( + /redactCurrentSurfaceForModel returned invalid JSON.*surrounding whitespace/, + ) + }) + + it('rejects task identifiers introduced by model-surface redaction', async () => { + let redactionCalled = false + const configured = proposer({ + response: { edits: [authoredEdit('finding-1')] }, + scenarioIdTransform: () => 'task-1', + redactCurrentSurfaceForModel: (surface) => { + redactionCalled = true + return { ...surface, modelContext: { note: 'Handle private-task' } } + }, + }) + + await expect( + configured.propose( + context({ + finding: finding(), + currentSurface: '{"prompt":{"systemPrompt":"Base"}}', + baselineOutcome: measuredOutcome('baseline', 0.4, 'private-task'), + }), + ), + ).rejects.toThrow(/raw scenario identifier/) + expect(redactionCalled).toBe(true) + }) + it('shows the author measured baseline, incumbent, and exact parent deltas', async () => { const source = finding() const capture: CapturedRequest = {}