Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval-

### Added

- `llmPolicyEditProposer({ redactCurrentSurfaceForModel })` can remove credentials and unrelated fields from the current surface sent to the model while applying validated edits to the complete original surface.
- `CostLedger.listPending()` exposes immutable pending paid calls and distinguishes calls that are active, late after cancellation, or interrupted by a prior process so durable workflows can reconcile exact reservations before resuming.
- `traceAnalystProposer()` accepts an opt-in `resolvePriorFindings` callback that forwards canonical prior findings into the existing analyst registry.

Expand All @@ -20,6 +21,7 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval-

### Fixed

- `compositeProposer` restores each member's original labels when replaying history, so stateful members do not repeat candidates whose labels were decorated for provenance.
- `InsightReport.interRater.kappa` now reports quadratic weighted kappa instead of Pearson correlation.
Read `interRater.pearson` for the previous correlation measure; `icc` and `spearman` are now reported separately.
- Compute contextual-bandit doubly robust estimates with separate logged-action and target-policy value terms, expose how many rows use DR versus IPS or the deprecated scalar path, and carry both values through belief-state records.
Expand Down
4 changes: 3 additions & 1 deletion src/agent-profile-cell.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,15 @@ import { hashJson } from './pre-registration'

export type AgentProfileCellSchemaVersion = 'agent-profile-cell/v1'

export type AgentProfileJsonObject = { [key: string]: AgentProfileJson }

export type AgentProfileJson =
| string
| number
| boolean
| null
| AgentProfileJson[]
| { [key: string]: AgentProfileJson }
| AgentProfileJsonObject

export type AgentProfileDimensionValue = string | number | boolean | null

Expand Down
98 changes: 97 additions & 1 deletion src/campaign/proposers/composite.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { describe, expect, it } from 'vitest'
import type { CodeSurface, ProposeContext, SurfaceProposer } from '../types'
import { compositeProposer } from './composite'
import { parameterSweepProposer } from './fapo'

function ctxOf(populationSize: number): ProposeContext {
return {
Expand Down Expand Up @@ -103,6 +104,49 @@ describe('compositeProposer (N proposers, one generation pool)', () => {
expect((pool[0] as { surface: CodeSurface }).surface.worktreeRef).toBe(first.worktreeRef)
})

it('lets stateful members recognize their own prior labels', async () => {
const composite = compositeProposer({
proposers: [
parameterSweepProposer({
candidates: [
{ label: 'low', rationale: 'try low', patch: { effort: 'low' } },
{ label: 'high', rationale: 'try high', patch: { effort: 'high' } },
],
}),
],
})
const first = await composite.propose({
...ctxOf(1),
currentSurface: '{"effort":"medium"}',
})
const firstCandidate = first[0] as { surface: string; label: string; rationale: string }
const second = await composite.propose({
...ctxOf(2),
currentSurface: '{"effort":"medium"}',
history: [
{
generationIndex: 0,
promoted: [],
candidates: [
{
surfaceHash: 'first',
label: firstCandidate.label,
rationale: firstCandidate.rationale,
composite: 0.5,
ci95: [0.5, 0.5],
dimensions: {},
scenarios: [],
},
],
},
],
})

expect(firstCandidate.label).toBe('parameter-sweep:low')
expect(second).toHaveLength(1)
expect((second[0] as { label: string }).label).toBe('parameter-sweep:high')
})

it('isolates a failing member; throws only when ALL members fail', async () => {
const oneDown = compositeProposer({
proposers: [stub('boom', [], { fail: true }), stub('ok', ['x1', 'x2'])],
Expand Down Expand Up @@ -132,10 +176,62 @@ describe('compositeProposer (N proposers, one generation pool)', () => {
expect(unanimous.decide?.({ history: [] })?.stop).toBe(true)
})

it('restores member labels before asking members whether to stop', () => {
let observedLabel: string | undefined
const member: SurfaceProposer = {
kind: 'stateful',
propose: async () => [],
decide: ({ history }) => {
observedLabel = history[0]?.candidates[0]?.label
return { stop: true }
},
}
const composite = compositeProposer({ proposers: [member] })

composite.decide?.({
history: [
{
generationIndex: 0,
promoted: [],
candidates: [
{
surfaceHash: 'first',
label: 'stateful:attempt-1',
composite: 0.5,
ci95: [0.5, 0.5],
dimensions: {},
scenarios: [],
},
],
},
],
})

expect(observedLabel).toBe('attempt-1')
})

it('rejects member kinds that make history ownership ambiguous', () => {
expect(() =>
compositeProposer({ proposers: [stub('same', ['a']), stub('same', ['b'])] }),
).toThrow(/duplicate member kind 'same'/)
expect(() => compositeProposer({ proposers: [stub('a:b', ['a'])] })).toThrow(
/must not contain ':'/,
)
expect(() => compositeProposer({ proposers: [stub(' spaced ', ['a'])] })).toThrow(
/trimmed and non-empty/,
)
})

it('fails loud on empty membership or bad weights', () => {
expect(() => compositeProposer({ proposers: [] })).toThrow(/at least one/)
expect(() => compositeProposer({ proposers: [stub('a', ['x'])], weights: [0] })).toThrow(
/positive/,
/finite and positive/,
)
expect(() =>
compositeProposer({ proposers: [stub('a', ['x'])], weights: [Number.POSITIVE_INFINITY] }),
).toThrow(/finite and positive/)
expect(() =>
compositeProposer({ proposers: [stub('a', ['x'])], weights: [Number.NaN] }),
).toThrow(/finite and positive/)
})
})
81 changes: 47 additions & 34 deletions src/campaign/proposers/composite.ts
Original file line number Diff line number Diff line change
@@ -1,34 +1,9 @@
/**
* `compositeProposer` — run N proposers TOGETHER on the same surface.
*
* The question this answers ("why can't we combine GEPA + skillOpt + ACE + a
* trace-analyst?"): nothing in the loop cares where candidates come from — the
* generation's population is one pool and the Pareto frontier / promotion logic
* evaluates every candidate identically. The only missing piece was a proposer
* that fans the population budget out across members and merges their proposals.
* This is that piece.
*
* Semantics:
* - Budget: each member is asked for a share of `populationSize`
* (near-equal split by default, or explicit `weights`). Members may return
* fewer; the pool is topped up round-robin from members that can offer more
* is NOT attempted — proposers are not obligated to be re-entrant.
* - Provenance: every candidate's `label` is prefixed with its member's kind
* (`gepa:...`, `skill-opt:...`) so generation records and the promotion
* provenance attribute each winner to the proposer family that made it —
* the cheap, honest version of proposer-level credit assignment.
* - Dedup: identical surfaces from different members collapse to the first.
* - Failure isolation: one member throwing does not sink the generation; its
* error is logged into the surviving candidates' generation via a warning
* and the pool proceeds (a generation with zero candidates from all members
* failing still throws — that is a real failure).
* - Early stop: the composite stops only when EVERY member with a `decide`
* votes stop (a member without `decide` never votes stop).
*
* This is deliberately NOT joint multi-surface mutation: every member mutates
* the SAME `MutableSurface`. Joint profile-patch surfaces (prompt+skills+tools
* in one candidate) require the composite-surface contract and measured
* component attribution — see the experiment-optimal research brief.
* Split one generation's candidate budget across independent proposers.
* Candidate labels retain the originating proposer kind, duplicate surfaces
* collapse to the first result, and one failed proposer does not discard the
* other results. The composite stops only when every member with `decide`
* votes to stop.
*/

import { surfaceContentHash } from '../surface-identity'
Expand Down Expand Up @@ -57,9 +32,27 @@ export function compositeProposer<TFindings = unknown>(
const members = opts.proposers
if (members.length === 0)
throw new Error('compositeProposer: at least one member proposer required')
const memberKinds = new Set<string>()
for (const member of members) {
if (!member.kind || member.kind.trim() !== member.kind) {
throw new Error('compositeProposer: member kinds must be trimmed and non-empty')
}
if (member.kind.includes(':')) {
throw new Error(`compositeProposer: member kind '${member.kind}' must not contain ':'`)
}
if (memberKinds.has(member.kind)) {
throw new Error(`compositeProposer: duplicate member kind '${member.kind}'`)
}
memberKinds.add(member.kind)
}
const weights = opts.weights ?? members.map(() => 1)
if (weights.length !== members.length || weights.some((w) => !(w > 0))) {
throw new Error('compositeProposer: weights must match proposers length and be positive')
if (
weights.length !== members.length ||
weights.some((weight) => !Number.isFinite(weight) || weight <= 0)
) {
throw new Error(
'compositeProposer: weights must match proposers length and be finite and positive',
)
}

return {
Expand Down Expand Up @@ -91,7 +84,11 @@ export function compositeProposer<TFindings = unknown>(
const share = shares[i] ?? 0
if (!member || share === 0 || ctx.signal.aborted) continue
try {
const proposals = await member.propose({ ...ctx, populationSize: share })
const proposals = await member.propose({
...ctx,
history: historyForMember(ctx.history, member.kind),
populationSize: share,
})
for (const proposal of proposals) {
const isCandidate =
typeof proposal === 'object' && proposal !== null && 'surface' in proposal
Expand Down Expand Up @@ -133,7 +130,11 @@ export function compositeProposer<TFindings = unknown>(
decide(args: { history: GenerationRecord[] }): { stop: boolean; reason?: string } {
const votes = members
.filter((m) => typeof m.decide === 'function')
.map((m) => (m.decide as NonNullable<SurfaceProposer<TFindings>['decide']>)(args))
.map((m) =>
(m.decide as NonNullable<SurfaceProposer<TFindings>['decide']>)({
history: historyForMember(args.history, m.kind),
}),
)
if (votes.length === 0) return { stop: false }
const allStop = votes.every((v) => v.stop)
return allStop
Expand All @@ -149,3 +150,15 @@ export function compositeProposer<TFindings = unknown>(
},
}
}

function historyForMember(history: GenerationRecord[], memberKind: string): GenerationRecord[] {
const prefix = `${memberKind}:`
return history.map((generation) => ({
...generation,
candidates: generation.candidates.map((candidate) =>
candidate.label?.startsWith(prefix)
? { ...candidate, label: candidate.label.slice(prefix.length) }
: candidate,
),
}))
}
Loading
Loading