feat(judges): add reference equivalence scoring

src/analyst/chat-client.ts

@@ -170,44 +170,22 @@ function wrapLlmClient(
   return {
     transport,
     defaultModel,
-    chat: async (req, callOpts) => {
+    chat: (req, callOpts) => {
       const resolved = resolveModel(req, defaultModel)
-      // LlmClient.call doesn't accept an external AbortSignal today (it
-      // owns its own AbortController for the per-attempt timeout). We
-      // race the response against the caller's signal so awaiting code
-      // unblocks on abort. The in-flight HTTP request still runs to its
-      // own timeoutMs — when LlmClient grows a signal parameter, wire
-      // it directly here and drop the race.
-      const call = inner.call({
+      const request: LlmCallRequest = {
         model: resolved.model!,
         messages: req.messages,
         jsonMode: req.jsonMode,
         jsonSchema: req.jsonSchema,
         temperature: req.temperature,
         maxTokens: req.maxTokens,
         timeoutMs: req.timeoutMs,
-      })
-      if (!callOpts?.signal) return await call
-      return await Promise.race([call, abortAsRejection(callOpts.signal)])
+      }
+      return inner.call(request, callOpts?.signal ? { signal: callOpts.signal } : undefined)
     },
   }
 }
 
-function abortAsRejection(signal: AbortSignal): Promise<never> {
-  if (signal.aborted) return Promise.reject(toAbortError(signal))
-  return new Promise<never>((_, reject) => {
-    signal.addEventListener('abort', () => reject(toAbortError(signal)), { once: true })
-  })
-}
-
-function toAbortError(signal: AbortSignal): Error {
-  const reason = (signal as { reason?: unknown }).reason
-  if (reason instanceof Error) return reason
-  const e = new Error('ChatClient.chat: aborted')
-  e.name = 'AbortError'
-  return e
-}
-
 function resolveModel(req: ChatRequest, defaultModel: string | undefined): ChatRequest {
   if (req.model) return req
   if (!defaultModel) {

src/campaign/index.ts

@@ -37,6 +37,11 @@ export {
 // ── Judge builders (single-call bridge to a canonical JudgeConfig) ────
 export type { LlmJudgeDimension, LlmJudgeOptions } from '../llm-judge'
 export { llmJudge } from '../llm-judge'
+export type {
+  ReferenceEquivalenceJudgeOptions,
+  ReferenceEquivalenceScenario,
+} from '../reference-equivalence-judge'
+export { createReferenceEquivalenceJudge } from '../reference-equivalence-judge'
 // ── Meta-loop: optimize the analyst's OWN prompt as a surface ─────────
 export {
   type AnalystArtifact,

src/campaign/run-campaign.ts

@@ -11,6 +11,8 @@
 
 import { join } from 'node:path'
 import { BackendIntegrityError, type BackendIntegrityReport } from '../integrity/backend-integrity'
+import { JudgeParseError } from '../judges'
+import type { LlmCallMetadata } from '../llm-client'
 import { confidenceInterval } from '../statistics'
 import { contentHash } from '../verdict-cache'
 import { resolveRunDir } from './run-dir'
@@ -191,8 +193,7 @@ export async function runCampaign<TScenario extends Scenario, TArtifact>(
             dispatchTimeoutMs: opts.dispatchTimeoutMs,
           })
           cellsRef.push(result.cell)
-          enforceCellUsage(result.cell, opts.expectUsage ?? 'warn')
-          totalCostUsd += result.cell.costUsd
+          totalCostUsd += totalCellCost(result.cell)
           Object.assign(artifactsByPath, result.artifactsByPath)
           if (opts.costCeiling !== undefined && totalCostUsd >= opts.costCeiling) {
             costCeilingReached = true
@@ -273,6 +274,7 @@ async function executeCell<TScenario extends Scenario, TArtifact>(
       manifestHash: args.manifestHash,
     })
     if (cached.status === 'hit') {
+      enforceDispatchUsage(cached.cell, args.opts.expectUsage ?? 'warn')
       return { cell: { ...cached.cell, cached: true }, artifactsByPath: {} }
     }
   }
@@ -381,20 +383,42 @@ async function executeCell<TScenario extends Scenario, TArtifact>(
     args.signal.removeEventListener('abort', onCampaignAbort)
   }
 
+  try {
+    enforceDispatchUsage(
+      {
+        cellId: args.slot.cellId,
+        artifact,
+        costUsd: costSoFar,
+        tokenUsage: { ...tokensSoFar },
+      },
+      args.opts.expectUsage ?? 'warn',
+    )
+  } catch (err) {
+    await trace.flush()
+    throw err
+  }
+
   // Run judges (only if we have an artifact). A judge that throws invalidates
   // the cell — recorded as `error`, NOT folded into a fake composite:0 (a fake
   // zero is indistinguishable from a real zero and poisons every aggregate).
   const judgeScores: Record<string, JudgeScore> = {}
+  let failedJudgeCall: LlmCallMetadata | undefined
   if (artifact !== undefined) {
     for (const judge of args.opts.judges ?? []) {
       if (judge.appliesTo && !judge.appliesTo(args.slot.scenario)) continue
       try {
-        judgeScores[judge.name] = await runJudgeCell(judge, {
+        const score = await runJudgeCell(judge, {
           artifact,
           scenario: args.slot.scenario,
           signal: args.signal,
         })
+        traceJudgeCost(trace, judge.name, score.llmCall)
+        judgeScores[judge.name] = score
       } catch (err) {
+        if (err instanceof JudgeParseError) {
+          failedJudgeCall = err.llmCall
+          traceJudgeCost(trace, judge.name, err.llmCall)
+        }
         errorMessage = `judge '${judge.name}' failed: ${err instanceof Error ? err.message : String(err)}`
         break
       }
@@ -412,6 +436,7 @@ async function executeCell<TScenario extends Scenario, TArtifact>(
     judgeScores,
     costUsd: costSoFar,
     tokenUsage: { ...tokensSoFar },
+    ...(failedJudgeCall ? { failedJudgeCall } : {}),
     ...(resolvedModel ? { resolvedModel } : {}),
     durationMs: Date.now() - startMs,
     seed: args.slot.cellSeed,
@@ -540,17 +565,16 @@ export function planCampaignRun<TScenario extends Scenario, TArtifact>(
 }
 
 /**
- * Per-cell stub guard. A cell that produced an artifact (no error) but reported
- * `costUsd === 0` AND zero tokens means the dispatch never called `ctx.cost` —
+ * Per-dispatch stub guard. An artifact produced with `costUsd === 0` AND zero
+ * tokens means the dispatch never called `ctx.cost` —
  * i.e. it ran against a stub or silently dropped its usage. `'warn'` logs it,
- * `'assert'` throws (fail-fast), `'off'` skips. An errored/skipped cell or a
- * deterministic judge-only run that genuinely made no LLM call is not flagged.
+ * `'assert'` throws (fail-fast), and `'off'` skips the check.
  */
-function enforceCellUsage<TArtifact>(
-  cell: CampaignCellResult<TArtifact>,
+function enforceDispatchUsage<TArtifact>(
+  cell: Pick<CampaignCellResult<TArtifact>, 'cellId' | 'artifact' | 'costUsd' | 'tokenUsage'>,
   mode: 'assert' | 'warn' | 'off',
 ): void {
-  if (mode === 'off' || cell.error) return
+  if (mode === 'off') return
   if (cell.artifact === null || cell.artifact === undefined) return
   const zeroTokens = cell.tokenUsage.input === 0 && cell.tokenUsage.output === 0
   if (cell.costUsd !== 0 || !zeroTokens) return
@@ -580,6 +604,24 @@ async function runJudgeCell<TArtifact, TScenario extends Scenario>(
   return judge.score(input)
 }
 
+function traceJudgeCost(
+  trace: CampaignTraceWriter,
+  judgeName: string,
+  call: LlmCallMetadata | undefined,
+): void {
+  if (call?.costUsd != null) {
+    trace.span(`cost.judge.${judgeName}`, { amountUsd: call.costUsd }).end()
+  }
+}
+
+function totalCellCost<TArtifact>(cell: CampaignCellResult<TArtifact>): number {
+  const scored = Object.values(cell.judgeScores).reduce(
+    (total, score) => total + (score.llmCall?.costUsd ?? 0),
+    0,
+  )
+  return cell.costUsd + scored + (cell.failedJudgeCall?.costUsd ?? 0)
+}
+
 function defaultBuildTraceWriter(
   storage: CampaignStorage,
 ): (cellId: string, dir: string) => CampaignTraceWriter {
@@ -752,7 +794,7 @@ function computeAggregates<TArtifact>(
   return {
     byJudge,
     byScenario,
-    totalCostUsd: cells.reduce((a, c) => a + c.costUsd, 0),
+    totalCostUsd: cells.reduce((total, cell) => total + totalCellCost(cell), 0),
     cellsExecuted: cells.filter((c) => !c.error).length,
     cellsSkipped: cells.filter((c) => c.error?.startsWith('skipped:')).length,
     cellsCached: cells.filter((c) => c.cached).length,

src/campaign/types.ts

@@ -17,6 +17,7 @@
  */
 
 import type { PolicyEditCandidateRecord } from '../analyst/policy-edit'
+import type { LlmCallMetadata } from '../llm-client'
 import type { RunTokenUsage } from '../run-record'
 
 /** Stable identifier + kind tag for any scenario. Consumers
@@ -118,6 +119,8 @@ export interface JudgeScore {
   dimensions: Record<string, number>
   composite: number
   notes: string
+  /** Present when scoring made an LLM call. Campaigns add it to cell accounting. */
+  llmCall?: LlmCallMetadata
   /** Set when the judge itself failed (call error, unparseable output).
    *  `composite`/`dimensions` carry no signal — aggregators MUST exclude
    *  failed scores from means instead of folding them into zeros. */
@@ -575,6 +578,8 @@ export interface CampaignCellResult<TArtifact> {
    *  `{ input: 0, output: 0 }` when the dispatch reported none — which the
    *  backend-integrity guard reads as a stub. */
   tokenUsage: CampaignTokenUsage
+  /** Paid call from a judge whose response could not be scored. */
+  failedJudgeCall?: LlmCallMetadata
   /** The concrete model the backend resolved this cell to at runtime, reported
    *  by the dispatch via `ctx.cost.observeModel`. Set only when the dispatch
    *  reported it — a profile that declares a concrete model up front has no

src/contract/index.ts

@@ -126,6 +126,26 @@ export {
 } from '../campaign/presets/run-improvement-loop'
 export { type RunCampaignOptions, runCampaign } from '../campaign/run-campaign'
 
+// ── Reference-answer judge ───────────────────────────────────────────
+
+export {
+  type ChatClient,
+  type CreateChatClientOpts,
+  createChatClient,
+} from '../analyst/chat-client'
+export type {
+  ReferenceEquivalenceJudgeInput,
+  ReferenceEquivalenceJudgeOptions,
+  ReferenceEquivalenceJudgeResult,
+  ReferenceEquivalenceScenario,
+} from '../reference-equivalence-judge'
+export {
+  createReferenceEquivalenceJudge,
+  REFERENCE_EQUIVALENCE_INPUT_LIMITS,
+  REFERENCE_EQUIVALENCE_JUDGE_VERSION,
+  runReferenceEquivalenceJudge,
+} from '../reference-equivalence-judge'
+
 // ── Proposers ────────────────────────────────────────────────────────
 
 export {

src/index.ts

@@ -1114,6 +1114,7 @@ export {
   runKeywordCoverageJudgeUrl,
 } from './keyword-coverage-judge'
 export type {
+  LlmCallMetadata,
   LlmCallRequest,
   LlmCallResult,
   LlmClientOptions,
@@ -1153,6 +1154,18 @@ export type {
   MultiToolchainLayerConfig,
 } from './multi-toolchain-layer'
 export { mergeLayerResults, multiToolchainLayer } from './multi-toolchain-layer'
+export type {
+  ReferenceEquivalenceJudgeInput,
+  ReferenceEquivalenceJudgeOptions,
+  ReferenceEquivalenceJudgeResult,
+  ReferenceEquivalenceScenario,
+} from './reference-equivalence-judge'
+export {
+  createReferenceEquivalenceJudge,
+  REFERENCE_EQUIVALENCE_INPUT_LIMITS,
+  REFERENCE_EQUIVALENCE_JUDGE_VERSION,
+  runReferenceEquivalenceJudge,
+} from './reference-equivalence-judge'
 // ── Reference replay ─────────────────────────────────────────────────
 export {
   compareReferenceReplay,

src/judges.ts

@@ -1,7 +1,10 @@
 import type { TCloud } from '@tangle-network/tcloud'
 import { JudgeError } from './errors'
+import type { LlmCallMetadata } from './llm-client'
 import type { JudgeFn, JudgeInput, JudgeScore } from './types'
 
+type JudgeParseErrorOptions = { cause?: unknown; llmCall?: LlmCallMetadata }
+
 /**
  * A judge's LLM response could not be parsed into scored dimensions.
  * Thrown instead of fabricating a `{ dimension: 'parse_error', score: 0 }`
@@ -14,11 +17,14 @@ export class JudgeParseError extends JudgeError {
   readonly judgeName: string
   /** The raw (truncated) model response that failed to parse. */
   readonly raw: string
+  /** Paid-call metadata remains available even when the verdict is unusable. */
+  readonly llmCall?: LlmCallMetadata
 
-  constructor(judgeName: string, raw: string, options?: { cause?: unknown }) {
+  constructor(judgeName: string, raw: string, options?: JudgeParseErrorOptions) {
     super(`judge '${judgeName}' returned an unparseable response: ${raw.slice(0, 200)}`, options)
     this.judgeName = judgeName
     this.raw = raw
+    this.llmCall = options?.llmCall
   }
 }
 

src/llm-client.ts

@@ -99,6 +99,8 @@ export interface LlmCallResult {
   raw: Record<string, unknown>
 }
 
+export type LlmCallMetadata = Pick<LlmCallResult, 'usage' | 'costUsd' | 'model' | 'durationMs'>
+
 export class LlmCallError extends AgentEvalError {
   constructor(
     message: string,
@@ -691,17 +693,22 @@ export async function callLlmJson<T = unknown>(
   req: LlmCallRequest,
   opts: LlmClientOptions = {},
 ): Promise<{ value: T; result: LlmCallResult }> {
+  const result = await callLlmStructured(req, opts)
+  const value = parseJsonResult<T>(result)
+  return { value, result }
+}
+
+/** Shared schema-to-JSON-mode fallback that preserves the raw result. */
+async function callLlmStructured(
+  req: LlmCallRequest,
+  opts: LlmClientOptions = {},
+): Promise<LlmCallResult> {
   try {
-    const result = await callLlm({ ...req, jsonMode: req.jsonMode ?? !req.jsonSchema }, opts)
-    const value = parseJsonResult<T>(result)
-    return { value, result }
+    return await callLlm({ ...req, jsonMode: req.jsonMode ?? !req.jsonSchema }, opts)
   } catch (err) {
     if (err instanceof LlmCallError && isSchemaRejection(err.status, err.body) && req.jsonSchema) {
-      // Degrade to json_object + retry.
       const degradedReq: LlmCallRequest = { ...req, jsonMode: true, jsonSchema: undefined }
-      const result = await callLlm(degradedReq, opts)
-      const value = parseJsonResult<T>(result)
-      return { value, result }
+      return await callLlm(degradedReq, opts)
     }
     throw err
   }
@@ -886,7 +893,8 @@ export class LlmClient {
   constructor(private readonly opts: LlmClientOptions = {}) {}
 
   call(req: LlmCallRequest, per?: LlmClientOptions): Promise<LlmCallResult> {
-    return callLlm(req, { ...this.opts, ...per })
+    const options = { ...this.opts, ...per }
+    return req.jsonSchema ? callLlmStructured(req, options) : callLlm(req, options)
   }
 
   callJson<T = unknown>(