feat(statistics): add exact paired sign test

CHANGELOG.md

@@ -4,6 +4,19 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval-
 
 ---
 
+## [0.114.0] — 2026-07-11 — exact directional paired inference
+
+### Added
+
+- `pairedSignTest(differences, alternative)` computes an exact one-sided sign test for paired numeric differences, excludes zero ties, requires a predeclared `greater` or `less` direction, and reports every denominator.
+
+### Changed
+
+- McNemar's exact two-sided calculation now reuses the same log-space binomial-tail implementation without changing its public result contract.
+
+This release is additive.
+Existing consumers do not need to update unless they want the new statistic.
+
 ## [0.113.0] — 2026-07-10 — immutable code candidates
 
 ### Changed

clients/python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "agent-eval-rpc"
-version = "0.113.0"
+version = "0.114.0"
 description = "Python RPC client for @tangle-network/agent-eval — judge content against rubrics over HTTP or stdio RPC. Eval logic runs in the Node runtime; this package is a thin wire client."
 readme = "README.md"
 requires-python = ">=3.10"

clients/python/src/agent_eval_rpc/__init__.py

@@ -58,7 +58,7 @@
 try:
     __version__ = version("agent-eval-rpc")
 except PackageNotFoundError:
-    __version__ = "0.113.0"
+    __version__ = "0.114.0"
 
 __all__ = [
     "Client",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "0.113.0",
+  "version": "0.114.0",
   "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {

scripts/verify-package-exports.mjs

@@ -57,6 +57,10 @@ try {
       '--input-type=module',
       '--eval',
       `
+        const root = await import('@tangle-network/agent-eval')
+        if (!('pairedSignTest' in root)) throw new Error('missing root export pairedSignTest')
+        const signTest = root.pairedSignTest([1, 0.5], 'greater')
+        if (signTest.pValue !== 0.25) throw new Error('invalid packed pairedSignTest result')
       `,
     ],
     appDir,

src/index.ts

@@ -399,8 +399,10 @@ export type {
   McNemarResult,
   PairedBootstrapOptions,
   PairedBootstrapResult,
+  PairedSignTestResult,
   ProportionInterval,
   RiskDifferenceResult,
+  SignTestAlternative,
   WeightedCompositeInput,
   WeightedCompositeResult,
 } from './statistics'
@@ -424,6 +426,7 @@ export {
   pairedBootstrap,
   pairedMde,
   pairedRiskDifference,
+  pairedSignTest,
   pairedTTest,
   partialCredit,
   passAtK,

src/statistics.ts

@@ -1003,6 +1003,78 @@ export function pairedBootstrap(
   }
 }
 
+/** Pre-registered direction for a one-sided paired sign test. */
+export type SignTestAlternative = 'greater' | 'less'
+
+/** Exact one-sided sign-test result for paired numeric differences. */
+export interface PairedSignTestResult {
+  /** Total supplied differences, including zero ties. */
+  n: number
+  /** Strictly positive differences. */
+  positive: number
+  /** Strictly negative differences. */
+  negative: number
+  /** Zero differences excluded from the binomial test. */
+  ties: number
+  /** Non-zero differences used by the binomial test. */
+  nNonTies: number
+  /** Direction of the pre-registered alternative hypothesis. */
+  alternative: SignTestAlternative
+  /** Exact one-sided p-value under P(positive) = P(negative) = 0.5. */
+  pValue: number
+}
+
+/**
+ * Exact one-sided sign test over paired differences.
+ *
+ * Pass `after[i] - before[i]` for each matched item. `alternative = 'greater'`
+ * tests whether positive signs are more likely than negative signs and returns
+ * `P(Binomial(nNonTies, 0.5) >= positive)`. `alternative = 'less'` treats
+ * negative signs as successes instead. With a continuous difference
+ * distribution this is the usual directional median test. Exact zero
+ * differences are ties and do not enter the binomial denominator. All-tie and
+ * empty inputs return p = 1. Every input difference must be finite, and the
+ * direction must be chosen explicitly so a caller cannot select it after
+ * seeing the signs.
+ */
+export function pairedSignTest(
+  differences: readonly number[],
+  alternative: SignTestAlternative,
+): PairedSignTestResult {
+  if (alternative !== 'greater' && alternative !== 'less') {
+    throw new ValidationError(
+      `pairedSignTest: alternative must be 'greater' or 'less', got ${alternative}`,
+    )
+  }
+
+  let positive = 0
+  let negative = 0
+  let ties = 0
+  for (let i = 0; i < differences.length; i++) {
+    const difference = differences[i]!
+    if (!Number.isFinite(difference)) {
+      throw new ValidationError(
+        `pairedSignTest: difference at index ${i} must be finite, got ${difference}`,
+      )
+    }
+    if (difference > 0) positive++
+    else if (difference < 0) negative++
+    else ties++
+  }
+
+  const nNonTies = positive + negative
+  const successes = alternative === 'greater' ? positive : negative
+  return {
+    n: differences.length,
+    positive,
+    negative,
+    ties,
+    nNonTies,
+    alternative,
+    pValue: binomialHalfUpperTail(successes, nNonTies),
+  }
+}
+
 // ── Binomial proportion + paired-binary + coding-eval estimators ─────
 //
 // The paired family above (pairedBootstrap/pairedTTest/wilcoxonSignedRank)
@@ -1192,14 +1264,44 @@ export function passAtK(n: number, c: number, k: number): number {
 function binomialSignTwoSided(b: number, c: number): number {
   const nd = b + c
   if (nd === 0) return 1
-  const k = Math.min(b, c)
-  const logHalfN = nd * Math.log(0.5)
-  let tail = 0
-  for (let i = 0; i <= k; i++) {
-    const logChoose = lnGamma(nd + 1) - lnGamma(i + 1) - lnGamma(nd - i + 1)
-    tail += Math.exp(logChoose + logHalfN)
+  return Math.min(1, 2 * binomialHalfLowerTail(Math.min(b, c), nd))
+}
+
+/** P(X >= successes) for X ~ Binomial(n, 0.5). */
+function binomialHalfUpperTail(successes: number, n: number): number {
+  if (successes <= 0) return 1
+  if (successes > n) return 0
+
+  // Use the smaller side of the distribution. For lower thresholds the
+  // complement sums at most half the mass; for upper thresholds symmetry
+  // maps the upper tail to a lower tail without subtraction.
+  if (successes <= n / 2) {
+    return Math.max(0, 1 - binomialHalfLowerTail(successes - 1, n))
   }
-  return Math.min(1, 2 * tail)
+  return binomialHalfLowerTail(n - successes, n)
+}
+
+/** P(X <= maxSuccesses) for X ~ Binomial(n, 0.5), accumulated in log space. */
+function binomialHalfLowerTail(maxSuccesses: number, n: number): number {
+  if (maxSuccesses < 0) return 0
+  if (maxSuccesses >= n) return 1
+  if (maxSuccesses === 0) return 2 ** -n
+
+  const logHalfN = n * Math.log(0.5)
+  let logTail = Number.NEGATIVE_INFINITY
+  for (let i = 0; i <= maxSuccesses; i++) {
+    const logChoose = lnGamma(n + 1) - lnGamma(i + 1) - lnGamma(n - i + 1)
+    logTail = logAddExp(logTail, logChoose + logHalfN)
+  }
+  return Math.min(1, Math.exp(logTail))
+}
+
+function logAddExp(a: number, b: number): number {
+  if (a === Number.NEGATIVE_INFINITY) return b
+  if (b === Number.NEGATIVE_INFINITY) return a
+  const max = Math.max(a, b)
+  const min = Math.min(a, b)
+  return max + Math.log1p(Math.exp(min - max))
 }
 
 // ── Anytime-valid e-process (betting test-martingale) ────────────────

tests/statistics.test.ts

@@ -15,6 +15,7 @@ import {
   pairedBootstrap,
   pairedMde,
   pairedRiskDifference,
+  pairedSignTest,
   pairedTTest,
   partialCredit,
   passAtK,
@@ -676,6 +677,64 @@ describe('pairedRiskDifference — paired-binary effect size + CI', () => {
   })
 })
 
+describe('pairedSignTest — exact one-sided paired differences', () => {
+  it('returns p = 0.25 for two positive differences', () => {
+    const result = pairedSignTest([1, 0.5], 'greater')
+    expect(result).toEqual({
+      n: 2,
+      positive: 2,
+      negative: 0,
+      ties: 0,
+      nNonTies: 2,
+      alternative: 'greater',
+      pValue: 0.25,
+    })
+  })
+
+  it('ignores zero ties in the binomial denominator', () => {
+    const result = pairedSignTest([1, 0], 'greater')
+    expect(result.nNonTies).toBe(1)
+    expect(result.ties).toBe(1)
+    expect(result.pValue).toBe(0.5)
+  })
+
+  it('returns p = 1 when every difference is tied', () => {
+    const result = pairedSignTest([0, -0, 0], 'greater')
+    expect(result.nNonTies).toBe(0)
+    expect(result.ties).toBe(3)
+    expect(result.pValue).toBe(1)
+  })
+
+  it('supports the opposite pre-registered direction', () => {
+    expect(pairedSignTest([1, 0.5], 'less').pValue).toBe(1)
+    expect(pairedSignTest([-1, -0.5], 'less').pValue).toBe(0.25)
+  })
+
+  it('matches a multi-term exact binomial tail without combinatorial overflow', () => {
+    const result = pairedSignTest([1, 1, 1, -1], 'greater')
+    expect(result.pValue).toBeCloseTo(5 / 16, 15)
+  })
+
+  it('is symmetric under sign reversal and direction reversal', () => {
+    const differences = [1, -0.5, 0, 2, 3, -4]
+    const greater = pairedSignTest(differences, 'greater')
+    const reflected = pairedSignTest(
+      differences.map((difference) => -difference),
+      'less',
+    )
+    expect(reflected.pValue).toBeCloseTo(greater.pValue, 15)
+    expect(reflected.positive).toBe(greater.negative)
+    expect(reflected.negative).toBe(greater.positive)
+    expect(reflected.ties).toBe(greater.ties)
+  })
+
+  it('rejects non-finite differences and invalid directions', () => {
+    expect(() => pairedSignTest([1, Number.NaN], 'greater')).toThrow(/index 1.*finite/)
+    expect(() => pairedSignTest([Number.POSITIVE_INFINITY], 'less')).toThrow(/finite/)
+    expect(() => pairedSignTest([1], 'two-sided' as 'greater')).toThrow(/alternative/)
+  })
+})
+
 describe('passAtK — unbiased coding-eval estimator', () => {
   it('0 correct ⇒ 0, all correct ⇒ 1', () => {
     expect(passAtK(5, 0, 1)).toBe(0)