From b0543194af339e4802438dbdaaf9f266cf3f0ebf Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sat, 11 Jul 2026 02:28:06 -0600 Subject: [PATCH 1/3] feat(statistics): add exact paired sign test --- scripts/verify-package-exports.mjs | 4 + src/index.ts | 3 + src/statistics.ts | 116 +++++++++++++++++++++++++++-- tests/statistics.test.ts | 59 +++++++++++++++ 4 files changed, 175 insertions(+), 7 deletions(-) diff --git a/scripts/verify-package-exports.mjs b/scripts/verify-package-exports.mjs index 083b81ed..8734c5eb 100644 --- a/scripts/verify-package-exports.mjs +++ b/scripts/verify-package-exports.mjs @@ -57,6 +57,10 @@ try { '--input-type=module', '--eval', ` + const root = await import('@tangle-network/agent-eval') + if (!('pairedSignTest' in root)) throw new Error('missing root export pairedSignTest') + const signTest = root.pairedSignTest([1, 0.5], 'greater') + if (signTest.pValue !== 0.25) throw new Error('invalid packed pairedSignTest result') `, ], appDir, diff --git a/src/index.ts b/src/index.ts index ff68d6c2..08492a4b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -399,8 +399,10 @@ export type { McNemarResult, PairedBootstrapOptions, PairedBootstrapResult, + PairedSignTestResult, ProportionInterval, RiskDifferenceResult, + SignTestAlternative, WeightedCompositeInput, WeightedCompositeResult, } from './statistics' @@ -424,6 +426,7 @@ export { pairedBootstrap, pairedMde, pairedRiskDifference, + pairedSignTest, pairedTTest, partialCredit, passAtK, diff --git a/src/statistics.ts b/src/statistics.ts index 7e8c70f6..f38d28f2 100644 --- a/src/statistics.ts +++ b/src/statistics.ts @@ -1003,6 +1003,78 @@ export function pairedBootstrap( } } +/** Pre-registered direction for a one-sided paired sign test. */ +export type SignTestAlternative = 'greater' | 'less' + +/** Exact one-sided sign-test result for paired numeric differences. */ +export interface PairedSignTestResult { + /** Total supplied differences, including zero ties. */ + n: number + /** Strictly positive differences. */ + positive: number + /** Strictly negative differences. */ + negative: number + /** Zero differences excluded from the binomial test. */ + ties: number + /** Non-zero differences used by the binomial test. */ + nNonTies: number + /** Direction of the pre-registered alternative hypothesis. */ + alternative: SignTestAlternative + /** Exact one-sided p-value under P(positive) = P(negative) = 0.5. */ + pValue: number +} + +/** + * Exact one-sided sign test over paired differences. + * + * Pass `after[i] - before[i]` for each matched item. `alternative = 'greater'` + * tests whether positive signs are more likely than negative signs and returns + * `P(Binomial(nNonTies, 0.5) >= positive)`. `alternative = 'less'` treats + * negative signs as successes instead. With a continuous difference + * distribution this is the usual directional median test. Exact zero + * differences are ties and do not enter the binomial denominator. All-tie and + * empty inputs return p = 1. Every input difference must be finite, and the + * direction must be chosen explicitly so a caller cannot select it after + * seeing the signs. + */ +export function pairedSignTest( + differences: readonly number[], + alternative: SignTestAlternative, +): PairedSignTestResult { + if (alternative !== 'greater' && alternative !== 'less') { + throw new ValidationError( + `pairedSignTest: alternative must be 'greater' or 'less', got ${alternative}`, + ) + } + + let positive = 0 + let negative = 0 + let ties = 0 + for (let i = 0; i < differences.length; i++) { + const difference = differences[i]! + if (!Number.isFinite(difference)) { + throw new ValidationError( + `pairedSignTest: difference at index ${i} must be finite, got ${difference}`, + ) + } + if (difference > 0) positive++ + else if (difference < 0) negative++ + else ties++ + } + + const nNonTies = positive + negative + const successes = alternative === 'greater' ? positive : negative + return { + n: differences.length, + positive, + negative, + ties, + nNonTies, + alternative, + pValue: binomialHalfUpperTail(successes, nNonTies), + } +} + // ── Binomial proportion + paired-binary + coding-eval estimators ───── // // The paired family above (pairedBootstrap/pairedTTest/wilcoxonSignedRank) @@ -1192,14 +1264,44 @@ export function passAtK(n: number, c: number, k: number): number { function binomialSignTwoSided(b: number, c: number): number { const nd = b + c if (nd === 0) return 1 - const k = Math.min(b, c) - const logHalfN = nd * Math.log(0.5) - let tail = 0 - for (let i = 0; i <= k; i++) { - const logChoose = lnGamma(nd + 1) - lnGamma(i + 1) - lnGamma(nd - i + 1) - tail += Math.exp(logChoose + logHalfN) + return Math.min(1, 2 * binomialHalfLowerTail(Math.min(b, c), nd)) +} + +/** P(X >= successes) for X ~ Binomial(n, 0.5). */ +function binomialHalfUpperTail(successes: number, n: number): number { + if (successes <= 0) return 1 + if (successes > n) return 0 + + // Use the smaller side of the distribution. For lower thresholds the + // complement sums at most half the mass; for upper thresholds symmetry + // maps the upper tail to a lower tail without subtraction. + if (successes <= n / 2) { + return Math.max(0, 1 - binomialHalfLowerTail(successes - 1, n)) } - return Math.min(1, 2 * tail) + return binomialHalfLowerTail(n - successes, n) +} + +/** P(X <= maxSuccesses) for X ~ Binomial(n, 0.5), accumulated in log space. */ +function binomialHalfLowerTail(maxSuccesses: number, n: number): number { + if (maxSuccesses < 0) return 0 + if (maxSuccesses >= n) return 1 + if (maxSuccesses === 0) return 2 ** -n + + const logHalfN = n * Math.log(0.5) + let logTail = Number.NEGATIVE_INFINITY + for (let i = 0; i <= maxSuccesses; i++) { + const logChoose = lnGamma(n + 1) - lnGamma(i + 1) - lnGamma(n - i + 1) + logTail = logAddExp(logTail, logChoose + logHalfN) + } + return Math.min(1, Math.exp(logTail)) +} + +function logAddExp(a: number, b: number): number { + if (a === Number.NEGATIVE_INFINITY) return b + if (b === Number.NEGATIVE_INFINITY) return a + const max = Math.max(a, b) + const min = Math.min(a, b) + return max + Math.log1p(Math.exp(min - max)) } // ── Anytime-valid e-process (betting test-martingale) ──────────────── diff --git a/tests/statistics.test.ts b/tests/statistics.test.ts index e306efb1..80701cb0 100644 --- a/tests/statistics.test.ts +++ b/tests/statistics.test.ts @@ -15,6 +15,7 @@ import { pairedBootstrap, pairedMde, pairedRiskDifference, + pairedSignTest, pairedTTest, partialCredit, passAtK, @@ -676,6 +677,64 @@ describe('pairedRiskDifference — paired-binary effect size + CI', () => { }) }) +describe('pairedSignTest — exact one-sided paired differences', () => { + it('returns p = 0.25 for two positive differences', () => { + const result = pairedSignTest([1, 0.5], 'greater') + expect(result).toEqual({ + n: 2, + positive: 2, + negative: 0, + ties: 0, + nNonTies: 2, + alternative: 'greater', + pValue: 0.25, + }) + }) + + it('ignores zero ties in the binomial denominator', () => { + const result = pairedSignTest([1, 0], 'greater') + expect(result.nNonTies).toBe(1) + expect(result.ties).toBe(1) + expect(result.pValue).toBe(0.5) + }) + + it('returns p = 1 when every difference is tied', () => { + const result = pairedSignTest([0, -0, 0], 'greater') + expect(result.nNonTies).toBe(0) + expect(result.ties).toBe(3) + expect(result.pValue).toBe(1) + }) + + it('supports the opposite pre-registered direction', () => { + expect(pairedSignTest([1, 0.5], 'less').pValue).toBe(1) + expect(pairedSignTest([-1, -0.5], 'less').pValue).toBe(0.25) + }) + + it('matches a multi-term exact binomial tail without combinatorial overflow', () => { + const result = pairedSignTest([1, 1, 1, -1], 'greater') + expect(result.pValue).toBeCloseTo(5 / 16, 15) + }) + + it('is symmetric under sign reversal and direction reversal', () => { + const differences = [1, -0.5, 0, 2, 3, -4] + const greater = pairedSignTest(differences, 'greater') + const reflected = pairedSignTest( + differences.map((difference) => -difference), + 'less', + ) + expect(reflected.pValue).toBeCloseTo(greater.pValue, 15) + expect(reflected.positive).toBe(greater.negative) + expect(reflected.negative).toBe(greater.positive) + expect(reflected.ties).toBe(greater.ties) + }) + + it('rejects non-finite differences and invalid directions', () => { + expect(() => pairedSignTest([1, Number.NaN], 'greater')).toThrow(/index 1.*finite/) + expect(() => pairedSignTest([Number.POSITIVE_INFINITY], 'less')).toThrow(/finite/) + expect(() => pairedSignTest([1], 'two-sided' as 'greater')).toThrow(/alternative/) + }) +}) + describe('passAtK — unbiased coding-eval estimator', () => { it('0 correct ⇒ 0, all correct ⇒ 1', () => { expect(passAtK(5, 0, 1)).toBe(0) From 009d1a1e6d1f683d74bb3c6f43625045c673eb97 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sat, 11 Jul 2026 02:36:06 -0600 Subject: [PATCH 2/3] chore(release): 0.114.0 exact paired sign test --- CHANGELOG.md | 13 +++++++++++++ clients/python/pyproject.toml | 2 +- clients/python/src/agent_eval_rpc/__init__.py | 2 +- package.json | 2 +- 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3382905b..7f146ab5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- --- +## [0.114.0] — 2026-07-11 — exact directional paired inference + +### Added + +- `pairedSignTest(differences, alternative)` computes an exact one-sided sign test for paired numeric differences, excludes zero ties, requires a predeclared `greater` or `less` direction, and reports every denominator. + +### Changed + +- McNemar's exact two-sided calculation now reuses the same log-space binomial-tail implementation without changing its public result contract. + +This release is additive. +Existing consumers do not need to update unless they want the new statistic. + ## [0.113.0] — 2026-07-10 — immutable code candidates ### Changed diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 1a5c2e9d..5171cfb7 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agent-eval-rpc" -version = "0.113.0" +version = "0.114.0" description = "Python RPC client for @tangle-network/agent-eval — judge content against rubrics over HTTP or stdio RPC. Eval logic runs in the Node runtime; this package is a thin wire client." readme = "README.md" requires-python = ">=3.10" diff --git a/clients/python/src/agent_eval_rpc/__init__.py b/clients/python/src/agent_eval_rpc/__init__.py index 75948575..5f1d8ed2 100644 --- a/clients/python/src/agent_eval_rpc/__init__.py +++ b/clients/python/src/agent_eval_rpc/__init__.py @@ -58,7 +58,7 @@ try: __version__ = version("agent-eval-rpc") except PackageNotFoundError: - __version__ = "0.113.0" + __version__ = "0.114.0" __all__ = [ "Client", diff --git a/package.json b/package.json index c307de9a..41a8fc82 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-eval", - "version": "0.113.0", + "version": "0.114.0", "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.", "homepage": "https://github.com/tangle-network/agent-eval#readme", "repository": { From a0cf25fa0363523b4ed91633db56e35627de22cf Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sat, 11 Jul 2026 02:39:16 -0600 Subject: [PATCH 3/3] fix(release): align Python lock version --- clients/python/uv.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/python/uv.lock b/clients/python/uv.lock index a4e7b33d..cd68a476 100644 --- a/clients/python/uv.lock +++ b/clients/python/uv.lock @@ -4,7 +4,7 @@ requires-python = ">=3.10" [[package]] name = "agent-eval-rpc" -version = "0.113.0" +version = "0.114.0" source = { editable = "." } dependencies = [ { name = "httpx" },