test: fix the causes behind two disabled test families and re-enable them

src/codex/history-job.ts

@@ -23,7 +23,7 @@ import type {
   CodexHistoryWorkerOperation,
   HistoryWorkerResult,
 } from "./history-worker";
-import { historyBackupPathFor } from "./history-provider";
+import { currentHistoryDbBusyTimeoutMs, historyBackupPathFor } from "./history-provider";
 import type { CodexHistoryFailureReason, CodexHistoryVerifiedNoopProof } from "./history-provider";
 import { getCodexHome, resolveCodexStateDbPath } from "./paths";
 
@@ -437,6 +437,10 @@ export async function runCodexHistoryJob(
       canonicalStateDbPath: request.canonicalStateDbPath,
       canonicalBackupPath: request.canonicalBackupPath,
       ...(request.expectedDesiredEnabled === undefined ? {} : { expectedDesiredEnabled: request.expectedDesiredEnabled }),
+      // A Worker is a fresh module realm: it would otherwise open state_5.sqlite with this
+      // module's default rather than the timeout this process resolved. Production sends the
+      // same codex-rs-matching 5s the Worker would have used on its own.
+      busyTimeoutMs: currentHistoryDbBusyTimeoutMs(),
       env: {
         ...(process.env.CODEX_HOME ? { CODEX_HOME: process.env.CODEX_HOME } : {}),
         ...(process.env.OPENCODEX_HOME ? { OPENCODEX_HOME: process.env.OPENCODEX_HOME } : {}),

src/codex/history-provider.ts

@@ -76,6 +76,22 @@ export function setHistoryDbBusyTimeoutForTests(ms: number): void {
   historyDbBusyTimeoutMs = ms;
 }
 
+/**
+ * Carry that timeout across a realm boundary. A Worker starts from the default above and cannot
+ * observe a parent that shortened the window — the same reason its run message carries the homes
+ * explicitly — so `history-job.ts` sends this value and `history-worker.ts` adopts it. In
+ * production both sides already hold the codex-rs-matching 5s. A non-finite or negative value is
+ * refused rather than allowed to disable the wait the app expects.
+ */
+export function currentHistoryDbBusyTimeoutMs(): number {
+  return historyDbBusyTimeoutMs;
+}
+
+export function adoptHistoryDbBusyTimeout(ms: number): void {
+  if (!Number.isFinite(ms) || ms < 0) return;
+  historyDbBusyTimeoutMs = Math.floor(ms);
+}
+
 function openStateDb(stateDbPath: string): Database {
   const db = new Database(stateDbPath);
   try {

src/codex/history-worker.ts

@@ -33,6 +33,7 @@ import {
 } from "./internal/history-writer";
 import {
   snapshotCodexHistoryNoop,
+  adoptHistoryDbBusyTimeout,
   type CodexHistoryFailureReason,
   type CodexHistoryVerifiedNoopProof,
 } from "./history-provider";
@@ -62,6 +63,11 @@ export interface HistoryWorkerRunMessage {
   readonly canonicalBackupPath: string;
   /** When set, prove this transition's desired direction while H is held. */
   readonly expectedDesiredEnabled?: boolean;
+  /**
+   * The parent realm's `state_5.sqlite` busy timeout. A Worker cannot observe a parent that
+   * resolved a different window, for the same reason the homes below are explicit.
+   */
+  readonly busyTimeoutMs?: number;
   /** Env snapshot: a Worker may not observe parent mutations on every platform. */
   readonly env?: { readonly CODEX_HOME?: string; readonly OPENCODEX_HOME?: string };
 }
@@ -107,7 +113,11 @@ export function isHistoryWorkerRunMessage(data: unknown): data is HistoryWorkerR
     && nonEmpty(message.canonicalCodexHome)
     && nonEmpty(message.canonicalStateDbPath)
     && nonEmpty(message.canonicalBackupPath)
-    && (message.expectedDesiredEnabled === undefined || typeof message.expectedDesiredEnabled === "boolean");
+    && (message.expectedDesiredEnabled === undefined || typeof message.expectedDesiredEnabled === "boolean")
+    && (message.busyTimeoutMs === undefined
+      || (typeof message.busyTimeoutMs === "number"
+        && Number.isFinite(message.busyTimeoutMs)
+        && message.busyTimeoutMs >= 0));
 }
 
 /**
@@ -207,6 +217,9 @@ if (typeof self !== "undefined" && typeof (self as { onmessage?: unknown }) ===
     try {
       if (message.env?.CODEX_HOME) process.env.CODEX_HOME = message.env.CODEX_HOME;
       if (message.env?.OPENCODEX_HOME) process.env.OPENCODEX_HOME = message.env.OPENCODEX_HOME;
+      // Before any DB open: the timeout has to be in force for the first `openStateDb`, not
+      // after the writer has already waited out this realm's default.
+      if (message.busyTimeoutMs !== undefined) adoptHistoryDbBusyTimeout(message.busyTimeoutMs);
       self.postMessage(runHistoryUnitUnderLock(message));
     } catch (error) {
       self.postMessage({

tests/codex-integration/codex-composed-acceptance.test.ts

@@ -65,6 +65,7 @@ import {
   resolveEffectiveUserIdentity,
 } from "../../src/codex/user-identity";
 import { claimOwnedServiceHome, withOwnedServiceHomePreload } from "../helpers/owned-service-home";
+import { HISTORY_BUSY_TIMEOUT_ENV } from "../helpers/history-busy-timeout-preload";
 import { INTERNAL_DEADLINE_MS, SERVER_BUDGET_MS } from "../helpers/test-budget";
 import { repoRoot as resolveRepoRoot } from "../helpers/repo-root";
 
@@ -80,6 +81,8 @@ const HELD_REQUEST_BUDGET_MS = SERVER_BUDGET_MS + INTERNAL_DEADLINE_MS;
 
 const repoRoot = resolveRepoRoot();
 const cliPath = resolve(repoRoot, "src/cli/index.ts");
+/** Preload that shortens only a spawned child's SQLite busy wait; see the helper's header. */
+const historyBusyTimeoutPreload = resolve(repoRoot, "tests/helpers/history-busy-timeout-preload.ts");
 const lockChildPath = resolve(repoRoot, "tests/helpers/codex-write-lock-child.ts");
 const roots: Fixture[] = [];
 
@@ -245,6 +248,7 @@ class Fixture {
     home = this.homeA,
     userprofile = this.userprofileA,
     includeServiceProbe = false,
+    extra: Record<string, string> = {},
   ): Record<string, string> {
     // Do not inherit ambient homes or proxy configuration.  `process.execPath`
     // is absolute, so a PATH is intentionally unnecessary for CLI children.
@@ -270,6 +274,7 @@ class Fixture {
       // lookup timed out" while powershell.exe is still starting.
       ...(process.env.CI === "true" ? { CI: "true" } : {}),
       ...(includeServiceProbe ? this.serviceManagerEnv : {}),
+      ...extra,
     };
   }
 
@@ -294,10 +299,18 @@ class Fixture {
     }, null, 2));
   }
 
-  spawnCli(argv: string[], home = this.homeA, userprofile = this.userprofileA) {
-    const child = Bun.spawn([process.execPath, ...withOwnedServiceHomePreload([cliPath, ...argv], this.serviceManagerPreloadPath)], {
+  spawnCli(
+    argv: string[],
+    home = this.homeA,
+    userprofile = this.userprofileA,
+    options: { readonly preloadPaths?: readonly string[]; readonly env?: Record<string, string> } = {},
+  ) {
+    // Extra preloads go ahead of the service-probe wiring so each stays a separate argv pair,
+    // which is what keeps a checkout path containing spaces safe on Windows.
+    const preloadArgs = (options.preloadPaths ?? []).flatMap(path => ["--preload", path]);
+    const child = Bun.spawn([process.execPath, ...preloadArgs, ...withOwnedServiceHomePreload([cliPath, ...argv], this.serviceManagerPreloadPath)], {
       cwd: this.root,
-      env: this.env(home, userprofile, true),
+      env: this.env(home, userprofile, true, options.env ?? {}),
       stdout: "pipe",
       stderr: "pipe",
     });
@@ -310,8 +323,9 @@ class Fixture {
     home = this.homeA,
     userprofile = this.userprofileA,
     timeoutMs = watchdogMs(15_000),
+    options: { readonly preloadPaths?: readonly string[]; readonly env?: Record<string, string> } = {},
   ): Promise<CliResult> {
-    const child = this.spawnCli(argv, home, userprofile);
+    const child = this.spawnCli(argv, home, userprofile, options);
     const completed = await Promise.race([
       Promise.all([new Response(child.stdout).text(), new Response(child.stderr).text(), child.exited]),
       new Promise<never>((_, reject) => setTimeout(() => reject(new Error(`CLI watchdog: ocx ${argv.join(" ")}`)), timeoutMs)),
@@ -822,9 +836,16 @@ describe("WP13 composed toggle acceptance", () => {
   }, CASE_TIMEOUT_MS);
 
   /** RED: report restore success after a blocked history worker; config recovery must not hide history contention. */
-  // This verifies a platform-independent busy-envelope contract. Its deliberate SQLite
-  // contention plus real CLI startup is not a Windows latency assertion.
-  test.skipIf(process.platform === "win32")("Restore truth: JSON distinguishes a busy history restore from native artifact recovery", async () => {
+  // This verifies a platform-independent busy-envelope contract, and it now runs everywhere.
+  // It was skipped on win32 after run 32344670867 killed it at the 45 s CLI watchdog
+  // (45197 ms, "CLI watchdog: ocx restore --json") on a shard where neighbouring cases took
+  // 54-106 s. Nothing about the contract failed there: no envelope, no SQLite error, no
+  // assertion — the child was still waiting. The waiting was production's own busy budget
+  // (5 s per attempt, two attempts, 500 ms apart) paid inside a real CLI child, and that wait
+  // is not the assertion. The child now gets the same shortened busy timeout the in-process
+  // history tests use, so the contended phase costs ~1 s instead of ~10.5 s while the lock,
+  // the retry count, and every assertion below stay exactly as they were.
+  test("Restore truth: JSON distinguishes a busy history restore from native artifact recovery", async () => {
     const fx = fixture();
     fx.writeConfig({ clientIntegrations: { codex: false } });
     const original = 'model = "gpt-5"\n';
@@ -879,13 +900,16 @@ describe("WP13 composed toggle acceptance", () => {
     `], { cwd: repoRoot, env: fx.env(), stdout: "pipe", stderr: "pipe" });
     fx.children.push(holder);
     await waitFor(() => existsSync(held) ? true : null, "history BEGIN IMMEDIATE");
-    // The contended restore deliberately waits out PRODUCTION's retry budget:
-    // a 5 s SQLite busy timeout per attempt, two attempts, plus the delay
-    // between them — ~11 s of intentional waiting before it can report `busy`.
-    // A 15 s watchdog left almost no margin and fired on a loaded macOS runner
-    // (dev CI run 31105071651). Give the wait its budget plus real headroom;
-    // the case's own 45 s test timeout still bounds it.
-    const blocked = await fx.runCli(["restore", "--json"], fx.homeA, fx.userprofileA, watchdogMs(30_000));
+    // The contended restore still exhausts PRODUCTION's retry budget — two attempts against a
+    // lock that never releases — but each attempt's SQLite busy timeout is shortened from 5 s
+    // to 250 ms in this child only. What is being proven is the envelope, not the length of
+    // the wait, and the full-length wait is what fired the watchdog on Windows (run
+    // 32344670867) and earlier on a loaded macOS runner (run 31105071651). The child's history
+    // Worker inherits the value through its run message, since a Worker is a separate realm.
+    const blocked = await fx.runCli(["restore", "--json"], fx.homeA, fx.userprofileA, watchdogMs(30_000), {
+      preloadPaths: [historyBusyTimeoutPreload],
+      env: { [HISTORY_BUSY_TIMEOUT_ENV]: "250" },
+    });
     expect(blocked.exitCode, JSON.stringify(blocked)).toBe(1);
     const envelope = JSON.parse(blocked.stdout) as { success: boolean; artifacts: { history: { state: string; reason?: string } } };
     expect(envelope).toMatchObject({ success: false, artifacts: { history: { state: "failed", reason: "busy" } } });

tests/codex-integration/codex-history-worker.test.ts

@@ -5,7 +5,7 @@ import { dirname, join, resolve } from "node:path";
 
 import { Database } from "bun:sqlite";
 
-import { historyBackupPathFor, setBeforeHistoryBackupConsumeForTests, setHistoryDbBusyTimeoutForTests } from "../../src/codex/history-provider";
+import { adoptHistoryDbBusyTimeout, currentHistoryDbBusyTimeoutMs, historyBackupPathFor, setBeforeHistoryBackupConsumeForTests, setHistoryDbBusyTimeoutForTests } from "../../src/codex/history-provider";
 import {
   isHistoryWorkerRunMessage,
   runHistoryUnitUnderLock,
@@ -129,6 +129,35 @@ test("the run message is structured-clone safe and fully explicit", () => {
   expect(isHistoryWorkerRunMessage({ ...message, operation: "delete-everything" })).toBe(false);
 });
 
+/**
+ * The busy timeout travels with the message because a Worker is a separate realm: without it the
+ * Worker opens `state_5.sqlite` with its own module default and ignores a parent that resolved a
+ * shorter window, which is what forced a composed acceptance case to skip on Windows.
+ */
+test("the run message carries the parent's busy timeout and refuses a malformed one", () => {
+  const fixture = makeFixture("ocx-history-worker-busy-timeout-");
+  const message = runMessage(fixture);
+  const inherited = currentHistoryDbBusyTimeoutMs();
+
+  expect(isHistoryWorkerRunMessage({ ...message, busyTimeoutMs: 0 })).toBe(true);
+  expect(isHistoryWorkerRunMessage({ ...message, busyTimeoutMs: inherited })).toBe(true);
+  for (const bad of [-1, Number.NaN, Number.POSITIVE_INFINITY, "250", null]) {
+    expect(isHistoryWorkerRunMessage({ ...message, busyTimeoutMs: bad })).toBe(false);
+  }
+
+  // Adoption refuses the same values rather than disabling the wait the app expects.
+  try {
+    for (const bad of [-1, Number.NaN, Number.POSITIVE_INFINITY]) {
+      adoptHistoryDbBusyTimeout(bad);
+      expect(currentHistoryDbBusyTimeoutMs()).toBe(inherited);
+    }
+    adoptHistoryDbBusyTimeout(1_234);
+    expect(currentHistoryDbBusyTimeoutMs()).toBe(1_234);
+  } finally {
+    adoptHistoryDbBusyTimeout(inherited);
+  }
+});
+
 test("skip is a recorded outcome, not an absence, and writes nothing", () => {
   const fixture = makeFixture("ocx-history-worker-skip-");
   const before = readFileSync(fixture.rollout, "utf8");

tests/codex-integration/model-metadata-sync.test.ts

@@ -22,9 +22,19 @@ const GENERATED = repoPath("src/generated/model-metadata.ts");
 const SOURCE = repoPath("scripts/model-metadata.source.json");
 
 describe("generated model metadata stays in sync with its source", () => {
-  test.skipIf(!existsSync(SOURCE))(
+  test(
     "regenerating reproduces the committed file byte for byte",
     async () => {
+      // The snapshot is tracked in this repository, so a missing input is a broken checkout,
+      // not an environment variation. This used to be `test.skipIf(!existsSync(SOURCE))`, which
+      // meant the drift gate this file exists to provide disappeared without a trace the moment
+      // the input went missing — including in CI, where nothing else compares the generated file
+      // against its source.
+      expect(
+        existsSync(SOURCE),
+        `${SOURCE} is missing; the generator input is repository-owned, so this checkout is incomplete`,
+      ).toBe(true);
+
       const outDir = mkdtempSync(join(tmpdir(), "model-metadata-sync-"));
       const outPath = join(outDir, "model-metadata.ts");
 

tests/helpers/history-busy-timeout-preload.ts

@@ -0,0 +1,29 @@
+/**
+ * Test-only seam for a spawned `ocx` child: shorten the `state_5.sqlite` busy timeout.
+ *
+ * A test that proves cross-process SQLite contention has to hold a real lock against a real CLI
+ * child, and the child then pays the production busy timeout twice over — 5 s per attempt plus
+ * the retry delay, about 10.5 s of pure waiting. That wait is not the assertion; the JSON
+ * envelope is. On a contended windows-latest shard the waiting alone pushed the child past its
+ * watchdog, which is how the composed restore-busy case came to be skipped on Windows.
+ *
+ * In-process tests already shrink this window with `setHistoryDbBusyTimeoutForTests`. A child
+ * process cannot be reached that way, so it gets the same knob through `--preload`. Inert unless
+ * `OCX_TEST_HISTORY_BUSY_TIMEOUT_MS` is set on that child's environment, and no production
+ * module reads that variable or imports this file. The retry count, the lock, the failure
+ * classification, and the reported envelope are all untouched: only the length of a sleep changes.
+ */
+import { setHistoryDbBusyTimeoutForTests } from "../../src/codex/history-provider";
+
+export const HISTORY_BUSY_TIMEOUT_ENV = "OCX_TEST_HISTORY_BUSY_TIMEOUT_MS";
+
+const raw = process.env[HISTORY_BUSY_TIMEOUT_ENV];
+if (raw !== undefined) {
+  const ms = Number(raw);
+  // A malformed value must not silently leave the production 5 s in place while the test
+  // believes it was shortened, so it fails the child loudly instead.
+  if (!Number.isFinite(ms) || ms < 0) {
+    throw new Error(`${HISTORY_BUSY_TIMEOUT_ENV} must be a non-negative number, received: ${raw}`);
+  }
+  setHistoryDbBusyTimeoutForTests(ms);
+}

tests/server/server-startup-reconcile-resilience.test.ts

@@ -33,7 +33,8 @@ import type { OcxConfig } from "../../src/types";
  * Sandboxed agent environments deny `Bun.serve` outright ("Is port 0 in use?", EADDRINUSE on
  * every port), which is an environment artifact and not a regression — the same class already
  * documented for tests/server/server-combo-failover-e2e.test.ts. Probe once so the
- * listener-bound assertion is hosted-CI-only while the boot-sequence assertions always run.
+ * listener-bound assertions degrade gracefully there while the boot-sequence assertions always
+ * run. The probe result only suppresses a case outside CI (see `SKIP_LISTENER`).
  *
  * The probe has to be `Bun.serve` itself: a `node:net` listener still binds in an environment
  * where Bun's does not, so probing with the wrong API reports a false green and the skip never
@@ -49,9 +50,22 @@ function canBindLoopback(): boolean {
   }
 }
 
+const IS_CI = process.env.CI === "true";
 const CAN_BIND = canBindLoopback();
 
-test.skipIf(!CAN_BIND)("startServer persists the Astra-first legacy roster upgrade", async () => {
+/**
+ * The graceful skip is for a restricted local sandbox only. In hosted CI a runner that cannot
+ * bind loopback is a broken runner, not an environment variation, and the unconditional
+ * `skipIf(!CAN_BIND)` deleted four startup assertions there with no trace in the summary. Under
+ * CI the cases run and fail on the real bind error instead.
+ */
+const SKIP_LISTENER = !CAN_BIND && !IS_CI;
+
+test.skipIf(!IS_CI)("hosted CI can bind a loopback listener for the startup cases", () => {
+  expect(CAN_BIND).toBe(true);
+});
+
+test.skipIf(SKIP_LISTENER)("startServer persists the Astra-first legacy roster upgrade", async () => {
   saveConfig({
     ...staleConfig(),
     subagentModels: ["gpt-5.5", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna", "gpt-5.4-mini"],
@@ -66,7 +80,7 @@ test.skipIf(!CAN_BIND)("startServer persists the Astra-first legacy roster upgra
   }
 });
 
-test.skipIf(!CAN_BIND)("startServer migrates old Grok Chat choices once and preserves later opt-in", async () => {
+test.skipIf(SKIP_LISTENER)("startServer migrates old Grok Chat choices once and preserves later opt-in", async () => {
   saveConfig({
     ...staleConfig(), defaultProvider: "xai",
     providers: { xai: {
@@ -93,7 +107,7 @@ test.skipIf(!CAN_BIND)("startServer migrates old Grok Chat choices once and pres
   } finally { await restarted.stop(true); }
 });
 
-test.skipIf(!CAN_BIND)("preset reconciliation cannot undo an in-memory Grok migration after its write fails", async () => {
+test.skipIf(SKIP_LISTENER)("preset reconciliation cannot undo an in-memory Grok migration after its write fails", async () => {
   saveConfig({
     ...staleConfig(), defaultProvider: "xai",
     providers: { xai: {
@@ -205,9 +219,10 @@ test("a fresh install with no config file at all does not throw on the boot path
   }
 });
 
-// Hosted-CI-only: binds a listener, which sandboxed agent environments refuse (see
-// canBindLoopback above). The three assertions above cover the same claim without a port.
-test.skipIf(!CAN_BIND)(
+// Binds a listener, which a sandboxed agent environment refuses (see canBindLoopback above);
+// outside CI that skips this case and the three assertions above still cover the boot sequence
+// without a port. In CI it always runs.
+test.skipIf(SKIP_LISTENER)(
   "startServer completes and serves /healthz when the config disappears before reconcile",
   async () => {
     saveConfig(staleConfig());