diff --git a/packages/cli/src/telemetry/events.ts b/packages/cli/src/telemetry/events.ts index d8c42e6ec03..1e8efe3fb6c 100644 --- a/packages/cli/src/telemetry/events.ts +++ b/packages/cli/src/telemetry/events.ts @@ -341,8 +341,8 @@ export function trackRenderComplete( catalogUsage?: CatalogUsage; workers?: number; // Worker auto-sizing provenance (RenderPerfSummary.workerSizing). Answers - // "why N workers?" fleet-wide, and validates the advisory per-worker heap - // budget before it's enforced (field OOM: 6 auto workers on a 24GB/4GB-heap + // "why N workers?" fleet-wide, and reports the per-worker heap budget + // that caps auto sizing (field OOM: 6 auto workers on a 24GB/4GB-heap // machine — see computeWorkerSizing in @hyperframes/engine). workersBoundBy?: string; workersCpuBased?: number; diff --git a/packages/engine/src/services/parallelCoordinator.test.ts b/packages/engine/src/services/parallelCoordinator.test.ts index 414902db5aa..365ce3ffbca 100644 --- a/packages/engine/src/services/parallelCoordinator.test.ts +++ b/packages/engine/src/services/parallelCoordinator.test.ts @@ -1,4 +1,6 @@ -import { describe, it, expect, vi } from "vitest"; +import { cpus, totalmem } from "os"; +import { getHeapStatistics } from "v8"; +import { afterEach, describe, it, expect, vi } from "vitest"; import { calculateOptimalWorkers, computeWorkerSizing, @@ -128,7 +130,43 @@ describe("calculateOptimalWorkers", () => { }); }); +vi.mock("os", async (importOriginal) => { + const os = await importOriginal(); + return { ...os, default: os, cpus: vi.fn(os.cpus), totalmem: vi.fn(os.totalmem) }; +}); +vi.mock("v8", async (importOriginal) => { + const v8 = await importOriginal(); + return { ...v8, default: v8, getHeapStatistics: vi.fn(v8.getHeapStatistics) }; +}); + describe("computeWorkerSizing", () => { + afterEach(() => { + vi.mocked(cpus).mockRestore(); + vi.mocked(totalmem).mockRestore(); + vi.mocked(getHeapStatistics).mockRestore(); + }); + + // The field case: a 24GB 14-core Mac, Node's default ~4GB heap, a long 1080x1920 project. + it("never auto-picks more workers than the V8 heap can feed", () => { + vi.mocked(cpus).mockReturnValue( + Array.from({ length: 14 }, () => ({}) as ReturnType[number]), + ); + vi.mocked(totalmem).mockReturnValue(24 * 1024 ** 3); + vi.mocked(getHeapStatistics).mockReturnValue({ + heap_size_limit: 4192 * 1024 ** 2, + } as ReturnType); + const sizing = computeWorkerSizing(1300, undefined, { concurrency: "auto" }); + expect(sizing.heapBasedWorkers).toBe(4); + expect(sizing.workers).toBe(4); + expect(sizing.boundBy).toBe("heap"); + expect(sizing.exceedsHeapAdvisory).toBe(false); + expect(computeWorkerSizing(1300, 6, { concurrency: "auto" }).workers).toBe(6); + vi.mocked(getHeapStatistics).mockReturnValue({ + heap_size_limit: 8192 * 1024 ** 2, + } as ReturnType); + expect(computeWorkerSizing(1300, undefined, { concurrency: "auto" }).workers).toBe(5); + }); + it("matches calculateOptimalWorkers and reports every constraint", () => { const config = { concurrency: "auto" as const }; const sizing = computeWorkerSizing(900, undefined, config); diff --git a/packages/engine/src/services/parallelCoordinator.ts b/packages/engine/src/services/parallelCoordinator.ts index 457f22b6317..dc8240af104 100644 --- a/packages/engine/src/services/parallelCoordinator.ts +++ b/packages/engine/src/services/parallelCoordinator.ts @@ -141,10 +141,8 @@ const MEMORY_PER_WORKER_MB = 1536; const HEAP_RESERVED_MB = 1024; // Parent-process V8 heap consumed per worker (protocol buffers + in-flight // frame buffers). Derived from the field OOM: 6 workers exhausted a ~4GB -// default heap ⇒ >~500MB/worker + base. ponytail: advisory-only until the -// workers_heap_* telemetry added alongside this constant validates the figure -// — enforcing a guessed budget could silently cut worker counts fleet-wide. -// TODO(PRINFRA-341): decide enforcement after ~2 weeks of fleet soak. +// default heap ⇒ >~500MB/worker + base. Caps auto sizing; an explicit +// `--workers N` is still the operator's call. const HEAP_PER_WORKER_MB = 640; const MIN_WORKERS = 1; const MAX_WORKER_DIAGNOSTIC_LINES = 8; @@ -275,7 +273,8 @@ export type WorkerSizingBound = | "frames" | "max_workers" | "min_parallel_floor" - | "contention"; + | "contention" + | "heap"; /** * Full provenance of a worker-sizing decision. Threaded into render @@ -290,9 +289,8 @@ export interface WorkerSizing { frameBasedWorkers: number; effectiveMaxWorkers: number; /** - * ADVISORY, not enforced (see HEAP_PER_WORKER_MB): how many workers the - * parent process's V8 heap could feed. Compare against `workers` in - * telemetry to validate the budget before enforcement. + * How many workers the parent process's V8 heap can feed (see + * HEAP_PER_WORKER_MB); auto sizing never exceeds it. */ heapBasedWorkers: number; /** V8 `heap_size_limit` for the parent process, MB. */ @@ -300,7 +298,7 @@ export interface WorkerSizing { totalMemoryMb: number; cpuCount: number; captureCostMultiplier: number; - /** true when the chosen count exceeds the advisory heap budget. */ + /** true when the chosen count exceeds the heap budget (only an explicit request can). */ exceedsHeapAdvisory: boolean; } @@ -405,6 +403,11 @@ export function computeWorkerSizing( } } + if (finalWorkers > heapBasedWorkers) { + finalWorkers = heapBasedWorkers; + boundBy = "heap"; + } + return finish(finalWorkers, boundBy, effectiveMaxWorkers); } diff --git a/packages/producer/src/services/render/captureCost.ts b/packages/producer/src/services/render/captureCost.ts index db00538d92b..c7ceb1c7286 100644 --- a/packages/producer/src/services/render/captureCost.ts +++ b/packages/producer/src/services/render/captureCost.ts @@ -116,11 +116,8 @@ function combineCaptureCostEstimates( * - Auto-sized renders only (`requestedWorkers === undefined`) — the field * failure was auto sizing, and an explicit `--workers N` is the operator's * own call. - * - Not enforced as a cap yet — the per-worker budget constant is derived - * from one field report; the `workers_heap_*` telemetry emitted with the - * sizing decides whether to enforce (see the TODO on HEAP_PER_WORKER_MB in - * @hyperframes/engine's parallelCoordinator). The message gives the - * operator the actionable knobs today. + * - Auto sizing is capped at the heap budget (computeWorkerSizing), so the + * warning cannot fire today; it stays for a budget that stops being a cap. * * Pure so the message shape + firing condition are unit-testable with a * synthetic `WorkerSizing` (the real one depends on the host's heap). diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index f26bcbf7dae..d590f14c6f5 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -455,9 +455,8 @@ export interface RenderPerfSummary { /** * Provenance of the auto worker-sizing decision (undefined when the * htmlInCanvas / low-memory pins short-circuited sizing). `boundBy` names - * the binding constraint; the heap fields are the advisory budget being - * validated by fleet telemetry before enforcement — see - * `computeWorkerSizing` in @hyperframes/engine. + * the binding constraint; the heap fields are the budget that caps auto + * sizing — see `computeWorkerSizing` in @hyperframes/engine. */ workerSizing?: WorkerSizing; chunkedEncode: boolean;