fix(engine): motion blur works over video by holding each video frame across samples

packages/engine/src/services/frameCapture-gpuCompletion.test.ts

@@ -33,7 +33,9 @@ describe("WebGPU frame completion", () => {
 
   it("runs the completion wait after video injection and before screenshot capture", () => {
     const source = readFileSync(new URL("./frameCapture.ts", import.meta.url), "utf8");
-    const injection = source.indexOf("await session.onBeforeCapture(page, quantizedTime)");
+    const injection = source.indexOf(
+      "await session.onBeforeCapture(page, quantizedTime, heldVideoTime)",
+    );
     const completion = source.indexOf("await waitForPendingSeekCompletion(page)");
     const screenshot = source.indexOf("async function captureFrameCore");
     expect(injection).toBeGreaterThan(-1);

packages/engine/src/services/frameCapture-motionBlur.test.ts

@@ -317,10 +317,59 @@ describe("resolveSessionMotionBlur rejects what accumulation cannot render", ()
     expect(() => resolveSessionMotionBlur(session)).toThrow(/format "png"/);
   });
 
-  it("rejects injected video frames, which cannot follow a sub-frame seek", () => {
-    expect(() =>
-      resolveSessionMotionBlur(withOptions({ onBeforeCapture: async () => {} })),
-    ).toThrow(/video/);
+  it("accepts a session that injects video frames (#5144)", () => {
+    const plan = resolveSessionMotionBlur(withOptions({ onBeforeCapture: async () => {} }));
+    expect(plan).not.toBeUndefined();
+  });
+});
+
+describe("injected video is held for the whole shutter window (#5144)", () => {
+  function recordInjections() {
+    const calls: Array<{ time: number; held: number | undefined; seeksBefore: number }> = [];
+    const hook: CaptureSession["onBeforeCapture"] = async (_page, time, held) => {
+      calls.push({ time, held, seeksBefore: seeks.length });
+    };
+    return { calls, hook };
+  }
+  const sampleSeekTimes = () =>
+    seeks.filter((s) => s.subFrameDivisions !== undefined).map((s) => s.time);
+
+  it("injects at the frame time first, then each sample at its own time holding the frame", async () => {
+    const { calls, hook } = recordInjections();
+
+    await captureFrameToBuffer(makeSession({ onBeforeCapture: hook }), 10, 10 / 30);
+
+    // The first call comes right after the eventful seek, before any sample seek.
+    expect(calls[0]).toEqual({ time: 10 / 30, held: 10 / 30, seeksBefore: 1 });
+    const sampleCalls = calls.slice(1);
+    expect(sampleCalls.map((call) => call.time)).toEqual(sampleSeekTimes());
+    expect(new Set(sampleCalls.map((call) => call.time)).size).toBe(16);
+    expect(sampleCalls.map((call) => call.held)).toEqual(Array(16).fill(10 / 30));
+  });
+
+  it("holds the frame time for the adaptive probes too", async () => {
+    const { calls, hook } = recordInjections();
+
+    await captureFrameToBuffer(
+      makeSession({ onBeforeCapture: hook, motionBlur: resolveMotionBlurPlan({}) ?? undefined }),
+      10,
+      10 / 30,
+    );
+
+    expect(calls.map((call) => call.time)).toEqual([10 / 30, ...sampleSeekTimes()]);
+    expect(calls.map((call) => call.held)).toEqual(Array(calls.length).fill(10 / 30));
+  });
+
+  it("holds nothing on a frame without blur", async () => {
+    const { calls, hook } = recordInjections();
+
+    await captureFrameToBuffer(
+      makeSession({ onBeforeCapture: hook, motionBlur: undefined }),
+      10,
+      10 / 30,
+    );
+
+    expect(calls).toEqual([{ time: 10 / 30, held: undefined, seeksBefore: 1 }]);
   });
 });
 

packages/engine/src/services/frameCapture.ts

@@ -102,8 +102,10 @@ export { isMemoryExhaustionError, isTransientBrowserError } from "./captureFailu
 
 export type { CaptureOptions, CaptureResult, CaptureBufferResult, CapturePerfSummary };
 
-/** Called after seeking, before screenshot. Use for video frame injection or other pre-capture work. */
-export type BeforeCaptureHook = (page: Page, time: number) => Promise<void>;
+/** Called after seeking, before screenshot. Use for video frame injection or other pre-capture work.
+ * `heldVideoTime` (motion-blur samples only) is the output frame's time: a video on screen at
+ * both times shows its frame for `heldVideoTime`, so footage does not smear. */
+export type BeforeCaptureHook = (page: Page, time: number, heldVideoTime?: number) => Promise<void>;
 
 export interface CaptureSession {
   browser: Browser;
@@ -2805,6 +2807,7 @@ async function prepareFrameForCapture(
   frameIndex: number,
   time: number,
   seekOptions?: HfSeekOptions,
+  heldVideoTime?: number,
 ): Promise<{
   quantizedTime: number;
   seekMs: number;
@@ -2835,7 +2838,7 @@ async function prepareFrameForCapture(
   // replacements for <video> elements.
   const beforeCaptureStart = Date.now();
   if (session.onBeforeCapture) {
-    await session.onBeforeCapture(page, quantizedTime);
+    await session.onBeforeCapture(page, quantizedTime, heldVideoTime);
   }
   await waitForPendingSeekCompletion(page);
   await page.evaluate(async () => {
@@ -3867,9 +3870,7 @@ export async function withFrameDeadline<T>(
  *
  * Called once initialization has settled the capture mode. `format: "png"` is required
  * because samples are averaged pixel by pixel: JPEG samples would be averaged after
- * lossy quantization and the blended frame is re-encoded as PNG. `<video>` content is
- * out of scope because it is supplied by the before-capture frame-injection hook rather
- * than by the timeline seek, so it cannot follow a sub-frame time.
+ * lossy quantization and the blended frame is re-encoded as PNG.
  */
 export function resolveSessionMotionBlur(session: CaptureSession): MotionBlurPlan | undefined {
   const plan = resolveMotionBlurPlan(session.options.motionBlur);
@@ -3884,11 +3885,6 @@ export function resolveSessionMotionBlur(session: CaptureSession): MotionBlurPla
       `[MotionBlur] sub-frame motion blur requires format "png", got "${session.options.format ?? "jpeg"}"`,
     );
   }
-  if (session.onBeforeCapture) {
-    throw new Error(
-      "[MotionBlur] sub-frame motion blur cannot run with injected video frames: video content is extracted per output frame and does not follow a sub-frame seek",
-    );
-  }
   return plan;
 }
 
@@ -3907,13 +3903,15 @@ async function captureFrameSurface(
   frameIndex: number,
   time: number,
   seekOptions?: HfSeekOptions,
+  heldVideoTime?: number,
 ): Promise<CapturedSurface> {
   const { page, options } = session;
   const { quantizedTime, seekMs, beforeCaptureMs } = await prepareFrameForCapture(
     session,
     frameIndex,
     time,
     seekOptions,
+    heldVideoTime,
   );
 
   const screenshotStart = Date.now();
@@ -4100,6 +4098,7 @@ async function resolveAdaptiveSampleCount(
   plan: MotionBlurPlan,
   fps: number,
   sampleSeek: HfSeekOptions,
+  frameTime: number,
 ): Promise<{
   samplesPerFrame: number;
   seekMs: number;
@@ -4115,8 +4114,8 @@ async function resolveAdaptiveSampleCount(
     };
   }
   const { windowStart, windowEnd } = motionBlurProbeTimes(plan, absFrameIndex, fps);
-  const probeA = await captureFrameSurface(session, frameIndex, windowStart, sampleSeek);
-  const probeB = await captureFrameSurface(session, frameIndex, windowEnd, sampleSeek);
+  const probeA = await captureFrameSurface(session, frameIndex, windowStart, sampleSeek, frameTime);
+  const probeB = await captureFrameSurface(session, frameIndex, windowEnd, sampleSeek, frameTime);
   return {
     samplesPerFrame: adaptiveSampleCount(probeDiffMagnitude(probeA.buffer, probeB.buffer)),
     seekMs: probeA.seekMs + probeB.seekMs,
@@ -4148,6 +4147,13 @@ async function captureAccumulatedFrame(
   const eventfulSeekStart = Date.now();
   await seekPageTimeline(session.page, frameTime, undefined);
   const totals = { seekMs: Date.now() - eventfulSeekStart, beforeCaptureMs: 0, screenshotMs: 0 };
+  if (session.onBeforeCapture) {
+    // The injector copies a video's own style only when its frame changes: inject at frameTime
+    // first so the held frame carries frameTime's style, not the first sample's.
+    const injectStart = Date.now();
+    await session.onBeforeCapture(session.page, frameTime, frameTime);
+    totals.beforeCaptureMs += Date.now() - injectStart;
+  }
 
   const sampleSeek: HfSeekOptions = {
     suppressEvents: true,
@@ -4163,6 +4169,7 @@ async function captureAccumulatedFrame(
       plan,
       fps,
       sampleSeek,
+      frameTime,
     );
     samplesPerFrame = chosen.samplesPerFrame;
     totals.seekMs += chosen.seekMs;
@@ -4172,7 +4179,13 @@ async function captureAccumulatedFrame(
 
   const accumulator = new MotionBlurAccumulator(plan.blend);
   for (const sampleTime of motionBlurSampleTimes(plan, absFrameIndex, fps, samplesPerFrame)) {
-    const sample = await captureFrameSurface(session, frameIndex, sampleTime, sampleSeek);
+    const sample = await captureFrameSurface(
+      session,
+      frameIndex,
+      sampleTime,
+      sampleSeek,
+      frameTime,
+    );
     totals.seekMs += sample.seekMs;
     totals.beforeCaptureMs += sample.beforeCaptureMs;
     totals.screenshotMs += sample.screenshotMs;

packages/engine/src/services/videoFrameInjector.test.ts

@@ -322,6 +322,60 @@ describe("createVideoFrameInjector cache hygiene against page-side skips", () =>
   });
 });
 
+describe("createVideoFrameInjector holds video frames for motion blur (#5144)", () => {
+  // Scene A's video plays until 1 s, scene B's from 1 s; frame index = 10 × time.
+  const clips = [
+    { videoId: "a", start: 0, end: 1 },
+    { videoId: "b", start: 1, end: 2 },
+  ];
+  const table = {
+    frameDirs: () => [],
+    getActiveFramePayloads: (time: number) =>
+      new Map(
+        clips
+          .filter((clip) => time >= clip.start && time < clip.end)
+          .map((clip) => {
+            const frameIndex = Math.floor(time * 10);
+            return [clip.videoId, { framePath: `/${clip.videoId}/${frameIndex}`, frameIndex }];
+          }),
+      ),
+  } as unknown as FrameLookupTable;
+  const page = { evaluate: vi.fn(async () => undefined) } as unknown as Page;
+
+  beforeEach(() => {
+    injectVideoFramesBatchMock.mockReset();
+    injectVideoFramesBatchMock.mockImplementation(async (_page, updates) =>
+      updates.map((u) => u.videoId),
+    );
+    syncVideoFrameVisibilityMock.mockReset();
+    syncVideoFrameVisibilityMock.mockResolvedValue(undefined);
+  });
+
+  it("shows the held frame of a video on screen at both times", async () => {
+    const hook = createVideoFrameInjector(table, { frameSrcResolver: inlineResolver });
+
+    // The sample alone would pick frame 14; the held frame time picks 15.
+    await hook!(page, 1.42, 1.5);
+
+    expect(syncVideoFrameVisibilityMock).toHaveBeenLastCalledWith(page, ["b"]);
+    expect(injectVideoFramesBatchMock.mock.calls[0]?.[1]).toEqual([
+      { videoId: "b", dataUri: inlineResolver("/b/15") },
+    ]);
+  });
+
+  it("follows the sample time across a cut, so neither scene's video drops out", async () => {
+    const hook = createVideoFrameInjector(table, { frameSrcResolver: inlineResolver });
+
+    // Frame 1.0 is scene B's first frame; a sample just before it still shows scene A.
+    await hook!(page, 0.98, 1.0);
+
+    expect(syncVideoFrameVisibilityMock).toHaveBeenLastCalledWith(page, ["a"]);
+    expect(injectVideoFramesBatchMock.mock.calls[0]?.[1]).toEqual([
+      { videoId: "a", dataUri: inlineResolver("/a/9") },
+    ]);
+  });
+});
+
 describe("createVideoFrameInjector extraction-cache lease renewal", () => {
   // Regression: a render can hold a compiled-dir symlink into a shared
   // extraction-cache entry far longer than the entry's one-time cache-hit

packages/engine/src/services/videoFrameInjector.ts

@@ -204,9 +204,14 @@ export function createVideoFrameInjector(
   }
 
   // fallow-ignore-next-line complexity
-  return async (page: Page, time: number) => {
+  return async (page: Page, time: number, heldVideoTime?: number) => {
     for (const cacheDir of frameLookup.frameDirs()) renewCacheLease(cacheDir);
     const activePayloads = frameLookup.getActiveFramePayloads(time);
+    if (heldVideoTime !== undefined) {
+      for (const [videoId, payload] of frameLookup.getActiveFramePayloads(heldVideoTime)) {
+        if (activePayloads.has(videoId)) activePayloads.set(videoId, payload);
+      }
+    }
 
     const updates: Array<{ videoId: string; dataUri: string; frameIndex: number }> = [];
     const activeIds = new Set<string>();

packages/producer/scripts/test-classification.mjs

@@ -30,6 +30,7 @@ const INTEGRATION_TEST_FILES = new Set([
   "src/services/htmlCompiler.mediaType.test.ts",
   "src/services/htmlCompiler.naturalDuration.test.ts",
   "src/services/render/hlsRender.integration.test.ts",
+  "src/services/render/motionBlurVideo.integration.test.ts",
   "src/services/render/renderCancel.integration.test.ts",
   "src/services/render/stages/compileStage.mediaType.test.ts",
   "src/services/render/stages/gifEncodeArgs.test.ts",