From c21aaef9eefd914846d179d176d713afc27ee84b Mon Sep 17 00:00:00 2001
From: Miguel Angel Simon Sierra <miguel.sierra@heygen.com>
Date: Sun, 11 Oct 2026 00:16:48 -0700
Subject: [PATCH 1/3] fix(engine): exports place sound and picture right when a
 source's audio and video start apart

---
 packages/engine/src/services/audioFxRender.ts |   2 +-
 packages/engine/src/services/audioMixer.ts    | 161 ++++++------------
 .../src/services/audioVolumeEnvelope.ts       |   2 +-
 .../src/services/streamStartOffset.test.ts    | 156 +++++++++++++++++
 .../src/services/videoFrameExtractor.ts       |   7 +
 packages/engine/src/utils/ffprobe.ts          |  28 +++
 6 files changed, 242 insertions(+), 114 deletions(-)
 create mode 100644 packages/engine/src/services/streamStartOffset.test.ts

diff --git a/packages/engine/src/services/audioFxRender.ts b/packages/engine/src/services/audioFxRender.ts
index 1f97d2bf3f3..48809831708 100644
--- a/packages/engine/src/services/audioFxRender.ts
+++ b/packages/engine/src/services/audioFxRender.ts
@@ -181,7 +181,7 @@ export function writeWav(
  *
  * The graph used to fold everything to mono, which collapsed a stereo bed's
  * width for the render only — and cost ~3 dB through the very mono-to-stereo
- * rematrix that `prepareAudioTrack`'s pan filter exists to avoid. Preview kept
+ * rematrix that `extractAudioSegment`'s pan filter exists to avoid. Preview kept
  * the track stereo, so the two diverged the moment any effect was enabled.
  */
 function deinterleave(samples: Float32Array, channels: number): Float32Array[] {
diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.ts
index a5292556586..0f7f8a52e89 100644
--- a/packages/engine/src/services/audioMixer.ts
+++ b/packages/engine/src/services/audioMixer.ts
@@ -9,7 +9,7 @@ import { isSelfOrAncestorHidden, memberGroupKey, isMemberGroupHidden } from "./m
 import { closeSync, existsSync, mkdirSync, mkdtempSync, openSync, rmSync, writeFileSync } from "fs";
 import { join, dirname, isAbsolute, relative } from "path";
 import { parseHTML } from "linkedom";
-import { extractAudioMetadata } from "../utils/ffprobe.js";
+import { extractAudioMetadata, type AudioMetadata } from "../utils/ffprobe.js";
 import { isNotMediaPayload } from "../utils/notMediaPayload.js";
 import { clampAudioGain } from "@hyperframes/core/audio-gain";
 import { clampFadesToDuration, readElementFades } from "@hyperframes/core/audio-fade";
@@ -129,12 +129,13 @@ const MAX_RAMP_SLICES = 240;
 function buildRampFilterComplex(
   lane: RateSpec & object,
   duration: number,
+  headFilter: string | null,
   tailFilter: string | null,
 ): string {
   const slices = Math.min(MAX_RAMP_SLICES, Math.max(1, Math.ceil(duration / RAMP_SLICE_SECONDS)));
   const step = duration / slices;
   const split = Array.from({ length: slices }, (_, i) => `[s${i}]`).join("");
-  const parts = [`[0:a]asplit=${slices}${split}`];
+  const parts = [`[0:a]${headFilter ? `${headFilter},` : ""}asplit=${slices}${split}`];
   for (let i = 0; i < slices; i += 1) {
     const from = sourceTimeAt(lane, i * step);
     const to = sourceTimeAt(lane, (i + 1) * step);
@@ -156,27 +157,20 @@ function buildRampFilterComplex(
 }
 
 function preparedAudioOutputArgs(
-  srcPath: string,
+  metadata: AudioMetadata | null,
   playbackRate: RateSpec,
-  duration = 0,
-): Promise<string[]> {
-  return stereoOutputArgs(srcPath).then((channelArgs) => {
-    const filters: string[] = [];
-    const outputArgs: string[] = [];
-    if (channelArgs[0] === "-af" && channelArgs[1]) {
-      filters.push(channelArgs[1]);
-    } else {
-      outputArgs.push(...channelArgs);
-    }
-    if (typeof playbackRate === "object") {
-      const graph = buildRampFilterComplex(playbackRate, duration, filters.join(",") || null);
-      return ["-filter_complex", graph, "-map", "[out]", ...outputArgs];
-    }
-    const atempo = buildAtempoFilter(playbackRate);
-    if (atempo) filters.push(atempo);
-    if (filters.length > 0) outputArgs.push("-af", filters.join(","));
-    return outputArgs;
-  });
+  duration: number,
+  headFilter: string | null,
+): string[] {
+  const channelFilter = metadata?.channels === 1 ? STEREO_CHANNEL_FILTER : null;
+  const outputArgs = channelFilter ? [] : ["-ac", "2"];
+  if (typeof playbackRate === "object") {
+    const graph = buildRampFilterComplex(playbackRate, duration, headFilter, channelFilter);
+    return ["-filter_complex", graph, "-map", "[out]", ...outputArgs];
+  }
+  const filters = [headFilter, channelFilter, buildAtempoFilter(playbackRate)].filter(Boolean);
+  if (filters.length > 0) outputArgs.push("-af", filters.join(","));
+  return outputArgs;
 }
 
 function escapeExpressionCommas(expression: string): string {
@@ -217,16 +211,6 @@ const VOLUME_SIMPLIFY_EPSILON = 0.005;
 // native stereo sources have FL/FR and pass through unchanged.
 const STEREO_CHANNEL_FILTER = "pan=stereo|FL=FL+FC|FR=FR+FC";
 
-async function stereoOutputArgs(srcPath: string): Promise<string[]> {
-  try {
-    const { channels } = await extractAudioMetadata(srcPath);
-    if (channels === 1) return ["-af", STEREO_CHANNEL_FILTER];
-  } catch {
-    // Preserve the previous FFmpeg conversion path when metadata probing fails.
-  }
-  return ["-ac", "2"];
-}
-
 /**
  * Reduce a sorted keyframe list to a perceptually-equivalent piecewise-linear
  * envelope with a bounded segment count.
@@ -654,65 +638,15 @@ export function parseAudioElements(html: string): AudioElement[] {
   return elements;
 }
 
-async function extractAudioFromVideo(
-  videoPath: string,
-  outputPath: string,
-  options?: { startTime?: number; duration?: number; playbackRate?: RateSpec },
-  signal?: AbortSignal,
-  config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout">>,
-): Promise<ExtractResult> {
-  const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
-  const outputDir = dirname(outputPath);
-  if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
-
-  const playbackRate = normalizeRateSpec(options?.playbackRate);
-  const args: string[] = [];
-  if (options?.startTime !== undefined) args.push("-ss", String(options.startTime));
-  if (options?.duration !== undefined) {
-    args.push("-t", String(sourceTimeAt(playbackRate, options.duration)));
-  }
-  args.push("-i", videoPath);
-  const outputArgs = await preparedAudioOutputArgs(videoPath, playbackRate, options?.duration);
-  args.push("-vn", "-acodec", "pcm_s16le", "-ar", "48000", ...outputArgs);
-  if (playbackRate !== 1 && options?.duration !== undefined) {
-    args.push("-t", String(options.duration));
-  }
-  args.push("-y", outputPath);
-
-  const result = await runFfmpeg(args, { signal, timeout: ffmpegProcessTimeout });
-
-  if (signal?.aborted) {
-    const failure: AudioProcessingFailure = {
-      stage: "cancelled",
-      reason: "cancelled",
-      owner: "user",
-      retryable: false,
-      detail: "Audio extract cancelled",
-    };
-    return {
-      success: false,
-      outputPath,
-      durationMs: result.durationMs,
-      error: failure.detail,
-      failure,
-    };
-  }
-  if (!result.success) {
-    const failure = ffmpegFailure("extract", result);
-    return {
-      success: false,
-      outputPath,
-      durationMs: result.durationMs,
-      error: failure.detail,
-      failure,
-    };
-  }
-  return { success: true, outputPath, durationMs: result.durationMs };
-}
-
-async function prepareAudioTrack(
+/**
+ * Decode `duration` seconds of a file's audio from `mediaStart` in Chrome's media time. Before
+ * every stream has started, an input seek snaps the audio to the video's first keyframe and the
+ * WAV drops a late audio stream's offset, so that span is read by timestamp instead.
+ */
+async function extractAudioSegment(
   srcPath: string,
   outputPath: string,
+  stage: "extract" | "prepare",
   mediaStart: number,
   duration: number,
   playbackRate: RateSpec = 1,
@@ -723,21 +657,24 @@ async function prepareAudioTrack(
   const outputDir = dirname(outputPath);
   if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
   const normalizedPlaybackRate = normalizeRateSpec(playbackRate);
-  const outputArgs = await preparedAudioOutputArgs(srcPath, normalizedPlaybackRate, duration);
+  const sourceDuration = sourceTimeAt(normalizedPlaybackRate, duration);
+  // A failed probe keeps FFmpeg's default channel conversion and the input seek.
+  const metadata = await extractAudioMetadata(srcPath).catch(() => null);
+  const inputSeek = mediaStart >= (metadata?.latestStreamLeadSeconds ?? 0);
+  const timestampTrim = inputSeek
+    ? null
+    : `aresample=async=1:first_pts=0,atrim=start=${formatFilterNumber(mediaStart)}:end=${formatFilterNumber(mediaStart + sourceDuration)},asetpts=PTS-STARTPTS`;
+  const outputArgs = preparedAudioOutputArgs(
+    metadata,
+    normalizedPlaybackRate,
+    duration,
+    timestampTrim,
+  );
 
-  const args = [
-    "-ss",
-    String(mediaStart),
-    "-t",
-    String(sourceTimeAt(normalizedPlaybackRate, duration)),
-    "-i",
-    srcPath,
-    "-acodec",
-    "pcm_s16le",
-    "-ar",
-    "48000",
-    ...outputArgs,
-  ];
+  const args = inputSeek
+    ? ["-ss", String(mediaStart), "-t", String(sourceDuration), "-i", srcPath]
+    : ["-i", srcPath];
+  args.push("-vn", "-acodec", "pcm_s16le", "-ar", "48000", ...outputArgs);
   if (normalizedPlaybackRate !== 1) args.push("-t", String(duration));
   args.push("-y", outputPath);
 
@@ -749,7 +686,7 @@ async function prepareAudioTrack(
       reason: "cancelled",
       owner: "user",
       retryable: false,
-      detail: "Audio prepare cancelled",
+      detail: `Audio ${stage} cancelled`,
     };
     return {
       success: false,
@@ -759,7 +696,7 @@ async function prepareAudioTrack(
       failure,
     };
   }
-  const failure = !result.success ? ffmpegFailure("prepare", result) : undefined;
+  const failure = !result.success ? ffmpegFailure(stage, result) : undefined;
   return {
     success: result.success,
     outputPath,
@@ -1243,14 +1180,13 @@ export async function processCompositionAudio(
         let audioSrcPath = srcPath;
         if (element.type === "video") {
           const extractedPath = join(workDir, `${element.id}-extracted.wav`);
-          const extractResult = await extractAudioFromVideo(
+          const extractResult = await extractAudioSegment(
             srcPath,
             extractedPath,
-            {
-              startTime: element.mediaStart,
-              duration: element.end - element.start,
-              playbackRate: element.playbackRate,
-            },
+            "extract",
+            element.mediaStart,
+            element.end - element.start,
+            element.playbackRate,
             effectiveSignal,
             config,
           );
@@ -1272,9 +1208,10 @@ export async function processCompositionAudio(
           audioSrcPath = extractedPath;
         } else {
           const trimmedPath = join(workDir, `${element.id}-trimmed.wav`);
-          const prepResult = await prepareAudioTrack(
+          const prepResult = await extractAudioSegment(
             srcPath,
             trimmedPath,
+            "prepare",
             element.mediaStart,
             element.end - element.start,
             element.playbackRate,
diff --git a/packages/engine/src/services/audioVolumeEnvelope.ts b/packages/engine/src/services/audioVolumeEnvelope.ts
index f7459f2e255..c1d9f778451 100644
--- a/packages/engine/src/services/audioVolumeEnvelope.ts
+++ b/packages/engine/src/services/audioVolumeEnvelope.ts
@@ -11,7 +11,7 @@
  * output (and the golden baselines) only change where a fade is actually applied.
  *
  * The prepared tracks are always `pcm_s16le`, 48 kHz, stereo (see
- * `prepareAudioTrack` / `extractAudioFromVideo`). Anything else is rejected so
+ * `extractAudioSegment`). Anything else is rejected so
  * the caller can fall back to the expression path rather than corrupting audio.
  */
 
diff --git a/packages/engine/src/services/streamStartOffset.test.ts b/packages/engine/src/services/streamStartOffset.test.ts
new file mode 100644
index 00000000000..cd711affbd3
--- /dev/null
+++ b/packages/engine/src/services/streamStartOffset.test.ts
@@ -0,0 +1,156 @@
+import { spawnSync } from "node:child_process";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
+import { processCompositionAudio } from "./audioMixer.js";
+import { extractVideoFramesRange, getFrameAtTime } from "./videoFrameExtractor.js";
+
+const FFMPEG = getFfmpegBinary();
+const HAS_FFMPEG = spawnSync(FFMPEG, ["-version"]).status === 0;
+const SAMPLE_RATE = 48_000;
+const LEAD_SECONDS = 7;
+const CLIP_SECONDS = 12;
+
+function ffmpeg(args: string[]): void {
+  const result = spawnSync(FFMPEG, ["-nostdin", "-v", "error", "-y", ...args], {
+    encoding: "utf-8",
+  });
+  if (result.status !== 0) throw new Error(result.stderr);
+}
+
+/** Per output second, the source second heard (the tone is 200 + 100 * source second Hz), or null for silence. */
+function sourceSecondsHeard(path: string): Array<number | null> {
+  const decoded = spawnSync(
+    FFMPEG,
+    ["-v", "error", "-i", path, "-ac", "1", "-ar", String(SAMPLE_RATE), "-f", "s16le", "-"],
+    { maxBuffer: 1 << 28 },
+  );
+  if (decoded.status !== 0) throw new Error(decoded.stderr.toString());
+  const pcm = decoded.stdout;
+  const heard: Array<number | null> = [];
+  const edge = 2_000;
+  for (let second = 0; second < CLIP_SECONDS; second += 1) {
+    let crossings = 0;
+    let energy = 0;
+    let previous = 0;
+    for (let i = second * SAMPLE_RATE + edge; i < (second + 1) * SAMPLE_RATE - edge; i += 1) {
+      const sample = pcm.readInt16LE(i * 2);
+      energy += sample * sample;
+      if (i > second * SAMPLE_RATE + edge && previous < 0 !== sample < 0) crossings += 1;
+      previous = sample;
+    }
+    const windowSeconds = (SAMPLE_RATE - 2 * edge) / SAMPLE_RATE;
+    const rms = Math.sqrt(energy / (SAMPLE_RATE - 2 * edge));
+    heard.push(rms < 30 ? null : Math.round((crossings / 2 / windowSeconds - 200) / 100));
+  }
+  return heard;
+}
+
+describe.skipIf(!HAS_FFMPEG)(
+  "sources whose audio and video streams start at different times",
+  () => {
+    const dir = mkdtempSync(join(tmpdir(), "hf-stream-start-"));
+
+    beforeAll(() => {
+      ffmpeg([
+        "-f",
+        "lavfi",
+        "-i",
+        "testsrc=size=64x36:rate=30:duration=20",
+        "-c:v",
+        "libx264",
+        "-g",
+        "300",
+        "-pix_fmt",
+        "yuv420p",
+        join(dir, "picture.mp4"),
+      ]);
+      ffmpeg([
+        "-f",
+        "lavfi",
+        "-i",
+        `aevalsrc='0.1*sin(2*PI*(200+100*floor(t))*t)':s=${SAMPLE_RATE}:d=20`,
+        "-c:a",
+        "aac",
+        join(dir, "steps.m4a"),
+      ]);
+      const lead = String(LEAD_SECONDS);
+      const picture = join(dir, "picture.mp4");
+      const steps = join(dir, "steps.m4a");
+      const mux = ["-map", "0:v", "-map", "1:a", "-c", "copy"];
+      ffmpeg(["-itsoffset", lead, "-i", picture, "-i", steps, ...mux, join(dir, "video-late.mp4")]);
+      ffmpeg(["-i", picture, "-itsoffset", lead, "-i", steps, ...mux, join(dir, "audio-late.mp4")]);
+    }, 60_000);
+
+    afterAll(() => rmSync(dir, { recursive: true, force: true }));
+
+    it.each([
+      { source: "video-late.mp4", audioLead: 0, type: "video", mediaStart: 0 },
+      { source: "video-late.mp4", audioLead: 0, type: "video", mediaStart: 3 },
+      { source: "video-late.mp4", audioLead: 0, type: "audio", mediaStart: 0 },
+      { source: "audio-late.mp4", audioLead: LEAD_SECONDS, type: "video", mediaStart: 0 },
+      { source: "audio-late.mp4", audioLead: LEAD_SECONDS, type: "video", mediaStart: 3 },
+      { source: "audio-late.mp4", audioLead: LEAD_SECONDS, type: "audio", mediaStart: 0 },
+    ] as const)(
+      "$source as <$type> from $mediaStart s plays each second where the browser plays it",
+      async ({ source, audioLead, type, mediaStart }) => {
+        const workDir = mkdtempSync(join(dir, "work-"));
+        const outputPath = `${workDir}.m4a`;
+        const result = await processCompositionAudio(
+          [
+            {
+              id: "clip",
+              src: source,
+              start: 0,
+              end: CLIP_SECONDS,
+              mediaStart,
+              layer: 0,
+              volume: 1,
+              type,
+            },
+          ],
+          dir,
+          workDir,
+          outputPath,
+          CLIP_SECONDS,
+        );
+        expect(result.success, JSON.stringify(result)).toBe(true);
+        const expected = Array.from({ length: CLIP_SECONDS }, (_, second) => {
+          const mediaTime = mediaStart + second;
+          return mediaTime < audioLead ? null : mediaTime - audioLead;
+        });
+        expect(sourceSecondsHeard(outputPath)).toEqual(expected);
+      },
+      30_000,
+    );
+
+    // Chrome, seeking a late video stream, paints nothing until its first frame is under 1 s away.
+    it("shows no video frame while a late video stream's first frame is a second or more away", async () => {
+      const extracted = await extractVideoFramesRange(
+        join(dir, "video-late.mp4"),
+        "video-late",
+        0,
+        CLIP_SECONDS,
+        { fps: 30, outputDir: join(dir, "frames") },
+      );
+      expect(getFrameAtTime(extracted, 1, 0)).toBeNull();
+      expect(getFrameAtTime(extracted, LEAD_SECONDS - 1.1, 0)).toBeNull();
+      expect(getFrameAtTime(extracted, LEAD_SECONDS - 0.9, 0)).not.toBeNull();
+      expect(getFrameAtTime(extracted, 8, 0)).not.toBeNull();
+    }, 30_000);
+
+    it("keeps every frame of a video stream that starts first", async () => {
+      const extracted = await extractVideoFramesRange(
+        join(dir, "audio-late.mp4"),
+        "audio-late",
+        0,
+        CLIP_SECONDS,
+        { fps: 30, outputDir: join(dir, "frames") },
+      );
+      expect(getFrameAtTime(extracted, 0, 0)).not.toBeNull();
+      expect(getFrameAtTime(extracted, 1, 0)).not.toBeNull();
+    }, 30_000);
+  },
+);
diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts
index e5c9aac861c..ed364184935 100644
--- a/packages/engine/src/services/videoFrameExtractor.ts
+++ b/packages/engine/src/services/videoFrameExtractor.ts
@@ -2613,6 +2613,8 @@ export async function extractAllVideoFrames(
   };
 }
 
+const BROWSER_SHOWS_UPCOMING_FIRST_FRAME_WITHIN_SECONDS = 1;
+
 function getFrameIndexAtTime(
   extracted: ExtractedFrames,
   globalTime: number,
@@ -2646,6 +2648,11 @@ function getFrameIndexAtTime(
   // e.g. 0.28 * 25 === 6.999999999999999 instead of 7.
   const frameIndex = Math.floor(sourceTime * extracted.fps + 1e-9);
   if (frameIndex < 0 || extracted.totalFrames <= 0) return null;
+  const secondsUntilFirstVideoFrame =
+    (extracted.metadata.videoStreamLeadSeconds ?? 0) - mediaStart - sourceTime;
+  if (secondsUntilFirstVideoFrame >= BROWSER_SHOWS_UPCOMING_FIRST_FRAME_WITHIN_SECONDS - 1e-6) {
+    return null;
+  }
   if (frameIndex >= extracted.totalFrames) {
     return loop || holdLastFrame ? extracted.totalFrames - 1 : null;
   }
diff --git a/packages/engine/src/utils/ffprobe.ts b/packages/engine/src/utils/ffprobe.ts
index ad63d91fff3..76eb27acff9 100644
--- a/packages/engine/src/utils/ffprobe.ts
+++ b/packages/engine/src/utils/ffprobe.ts
@@ -184,6 +184,8 @@ export interface VideoMetadata {
    * timestamps are absolute, so callers crossing those APIs must normalize by
    * this value. Absent only in legacy/manually-constructed metadata. */
   videoStreamStartSeconds?: number;
+  /** Seconds after the file's media time 0 (its earliest audio/video stream start) at which video begins. */
+  videoStreamLeadSeconds?: number;
   width: number;
   height: number;
   fps: number;
@@ -214,6 +216,8 @@ export interface AudioMetadata {
   channels: number;
   audioCodec: string;
   bitrate?: number;
+  /** Seconds after the file's media time 0 at which its last audio or video stream starts. */
+  latestStreamLeadSeconds?: number;
 }
 
 interface FFProbeStream {
@@ -820,6 +824,7 @@ export async function extractMediaMetadata(filePath: string): Promise<VideoMetad
       durationSeconds: containerDuration,
       videoStreamDurationSeconds: effectiveStreamDuration,
       videoStreamStartSeconds: streamStart,
+      videoStreamLeadSeconds: streamLeadSeconds(output, videoStream),
       width: videoStream.width || stillImage()?.width || 0,
       height: videoStream.height || stillImage()?.height || 0,
       fps,
@@ -989,6 +994,7 @@ export async function extractAudioMetadata(
       channels: audioStream.channels || 2,
       audioCodec,
       bitrate: output.format.bit_rate ? parseInt(output.format.bit_rate) : undefined,
+      latestStreamLeadSeconds: latestStreamLeadSeconds(output),
     };
   })();
 
@@ -997,6 +1003,28 @@ export async function extractAudioMetadata(
   return probePromise;
 }
 
+/** Chrome's media time 0 is the earliest of these; a stream starting later plays that much in. */
+function audioVideoStreamStarts(output: FFProbeOutput | null): number[] {
+  return (output?.streams ?? [])
+    .filter(
+      (s) =>
+        (s.codec_type === "audio" || s.codec_type === "video") && s.disposition?.attached_pic !== 1,
+    )
+    .map((s) => parseFloat(s.start_time ?? ""))
+    .filter(Number.isFinite);
+}
+
+function streamLeadSeconds(output: FFProbeOutput | null, stream: FFProbeStream): number {
+  const starts = audioVideoStreamStarts(output);
+  const start = parseFloat(stream.start_time ?? "");
+  return Number.isFinite(start) && starts.length > 0 ? Math.max(0, start - Math.min(...starts)) : 0;
+}
+
+function latestStreamLeadSeconds(output: FFProbeOutput): number {
+  const starts = audioVideoStreamStarts(output);
+  return starts.length > 0 ? Math.max(...starts) - Math.min(...starts) : 0;
+}
+
 function audioStartSeconds(stream: FFProbeStream, format: FFProbeFormat): number {
   const start = parseFloat(stream.start_time ?? format.start_time ?? "");
   return Number.isFinite(start) ? start : 0;

From 3371468d4fc0a782fea7682b85c80f659d142a3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miguel=20=C3=81ngel?= <miguel.sierra@heygen.com>
Date: Sun, 11 Oct 2026 05:59:36 -0400
Subject: [PATCH 2/3] fix(engine): a late video stream plays to its end and
 distributed and HDR renders hide its lead

---
 packages/engine/src/index.ts                  |  1 +
 packages/engine/src/services/audioFxRender.ts |  9 +--
 packages/engine/src/services/audioMixer.ts    | 31 ++------
 .../src/services/streamStartOffset.test.ts    | 78 ++++++++++++++++---
 .../src/services/videoFrameExtractor.ts       | 47 +++++++----
 .../src/services/distributed/shared.ts        |  4 +
 .../distributed/videoMetadata.test.ts         | 12 ++-
 .../src/services/hdrCompositor.test.ts        | 39 +++++++++-
 .../producer/src/services/hdrCompositor.ts    |  5 ++
 .../render/stages/captureHdrResources.ts      | 20 +++--
 10 files changed, 178 insertions(+), 68 deletions(-)

diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts
index 69d284b9cf5..8a69146f7a2 100644
--- a/packages/engine/src/index.ts
+++ b/packages/engine/src/index.ts
@@ -213,6 +213,7 @@ export {
   extractionFrameCountForDuration,
   resolveProjectRelativeSrc,
   getFrameAtTime,
+  isVideoHiddenBeforeStreamStart,
   createFrameLookupTable,
   FrameLookupTable,
   analyzeClipMediaFit,
diff --git a/packages/engine/src/services/audioFxRender.ts b/packages/engine/src/services/audioFxRender.ts
index 48809831708..ad1dc0edc23 100644
--- a/packages/engine/src/services/audioFxRender.ts
+++ b/packages/engine/src/services/audioFxRender.ts
@@ -176,14 +176,7 @@ export function writeWav(
   writeFileSync(path, buf);
 }
 
-/**
- * Split an interleaved buffer into one array per channel.
- *
- * The graph used to fold everything to mono, which collapsed a stereo bed's
- * width for the render only — and cost ~3 dB through the very mono-to-stereo
- * rematrix that `extractAudioSegment`'s pan filter exists to avoid. Preview kept
- * the track stereo, so the two diverged the moment any effect was enabled.
- */
+/** Split an interleaved buffer into one array per channel, keeping stereo width as preview does. */
 function deinterleave(samples: Float32Array, channels: number): Float32Array[] {
   if (channels <= 1) return [samples];
   const frames = Math.floor(samples.length / channels);
diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.ts
index 0f7f8a52e89..cdbdaf63e45 100644
--- a/packages/engine/src/services/audioMixer.ts
+++ b/packages/engine/src/services/audioMixer.ts
@@ -1,10 +1,4 @@
 // fallow-ignore-file complexity code-duplication
-/**
- * Audio Mixer Service
- *
- * Processes and mixes audio tracks using FFmpeg.
- */
-
 import { isSelfOrAncestorHidden, memberGroupKey, isMemberGroupHidden } from "./mediaHidden.js";
 import { closeSync, existsSync, mkdirSync, mkdtempSync, openSync, rmSync, writeFileSync } from "fs";
 import { join, dirname, isAbsolute, relative } from "path";
@@ -61,19 +55,8 @@ import type { AudioVolumeKeyframe } from "./audioMixer.types.js";
 
 export type { AudioElement, MixResult } from "./audioMixer.types.js";
 
-/**
- * Filename every caller must use for the mixed-audio artifact.
- *
- * The extension is load-bearing, not cosmetic: FFmpeg picks the muxer from it,
- * and the mix is AAC-encoded. A raw ADTS `.aac` stream has nowhere to record
- * the encoder's priming delay, so those leading samples decode as real silence
- * and shift the whole track ~1024 samples (21.33 ms at 48 kHz) late against a
- * frame-accurate video track. An MP4-family container carries the delay as an
- * edit list, which every decoder then strips, so the mix lands on its authored
- * start. Keep the choice here rather than at each call site: the same file is
- * muxed into the video, shipped in a distributed plan, and handed to users as
- * the PNG-sequence sidecar, and all three have to agree.
- */
+/** An MP4-family container records AAC priming as an edit list; a raw .aac plays ~21 ms late.
+ * The video mux, the distributed plan and the PNG-sequence sidecar all read this one name. */
 export const MIXED_AUDIO_FILENAME = "audio.m4a";
 
 function clampVolume(volume: number): number {
@@ -638,11 +621,8 @@ export function parseAudioElements(html: string): AudioElement[] {
   return elements;
 }
 
-/**
- * Decode `duration` seconds of a file's audio from `mediaStart` in Chrome's media time. Before
- * every stream has started, an input seek snaps the audio to the video's first keyframe and the
- * WAV drops a late audio stream's offset, so that span is read by timestamp instead.
- */
+/** Decode `duration` s of audio from `mediaStart` in Chrome's media time. Until every stream has
+ * started, an input seek snaps audio to the video's first keyframe, so read by timestamp instead. */
 async function extractAudioSegment(
   srcPath: string,
   outputPath: string,
@@ -661,9 +641,10 @@ async function extractAudioSegment(
   // A failed probe keeps FFmpeg's default channel conversion and the input seek.
   const metadata = await extractAudioMetadata(srcPath).catch(() => null);
   const inputSeek = mediaStart >= (metadata?.latestStreamLeadSeconds ?? 0);
+  const start = formatFilterNumber(mediaStart);
   const timestampTrim = inputSeek
     ? null
-    : `aresample=async=1:first_pts=0,atrim=start=${formatFilterNumber(mediaStart)}:end=${formatFilterNumber(mediaStart + sourceDuration)},asetpts=PTS-STARTPTS`;
+    : `atrim=start=${start}:end=${formatFilterNumber(mediaStart + sourceDuration)},asetpts=PTS-${start}/TB,aresample=async=1:first_pts=0`;
   const outputArgs = preparedAudioOutputArgs(
     metadata,
     normalizedPlaybackRate,
diff --git a/packages/engine/src/services/streamStartOffset.test.ts b/packages/engine/src/services/streamStartOffset.test.ts
index cd711affbd3..d00d40bf45d 100644
--- a/packages/engine/src/services/streamStartOffset.test.ts
+++ b/packages/engine/src/services/streamStartOffset.test.ts
@@ -5,13 +5,19 @@ import { join } from "node:path";
 import { afterAll, beforeAll, describe, expect, it } from "vitest";
 import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
 import { processCompositionAudio } from "./audioMixer.js";
-import { extractVideoFramesRange, getFrameAtTime } from "./videoFrameExtractor.js";
+import { extractMediaMetadata } from "../utils/ffprobe.js";
+import {
+  extractVideoFramesRange,
+  getFrameAtTime,
+  resolveVideoExtractionWindow,
+} from "./videoFrameExtractor.js";
 
 const FFMPEG = getFfmpegBinary();
 const HAS_FFMPEG = spawnSync(FFMPEG, ["-version"]).status === 0;
 const SAMPLE_RATE = 48_000;
 const LEAD_SECONDS = 7;
 const CLIP_SECONDS = 12;
+const PICTURE_SECONDS = 20;
 
 function ffmpeg(args: string[]): void {
   const result = spawnSync(FFMPEG, ["-nostdin", "-v", "error", "-y", ...args], {
@@ -21,7 +27,7 @@ function ffmpeg(args: string[]): void {
 }
 
 /** Per output second, the source second heard (the tone is 200 + 100 * source second Hz), or null for silence. */
-function sourceSecondsHeard(path: string): Array<number | null> {
+function sourceSecondsHeard(path: string, seconds: number): Array<number | null> {
   const decoded = spawnSync(
     FFMPEG,
     ["-v", "error", "-i", path, "-ac", "1", "-ar", String(SAMPLE_RATE), "-f", "s16le", "-"],
@@ -31,7 +37,7 @@ function sourceSecondsHeard(path: string): Array<number | null> {
   const pcm = decoded.stdout;
   const heard: Array<number | null> = [];
   const edge = 2_000;
-  for (let second = 0; second < CLIP_SECONDS; second += 1) {
+  for (let second = 0; second < seconds; second += 1) {
     let crossings = 0;
     let energy = 0;
     let previous = 0;
@@ -43,7 +49,7 @@ function sourceSecondsHeard(path: string): Array<number | null> {
     }
     const windowSeconds = (SAMPLE_RATE - 2 * edge) / SAMPLE_RATE;
     const rms = Math.sqrt(energy / (SAMPLE_RATE - 2 * edge));
-    heard.push(rms < 30 ? null : Math.round((crossings / 2 / windowSeconds - 200) / 100));
+    heard.push(rms < 30 ? null : Math.round((crossings / 2 / windowSeconds - 200) / 100) + 0);
   }
   return heard;
 }
@@ -58,7 +64,7 @@ describe.skipIf(!HAS_FFMPEG)(
         "-f",
         "lavfi",
         "-i",
-        "testsrc=size=64x36:rate=30:duration=20",
+        `testsrc=size=64x36:rate=30:duration=${PICTURE_SECONDS}`,
         "-c:v",
         "libx264",
         "-g",
@@ -82,20 +88,39 @@ describe.skipIf(!HAS_FFMPEG)(
       const mux = ["-map", "0:v", "-map", "1:a", "-c", "copy"];
       ffmpeg(["-itsoffset", lead, "-i", picture, "-i", steps, ...mux, join(dir, "video-late.mp4")]);
       ffmpeg(["-i", picture, "-itsoffset", lead, "-i", steps, ...mux, join(dir, "audio-late.mp4")]);
+      // MPEG-TS starts its clock at 1.4 s, so no stream starts at 0.
+      ffmpeg(["-i", join(dir, "video-late.mp4"), "-c", "copy", join(dir, "video-late.ts")]);
     }, 60_000);
 
     afterAll(() => rmSync(dir, { recursive: true, force: true }));
 
+    const flatLane = {
+      target: "rate",
+      points: [
+        { t: 0, v: 1 },
+        { t: CLIP_SECONDS, v: 1 },
+      ],
+    };
     it.each([
       { source: "video-late.mp4", audioLead: 0, type: "video", mediaStart: 0 },
       { source: "video-late.mp4", audioLead: 0, type: "video", mediaStart: 3 },
       { source: "video-late.mp4", audioLead: 0, type: "audio", mediaStart: 0 },
+      { source: "video-late.ts", audioLead: 0, type: "video", mediaStart: 0 },
+      { source: "video-late.mp4", audioLead: 0, type: "video", mediaStart: 3, rate: flatLane },
       { source: "audio-late.mp4", audioLead: LEAD_SECONDS, type: "video", mediaStart: 0 },
       { source: "audio-late.mp4", audioLead: LEAD_SECONDS, type: "video", mediaStart: 3 },
       { source: "audio-late.mp4", audioLead: LEAD_SECONDS, type: "audio", mediaStart: 0 },
+      {
+        source: "audio-late.mp4",
+        audioLead: LEAD_SECONDS,
+        type: "video",
+        mediaStart: 0,
+        seconds: 5,
+      },
     ] as const)(
-      "$source as <$type> from $mediaStart s plays each second where the browser plays it",
-      async ({ source, audioLead, type, mediaStart }) => {
+      "%#: $source as <$type> from $mediaStart s plays each second where the browser plays it",
+      async ({ source, audioLead, type, mediaStart, ...rest }) => {
+        const seconds = "seconds" in rest ? rest.seconds : CLIP_SECONDS;
         const workDir = mkdtempSync(join(dir, "work-"));
         const outputPath = `${workDir}.m4a`;
         const result = await processCompositionAudio(
@@ -104,8 +129,9 @@ describe.skipIf(!HAS_FFMPEG)(
               id: "clip",
               src: source,
               start: 0,
-              end: CLIP_SECONDS,
+              end: seconds,
               mediaStart,
+              ...("rate" in rest ? { playbackRate: rest.rate } : {}),
               layer: 0,
               volume: 1,
               type,
@@ -114,14 +140,14 @@ describe.skipIf(!HAS_FFMPEG)(
           dir,
           workDir,
           outputPath,
-          CLIP_SECONDS,
+          seconds,
         );
         expect(result.success, JSON.stringify(result)).toBe(true);
-        const expected = Array.from({ length: CLIP_SECONDS }, (_, second) => {
+        const expected = Array.from({ length: seconds }, (_, second) => {
           const mediaTime = mediaStart + second;
           return mediaTime < audioLead ? null : mediaTime - audioLead;
         });
-        expect(sourceSecondsHeard(outputPath)).toEqual(expected);
+        expect(sourceSecondsHeard(outputPath, seconds)).toEqual(expected);
       },
       30_000,
     );
@@ -152,5 +178,35 @@ describe.skipIf(!HAS_FFMPEG)(
       expect(getFrameAtTime(extracted, 0, 0)).not.toBeNull();
       expect(getFrameAtTime(extracted, 1, 0)).not.toBeNull();
     }, 30_000);
+
+    it("hides a late video stream's lead in a transport whose clock starts at 1.4 s", async () => {
+      const extracted = await extractVideoFramesRange(
+        join(dir, "video-late.ts"),
+        "video-late-ts",
+        0,
+        CLIP_SECONDS,
+        { fps: 30, outputDir: join(dir, "frames") },
+      );
+      expect(getFrameAtTime(extracted, 1, 0)).toBeNull();
+      expect(getFrameAtTime(extracted, LEAD_SECONDS - 0.5, 0)).not.toBeNull();
+    }, 30_000);
+
+    it("plays a late video stream to its end on the media timeline and loops on its full length", async () => {
+      const source = join(dir, "video-late.mp4");
+      const mediaSeconds = LEAD_SECONDS + PICTURE_SECONDS;
+      const window = resolveVideoExtractionWindow(
+        { start: 0, end: mediaSeconds, mediaStart: 0, loop: false },
+        await extractMediaMetadata(source),
+        mediaSeconds,
+      );
+      expect(window.durationSeconds).toBeCloseTo(mediaSeconds, 1);
+
+      const extracted = await extractVideoFramesRange(source, "video-late-full", 0, mediaSeconds, {
+        fps: 10,
+        outputDir: join(dir, "frames"),
+      });
+      expect(getFrameAtTime(extracted, PICTURE_SECONDS + 1.5, 0, true)).not.toBeNull();
+      expect(getFrameAtTime(extracted, mediaSeconds + 1.5, 0, true)).toBeNull();
+    }, 30_000);
   },
 );
diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts
index ed364184935..0a38fa38521 100644
--- a/packages/engine/src/services/videoFrameExtractor.ts
+++ b/packages/engine/src/services/videoFrameExtractor.ts
@@ -1229,9 +1229,10 @@ export async function extractVideoFramesRange(
   if (framePaths.size > 0 && duration > 0 && !options.finalFrameOnly && !options.frameRanges) {
     const expectedAtFps = extractionFrameCountForDuration(duration, normalizedFps, metadata.isVFR);
     if (expectedAtFps > 0 && framePaths.size <= expectedAtFps * 0.6) {
-      const correctedDuration = startTime + framePaths.size / fps;
-      if (correctedDuration < metadata.videoStreamDurationSeconds) {
-        metadata.videoStreamDurationSeconds = correctedDuration;
+      const correctedStreamDuration =
+        startTime + framePaths.size / fps - (metadata.videoStreamLeadSeconds ?? 0);
+      if (correctedStreamDuration < metadata.videoStreamDurationSeconds) {
+        metadata.videoStreamDurationSeconds = correctedStreamDuration;
       }
     }
   }
@@ -1302,7 +1303,7 @@ function resolveSegmentDuration(
 export function resolvePlayableVideoDuration(metadata: VideoMetadata): number {
   return Number.isFinite(metadata.videoStreamDurationSeconds) &&
     metadata.videoStreamDurationSeconds > 0
-    ? metadata.videoStreamDurationSeconds
+    ? (metadata.videoStreamLeadSeconds ?? 0) + metadata.videoStreamDurationSeconds
     : metadata.durationSeconds;
 }
 
@@ -1503,14 +1504,17 @@ export async function resolveFinalFrameExtractionWindow(
 ): Promise<TimelineExtractionWindow> {
   if (!window.ensureFinalFrame) return window;
   const playableDuration = resolvePlayableVideoDuration(metadata);
-  const finalFrameTimestamp = await extractFinalVideoFrameTimestamp(
-    videoPath,
-    {
-      videoStreamDurationSeconds: playableDuration,
-      videoStreamStartSeconds: metadata.videoStreamStartSeconds,
-    },
-    signal,
-  );
+  const videoStreamLead = metadata.videoStreamLeadSeconds ?? 0;
+  const finalFrameTimestamp =
+    videoStreamLead +
+    (await extractFinalVideoFrameTimestamp(
+      videoPath,
+      {
+        videoStreamDurationSeconds: playableDuration - videoStreamLead,
+        videoStreamStartSeconds: metadata.videoStreamStartSeconds,
+      },
+      signal,
+    ));
   if (window.mediaStart < finalFrameTimestamp - 1e-9) return window;
 
   const sourceRemaining = playableDuration - video.mediaStart;
@@ -2615,6 +2619,16 @@ export async function extractAllVideoFrames(
 
 const BROWSER_SHOWS_UPCOMING_FIRST_FRAME_WITHIN_SECONDS = 1;
 
+export function isVideoHiddenBeforeStreamStart(
+  videoStreamLeadSeconds: number | undefined,
+  mediaTime: number,
+): boolean {
+  return (
+    (videoStreamLeadSeconds ?? 0) - mediaTime >=
+    BROWSER_SHOWS_UPCOMING_FIRST_FRAME_WITHIN_SECONDS - 1e-6
+  );
+}
+
 function getFrameIndexAtTime(
   extracted: ExtractedFrames,
   globalTime: number,
@@ -2648,9 +2662,12 @@ function getFrameIndexAtTime(
   // e.g. 0.28 * 25 === 6.999999999999999 instead of 7.
   const frameIndex = Math.floor(sourceTime * extracted.fps + 1e-9);
   if (frameIndex < 0 || extracted.totalFrames <= 0) return null;
-  const secondsUntilFirstVideoFrame =
-    (extracted.metadata.videoStreamLeadSeconds ?? 0) - mediaStart - sourceTime;
-  if (secondsUntilFirstVideoFrame >= BROWSER_SHOWS_UPCOMING_FIRST_FRAME_WITHIN_SECONDS - 1e-6) {
+  if (
+    isVideoHiddenBeforeStreamStart(
+      extracted.metadata.videoStreamLeadSeconds,
+      mediaStart + sourceTime,
+    )
+  ) {
     return null;
   }
   if (frameIndex >= extracted.totalFrames) {
diff --git a/packages/producer/src/services/distributed/shared.ts b/packages/producer/src/services/distributed/shared.ts
index ef574f8d8fe..6d5171bd0e3 100644
--- a/packages/producer/src/services/distributed/shared.ts
+++ b/packages/producer/src/services/distributed/shared.ts
@@ -239,6 +239,10 @@ function readVideoMetadata(
       record.videoStreamStartSeconds === undefined
         ? 0
         : readFiniteNumber(record.videoStreamStartSeconds, `${field}.videoStreamStartSeconds`),
+    videoStreamLeadSeconds:
+      record.videoStreamLeadSeconds === undefined
+        ? 0
+        : readFiniteNumber(record.videoStreamLeadSeconds, `${field}.videoStreamLeadSeconds`),
     width: readPositiveInteger(record.width, `${field}.width`),
     height: readPositiveInteger(record.height, `${field}.height`),
     fps: readFiniteNumber(record.fps, `${field}.fps`),
diff --git a/packages/producer/src/services/distributed/videoMetadata.test.ts b/packages/producer/src/services/distributed/videoMetadata.test.ts
index c62719e4c87..bf1fa104ee8 100644
--- a/packages/producer/src/services/distributed/videoMetadata.test.ts
+++ b/packages/producer/src/services/distributed/videoMetadata.test.ts
@@ -94,7 +94,7 @@ describe("distributed video metadata", () => {
     expect(sourceDerived.videos[0]?.mediaStart).toBe(1);
   });
 
-  it("round-trips a non-zero video stream start and defaults legacy plans to zero", () => {
+  it("round-trips a non-zero video stream start and lead and defaults legacy plans to zero", () => {
     const withStart = buildPlanVideosJson({
       videos: [video()],
       extracted: [
@@ -102,16 +102,22 @@ describe("distributed video metadata", () => {
           metadata: {
             ...extractedMetadata().metadata,
             videoStreamStartSeconds: 5,
+            videoStreamLeadSeconds: 7,
           },
         }),
       ],
       compositionEnd: 8,
     });
-    expect(parsePlanVideosJson(withStart).extracted[0]?.metadata.videoStreamStartSeconds).toBe(5);
+    const parsed = parsePlanVideosJson(withStart).extracted[0]?.metadata;
+    expect(parsed?.videoStreamStartSeconds).toBe(5);
+    expect(parsed?.videoStreamLeadSeconds).toBe(7);
 
     const legacy = structuredClone(withStart);
     delete legacy.extracted[0]?.metadata.videoStreamStartSeconds;
-    expect(parsePlanVideosJson(legacy).extracted[0]?.metadata.videoStreamStartSeconds).toBe(0);
+    delete legacy.extracted[0]?.metadata.videoStreamLeadSeconds;
+    const legacyParsed = parsePlanVideosJson(legacy).extracted[0]?.metadata;
+    expect(legacyParsed?.videoStreamStartSeconds).toBe(0);
+    expect(legacyParsed?.videoStreamLeadSeconds).toBe(0);
   });
 
   it.each([Number.NaN, Number.POSITIVE_INFINITY, 0, 2])(
diff --git a/packages/producer/src/services/hdrCompositor.test.ts b/packages/producer/src/services/hdrCompositor.test.ts
index 774301c9cb1..5973b9c1916 100644
--- a/packages/producer/src/services/hdrCompositor.test.ts
+++ b/packages/producer/src/services/hdrCompositor.test.ts
@@ -1,6 +1,9 @@
+import { mkdtempSync, openSync, closeSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 import { describe, expect, it } from "vitest";
 import type { ElementStackingInfo } from "@hyperframes/engine";
-import { selectDomLayerShowIds } from "./hdrCompositor.js";
+import { blitHdrVideoLayer, selectDomLayerShowIds } from "./hdrCompositor.js";
 
 function makeEl(id: string, overrides?: Partial<ElementStackingInfo>): ElementStackingInfo {
   return {
@@ -62,3 +65,37 @@ describe("selectDomLayerShowIds", () => {
     ).toEqual(["active-overlay"]);
   });
 });
+
+describe("blitHdrVideoLayer", () => {
+  it.each([
+    { lead: undefined, painted: true },
+    { lead: 7, painted: false },
+  ])("paints at 1 s of a video whose stream starts $lead s in: $painted", ({ lead, painted }) => {
+    const dir = mkdtempSync(join(tmpdir(), "hf-hdr-lead-"));
+    const rawPath = join(dir, "frames.rgb48le");
+    const frameSize = 2 * 2 * 6;
+    writeFileSync(rawPath, Buffer.alloc(frameSize * 2, 0x80));
+    const fd = openSync(rawPath, "r");
+    try {
+      const canvas = Buffer.alloc(frameSize);
+      const el = makeEl("v", { width: 2, height: 2, layoutWidth: 2, layoutHeight: 2, isHdr: true });
+      const source = {
+        dir,
+        rawPath,
+        fd,
+        width: 2,
+        height: 2,
+        frameSize,
+        frameCount: 2,
+        scratch: Buffer.alloc(frameSize),
+        mediaStart: 0,
+        videoStreamLeadSeconds: lead,
+      };
+      blitHdrVideoLayer(canvas, el, 1, 1, new Map([["v", source]]), new Map([["v", 0]]), 2, 2);
+      expect(canvas.some((byte) => byte !== 0)).toBe(painted);
+    } finally {
+      closeSync(fd);
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+});
diff --git a/packages/producer/src/services/hdrCompositor.ts b/packages/producer/src/services/hdrCompositor.ts
index cff19df7627..13e474dce7c 100644
--- a/packages/producer/src/services/hdrCompositor.ts
+++ b/packages/producer/src/services/hdrCompositor.ts
@@ -29,6 +29,7 @@ import {
   blitRgb48leAffine,
   parseTransformMatrix,
   convertTransfer,
+  isVideoHiddenBeforeStreamStart,
 } from "@hyperframes/engine";
 import type { ProducerLogger } from "../logger.js";
 import { type HdrImageTransferCache } from "./hdrImageTransferCache.js";
@@ -141,6 +142,8 @@ export interface HdrVideoFrameSource {
   scratch: Buffer;
   /** The raw file contains one playable source cycle and must wrap at EOF. */
   loop?: boolean;
+  mediaStart?: number;
+  videoStreamLeadSeconds?: number;
 }
 
 export function closeHdrVideoFrameSource(source: HdrVideoFrameSource, log?: ProducerLogger): void {
@@ -198,6 +201,8 @@ export function blitHdrVideoLayer(
     frameSource.loop,
   );
   if (effectiveIndex === null) return;
+  const mediaTime = (frameSource.mediaStart ?? 0) + effectiveIndex / fps;
+  if (isVideoHiddenBeforeStreamStart(frameSource.videoStreamLeadSeconds, mediaTime)) return;
   const frameOffset = effectiveIndex * frameSource.frameSize;
 
   try {
diff --git a/packages/producer/src/services/render/stages/captureHdrResources.ts b/packages/producer/src/services/render/stages/captureHdrResources.ts
index 2634122b76a..c53cf21f0f1 100644
--- a/packages/producer/src/services/render/stages/captureHdrResources.ts
+++ b/packages/producer/src/services/render/stages/captureHdrResources.ts
@@ -419,7 +419,10 @@ export async function extractHdrVideoFrames(args: {
   const out = new Map<string, HdrVideoFrameSource>();
   mkdirSync(framesDir, { recursive: true });
   const plannedVideos: Array<{ durationSeconds: number; width: number; height: number }> = [];
-  const extractionWindows = new Map<string, HdrExtractionWindow>();
+  const extractionWindows = new Map<
+    string,
+    { window: HdrExtractionWindow; videoStreamLeadSeconds?: number }
+  >();
   for (const [videoId, srcPath] of prep.hdrVideoSrcPaths) {
     const video = composition.videos.find((v) => v.id === videoId);
     if (!video) continue;
@@ -434,7 +437,10 @@ export async function extractHdrVideoFrames(args: {
       initialWindow,
       abortSignal,
     );
-    extractionWindows.set(videoId, window);
+    extractionWindows.set(videoId, {
+      window,
+      videoStreamLeadSeconds: metadata.videoStreamLeadSeconds,
+    });
     prep.hdrVideoStartTimes.set(videoId, window.compositionStart);
     plannedVideos.push({
       durationSeconds: window.durationSeconds,
@@ -453,14 +459,16 @@ export async function extractHdrVideoFrames(args: {
   try {
     for (const [videoId, srcPath] of prep.hdrVideoSrcPaths) {
       const video = composition.videos.find((v) => v.id === videoId);
-      const window = extractionWindows.get(videoId);
-      if (!video || !window) continue;
+      const planned = extractionWindows.get(videoId);
+      if (!video || !planned) continue;
+      const { window } = planned;
       mkdirSync(framesDir, { recursive: true });
       const frameDir = mkdtempSync(join(framesDir, `hdr_${tempDirSafePrefix(videoId)}-`));
       createdFrameDirs.add(frameDir);
       const dims = prep.hdrExtractionDims.get(videoId) ?? { width, height };
       const rawPath = join(frameDir, "frames.rgb48le");
-      const extractionStart = String(window.extractionMediaStart ?? window.mediaStart);
+      const extractionMediaStart = window.extractionMediaStart ?? window.mediaStart;
+      const extractionStart = String(extractionMediaStart);
       const ffmpegArgs: string[] = [];
       if (window.finalFrameOnly) {
         // Decode before seeking for the one-frame path. Input-side seeking can
@@ -524,6 +532,8 @@ export async function extractHdrVideoFrames(args: {
           frameCount,
           scratch: Buffer.allocUnsafe(frameSize),
           loop: video.loop,
+          mediaStart: extractionMediaStart,
+          videoStreamLeadSeconds: planned.videoStreamLeadSeconds,
         });
         handedOff = true;
       } finally {

From a73d397aff87c0f63e097bbafa5fd063e1c4ee6c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miguel=20=C3=81ngel?= <miguel.sierra@heygen.com>
Date: Sun, 11 Oct 2026 08:02:48 -0400
Subject: [PATCH 3/3] test(engine): pin a late video stream's trimmed start,
 held tail and HDR trim

---
 .../src/services/streamStartOffset.test.ts    | 34 +++++++++
 .../src/services/hdrCompositor.test.ts        | 70 +++++++++++--------
 2 files changed, 74 insertions(+), 30 deletions(-)

diff --git a/packages/engine/src/services/streamStartOffset.test.ts b/packages/engine/src/services/streamStartOffset.test.ts
index d00d40bf45d..2d87f1f7c76 100644
--- a/packages/engine/src/services/streamStartOffset.test.ts
+++ b/packages/engine/src/services/streamStartOffset.test.ts
@@ -9,6 +9,7 @@ import { extractMediaMetadata } from "../utils/ffprobe.js";
 import {
   extractVideoFramesRange,
   getFrameAtTime,
+  resolveFinalFrameExtractionWindow,
   resolveVideoExtractionWindow,
 } from "./videoFrameExtractor.js";
 
@@ -208,5 +209,38 @@ describe.skipIf(!HAS_FFMPEG)(
       expect(getFrameAtTime(extracted, PICTURE_SECONDS + 1.5, 0, true)).not.toBeNull();
       expect(getFrameAtTime(extracted, mediaSeconds + 1.5, 0, true)).toBeNull();
     }, 30_000);
+
+    it.each([6.5, 10])(
+      "shows a late video stream from a trim at %s s on the media timeline",
+      async (mediaStart) => {
+        const extracted = await extractVideoFramesRange(
+          join(dir, "video-late.mp4"),
+          `video-late-from-${mediaStart}`,
+          mediaStart,
+          5,
+          { fps: 10, outputDir: join(dir, "frames") },
+        );
+        expect(getFrameAtTime(extracted, 0.05, 0, false, mediaStart)).not.toBeNull();
+      },
+      30_000,
+    );
+
+    it("holds a late video stream's last frame past its end", async () => {
+      const source = join(dir, "video-late.mp4");
+      const metadata = await extractMediaMetadata(source);
+      const video = {
+        start: 0,
+        end: 5,
+        mediaStart: LEAD_SECONDS + PICTURE_SECONDS + 0.5,
+        loop: false,
+      };
+      const window = await resolveFinalFrameExtractionWindow(
+        source,
+        video,
+        metadata,
+        resolveVideoExtractionWindow(video, metadata, 5),
+      );
+      expect(window.extractionMediaStart).toBeGreaterThan(LEAD_SECONDS + PICTURE_SECONDS - 0.1);
+    }, 30_000);
   },
 );
diff --git a/packages/producer/src/services/hdrCompositor.test.ts b/packages/producer/src/services/hdrCompositor.test.ts
index 5973b9c1916..1680f9a90f2 100644
--- a/packages/producer/src/services/hdrCompositor.test.ts
+++ b/packages/producer/src/services/hdrCompositor.test.ts
@@ -68,34 +68,44 @@ describe("selectDomLayerShowIds", () => {
 
 describe("blitHdrVideoLayer", () => {
   it.each([
-    { lead: undefined, painted: true },
-    { lead: 7, painted: false },
-  ])("paints at 1 s of a video whose stream starts $lead s in: $painted", ({ lead, painted }) => {
-    const dir = mkdtempSync(join(tmpdir(), "hf-hdr-lead-"));
-    const rawPath = join(dir, "frames.rgb48le");
-    const frameSize = 2 * 2 * 6;
-    writeFileSync(rawPath, Buffer.alloc(frameSize * 2, 0x80));
-    const fd = openSync(rawPath, "r");
-    try {
-      const canvas = Buffer.alloc(frameSize);
-      const el = makeEl("v", { width: 2, height: 2, layoutWidth: 2, layoutHeight: 2, isHdr: true });
-      const source = {
-        dir,
-        rawPath,
-        fd,
-        width: 2,
-        height: 2,
-        frameSize,
-        frameCount: 2,
-        scratch: Buffer.alloc(frameSize),
-        mediaStart: 0,
-        videoStreamLeadSeconds: lead,
-      };
-      blitHdrVideoLayer(canvas, el, 1, 1, new Map([["v", source]]), new Map([["v", 0]]), 2, 2);
-      expect(canvas.some((byte) => byte !== 0)).toBe(painted);
-    } finally {
-      closeSync(fd);
-      rmSync(dir, { recursive: true, force: true });
-    }
-  });
+    { lead: undefined, mediaStart: 0, painted: true },
+    { lead: 7, mediaStart: 0, painted: false },
+    { lead: 7, mediaStart: 6.5, painted: true },
+  ])(
+    "paints at 1 s of a clip from $mediaStart s whose stream starts $lead s in: $painted",
+    ({ lead, mediaStart, painted }) => {
+      const dir = mkdtempSync(join(tmpdir(), "hf-hdr-lead-"));
+      const rawPath = join(dir, "frames.rgb48le");
+      const frameSize = 2 * 2 * 6;
+      writeFileSync(rawPath, Buffer.alloc(frameSize * 2, 0x80));
+      const fd = openSync(rawPath, "r");
+      try {
+        const canvas = Buffer.alloc(frameSize);
+        const el = makeEl("v", {
+          width: 2,
+          height: 2,
+          layoutWidth: 2,
+          layoutHeight: 2,
+          isHdr: true,
+        });
+        const source = {
+          dir,
+          rawPath,
+          fd,
+          width: 2,
+          height: 2,
+          frameSize,
+          frameCount: 2,
+          scratch: Buffer.alloc(frameSize),
+          mediaStart,
+          videoStreamLeadSeconds: lead,
+        };
+        blitHdrVideoLayer(canvas, el, 1, 1, new Map([["v", source]]), new Map([["v", 0]]), 2, 2);
+        expect(canvas.some((byte) => byte !== 0)).toBe(painted);
+      } finally {
+        closeSync(fd);
+        rmSync(dir, { recursive: true, force: true });
+      }
+    },
+  );
 });