Skip to content

Commit b46ebed

Browse files
fix(engine): a 29.97 fps camera clip exports smoothly at 30 fps (#5849)
* fix(engine): a 29.97 fps camera clip exports smoothly at 30 fps Extraction samples each output frame from the last source frame starting at or before it. Millisecond container ticks move each frame by up to half a millisecond, while 29.97 fps drifts against 30 fps by 0.033 ms per frame, so the rounding decided the slot and the export alternated repeats and skips. Constant-rate frames are now re-spaced on the stream's own frame grid from the first decoded frame before sampling. The cache schema moves to v8 so exports stop reusing frames extracted the old way. * fix(engine): every extraction process snaps frames to one absolute grid The grid was anchored on each process's first decoded frame, which carries its own millisecond rounding, so a segment or distributed chunk could place the 29.97 -> 30 repeat elsewhere and skip a frame at its seam. Frames are now snapped to the stream's own grid in file time (this process's seek plus the video's lead), the same in every process.
1 parent a971c2b commit b46ebed

4 files changed

Lines changed: 130 additions & 17 deletions

File tree

‎packages/engine/src/services/extractionCache.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ function seedPartialDir(entry: { dir: string; keyHash: string }, frameContent: s
6868

6969
describe("extractionCache constants", () => {
7070
it("exposes the v7 schema prefix", () => {
71-
expect(SCHEMA_PREFIX).toBe("hfcache-v7-");
71+
expect(SCHEMA_PREFIX).toBe("hfcache-v8-");
7272
});
7373

7474
it("exposes the frame filename prefix shared with the extractor", () => {

‎packages/engine/src/services/extractionCache.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ export const GC_MARKER = ".hf-last-gc";
6262
* Current schema version. Bump it whenever extraction writes different frames for the same key,
6363
* or warm entries keep serving the old frames across a deploy. Each bump's commit says why.
6464
*/
65-
export const SCHEMA_PREFIX = "hfcache-v7-";
65+
export const SCHEMA_PREFIX = "hfcache-v8-";
6666

6767
/** Truncated hex chars of SHA-256 used for the entry directory name. */
6868
const KEY_HEX_CHARS = 16;

‎packages/engine/src/services/videoFrameExtractor.test.ts‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1653,6 +1653,101 @@ describe.skipIf(!HAS_FFMPEG)("video frame extraction format", () => {
16531653
60_000,
16541654
);
16551655

1656+
it("samples a 29.97 fps source with millisecond timestamps at 30 fps without skipping a frame", async () => {
1657+
// Each frame writes its index as eight black or white blocks; the mp4 keeps 1/1000 s ticks.
1658+
const fixture = join(FIXTURE_DIR, "ntsc-ms-ticks.mp4");
1659+
const synth = await runFfmpeg([
1660+
...["-y", "-v", "error", "-f", "lavfi", "-i"],
1661+
"color=c=black:s=64x8:r=30000/1001:d=2,format=yuv420p,geq=lum='if(mod(floor(N/pow(2\\,floor(X/8)))\\,2)\\,235\\,16)':cb=128:cr=128",
1662+
...["-c:v", "libx264", "-qp", "0", "-g", "30", "-bf", "0", "-video_track_timescale", "1000"],
1663+
fixture,
1664+
]);
1665+
if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
1666+
1667+
const result = await extractVideoFramesRange(fixture, "ntsc-ms-ticks", 0, 1.5, {
1668+
fps: 30,
1669+
outputDir: join(FIXTURE_DIR, "out-ntsc-ms"),
1670+
format: "png",
1671+
});
1672+
const indexOf = (file: string): number => {
1673+
const row = spawnSync("ffmpeg", [
1674+
"-v",
1675+
"error",
1676+
"-i",
1677+
file,
1678+
"-vf",
1679+
"format=gray,crop=64:1:0:4",
1680+
"-f",
1681+
"rawvideo",
1682+
"-",
1683+
]).stdout;
1684+
return [...Array(8).keys()].reduce(
1685+
(n, bit) => n + (row[bit * 8 + 4]! > 128 ? 1 << bit : 0),
1686+
0,
1687+
);
1688+
};
1689+
const shown = [...result.framePaths.keys()]
1690+
.sort((a, b) => a - b)
1691+
.map((i) => indexOf(result.framePaths.get(i)!));
1692+
const steps = shown.slice(1).map((n, i) => n - shown[i]!);
1693+
1694+
expect(shown.length).toBeGreaterThanOrEqual(44);
1695+
expect(
1696+
steps.filter((step) => step > 1),
1697+
`source frames ${shown.join(",")}`,
1698+
).toEqual([]);
1699+
expect(steps.filter((step) => step === 0).length).toBeLessThanOrEqual(1);
1700+
}, 60_000);
1701+
1702+
it("samples a millisecond-tick 29.97 fps source the same in segments as in one pass", async () => {
1703+
const fixture = join(FIXTURE_DIR, "ntsc-ms-ticks-8s.mp4");
1704+
const synth = await runFfmpeg([
1705+
...["-y", "-v", "error", "-f", "lavfi", "-i"],
1706+
"color=c=black:s=64x8:r=30000/1001:d=8,format=yuv420p,geq=lum='if(mod(floor(N/pow(2\\,floor(X/8)))\\,2)\\,235\\,16)':cb=128:cr=128",
1707+
...["-c:v", "libx264", "-qp", "0", "-g", "30", "-bf", "0", "-video_track_timescale", "1000"],
1708+
fixture,
1709+
]);
1710+
if (!synth.success) throw new Error(`fixture synthesis failed: ${synth.stderr.slice(-400)}`);
1711+
const indexOf = (file: string): number => {
1712+
const row = spawnSync("ffmpeg", [
1713+
"-v",
1714+
"error",
1715+
"-i",
1716+
file,
1717+
"-vf",
1718+
"format=gray,crop=64:1:0:4",
1719+
"-f",
1720+
"rawvideo",
1721+
"-",
1722+
]).stdout;
1723+
return [...Array(8).keys()].reduce(
1724+
(n, bit) => n + (row[bit * 8 + 4]! > 128 ? 1 << bit : 0),
1725+
0,
1726+
);
1727+
};
1728+
const options = {
1729+
fps: 30,
1730+
outputDir: join(FIXTURE_DIR, "out-ntsc-seams"),
1731+
format: "png" as const,
1732+
};
1733+
const single = await extractVideoFramesRange(fixture, "ntsc-single", 0, 7.5, options);
1734+
1735+
const seams = [4, 17, 29, 41, 58, 73, 90, 107, 121, 139, 156, 170, 188, 201];
1736+
const differing: string[] = [];
1737+
for (const firstFrame of seams) {
1738+
const segment = await extractVideoFramesRange(fixture, `ntsc-seam-${firstFrame}`, 0, 7.5, {
1739+
...options,
1740+
frameRanges: [{ firstFrame, frames: 12 }],
1741+
});
1742+
for (const [slot, path] of segment.framePaths) {
1743+
const expected = indexOf(single.framePaths.get(slot)!);
1744+
const shown = indexOf(path);
1745+
if (shown !== expected) differing.push(`slot ${slot}: ${shown} vs ${expected}`);
1746+
}
1747+
}
1748+
expect(differing).toEqual([]);
1749+
}, 120_000);
1750+
16561751
// ffmpeg < 6.1 drops frame durations from CFR resampling once any -vf is set, so the colour
16571752
// filters would cut the still that ends this window short.
16581753
it.each([

‎packages/engine/src/services/videoFrameExtractor.ts‎

Lines changed: 33 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -836,14 +836,22 @@ function chromeGuessForUntaggedMatrix(metadata: VideoMetadata): string[] {
836836
* same absolute times as a single pass, so their frames are byte-identical to it.
837837
*/
838838
const EXTRACTION_SEGMENT_SECONDS = 120;
839+
/** Frame times since file start are `PTS*TB + seek`; snapping them to `lead + k/FR` gives every process one grid. */
840+
function undoTickRoundingOnTheSourceFrameGrid(seek: number, lead: number): string[] {
841+
const sinceFileStart = `(PTS*TB+${seek}-${lead})`;
842+
return [
843+
"settb=AVTB",
844+
`setpts=if(isnan(FR)\\,PTS\\,(${lead}+round(${sinceFileStart}*FR)/FR-${seek})/TB)`,
845+
];
846+
}
839847

840848
// Caps extraction ffmpeg runs across all clips; uncapped, dozens of clips starved each other into timeouts.
841849
const EXTRACTION_SLOTS = Math.max(1, Math.floor(cpus().length / 2));
842850
const inExtractionSlot = createConcurrencyLimit(EXTRACTION_SLOTS);
843851

844852
interface SegmentedExtraction {
845853
decodeArgs: string[];
846-
filterAndEncodeArgs: string[];
854+
filterAndEncodeArgs: (seek: number) => string[];
847855
videoPath: string;
848856
startTime: number;
849857
fps: number;
@@ -879,14 +887,15 @@ async function runSegmentedExtraction(job: SegmentedExtraction): Promise<RunFfmp
879887
while (next < segmentCount && !signal.aborted) {
880888
const index = next++;
881889
const { firstFrame, frames } = segments[index]!;
890+
const seek = startTime + firstFrame / fps;
882891
const result = await inExtractionSlot(
883892
() =>
884893
runFfmpeg(
885894
[
886895
...decodeArgs,
887896
"-noaccurate_seek",
888897
"-ss",
889-
String(startTime + firstFrame / fps),
898+
String(seek),
890899
"-i",
891900
videoPath,
892901
// Read one frame past the segment and keep exactly its own frames.
@@ -896,7 +905,7 @@ async function runSegmentedExtraction(job: SegmentedExtraction): Promise<RunFfmp
896905
String(frames),
897906
"-start_number",
898907
String(firstFrame + 1),
899-
...filterAndEncodeArgs,
908+
...filterAndEncodeArgs(seek),
900909
],
901910
{ ...job.runOptions, signal },
902911
),
@@ -1019,15 +1028,20 @@ export async function extractVideoFramesRange(
10191028
if (decodeWithVideoToolbox) {
10201029
vfFilters.push("format=nv12");
10211030
}
1022-
if (sampleAtOutputFps) {
1023-
// Each slot takes the last frame starting at or before its time. Ticks of 2 ms or finer
1024-
// first shift half a tick, absorbing container rounding (WebM/MKV use whole ms).
1025-
vfFilters.push(
1026-
"settb=intb/2",
1027-
"setpts=PTS-lte(TB\\,0.001)",
1028-
`fps=${ffmpegFps}:start_time=0:round=up`,
1029-
);
1030-
}
1031+
const samplingAt = vfFilters.length;
1032+
// Each slot takes the last frame starting at or before its time. Ticks of 2 ms or finer
1033+
// first shift half a tick, absorbing container rounding (WebM/MKV use whole ms).
1034+
const samplingFilters = (seek: number): string[] =>
1035+
sampleAtOutputFps
1036+
? [
1037+
...(metadata.isVFR
1038+
? []
1039+
: undoTickRoundingOnTheSourceFrameGrid(seek, metadata.videoStreamLeadSeconds ?? 0)),
1040+
"settb=intb/2",
1041+
"setpts=PTS-lte(TB\\,0.001)",
1042+
`fps=${ffmpegFps}:start_time=0:round=up`,
1043+
]
1044+
: [];
10311045
if (options.sdrToHdrTransfer) {
10321046
// Ordering intent: fps sampling runs BEFORE the colorspace remap so only
10331047
// kept frames are converted. The remap is pointwise per-frame, so the
@@ -1055,7 +1069,11 @@ export async function extractVideoFramesRange(
10551069
encodeArgs.push("-y", outputPattern);
10561070

10571071
const runOptions = { signal, timeout: ffmpegProcessTimeout };
1058-
const filterArgs = vfFilters.length > 0 ? ["-vf", vfFilters.join(",")] : [];
1072+
const filterArgsFor = (seek: number): string[] => {
1073+
const filters = [...vfFilters];
1074+
filters.splice(samplingAt, 0, ...samplingFilters(seek));
1075+
return filters.length > 0 ? ["-vf", filters.join(",")] : [];
1076+
};
10591077
const segmentFrames = Math.round(EXTRACTION_SEGMENT_SECONDS * fps);
10601078
const totalFrames = extractionFrameCountForDuration(duration, normalizedFps, metadata.isVFR);
10611079
// A VFR window rounds its frame count up, and only the segments' `-frames:v` bound
@@ -1066,7 +1084,7 @@ export async function extractVideoFramesRange(
10661084
sampleAtOutputFps && (options.frameRanges || boundVfrFrameCount || totalFrames > segmentFrames)
10671085
? await runSegmentedExtraction({
10681086
decodeArgs,
1069-
filterAndEncodeArgs: [...filterArgs, ...encodeArgs],
1087+
filterAndEncodeArgs: (seek) => [...filterArgsFor(seek), ...encodeArgs],
10701088
videoPath,
10711089
startTime,
10721090
fps,
@@ -1077,7 +1095,7 @@ export async function extractVideoFramesRange(
10771095
runOptions,
10781096
})
10791097
: await inExtractionSlot(
1080-
() => runFfmpeg([...args, ...filterArgs, ...encodeArgs], runOptions),
1098+
() => runFfmpeg([...args, ...filterArgsFor(startTime), ...encodeArgs], runOptions),
10811099
signal,
10821100
);
10831101
if (processResult.failureReason === "external_interruption") {

0 commit comments

Comments
 (0)