From d39c7dd4233ab3a73ed30508fa80697dd548aefd Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Sat, 3 Oct 2026 02:25:54 -0700 Subject: [PATCH 1/3] fix(core): music keeps playing smoothly across video cuts in preview While no Web Audio buffer plays, the preview playhead follows an audio element. It followed the first one in page order, so a sub-composition whose shots each carry a short sound clip took the playhead away from the music bed. At every cut the next shot's sound starts late, the playhead waited for it, the music played on, and media sync then pulled the music back: a jump at each cut. The followed clip still keeps the playhead; otherwise the in-window clip that runs longest takes it. --- .../core/src/runtime/audioClockSource.test.ts | 42 +++++++++++++ packages/core/src/runtime/init.ts | 60 ++++++++++--------- 2 files changed, 75 insertions(+), 27 deletions(-) diff --git a/packages/core/src/runtime/audioClockSource.test.ts b/packages/core/src/runtime/audioClockSource.test.ts index 3ff15d50927..ac3acbf995c 100644 --- a/packages/core/src/runtime/audioClockSource.test.ts +++ b/packages/core/src/runtime/audioClockSource.test.ts @@ -167,6 +167,48 @@ describe("the audio the playhead follows", () => { expect(lastTime).toBeGreaterThan(1.4); }); + it("follows the music across cuts between short clips earlier in the page", async () => { + mount( + `` + + `` + + `` + + ``, + ); + const shots = ["shot1", "shot2", "shot3"].map( + (id) => document.getElementById(id) as HTMLAudioElement, + ); + const music = document.getElementById("music") as HTMLAudioElement; + let musicTime = 0; + const musicSeeks: number[] = []; + Object.defineProperty(music, "currentTime", { + configurable: true, + get: () => musicTime, + set: (value: number) => { + musicSeeks.push(value); + musicTime = value; + }, + }); + initSandboxRuntimeModular(); + await flush(); + window.__player?.play(); + await flush(); + const playedAt = nowMs; + musicSeeks.length = 0; // Play itself lands every clip on the playhead + Object.assign(music, { paused: false }); + for (let frame = 1; frame <= 170; frame++) { + const t = (nowMs + 1000 / 60 - playedAt) / 1000; + musicTime = t; // the music plays straight through every cut + shots.forEach((shot, i) => { + if (t < i) return; + // Each shot's sound starts 250 ms late, as a cold clip does at a cut. + Object.assign(shot, { paused: t >= i + 1, currentTime: Math.max(0, t - i - 0.25) }); + }); + stepFrames(1); + expect(window.__player!.getTime()).toBeCloseTo(t, 3); + } + expect(musicSeeks).toEqual([]); + }); + it("follows the next clip when one earlier in the page failed to load", async () => { mount( `` + diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index 8a7f06ca445..f4a57200d04 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -4534,6 +4534,25 @@ export function initSandboxRuntimeModular(): void { state.transportRafId = window.requestAnimationFrame(transportTick); }; + const followedOrLongestRunningAudio = ( + followed: HTMLMediaElement | null, + ): { el: HTMLMediaElement; start: number } | null => { + let leader: { el: HTMLMediaElement; start: number; runsUntil: number } | null = null; + for (const el of document.querySelectorAll("audio[data-start]")) { + if (!isMediaElement(el) || !el.isConnected) continue; + if (isSilencedByHidden(el) || isUnplayable(el)) continue; + if (!el.hasAttribute("src") && !el.querySelector("source[src]")) continue; + const start = resolveAbsoluteMediaStartSeconds(el); + const durAttr = parseStrictFiniteTimingNumber(el.dataset.duration); + const end = durAttr != null && durAttr > 0 ? start + durAttr : Infinity; + if (!Number.isFinite(start) || !isInClipWindow(state.currentTime, start, end)) continue; + if (el === followed) return { el, start }; + const runsUntil = start + (resolveMediaElementDurationSeconds(el) ?? Infinity); + if (!leader || runsUntil > leader.runsUntil) leader = { el, start, runsUntil }; + } + return leader; + }; + const transportTick = () => { if (state.tornDown || inTransportTick) return; inTransportTick = true; @@ -4640,34 +4659,21 @@ export function initSandboxRuntimeModular(): void { clock.attachAudioSource({ currentTimeSeconds: webAudioTime }); } } else { - const audioEls = document.querySelectorAll("audio[data-start]"); - const followed = clock.audioElement(); + const leader = followedOrLongestRunningAudio(clock.audioElement()); let foundActive = false; - for (const rawEl of followed ? [followed, ...audioEls] : audioEls) { - if (!isMediaElement(rawEl) || !rawEl.isConnected) continue; - if (isSilencedByHidden(rawEl) || isUnplayable(rawEl)) continue; - if (!rawEl.hasAttribute("src") && !rawEl.querySelector("source[src]")) continue; - const start = resolveAbsoluteMediaStartSeconds(rawEl); - const durAttr = parseStrictFiniteTimingNumber(rawEl.dataset.duration); - const end = durAttr != null && durAttr > 0 ? start + durAttr : Infinity; - const mediaStart = readElementPlaybackStart(rawEl); - if (Number.isFinite(start) && isInClipWindow(state.currentTime, start, end)) { - if (!rawEl.paused) { - clock.attachAudioSource({ - el: rawEl, - compositionStart: start, - mediaStart, - rate: readElementRateSpec(rawEl), - }); - foundActive = true; - } else if (rawEl.readyState < HTMLMediaElement.HAVE_FUTURE_DATA) { - // Audio is buffering — freeze visuals at last known position - // instead of falling through to monotonic (which runs ahead). - clock.attachAudioSource({ currentTimeSeconds: state.currentTime }); - foundActive = true; - } - break; - } + if (leader && !leader.el.paused) { + clock.attachAudioSource({ + el: leader.el, + compositionStart: leader.start, + mediaStart: readElementPlaybackStart(leader.el), + rate: readElementRateSpec(leader.el), + }); + foundActive = true; + } else if (leader && leader.el.readyState < HTMLMediaElement.HAVE_FUTURE_DATA) { + // Audio is buffering — freeze visuals at last known position + // instead of falling through to monotonic (which runs ahead). + clock.attachAudioSource({ currentTimeSeconds: state.currentTime }); + foundActive = true; } if (!foundActive && clock.hasAudioSource()) { clock.detachAudioSource(); From 2cf3fb0659bbcae86f714119802863c27440fb57 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Sat, 3 Oct 2026 02:36:58 -0700 Subject: [PATCH 2/3] refactor(core): check the followed clip first and drop the clock pick's flag --- .../core/src/runtime/audioClockSource.test.ts | 9 ++++-- packages/core/src/runtime/init.ts | 29 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/packages/core/src/runtime/audioClockSource.test.ts b/packages/core/src/runtime/audioClockSource.test.ts index ac3acbf995c..f959e55812f 100644 --- a/packages/core/src/runtime/audioClockSource.test.ts +++ b/packages/core/src/runtime/audioClockSource.test.ts @@ -192,16 +192,19 @@ describe("the audio the playhead follows", () => { await flush(); window.__player?.play(); await flush(); + const coldStartLagSeconds = 0.25; // how late each shot's sound starts, as a cold clip does at a cut const playedAt = nowMs; musicSeeks.length = 0; // Play itself lands every clip on the playhead Object.assign(music, { paused: false }); for (let frame = 1; frame <= 170; frame++) { const t = (nowMs + 1000 / 60 - playedAt) / 1000; - musicTime = t; // the music plays straight through every cut + musicTime = t; shots.forEach((shot, i) => { if (t < i) return; - // Each shot's sound starts 250 ms late, as a cold clip does at a cut. - Object.assign(shot, { paused: t >= i + 1, currentTime: Math.max(0, t - i - 0.25) }); + Object.assign(shot, { + paused: t >= i + 1, + currentTime: Math.max(0, t - i - coldStartLagSeconds), + }); }); stepFrames(1); expect(window.__player!.getTime()).toBeCloseTo(t, 3); diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index f4a57200d04..722a020e3d3 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -4537,8 +4537,9 @@ export function initSandboxRuntimeModular(): void { const followedOrLongestRunningAudio = ( followed: HTMLMediaElement | null, ): { el: HTMLMediaElement; start: number } | null => { - let leader: { el: HTMLMediaElement; start: number; runsUntil: number } | null = null; - for (const el of document.querySelectorAll("audio[data-start]")) { + const audioEls = document.querySelectorAll("audio[data-start]"); + let longest: { el: HTMLMediaElement; start: number; runsUntil: number } | null = null; + for (const el of followed ? [followed, ...audioEls] : audioEls) { if (!isMediaElement(el) || !el.isConnected) continue; if (isSilencedByHidden(el) || isUnplayable(el)) continue; if (!el.hasAttribute("src") && !el.querySelector("source[src]")) continue; @@ -4548,9 +4549,9 @@ export function initSandboxRuntimeModular(): void { if (!Number.isFinite(start) || !isInClipWindow(state.currentTime, start, end)) continue; if (el === followed) return { el, start }; const runsUntil = start + (resolveMediaElementDurationSeconds(el) ?? Infinity); - if (!leader || runsUntil > leader.runsUntil) leader = { el, start, runsUntil }; + if (!longest || runsUntil > longest.runsUntil) longest = { el, start, runsUntil }; } - return leader; + return longest; }; const transportTick = () => { @@ -4659,23 +4660,19 @@ export function initSandboxRuntimeModular(): void { clock.attachAudioSource({ currentTimeSeconds: webAudioTime }); } } else { - const leader = followedOrLongestRunningAudio(clock.audioElement()); - let foundActive = false; - if (leader && !leader.el.paused) { + const source = followedOrLongestRunningAudio(clock.audioElement()); + if (source && !source.el.paused) { clock.attachAudioSource({ - el: leader.el, - compositionStart: leader.start, - mediaStart: readElementPlaybackStart(leader.el), - rate: readElementRateSpec(leader.el), + el: source.el, + compositionStart: source.start, + mediaStart: readElementPlaybackStart(source.el), + rate: readElementRateSpec(source.el), }); - foundActive = true; - } else if (leader && leader.el.readyState < HTMLMediaElement.HAVE_FUTURE_DATA) { + } else if (source && source.el.readyState < HTMLMediaElement.HAVE_FUTURE_DATA) { // Audio is buffering — freeze visuals at last known position // instead of falling through to monotonic (which runs ahead). clock.attachAudioSource({ currentTimeSeconds: state.currentTime }); - foundActive = true; - } - if (!foundActive && clock.hasAudioSource()) { + } else if (clock.hasAudioSource()) { clock.detachAudioSource(); } } From ff4ae9408d32a4a1e359eba8805fff70e21476f4 Mon Sep 17 00:00:00 2001 From: Miguel Angel Simon Sierra Date: Sat, 3 Oct 2026 02:52:26 -0700 Subject: [PATCH 3/3] fix(core): an ended clip never leads the playhead, and the followed clip keeps it A bed whose file ends before its authored length no longer wins the pick and leaves the playhead on wall time. Pins the rule that a longer clip starting late cannot take the playhead from the followed one. --- .../core/src/runtime/audioClockSource.test.ts | 49 +++++++++++++++++++ packages/core/src/runtime/init.ts | 2 +- 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/packages/core/src/runtime/audioClockSource.test.ts b/packages/core/src/runtime/audioClockSource.test.ts index f959e55812f..ec3d5b5d60f 100644 --- a/packages/core/src/runtime/audioClockSource.test.ts +++ b/packages/core/src/runtime/audioClockSource.test.ts @@ -212,6 +212,55 @@ describe("the audio the playhead follows", () => { expect(musicSeeks).toEqual([]); }); + it("keeps following a playing voice when a longer bed starts late", async () => { + mount( + `` + + ``, + ); + const vo = document.getElementById("vo") as HTMLAudioElement; + const bed = document.getElementById("bed") as HTMLAudioElement; + initSandboxRuntimeModular(); + await flush(); + window.__player?.play(); + await flush(); + const playedAt = nowMs; + Object.assign(vo, { paused: false }); + for (let frame = 0; frame < 90; frame++) { + stepFrames(1); + const t = (nowMs - playedAt) / 1000; + vo.currentTime = t; + if (t >= 1) Object.assign(bed, { paused: false, currentTime: 0 }); // started, not moving yet + } + expect(window.__player!.getTime()).toBeGreaterThan(1.4); + }); + + it("follows a playing clip when the longest one ran out of source early", async () => { + mount( + `` + + ``, + ); + const shot = document.getElementById("shot") as HTMLAudioElement; + const bed = document.getElementById("bed") as HTMLAudioElement; + initSandboxRuntimeModular(); + await flush(); + window.__player?.play(); + await flush(); + const playedAt = nowMs; + Object.assign(shot, { paused: false }); + Object.assign(bed, { paused: false }); + for (let frame = 0; frame < 150; frame++) { + stepFrames(1); + const t = (nowMs - playedAt) / 1000; + shot.currentTime = t + 0.2; // ahead of the wall clock, so only the shot explains it + bed.currentTime = Math.min(t, 1); + if (t >= 1) { + Object.assign(bed, { paused: true }); + Object.defineProperty(bed, "ended", { value: true, configurable: true }); + } + } + expect(window.__player!.getTime()).toBeGreaterThan((nowMs - playedAt) / 1000 + 0.1); + }); + it("follows the next clip when one earlier in the page failed to load", async () => { mount( `` + diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index 722a020e3d3..0cd8738bee2 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -4541,7 +4541,7 @@ export function initSandboxRuntimeModular(): void { let longest: { el: HTMLMediaElement; start: number; runsUntil: number } | null = null; for (const el of followed ? [followed, ...audioEls] : audioEls) { if (!isMediaElement(el) || !el.isConnected) continue; - if (isSilencedByHidden(el) || isUnplayable(el)) continue; + if (isSilencedByHidden(el) || isUnplayable(el) || (el.ended && !el.loop)) continue; if (!el.hasAttribute("src") && !el.querySelector("source[src]")) continue; const start = resolveAbsoluteMediaStartSeconds(el); const durAttr = parseStrictFiniteTimingNumber(el.dataset.duration);