fix(core): music keeps playing smoothly across video cuts in preview

packages/core/src/runtime/audioClockSource.test.ts

@@ -167,6 +167,100 @@ describe("the audio the playhead follows", () => {
     expect(lastTime).toBeGreaterThan(1.4);
   });
 
+  it("follows the music across cuts between short clips earlier in the page", async () => {
+    mount(
+      `<audio id="shot1" data-start="0" data-duration="1" src="/assets/a.mp4"></audio>` +
+        `<audio id="shot2" data-start="1" data-duration="1" src="/assets/b.mp4"></audio>` +
+        `<audio id="shot3" data-start="2" data-duration="1" src="/assets/c.mp4"></audio>` +
+        `<audio id="music" data-start="0" data-duration="10" src="/assets/music.wav"></audio>`,
+    );
+    const shots = ["shot1", "shot2", "shot3"].map(
+      (id) => document.getElementById(id) as HTMLAudioElement,
+    );
+    const music = document.getElementById("music") as HTMLAudioElement;
+    let musicTime = 0;
+    const musicSeeks: number[] = [];
+    Object.defineProperty(music, "currentTime", {
+      configurable: true,
+      get: () => musicTime,
+      set: (value: number) => {
+        musicSeeks.push(value);
+        musicTime = value;
+      },
+    });
+    initSandboxRuntimeModular();
+    await flush();
+    window.__player?.play();
+    await flush();
+    const coldStartLagSeconds = 0.25; // how late each shot's sound starts, as a cold clip does at a cut
+    const playedAt = nowMs;
+    musicSeeks.length = 0; // Play itself lands every clip on the playhead
+    Object.assign(music, { paused: false });
+    for (let frame = 1; frame <= 170; frame++) {
+      const t = (nowMs + 1000 / 60 - playedAt) / 1000;
+      musicTime = t;
+      shots.forEach((shot, i) => {
+        if (t < i) return;
+        Object.assign(shot, {
+          paused: t >= i + 1,
+          currentTime: Math.max(0, t - i - coldStartLagSeconds),
+        });
+      });
+      stepFrames(1);
+      expect(window.__player!.getTime()).toBeCloseTo(t, 3);
+    }
+    expect(musicSeeks).toEqual([]);
+  });
+
+  it("keeps following a playing voice when a longer bed starts late", async () => {
+    mount(
+      `<audio id="vo" data-start="0" data-duration="5" src="/assets/vo.mp3"></audio>` +
+        `<audio id="bed" data-start="1" data-duration="60" src="/assets/bed.wav"></audio>`,
+    );
+    const vo = document.getElementById("vo") as HTMLAudioElement;
+    const bed = document.getElementById("bed") as HTMLAudioElement;
+    initSandboxRuntimeModular();
+    await flush();
+    window.__player?.play();
+    await flush();
+    const playedAt = nowMs;
+    Object.assign(vo, { paused: false });
+    for (let frame = 0; frame < 90; frame++) {
+      stepFrames(1);
+      const t = (nowMs - playedAt) / 1000;
+      vo.currentTime = t;
+      if (t >= 1) Object.assign(bed, { paused: false, currentTime: 0 }); // started, not moving yet
+    }
+    expect(window.__player!.getTime()).toBeGreaterThan(1.4);
+  });
+
+  it("follows a playing clip when the longest one ran out of source early", async () => {
+    mount(
+      `<audio id="shot" data-start="0" data-duration="3" src="/assets/shot.mp4"></audio>` +
+        `<audio id="bed" data-start="0" data-duration="10" src="/assets/bed.wav"></audio>`,
+    );
+    const shot = document.getElementById("shot") as HTMLAudioElement;
+    const bed = document.getElementById("bed") as HTMLAudioElement;
+    initSandboxRuntimeModular();
+    await flush();
+    window.__player?.play();
+    await flush();
+    const playedAt = nowMs;
+    Object.assign(shot, { paused: false });
+    Object.assign(bed, { paused: false });
+    for (let frame = 0; frame < 150; frame++) {
+      stepFrames(1);
+      const t = (nowMs - playedAt) / 1000;
+      shot.currentTime = t + 0.2; // ahead of the wall clock, so only the shot explains it
+      bed.currentTime = Math.min(t, 1);
+      if (t >= 1) {
+        Object.assign(bed, { paused: true });
+        Object.defineProperty(bed, "ended", { value: true, configurable: true });
+      }
+    }
+    expect(window.__player!.getTime()).toBeGreaterThan((nowMs - playedAt) / 1000 + 0.1);
+  });
+
   it("follows the next clip when one earlier in the page failed to load", async () => {
     mount(
       `<audio id="broken" data-start="0" data-duration="10" src="/assets/missing.mp3"></audio>` +

packages/core/src/runtime/init.ts

@@ -4534,6 +4534,26 @@ export function initSandboxRuntimeModular(): void {
     state.transportRafId = window.requestAnimationFrame(transportTick);
   };
 
+  const followedOrLongestRunningAudio = (
+    followed: HTMLMediaElement | null,
+  ): { el: HTMLMediaElement; start: number } | null => {
+    const audioEls = document.querySelectorAll("audio[data-start]");
+    let longest: { el: HTMLMediaElement; start: number; runsUntil: number } | null = null;
+    for (const el of followed ? [followed, ...audioEls] : audioEls) {
+      if (!isMediaElement(el) || !el.isConnected) continue;
+      if (isSilencedByHidden(el) || isUnplayable(el) || (el.ended && !el.loop)) continue;
+      if (!el.hasAttribute("src") && !el.querySelector("source[src]")) continue;
+      const start = resolveAbsoluteMediaStartSeconds(el);
+      const durAttr = parseStrictFiniteTimingNumber(el.dataset.duration);
+      const end = durAttr != null && durAttr > 0 ? start + durAttr : Infinity;
+      if (!Number.isFinite(start) || !isInClipWindow(state.currentTime, start, end)) continue;
+      if (el === followed) return { el, start };
+      const runsUntil = start + (resolveMediaElementDurationSeconds(el) ?? Infinity);
+      if (!longest || runsUntil > longest.runsUntil) longest = { el, start, runsUntil };
+    }
+    return longest;
+  };
+
   const transportTick = () => {
     if (state.tornDown || inTransportTick) return;
     inTransportTick = true;
@@ -4640,36 +4660,19 @@ export function initSandboxRuntimeModular(): void {
             clock.attachAudioSource({ currentTimeSeconds: webAudioTime });
           }
         } else {
-          const audioEls = document.querySelectorAll("audio[data-start]");
-          const followed = clock.audioElement();
-          let foundActive = false;
-          for (const rawEl of followed ? [followed, ...audioEls] : audioEls) {
-            if (!isMediaElement(rawEl) || !rawEl.isConnected) continue;
-            if (isSilencedByHidden(rawEl) || isUnplayable(rawEl)) continue;
-            if (!rawEl.hasAttribute("src") && !rawEl.querySelector("source[src]")) continue;
-            const start = resolveAbsoluteMediaStartSeconds(rawEl);
-            const durAttr = parseStrictFiniteTimingNumber(rawEl.dataset.duration);
-            const end = durAttr != null && durAttr > 0 ? start + durAttr : Infinity;
-            const mediaStart = readElementPlaybackStart(rawEl);
-            if (Number.isFinite(start) && isInClipWindow(state.currentTime, start, end)) {
-              if (!rawEl.paused) {
-                clock.attachAudioSource({
-                  el: rawEl,
-                  compositionStart: start,
-                  mediaStart,
-                  rate: readElementRateSpec(rawEl),
-                });
-                foundActive = true;
-              } else if (rawEl.readyState < HTMLMediaElement.HAVE_FUTURE_DATA) {
-                // Audio is buffering — freeze visuals at last known position
-                // instead of falling through to monotonic (which runs ahead).
-                clock.attachAudioSource({ currentTimeSeconds: state.currentTime });
-                foundActive = true;
-              }
-              break;
-            }
-          }
-          if (!foundActive && clock.hasAudioSource()) {
+          const source = followedOrLongestRunningAudio(clock.audioElement());
+          if (source && !source.el.paused) {
+            clock.attachAudioSource({
+              el: source.el,
+              compositionStart: source.start,
+              mediaStart: readElementPlaybackStart(source.el),
+              rate: readElementRateSpec(source.el),
+            });
+          } else if (source && source.el.readyState < HTMLMediaElement.HAVE_FUTURE_DATA) {
+            // Audio is buffering — freeze visuals at last known position
+            // instead of falling through to monotonic (which runs ahead).
+            clock.attachAudioSource({ currentTimeSeconds: state.currentTime });
+          } else if (clock.hasAudioSource()) {
             clock.detachAudioSource();
           }
         }