feat(core): audible videos in preview web audio, audio groups, meters and carve (2/8)

docs/reference/audio-effects.mdx

@@ -20,10 +20,10 @@ nothing to load beside the markup.
 | `data-fx-chain` | The effects, in signal order | JSON | `<audio>`, `<video>`, `<hf-audio-group>` | Yes |
 | `data-automation` | Envelopes on volume or an effect parameter | JSON | `<audio>`, `<video>`, `<hf-audio-group>` | Yes |
 | `data-fx-carve` | A voiceover carve's own settings | JSON | `<audio>`, `<video>` | No — see [below](#data-fx-carve) |
-| `data-audio-group` | The id of the group this clip belongs to | A plain id, not JSON | `<audio>` only | Yes |
+| `data-audio-group` | The id of the group this clip belongs to | A plain id, not JSON | `<audio>`, audible `<video>` | Yes |
 
-`data-audio-group` is the odd one: a bare string, and **ignored on `<video>`**.
-The other three are JSON. Of those, only `data-fx-chain` and `data-automation`
+`data-audio-group` is the odd one: a bare string, and **ignored on a muted
+`<video>`** (`muted`, or `data-has-audio="false"`). The other three are JSON. Of those, only `data-fx-chain` and `data-automation`
 can also sit on an `<hf-audio-group>`, where they apply to the group's whole bus;
 `data-fx-carve` belongs to a clip.
 
@@ -170,8 +170,9 @@ dangles.
 - `<hf-audio-group>` is optional metadata: `data-label`, `data-volume`,
   `data-hidden`, and its own `data-fx-chain` / `data-automation`. A group with
   members but no element still resolves, using its id as the label.
-- **Audio only.** `data-audio-group` on a `<video>` is ignored, and groups do
-  not nest.
+- **Audio and audible video.** A `<video>` with sound (`data-has-audio="true"`,
+  no `muted`) joins its group in preview and render; on a muted video
+  `data-audio-group` is ignored. Groups do not nest.
 - `data-audio-group=""` is no group at all.
 - The element must be inert. The runtime injects
   `hf-audio-group{display:none!important}` so an unknown custom element cannot

packages/core/package-subpaths.json

@@ -218,6 +218,12 @@
       "types": "./dist/clipFacts.d.ts",
       "environments": ["browser", "bun", "node"]
     },
+    "./audible-video": {
+      "source": "./src/audibleVideo.ts",
+      "runtime": "./dist/audibleVideo.js",
+      "types": "./dist/audibleVideo.d.ts",
+      "environments": ["browser", "bun", "node"]
+    },
     "./audio-gain": {
       "source": "./src/audioGain.ts",
       "runtime": "./dist/audioGain.js",

packages/core/package.json

@@ -232,6 +232,12 @@
       "import": "./src/clipFacts.ts",
       "types": "./src/clipFacts.ts"
     },
+    "./audible-video": {
+      "bun": "./src/audibleVideo.ts",
+      "node": "./dist/audibleVideo.js",
+      "import": "./src/audibleVideo.ts",
+      "types": "./src/audibleVideo.ts"
+    },
     "./audio-gain": {
       "bun": "./src/audioGain.ts",
       "node": "./dist/audioGain.js",
@@ -614,6 +620,10 @@
         "import": "./dist/clipFacts.js",
         "types": "./dist/clipFacts.d.ts"
       },
+      "./audible-video": {
+        "import": "./dist/audibleVideo.js",
+        "types": "./dist/audibleVideo.d.ts"
+      },
       "./audio-gain": {
         "import": "./dist/audioGain.js",
         "types": "./dist/audioGain.d.ts"

packages/core/src/audibleVideo.test.ts

@@ -0,0 +1,63 @@
+import { describe, expect, it } from "vitest";
+import {
+  AUDIBLE_MEDIA_SELECTOR,
+  audibleVideoNeedsWebAudio,
+  isAudibleVideoElement,
+} from "./audibleVideo";
+
+const el = (tagName: string, attrs: Record<string, string>) => ({
+  tagName,
+  hasAttribute: (n: string) => n in attrs,
+  getAttribute: (n: string) => (n in attrs ? (attrs[n] ?? null) : null),
+});
+
+describe("isAudibleVideoElement", () => {
+  it("unmuted video with data-has-audio=true or no attribute is audible", () => {
+    expect(isAudibleVideoElement(el("VIDEO", { "data-has-audio": "true" }))).toBe(true);
+    expect(isAudibleVideoElement(el("video", {}))).toBe(true);
+  });
+  it("muted, false or empty is not audible", () => {
+    expect(isAudibleVideoElement(el("VIDEO", { muted: "" }))).toBe(false);
+    expect(isAudibleVideoElement(el("VIDEO", { "data-has-audio": "false" }))).toBe(false);
+    expect(isAudibleVideoElement(el("VIDEO", { "data-has-audio": "" }))).toBe(false);
+  });
+  it("only the exact value true counts, so TRUE is not audible", () => {
+    expect(isAudibleVideoElement(el("VIDEO", { "data-has-audio": "TRUE" }))).toBe(false);
+  });
+  it("never true for non-video", () => {
+    expect(isAudibleVideoElement(el("AUDIO", {}))).toBe(false);
+  });
+});
+
+describe("AUDIBLE_MEDIA_SELECTOR", () => {
+  it("matches audio plus exactly the videos the predicate calls audible", () => {
+    document.body.innerHTML = `
+      <audio id="a" data-start="0"></audio>
+      <video id="v-true" data-start="0" data-has-audio="true"></video>
+      <video id="v-absent" data-start="0"></video>
+      <video id="v-muted" data-start="0" data-has-audio="true" muted></video>
+      <video id="v-false" data-start="0" data-has-audio="false"></video>
+      <video id="v-empty" data-start="0" data-has-audio=""></video>`;
+    const matched = Array.from(document.querySelectorAll(AUDIBLE_MEDIA_SELECTOR), (n) => n.id);
+    expect(matched).toEqual(["a", "v-true", "v-absent"]);
+    const audibleVideos = Array.from(document.querySelectorAll("video"))
+      .filter(isAudibleVideoElement)
+      .map((n) => n.id);
+    expect(audibleVideos).toEqual(matched.filter((id) => id.startsWith("v-")));
+  });
+});
+
+describe("audibleVideoNeedsWebAudio", () => {
+  it("keeps a unity, unprocessed video on native output", () => {
+    expect(audibleVideoNeedsWebAudio({})).toBe(false);
+    expect(audibleVideoNeedsWebAudio({ volume: 1, fxChain: "", automation: null })).toBe(false);
+    expect(audibleVideoNeedsWebAudio({ volume: Number.NaN })).toBe(false);
+  });
+
+  it.each([{ volume: 1.5 }, { fxChain: "[]" }, { automation: "{}" }, { audioGroup: "music" }])(
+    "routes a video carrying %o",
+    (fields) => {
+      expect(audibleVideoNeedsWebAudio(fields)).toBe(true);
+    },
+  );
+});

packages/core/src/audibleVideo.ts

@@ -0,0 +1,25 @@
+export const AUDIBLE_VIDEO_QUALIFIER =
+  ':not([muted]):is(:not([data-has-audio]),[data-has-audio="true"])';
+
+export const AUDIBLE_MEDIA_SELECTOR = `audio[data-start], video[data-start]${AUDIBLE_VIDEO_QUALIFIER}`;
+
+export function isAudibleVideoElement(el: {
+  tagName: string;
+  hasAttribute(name: string): boolean;
+  getAttribute(name: string): string | null;
+}): boolean {
+  if (el.tagName.toLowerCase() !== "video" || el.hasAttribute("muted")) return false;
+  const declared = el.getAttribute("data-has-audio");
+  return declared === null || declared === "true";
+}
+
+export function audibleVideoNeedsWebAudio(fields: {
+  volume?: number | null;
+  fxChain?: string | null;
+  automation?: string | null;
+  audioGroup?: string | null;
+}): boolean {
+  return (
+    (fields.volume ?? 1) > 1 || Boolean(fields.fxChain || fields.automation || fields.audioGroup)
+  );
+}

packages/core/src/audioGroups.test.ts

@@ -67,9 +67,12 @@ describe("resolveAudioGroups", () => {
     expect(resolveAudioGroups(document)[0].memberIds).toEqual(["vo-1"]);
   });
 
-  it("ignores a data-audio-group on a video element (audio only in v1)", () => {
-    document.body.innerHTML = `<video id="v-1" data-audio-group="voiceover"></video>`;
-    expect(resolveAudioGroups(document)).toEqual([]);
+  it("groups an audible video and ignores a muted or silent one", () => {
+    document.body.innerHTML = `
+      <video id="v-1" data-has-audio="true" data-audio-group="voiceover"></video>
+      <video id="v-2" muted data-audio-group="voiceover"></video>
+      <video id="v-3" data-has-audio="false" data-audio-group="voiceover"></video>`;
+    expect(resolveAudioGroups(document).map((g) => g.memberIds)).toEqual([["v-1"]]);
   });
 
   it("reads the group element's fx chain, automation, volume and hidden", () => {
@@ -115,11 +118,12 @@ describe("audioGroupOf", () => {
     expect(audioGroupOf(document.getElementById("vo-1") as Element)).toBeNull();
   });
 
-  it("ignores video membership so preview matches the audio-only render", () => {
-    document.body.innerHTML = `<video id="v-1" data-audio-group="voiceover"></video>`;
-    const el = document.getElementById("v-1") as Element;
-    expect(audioGroupOf(el)).toBeNull();
-    expect(resolveAudioGroups(document)).toEqual([]);
+  it("reads an audible video's membership and ignores a muted video's", () => {
+    document.body.innerHTML = `
+      <video id="v-1" data-has-audio="true" data-audio-group="voiceover"></video>
+      <video id="v-2" muted data-audio-group="voiceover"></video>`;
+    expect(audioGroupOf(document.getElementById("v-1") as Element)).toBe("voiceover");
+    expect(audioGroupOf(document.getElementById("v-2") as Element)).toBeNull();
   });
 
   it("normalizes an empty membership attribute to null", () => {

packages/core/src/audioGroups.ts

@@ -12,6 +12,7 @@
 import { HF_AUDIO_FX_ATTR } from "./audioFx.js";
 import { AUDIO_GROUP_RENDER_ID_ATTR, MEDIA_RENDER_ID_ATTR } from "./compiler/mediaRenderIds.js";
 import { HF_AUDIO_AUTOMATION_ATTR } from "./audioAutomation.js";
+import { isAudibleVideoElement } from "./audibleVideo.js";
 
 export const HF_AUDIO_GROUP_TAG = "hf-audio-group";
 export const HF_AUDIO_GROUP_ATTR = "data-audio-group";
@@ -129,7 +130,11 @@ export function isMemberGroupHidden(
 
 export function resolveAudioGroups(root: ParentNode): HfAudioGroup[] {
   const membersByGroup = new Map<string, string[]>();
-  for (const member of root.querySelectorAll(`audio[${HF_AUDIO_GROUP_ATTR}]`)) {
+  const candidates = root.querySelectorAll(
+    `audio[${HF_AUDIO_GROUP_ATTR}], video[${HF_AUDIO_GROUP_ATTR}]`,
+  );
+  for (const member of candidates) {
+    if (!isGroupableMember(member)) continue;
     // The render-stamped instance key when the compiler has been through
     // (`assignMediaRenderIds`), else the author id. An author id is unique only
     // per composition FILE, so a sub-composition declaring a bus AND its members
@@ -195,20 +200,25 @@ export function resolveCarveSourceIds(doc: Document, ids: readonly string[]): st
   return out;
 }
 
-/** The group an audio member belongs to, or null. Membership is audio-only in
- * v1, matching `resolveAudioGroups` and the render mixer; video and group-bus
+/** The group an audio or audible-video member belongs to, or null, matching
+ * `resolveAudioGroups` and the render mixer; muted video and group-bus
  * attributes are inert.
  *
  * Tolerant of objects that only partially implement `Element` (test doubles
  * for `HTMLMediaElement` commonly do) — anything missing `tagName` or
  * `getAttribute` simply has no group, mirroring `readChain`'s style in
  * `runtime/audioFx.ts`. */
 export function audioGroupOf(el: Element): string | null {
-  if (typeof el.tagName !== "string" || el.tagName.toLowerCase() !== "audio") return null;
-  if (typeof el.getAttribute !== "function") return null;
+  if (typeof el.tagName !== "string" || typeof el.getAttribute !== "function") return null;
+  if (!isGroupableMember(el)) return null;
   return el.getAttribute(HF_AUDIO_GROUP_ATTR) || null;
 }
 
+function isGroupableMember(el: Element): boolean {
+  if (el.tagName.toLowerCase() === "audio") return true;
+  return typeof el.hasAttribute === "function" && isAudibleVideoElement(el);
+}
+
 /**
  * Make `<hf-audio-group>` inert, once per document.
  *

packages/core/src/compiler/mediaRenderIds.test.ts

@@ -166,6 +166,16 @@ describe("audio group render ids", () => {
     expect(d.querySelector("audio")?.getAttribute(AUDIO_GROUP_RENDER_ID_ATTR)).toBe("vo");
   });
 
+  it("stamps an audible video member and skips a muted one", () => {
+    const d = doc(`<hf-audio-group id="vo"></hf-audio-group>
+      <video id="v" src="a.mp4" data-has-audio="true" data-audio-group="vo"></video>
+      <video id="m" src="b.mp4" muted data-audio-group="vo"></video>`);
+    assignMediaRenderIds(d);
+    const [audible, muted] = [...d.querySelectorAll("video")];
+    expect(audible?.getAttribute(AUDIO_GROUP_RENDER_ID_ATTR)).toBe("vo");
+    expect(muted?.hasAttribute(AUDIO_GROUP_RENDER_ID_ATTR)).toBe(false);
+  });
+
   it("leaves a member alone when no bus element declares its group", () => {
     const d = doc(`<audio id="a" src="a.wav" data-audio-group="ghost"></audio>`);
     assignMediaRenderIds(d);

packages/core/src/compiler/mediaRenderIds.ts

@@ -25,6 +25,8 @@
  * fix scene footage.
  */
 
+import { AUDIBLE_VIDEO_QUALIFIER } from "../audibleVideo.js";
+
 export const MEDIA_RENDER_ID_ATTR = "data-hf-render-id";
 
 /**
@@ -141,7 +143,7 @@ function assignAudioGroupRenderIds(document: DocumentLike, taken: Set<string>):
   if (busesById.size === 0) return;
 
   for (const member of document.querySelectorAll(
-    "audio[data-audio-group]",
+    `audio[data-audio-group], video[data-audio-group]${AUDIBLE_VIDEO_QUALIFIER}`,
   ) as Iterable<ScopedElementLike>) {
     const groupId = member.getAttribute("data-audio-group");
     const buses = groupId ? busesById.get(groupId) : undefined;

packages/core/src/editing/affordances.test.ts

@@ -140,6 +140,16 @@ describe("resolveEditingAffordances — sections", () => {
     expect(s).toMatchObject({ media: true, colorGrading: true });
   });
 
+  it("audible video: audioFx; silent video: no audioFx", () => {
+    expect(
+      resolveEditingAffordances(baseFacts({ tag: "video", hasAudio: true })).sections.audioFx,
+    ).toBe(true);
+    expect(
+      resolveEditingAffordances(baseFacts({ tag: "video", hasAudio: false })).sections.audioFx,
+    ).toBe(false);
+    expect(resolveEditingAffordances(baseFacts({ tag: "video" })).sections.audioFx).toBe(false);
+  });
+
   it("audio: media but not colorGrading", () => {
     const s = resolveEditingAffordances(baseFacts({ tag: "audio" })).sections;
     expect(s).toMatchObject({ media: true, colorGrading: false });

packages/core/src/editing/affordances.ts

@@ -69,6 +69,7 @@ export interface EditableElementFacts {
   hasTimingStart: boolean;
   /** count of GSAP tweens targeting this element */
   animationCount: number;
+  hasAudio?: boolean;
 }
 
 /**
@@ -218,7 +219,8 @@ export function resolveEditingSections(facts: EditableElementFacts): EditingSect
   return {
     text: facts.hasEditableText && !facts.isCompositionHost && !facts.isInsideLockedComposition,
     media: facts.tag === "video" || facts.tag === "audio" || facts.tag === "img",
-    audioFx: facts.tag === "audio" || isAudioBus,
+    audioFx:
+      facts.tag === "audio" || isAudioBus || (facts.tag === "video" && facts.hasAudio === true),
     colorGrading: facts.tag === "video" || facts.tag === "img",
     // A bus has no clip range at all — no `data-start`, no duration, and its
     // automation clock is composition time — so Start/Duration/End would be

packages/core/src/index.ts

@@ -183,6 +183,11 @@ export {
 } from "./compiler/subCompositionValidity";
 export { RUNTIME_BOOTSTRAP_ATTR, stripEmbeddedRuntimeScripts } from "./compiler/htmlDocument";
 export { queryByAttr } from "./utils/cssSelector";
+export {
+  AUDIBLE_MEDIA_SELECTOR,
+  audibleVideoNeedsWebAudio,
+  isAudibleVideoElement,
+} from "./audibleVideo";
 export { decodeUrlPathVariants } from "./utils/urlPath";
 export {
   clearGifFramesBeforeNext,