feat(studio): link a video and its audio with data-link (6/8)

packages/core/package-subpaths.json

@@ -200,6 +200,12 @@
       "types": "./dist/audioCarve.d.ts",
       "environments": ["browser", "bun", "node"]
     },
+    "./media-link": {
+      "source": "./src/mediaLink.ts",
+      "runtime": "./dist/mediaLink.js",
+      "types": "./dist/mediaLink.d.ts",
+      "environments": ["browser", "bun", "node"]
+    },
     "./audio-groups": {
       "source": "./src/audioGroups.ts",
       "runtime": "./dist/audioGroups.js",

packages/core/package.json

@@ -214,6 +214,12 @@
       "import": "./src/audioCarve.ts",
       "types": "./src/audioCarve.ts"
     },
+    "./media-link": {
+      "bun": "./src/mediaLink.ts",
+      "node": "./dist/mediaLink.js",
+      "import": "./src/mediaLink.ts",
+      "types": "./src/mediaLink.ts"
+    },
     "./audio-groups": {
       "bun": "./src/audioGroups.ts",
       "node": "./dist/audioGroups.js",
@@ -608,6 +614,10 @@
         "import": "./dist/audioCarve.js",
         "types": "./dist/audioCarve.d.ts"
       },
+      "./media-link": {
+        "import": "./dist/mediaLink.js",
+        "types": "./dist/mediaLink.d.ts"
+      },
       "./audio-groups": {
         "import": "./dist/audioGroups.js",
         "types": "./dist/audioGroups.d.ts"

packages/core/src/mediaLink.test.ts

@@ -0,0 +1,99 @@
+import { describe, expect, it } from "vitest";
+import {
+  MEDIA_LINK_ATTR,
+  linkTimingMismatches,
+  mintLinkId,
+  readLinkTiming,
+  relinkSplitHalves,
+} from "./mediaLink";
+
+const attrs = (values: Record<string, string>) => ({
+  getAttribute: (name: string) => values[name] ?? null,
+});
+
+describe("readLinkTiming", () => {
+  it("defaults media-start to 0 and playback-rate to 1", () => {
+    expect(readLinkTiming(attrs({ "data-start": "2", "data-duration": "6" }))).toEqual({
+      start: 2,
+      duration: 6,
+      mediaStart: 0,
+      playbackRate: 1,
+    });
+  });
+
+  it("reads data-playback-start as the media-start alias", () => {
+    expect(readLinkTiming(attrs({ "data-playback-start": "1.5" })).mediaStart).toBe(1.5);
+  });
+});
+
+describe("linkTimingMismatches", () => {
+  const base = {
+    "data-start": "2",
+    "data-duration": "6",
+    "data-media-start": "1",
+    "data-playback-rate": "1",
+  };
+
+  it("is empty for members in sync, treating absent defaults as equal", () => {
+    expect(
+      linkTimingMismatches([
+        attrs(base),
+        attrs({ "data-start": "2", "data-duration": "6", "data-media-start": "1" }),
+      ]),
+    ).toEqual([]);
+  });
+
+  it.each([
+    ["data-start", "start"],
+    ["data-duration", "duration"],
+    ["data-media-start", "media-start"],
+    ["data-playback-rate", "playback-rate"],
+  ])("names %s when it drifts", (attr, field) => {
+    expect(linkTimingMismatches([attrs(base), attrs({ ...base, [attr]: "3" })])).toEqual([field]);
+  });
+
+  it("ignores sub-millisecond float noise", () => {
+    expect(linkTimingMismatches([attrs(base), attrs({ ...base, "data-start": "2.0004" })])).toEqual(
+      [],
+    );
+  });
+});
+
+describe("mintLinkId", () => {
+  it("returns the first lk-N not already taken", () => {
+    expect(mintLinkId([])).toBe("lk-1");
+    expect(mintLinkId(["lk-1", "lk-2", "lk-4"])).toBe("lk-3");
+  });
+});
+
+describe("relinkSplitHalves", () => {
+  function doc(html: string): Document {
+    document.body.innerHTML = html;
+    return document;
+  }
+
+  it("gives each linked group's right halves one fresh shared id", () => {
+    const d = doc(`
+      <video id="v" data-link="lk-1"></video><video id="v-split" data-link="lk-1"></video>
+      <audio id="a" data-link="lk-1"></audio><audio id="a-split" data-link="lk-1"></audio>
+      <img id="i" /><img id="i-split" />`);
+    relinkSplitHalves(d, ["v-split", "a-split", "i-split"]);
+    const link = (id: string) => d.getElementById(id)?.getAttribute(MEDIA_LINK_ATTR) ?? null;
+    expect(link("v")).toBe("lk-1");
+    expect(link("a")).toBe("lk-1");
+    expect(link("v-split")).toBe("lk-2");
+    expect(link("a-split")).toBe("lk-2");
+    expect(link("i-split")).toBeNull();
+  });
+
+  it("mints distinct ids for distinct groups", () => {
+    const d = doc(`
+      <video id="v" data-link="lk-1"></video><video id="v2" data-link="lk-1"></video>
+      <video id="w" data-link="lk-7"></video><video id="w2" data-link="lk-7"></video>`);
+    relinkSplitHalves(d, ["v2", "w2"]);
+    const ids = ["v2", "w2"].map((id) => d.getElementById(id)?.getAttribute(MEDIA_LINK_ATTR));
+    expect(new Set(ids).size).toBe(2);
+    expect(ids).not.toContain("lk-1");
+    expect(ids).not.toContain("lk-7");
+  });
+});

packages/core/src/mediaLink.ts

@@ -0,0 +1,95 @@
+/**
+ * Linked clips: members sharing a `data-link` id are edited as one (a detached
+ * video + its audio). An editing contract only; playback and render ignore it.
+ */
+export const MEDIA_LINK_ATTR = "data-link";
+
+interface AttributeReader {
+  getAttribute(name: string): string | null;
+}
+
+export interface LinkTiming {
+  start: number;
+  duration: number;
+  mediaStart: number;
+  playbackRate: number;
+}
+
+const LINK_TIMING_FIELDS: ReadonlyArray<[keyof LinkTiming, string]> = [
+  ["start", "start"],
+  ["duration", "duration"],
+  ["mediaStart", "media-start"],
+  ["playbackRate", "playback-rate"],
+];
+
+const SYNC_TOLERANCE_S = 1e-3;
+
+function readNumber(el: AttributeReader, names: string[], fallback: number): number {
+  for (const name of names) {
+    const raw = el.getAttribute(name);
+    if (raw === null || raw.trim() === "") continue;
+    const value = Number(raw);
+    if (Number.isFinite(value)) return value;
+  }
+  return fallback;
+}
+
+export function readLinkTiming(el: AttributeReader): LinkTiming {
+  return {
+    start: readNumber(el, ["data-start"], 0),
+    duration: readNumber(el, ["data-duration"], 0),
+    mediaStart: readNumber(el, ["data-media-start", "data-playback-start"], 0),
+    playbackRate: readNumber(el, ["data-playback-rate"], 1),
+  };
+}
+
+/** The timing fields (`start`, `duration`, `media-start`, `playback-rate`) on which members disagree. */
+export function linkTimingMismatches(members: readonly AttributeReader[]): string[] {
+  const timings = members.map(readLinkTiming);
+  const first = timings[0];
+  if (!first) return [];
+  return LINK_TIMING_FIELDS.filter(([key]) =>
+    timings.some((timing) => Math.abs(timing[key] - first[key]) > SYNC_TOLERANCE_S),
+  ).map(([, field]) => field);
+}
+
+/** The composition a link group stays inside: the nearest inline composition or file host. */
+export function linkScopeOf(el: Element): Element | null {
+  return el.parentElement?.closest("[data-composition-id], [data-composition-file]") ?? null;
+}
+
+export function mintLinkId(taken: Iterable<string>): string {
+  const used = new Set(taken);
+  let n = 1;
+  while (used.has(`lk-${n}`)) n += 1;
+  return `lk-${n}`;
+}
+
+function takenLinkIds(doc: Document): string[] {
+  return Array.from(doc.querySelectorAll(`[id], [${MEDIA_LINK_ATTR}]`)).flatMap((el) => [
+    el.id,
+    el.getAttribute(MEDIA_LINK_ATTR) ?? "",
+  ]);
+}
+
+/**
+ * After a split, the right halves are clones still carrying the left halves'
+ * link id. Give each group's right halves one fresh shared id, so each half of
+ * a linked pair is its own pair.
+ */
+export function relinkSplitHalves(doc: Document, rightHalfIds: readonly string[]): void {
+  const taken = new Set(takenLinkIds(doc));
+  const renamed = new Map<string, string>();
+  for (const id of rightHalfIds) {
+    const el = doc.getElementById(id);
+    const oldLink = el?.getAttribute(MEDIA_LINK_ATTR);
+    if (!el || !oldLink) continue;
+    let fresh = renamed.get(oldLink);
+    if (!fresh) {
+      fresh = mintLinkId(taken);
+      taken.add(fresh);
+      renamed.set(oldLink, fresh);
+    }
+    el.setAttribute(MEDIA_LINK_ATTR, fresh);
+  }
+}

packages/lint/src/rules/linkedClips.test.ts

@@ -0,0 +1,83 @@
+import { describe, expect, it } from "vitest";
+import { lintHyperframeHtml } from "../hyperframeLinter.js";
+
+const VIDEO =
+  'id="talk" src="talk.mp4" muted data-link="lk-1" data-start="2" data-duration="6" data-media-start="1" data-track-index="0"';
+const AUDIO =
+  'id="talk-audio" src="talk.mp4" data-link="lk-1" data-start="2" data-duration="6" data-media-start="1" data-track-index="2"';
+
+function composition(body: string): string {
+  return `
+<html><body>
+  <div id="root" data-composition-id="c1" data-width="1920" data-height="1080">
+    ${body}
+  </div>
+  <script>window.__timelines = {};</script>
+</body></html>`;
+}
+
+async function linkFindings(body: string) {
+  const result = await lintHyperframeHtml(composition(body));
+  return result.findings.filter((f) => f.code.startsWith("linked_clip"));
+}
+
+describe("linked clip rules", () => {
+  it("an in-sync pair is clean", async () => {
+    expect(await linkFindings(`<video ${VIDEO}></video><audio ${AUDIO}></audio>`)).toEqual([]);
+  });
+
+  it("absent media-start and playback-rate equal their defaults", async () => {
+    const video = VIDEO.replace(' data-media-start="1"', ' data-media-start="0"');
+    const audio = AUDIO.replace(' data-media-start="1"', ' data-playback-rate="1"');
+    expect(await linkFindings(`<video ${video}></video><audio ${audio}></audio>`)).toEqual([]);
+  });
+
+  it.each([
+    ['data-start="2"', 'data-start="2.5"', "start"],
+    ['data-duration="6"', 'data-duration="5"', "duration"],
+    ['data-media-start="1"', 'data-media-start="0"', "media-start"],
+    ['data-media-start="1"', 'data-media-start="1" data-playback-rate="2"', "playback-rate"],
+  ])("warns when %s drifts to %s", async (from, to, field) => {
+    const findings = await linkFindings(
+      `<video ${VIDEO}></video><audio ${AUDIO.replace(from, to)}></audio>`,
+    );
+    expect(findings).toHaveLength(1);
+    expect(findings[0]?.code).toBe("linked_clips_out_of_sync");
+    expect(findings[0]?.severity).toBe("warning");
+    expect(findings[0]?.message).toContain(field);
+    expect(findings[0]?.message).toContain("lk-1");
+  });
+
+  it("warns on a link id with one member", async () => {
+    const findings = await linkFindings(`<video ${VIDEO}></video>`);
+    expect(findings).toHaveLength(1);
+    expect(findings[0]?.code).toBe("linked_clip_orphan");
+    expect(findings[0]?.severity).toBe("warning");
+    expect(findings[0]?.elementId).toBe("talk");
+  });
+
+  it("track index, volume and fades may differ freely", async () => {
+    const audio = `${AUDIO} data-volume="0.5" data-fade-in="1"`;
+    expect(await linkFindings(`<video ${VIDEO}></video><audio ${audio}></audio>`)).toEqual([]);
+  });
+
+  it("groups links per composition, so a nested composition may reuse an id", async () => {
+    const nested = `<div id="child" data-composition-id="child" data-start="0" data-duration="10">
+      <video ${VIDEO.replace('id="talk"', 'id="c-talk"').replace('data-start="2"', 'data-start="5"')}></video>
+      <audio ${AUDIO.replace('id="talk-audio"', 'id="c-audio"').replace('data-start="2"', 'data-start="5"')}></audio>
+    </div>`;
+    expect(await linkFindings(`<video ${VIDEO}></video><audio ${AUDIO}></audio>${nested}`)).toEqual(
+      [],
+    );
+  });
+
+  it("treats an inlined file host as its own composition too", async () => {
+    const hosted = `<div id="host" data-composition-file="child.html" data-start="0">
+      <video ${VIDEO.replace('id="talk"', 'id="h-talk"').replace('data-start="2"', 'data-start="5"')}></video>
+    </div>`;
+    const findings = await linkFindings(
+      `<video ${VIDEO}></video><audio ${AUDIO}></audio>${hosted}`,
+    );
+    expect(findings.map((f) => [f.code, f.elementId])).toEqual([["linked_clip_orphan", "h-talk"]]);
+  });
+});