From 35c72d8ab867b29e5c37c5ae845d5d0fc2be5b0e Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Tue, 29 Sep 2026 17:09:44 -0700 Subject: [PATCH 01/21] feat(core): add data-link media link helpers Co-Authored-By: Claude Opus 5.5 --- packages/core/package-subpaths.json | 6 ++ packages/core/package.json | 10 +++ packages/core/src/mediaLink.test.ts | 99 +++++++++++++++++++++++++++++ packages/core/src/mediaLink.ts | 90 ++++++++++++++++++++++++++ 4 files changed, 205 insertions(+) create mode 100644 packages/core/src/mediaLink.test.ts create mode 100644 packages/core/src/mediaLink.ts diff --git a/packages/core/package-subpaths.json b/packages/core/package-subpaths.json index f2626dd280..da3dfb8adf 100644 --- a/packages/core/package-subpaths.json +++ b/packages/core/package-subpaths.json @@ -200,6 +200,12 @@ "types": "./dist/audioCarve.d.ts", "environments": ["browser", "bun", "node"] }, + "./media-link": { + "source": "./src/mediaLink.ts", + "runtime": "./dist/mediaLink.js", + "types": "./dist/mediaLink.d.ts", + "environments": ["browser", "bun", "node"] + }, "./audio-groups": { "source": "./src/audioGroups.ts", "runtime": "./dist/audioGroups.js", diff --git a/packages/core/package.json b/packages/core/package.json index 836954ea54..62f017b31e 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -214,6 +214,12 @@ "import": "./src/audioCarve.ts", "types": "./src/audioCarve.ts" }, + "./media-link": { + "bun": "./src/mediaLink.ts", + "node": "./dist/mediaLink.js", + "import": "./src/mediaLink.ts", + "types": "./src/mediaLink.ts" + }, "./audio-groups": { "bun": "./src/audioGroups.ts", "node": "./dist/audioGroups.js", @@ -608,6 +614,10 @@ "import": "./dist/audioCarve.js", "types": "./dist/audioCarve.d.ts" }, + "./media-link": { + "import": "./dist/mediaLink.js", + "types": "./dist/mediaLink.d.ts" + }, "./audio-groups": { "import": "./dist/audioGroups.js", "types": "./dist/audioGroups.d.ts" diff --git a/packages/core/src/mediaLink.test.ts b/packages/core/src/mediaLink.test.ts new file mode 100644 index 0000000000..568c7f930d --- /dev/null +++ b/packages/core/src/mediaLink.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it } from "vitest"; +import { + MEDIA_LINK_ATTR, + linkTimingMismatches, + mintLinkId, + readLinkTiming, + relinkSplitHalves, +} from "./mediaLink"; + +const attrs = (values: Record) => ({ + getAttribute: (name: string) => values[name] ?? null, +}); + +describe("readLinkTiming", () => { + it("defaults media-start to 0 and playback-rate to 1", () => { + expect(readLinkTiming(attrs({ "data-start": "2", "data-duration": "6" }))).toEqual({ + start: 2, + duration: 6, + mediaStart: 0, + playbackRate: 1, + }); + }); + + it("reads data-playback-start as the media-start alias", () => { + expect(readLinkTiming(attrs({ "data-playback-start": "1.5" })).mediaStart).toBe(1.5); + }); +}); + +describe("linkTimingMismatches", () => { + const base = { + "data-start": "2", + "data-duration": "6", + "data-media-start": "1", + "data-playback-rate": "1", + }; + + it("is empty for members in sync, treating absent defaults as equal", () => { + expect( + linkTimingMismatches([ + attrs(base), + attrs({ "data-start": "2", "data-duration": "6", "data-media-start": "1" }), + ]), + ).toEqual([]); + }); + + it.each([ + ["data-start", "start"], + ["data-duration", "duration"], + ["data-media-start", "media-start"], + ["data-playback-rate", "playback-rate"], + ])("names %s when it drifts", (attr, field) => { + expect(linkTimingMismatches([attrs(base), attrs({ ...base, [attr]: "3" })])).toEqual([field]); + }); + + it("ignores sub-millisecond float noise", () => { + expect(linkTimingMismatches([attrs(base), attrs({ ...base, "data-start": "2.0004" })])).toEqual( + [], + ); + }); +}); + +describe("mintLinkId", () => { + it("returns the first lk-N not already taken", () => { + expect(mintLinkId([])).toBe("lk-1"); + expect(mintLinkId(["lk-1", "lk-2", "lk-4"])).toBe("lk-3"); + }); +}); + +describe("relinkSplitHalves", () => { + function doc(html: string): Document { + document.body.innerHTML = html; + return document; + } + + it("gives each linked group's right halves one fresh shared id", () => { + const d = doc(` + + + `); + relinkSplitHalves(d, ["v-split", "a-split", "i-split"]); + const link = (id: string) => d.getElementById(id)?.getAttribute(MEDIA_LINK_ATTR) ?? null; + expect(link("v")).toBe("lk-1"); + expect(link("a")).toBe("lk-1"); + expect(link("v-split")).toBe("lk-2"); + expect(link("a-split")).toBe("lk-2"); + expect(link("i-split")).toBeNull(); + }); + + it("mints distinct ids for distinct groups", () => { + const d = doc(` + + `); + relinkSplitHalves(d, ["v2", "w2"]); + const ids = ["v2", "w2"].map((id) => d.getElementById(id)?.getAttribute(MEDIA_LINK_ATTR)); + expect(new Set(ids).size).toBe(2); + expect(ids).not.toContain("lk-1"); + expect(ids).not.toContain("lk-7"); + }); +}); diff --git a/packages/core/src/mediaLink.ts b/packages/core/src/mediaLink.ts new file mode 100644 index 0000000000..0736c507ea --- /dev/null +++ b/packages/core/src/mediaLink.ts @@ -0,0 +1,90 @@ +/** + * Linked clips: members sharing a `data-link` id are edited as one (a detached + * video + its audio). An editing contract only; playback and render ignore it. + */ +export const MEDIA_LINK_ATTR = "data-link"; + +interface AttributeReader { + getAttribute(name: string): string | null; +} + +export interface LinkTiming { + start: number; + duration: number; + mediaStart: number; + playbackRate: number; +} + +const LINK_TIMING_FIELDS: ReadonlyArray<[keyof LinkTiming, string]> = [ + ["start", "start"], + ["duration", "duration"], + ["mediaStart", "media-start"], + ["playbackRate", "playback-rate"], +]; + +const SYNC_TOLERANCE_S = 1e-3; + +function readNumber(el: AttributeReader, names: string[], fallback: number): number { + for (const name of names) { + const raw = el.getAttribute(name); + if (raw === null || raw.trim() === "") continue; + const value = Number(raw); + if (Number.isFinite(value)) return value; + } + return fallback; +} + +export function readLinkTiming(el: AttributeReader): LinkTiming { + return { + start: readNumber(el, ["data-start"], 0), + duration: readNumber(el, ["data-duration"], 0), + mediaStart: readNumber(el, ["data-media-start", "data-playback-start"], 0), + playbackRate: readNumber(el, ["data-playback-rate"], 1), + }; +} + +/** The timing fields (`start`, `duration`, `media-start`, `playback-rate`) on which members disagree. */ +export function linkTimingMismatches(members: readonly AttributeReader[]): string[] { + const timings = members.map(readLinkTiming); + const first = timings[0]; + if (!first) return []; + return LINK_TIMING_FIELDS.filter(([key]) => + timings.some((timing) => Math.abs(timing[key] - first[key]) > SYNC_TOLERANCE_S), + ).map(([, field]) => field); +} + +export function mintLinkId(taken: Iterable): string { + const used = new Set(taken); + let n = 1; + while (used.has(`lk-${n}`)) n += 1; + return `lk-${n}`; +} + +function takenLinkIds(doc: Document): string[] { + return Array.from(doc.querySelectorAll(`[id], [${MEDIA_LINK_ATTR}]`)).flatMap((el) => [ + el.id, + el.getAttribute(MEDIA_LINK_ATTR) ?? "", + ]); +} + +/** + * After a split, the right halves are clones still carrying the left halves' + * link id. Give each group's right halves one fresh shared id, so each half of + * a linked pair is its own pair. + */ +export function relinkSplitHalves(doc: Document, rightHalfIds: readonly string[]): void { + const taken = new Set(takenLinkIds(doc)); + const renamed = new Map(); + for (const id of rightHalfIds) { + const el = doc.getElementById(id); + const oldLink = el?.getAttribute(MEDIA_LINK_ATTR); + if (!el || !oldLink) continue; + let fresh = renamed.get(oldLink); + if (!fresh) { + fresh = mintLinkId(taken); + taken.add(fresh); + renamed.set(oldLink, fresh); + } + el.setAttribute(MEDIA_LINK_ATTR, fresh); + } +} From b7c370b8cec3ded70ef4ef4fc69ad402aa538231 Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Tue, 29 Sep 2026 17:23:33 -0700 Subject: [PATCH 02/21] feat(lint): warn on out-of-sync and orphaned linked clips Co-Authored-By: Claude Opus 5.5 --- packages/lint/src/rules/linkedClips.test.ts | 63 +++++++++++++++++ packages/lint/src/rules/linkedClips.ts | 76 +++++++++++++++++++++ packages/lint/src/rules/media.ts | 4 ++ 3 files changed, 143 insertions(+) create mode 100644 packages/lint/src/rules/linkedClips.test.ts create mode 100644 packages/lint/src/rules/linkedClips.ts diff --git a/packages/lint/src/rules/linkedClips.test.ts b/packages/lint/src/rules/linkedClips.test.ts new file mode 100644 index 0000000000..fe0c70220b --- /dev/null +++ b/packages/lint/src/rules/linkedClips.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from "vitest"; +import { lintHyperframeHtml } from "../hyperframeLinter.js"; + +const VIDEO = + 'id="talk" src="talk.mp4" muted data-link="lk-1" data-start="2" data-duration="6" data-media-start="1" data-track-index="0"'; +const AUDIO = + 'id="talk-audio" src="talk.mp4" data-link="lk-1" data-start="2" data-duration="6" data-media-start="1" data-track-index="2"'; + +function composition(body: string): string { + return ` + +
+ ${body} +
+ +`; +} + +async function linkFindings(body: string) { + const result = await lintHyperframeHtml(composition(body)); + return result.findings.filter((f) => f.code.startsWith("linked_clip")); +} + +describe("linked clip rules", () => { + it("an in-sync pair is clean", async () => { + expect(await linkFindings(``)).toEqual([]); + }); + + it("absent media-start and playback-rate equal their defaults", async () => { + const video = VIDEO.replace(' data-media-start="1"', ' data-media-start="0"'); + const audio = AUDIO.replace(' data-media-start="1"', ' data-playback-rate="1"'); + expect(await linkFindings(``)).toEqual([]); + }); + + it.each([ + ['data-start="2"', 'data-start="2.5"', "start"], + ['data-duration="6"', 'data-duration="5"', "duration"], + ['data-media-start="1"', 'data-media-start="0"', "media-start"], + ['data-media-start="1"', 'data-media-start="1" data-playback-rate="2"', "playback-rate"], + ])("warns when %s drifts to %s", async (from, to, field) => { + const findings = await linkFindings( + ``, + ); + expect(findings).toHaveLength(1); + expect(findings[0]?.code).toBe("linked_clips_out_of_sync"); + expect(findings[0]?.severity).toBe("warning"); + expect(findings[0]?.message).toContain(field); + expect(findings[0]?.message).toContain("lk-1"); + }); + + it("warns on a link id with one member", async () => { + const findings = await linkFindings(``); + expect(findings).toHaveLength(1); + expect(findings[0]?.code).toBe("linked_clip_orphan"); + expect(findings[0]?.severity).toBe("warning"); + expect(findings[0]?.elementId).toBe("talk"); + }); + + it("track index, volume and fades may differ freely", async () => { + const audio = `${AUDIO} data-volume="0.5" data-fade-in="1"`; + expect(await linkFindings(``)).toEqual([]); + }); +}); diff --git a/packages/lint/src/rules/linkedClips.ts b/packages/lint/src/rules/linkedClips.ts new file mode 100644 index 0000000000..8e57da3cf9 --- /dev/null +++ b/packages/lint/src/rules/linkedClips.ts @@ -0,0 +1,76 @@ +import type { LintContext, HyperframeLintFinding, OpenTag } from "../context"; +import { readAttr, truncateSnippet } from "../utils"; + +const SYNC_TOLERANCE_S = 1e-3; + +const LINKED_TIMING_FIELDS: ReadonlyArray<{ field: string; attrs: string[]; fallback: number }> = [ + { field: "start", attrs: ["data-start"], fallback: 0 }, + { field: "duration", attrs: ["data-duration"], fallback: 0 }, + { field: "media-start", attrs: ["data-media-start", "data-playback-start"], fallback: 0 }, + { field: "playback-rate", attrs: ["data-playback-rate"], fallback: 1 }, +]; + +function readTimingField(tag: OpenTag, attrs: string[], fallback: number): number { + for (const attr of attrs) { + const raw = readAttr(tag.raw, attr); + if (raw === null || raw.trim() === "") continue; + const value = Number(raw); + if (Number.isFinite(value)) return value; + } + return fallback; +} + +function driftingFields(members: readonly OpenTag[]): string[] { + return LINKED_TIMING_FIELDS.filter(({ attrs, fallback }) => { + const values = members.map((tag) => readTimingField(tag, attrs, fallback)); + const first = values[0] ?? fallback; + return values.some((value) => Math.abs(value - first) > SYNC_TOLERANCE_S); + }).map(({ field }) => field); +} + +function groupByLink(tags: readonly OpenTag[]): Map { + const groups = new Map(); + for (const tag of tags) { + const link = readAttr(tag.raw, "data-link"); + if (!link) continue; + groups.set(link, [...(groups.get(link) ?? []), tag]); + } + return groups; +} + +const memberLabel = (tag: OpenTag) => { + const id = readAttr(tag.raw, "id"); + return id ? `#${id}` : `<${tag.name}>`; +}; + +export function findLinkedClipFindings(ctx: LintContext): HyperframeLintFinding[] { + const findings: HyperframeLintFinding[] = []; + for (const [link, members] of groupByLink(ctx.tags)) { + const first = members[0]; + if (!first) continue; + const elementId = readAttr(first.raw, "id") || undefined; + if (members.length === 1) { + findings.push({ + code: "linked_clip_orphan", + severity: "warning", + message: `${memberLabel(first)} is the only clip with data-link="${link}"; its linked partner is gone.`, + elementId, + fixHint: `Remove data-link="${link}" from ${memberLabel(first)}, or restore the partner clip.`, + snippet: truncateSnippet(first.raw), + }); + continue; + } + const drift = driftingFields(members); + if (drift.length === 0) continue; + findings.push({ + code: "linked_clips_out_of_sync", + severity: "warning", + message: `Linked clips ${members.map(memberLabel).join(", ")} (data-link="${link}") differ in ${drift.join(", ")}.`, + elementId, + fixHint: + "Linked clips are edited as one: give every member the same data-start, data-duration, data-media-start and data-playback-rate, or remove data-link from all of them to unlink.", + snippet: truncateSnippet(first.raw), + }); + } + return findings; +} diff --git a/packages/lint/src/rules/media.ts b/packages/lint/src/rules/media.ts index 695705b63d..62c45aab08 100644 --- a/packages/lint/src/rules/media.ts +++ b/packages/lint/src/rules/media.ts @@ -13,6 +13,7 @@ import { import { validateColorGradingContract } from "@hyperframes/parsers/color-grading-contract"; import { extractMediaSrcMutations } from "@hyperframes/parsers/composition"; import { parseHTML } from "linkedom"; +import { findLinkedClipFindings } from "./linkedClips"; /** * Does the GSAP call that names `#id` also set `volume` in the same call? @@ -831,6 +832,9 @@ export const mediaRules: Array<(ctx: LintContext) => HyperframeLintFinding[]> = // audio_group_carve_attr findAudioGroupCarveAttrFindings, + + // linked_clips_out_of_sync + linked_clip_orphan + findLinkedClipFindings, ]; /** From c5daf65b17f20a97185293575c32f7e9409b2577 Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Tue, 29 Sep 2026 22:56:41 -0700 Subject: [PATCH 03/21] feat(studio): select, drag and trim linked clips together Co-Authored-By: Claude Opus 5.5 --- .../components/editor/mediaAudioEdits.test.ts | 8 +- .../src/components/editor/mediaAudioEdits.ts | 4 +- .../propertyPanelFlatMediaSection.test.tsx | 6 +- .../player/components/audioClipLink.test.ts | 109 +++++++++++++----- .../src/player/components/audioClipLink.ts | 69 ++++++++--- .../components/timelineClipDragTypes.ts | 4 + .../components/timelineClipGestureHandlers.ts | 4 + .../components/timelineLinkSelection.ts | 10 ++ .../components/useTimelineClipContextMenu.ts | 2 + .../player/components/useTimelineClipDrag.ts | 13 +++ .../studio/src/player/lib/timelineDOM.test.ts | 28 +++++ packages/studio/src/player/lib/timelineDOM.ts | 4 + .../src/player/store/timelineElement.ts | 2 + 13 files changed, 206 insertions(+), 57 deletions(-) create mode 100644 packages/studio/src/player/components/timelineLinkSelection.ts diff --git a/packages/studio/src/components/editor/mediaAudioEdits.test.ts b/packages/studio/src/components/editor/mediaAudioEdits.test.ts index 9f2e0203b1..cdfa5eca1b 100644 --- a/packages/studio/src/components/editor/mediaAudioEdits.test.ts +++ b/packages/studio/src/components/editor/mediaAudioEdits.test.ts @@ -88,9 +88,9 @@ describe("cutoutOps", () => { describe("mintLinkId", () => { it("dedupes against element ids and existing links", () => { const doc = document.implementation.createHTMLDocument("t"); - expect(mintLinkId(doc)).toBe("link"); - doc.body.innerHTML = ''; - expect(mintLinkId(doc)).toBe("link-3"); + expect(mintLinkId(doc)).toBe("lk-1"); + doc.body.innerHTML = '
'; + expect(mintLinkId(doc)).toBe("lk-3"); }); }); @@ -176,7 +176,7 @@ describe("buildKeepSoundCutoutEdit", () => { { target: "rate", points: [{ t: 0, v: 2 }] }, ]); expect(audio.getAttribute("data-link")).toBe(video.getAttribute("data-link")); - expect(video.getAttribute("data-link")).toBe("link"); + expect(video.getAttribute("data-link")).toBe("lk-1"); expect(audio.parentElement).toBe(video.parentElement); }); diff --git a/packages/studio/src/components/editor/mediaAudioEdits.ts b/packages/studio/src/components/editor/mediaAudioEdits.ts index 48f9371633..70f85b0cd0 100644 --- a/packages/studio/src/components/editor/mediaAudioEdits.ts +++ b/packages/studio/src/components/editor/mediaAudioEdits.ts @@ -23,7 +23,7 @@ import type { CommitDomAttributeBatch } from "../../hooks/domEditCommitTypes"; import { buildDomEditPatchTarget, type DomEditSelection } from "./domEditing"; import { generateId } from "../../utils/generateId"; -const MEDIA_LINK_ATTR = "data-link"; +import { MEDIA_LINK_ATTR, mintLinkId as mintLinkIdFrom } from "@hyperframes/core/media-link"; const SOUND_KEPT_ON_LINKED_AUDIO_STAGE = "Background removed. Sound kept on a linked audio track."; const SOUND_NOT_KEPT_MESSAGE = @@ -105,7 +105,7 @@ export function mintLinkId(doc: Document): string { const link = el.getAttribute(MEDIA_LINK_ATTR); if (link) taken.add(link); } - return firstFreeName("link", taken); + return mintLinkIdFrom(taken); } function firstFreeTrackIndex(source: string): number { diff --git a/packages/studio/src/components/editor/propertyPanelFlatMediaSection.test.tsx b/packages/studio/src/components/editor/propertyPanelFlatMediaSection.test.tsx index 8507215654..b479e00f44 100644 --- a/packages/studio/src/components/editor/propertyPanelFlatMediaSection.test.tsx +++ b/packages/studio/src/components/editor/propertyPanelFlatMediaSection.test.tsx @@ -879,10 +879,10 @@ describe.each(["flat", "design"])("%s panel — cutout keeps the soun htmlOp("muted", "true"), dataOp("has-audio", null), dataOp("volume", null), - dataOp("link", "link"), + dataOp("link", "lk-1"), ]); const source = - '
'; + '
'; const prepared = options?.prepareContent?.(source) ?? ""; const audio = new DOMParser() .parseFromString(prepared, "text/html") @@ -890,7 +890,7 @@ describe.each(["flat", "design"])("%s panel — cutout keeps the soun expect(audio?.getAttribute("src")).toBe("assets/intro-loop.mp4"); expect(audio?.getAttribute("data-start")).toBe("2"); expect(audio?.getAttribute("data-duration")).toBe("3"); - expect(audio?.getAttribute("data-link")).toBe("link"); + expect(audio?.getAttribute("data-link")).toBe("lk-1"); expect(audio?.getAttribute("data-volume")).toBe("0.5"); expect(text).toContain("Background removed. Sound kept on a linked audio track."); }); diff --git a/packages/studio/src/player/components/audioClipLink.test.ts b/packages/studio/src/player/components/audioClipLink.test.ts index 1a1196bab9..b0b40a64e4 100644 --- a/packages/studio/src/player/components/audioClipLink.test.ts +++ b/packages/studio/src/player/components/audioClipLink.test.ts @@ -1,34 +1,81 @@ import { describe, expect, it } from "vitest"; -import { audioPillFlags, isLinkedVideoAudio } from "./audioClipLink"; - -const audio = { id: "bed", tag: "audio", src: "assets/City Ride.mp4?v=2" }; - -describe("isLinkedVideoAudio", () => { - it("links an audio pill to a video of the same file", () => { - expect( - isLinkedVideoAudio(audio, [ - audio, - { id: "picture", tag: "video", src: "/preview/assets/city ride.mp4" }, - ]), - ).toBe(true); - }); - - it("greys a hidden clip and a muted group without dropping the link", () => { - expect(audioPillFlags({ ...audio, hidden: true }, [])).toEqual({ - muted: true, - linked: false, - }); - expect(audioPillFlags({ ...audio, audioGroupHidden: true }, [])).toEqual({ - muted: true, - linked: false, - }); - }); - - it("leaves a standalone audio file unlinked", () => { - expect( - isLinkedVideoAudio({ id: "vo", tag: "audio", src: "assets/voice.wav" }, [ - { id: "picture", tag: "video", src: "assets/city.mp4" }, - ]), - ).toBe(false); +import { + audioPillFlags, + expandToLinkedMembers, + linkedGestureKeys, + linkedMembersOf, + mediaFileKey, +} from "./audioClipLink"; + +const video = { id: "talk", link: "lk-1" }; +const audio = { id: "talk-audio", link: "lk-1" }; +const other = { id: "music", link: "lk-2" }; +const otherAudio = { id: "music-audio", link: "lk-2" }; +const plain = { id: "title" }; +const elements = [video, audio, other, otherAudio, plain]; + +describe("audioPillFlags", () => { + it("is linked only by data-link, never by a shared file name", () => { + expect(audioPillFlags({ link: "lk-1" }).linked).toBe(true); + expect(audioPillFlags({}).linked).toBe(false); + }); + + it("greys a hidden clip and a muted group", () => { + expect(audioPillFlags({ hidden: true }).muted).toBe(true); + expect(audioPillFlags({ audioGroupHidden: true }).muted).toBe(true); + }); +}); + +describe("mediaFileKey", () => { + it("matches the same file across folders, query strings and case", () => { + expect(mediaFileKey("assets/City Ride.mp4?v=2")).toBe(mediaFileKey("/preview/city ride.mp4")); + }); +}); + +describe("expandToLinkedMembers", () => { + it("adds every partner of a linked clip", () => { + expect(expandToLinkedMembers(["talk"], elements)).toEqual(new Set(["talk", "talk-audio"])); + }); + + it("leaves unlinked clips alone and does not pull in other groups", () => { + expect(expandToLinkedMembers(["title"], elements)).toEqual(new Set(["title"])); + expect(expandToLinkedMembers(["talk-audio", "title"], elements)).toEqual( + new Set(["talk-audio", "talk", "title"]), + ); + }); + + it("prefers the store key over the id", () => { + const keyed = [ + { id: "a", key: "k-a", link: "lk-9" }, + { id: "b", key: "k-b", link: "lk-9" }, + ]; + expect(expandToLinkedMembers(["k-a"], keyed)).toEqual(new Set(["k-a", "k-b"])); + }); +}); + +describe("linkedMembersOf", () => { + it("returns the group, or the clip alone when unlinked", () => { + expect(linkedMembersOf(video, elements).map((el) => el.id)).toEqual(["talk", "talk-audio"]); + expect(linkedMembersOf(plain, elements)).toEqual([plain]); + }); +}); + +describe("linkedGestureKeys", () => { + it("drags an unselected linked clip with its partner", () => { + expect(linkedGestureKeys(new Set(["title"]), video, elements, false)).toEqual( + new Set(["talk", "talk-audio"]), + ); + }); + + it("keeps a selection that holds the grabbed clip and adds partners", () => { + expect(linkedGestureKeys(new Set(["talk", "title"]), video, elements, false)).toEqual( + new Set(["talk", "talk-audio", "title"]), + ); + }); + + it("Alt edits the grabbed clip alone", () => { + expect(linkedGestureKeys(new Set(["talk", "talk-audio"]), video, elements, true)).toEqual( + new Set(["talk"]), + ); }); }); diff --git a/packages/studio/src/player/components/audioClipLink.ts b/packages/studio/src/player/components/audioClipLink.ts index f3a17993c0..16f7b35a13 100644 --- a/packages/studio/src/player/components/audioClipLink.ts +++ b/packages/studio/src/player/components/audioClipLink.ts @@ -1,33 +1,68 @@ import type { TimelineElement } from "../store/timelineElement"; -function mediaFileKey(src: string | undefined): string | null { +type LinkedElement = Pick; + +const keyOf = (element: Pick) => element.key ?? element.id; + +/** Same file name, ignoring folder, query and case: how a detached audio names its video. */ +export function mediaFileKey(src: string | undefined): string | null { if (!src) return null; const path = src.split(/[?#]/, 1)[0] ?? ""; const segment = path.split(/[/\\]/).pop()?.trim().toLowerCase() ?? ""; return segment.length > 0 ? segment : null; } +function isLinked(element: Pick): boolean { + return typeof element.link === "string" && element.link.length > 0; +} + export function audioPillFlags( - audio: Pick, - elements: readonly Pick[], + audio: Pick, + _elements?: readonly unknown[], ): { muted: boolean; linked: boolean } { return { muted: audio.hidden === true || audio.audioGroupHidden === true, - linked: isLinkedVideoAudio(audio, elements), + linked: isLinked(audio), }; } -/** An audio pill whose file is also a video clip on this timeline. */ -export function isLinkedVideoAudio( - audio: Pick, - elements: readonly Pick[], -): boolean { - if (audio.tag.trim().toLowerCase() !== "audio") return false; - const key = mediaFileKey(audio.src); - if (!key) return false; - return elements.some((element) => { - if (element.id === audio.id) return false; - if (element.tag.trim().toLowerCase() !== "video") return false; - return mediaFileKey(element.src) === key; - }); +/** Every member of `element`'s link group, itself included; just `[element]` when unlinked. */ +export function linkedMembersOf(element: T, elements: readonly T[]): T[] { + if (!isLinked(element)) return [element]; + const members = elements.filter((candidate) => candidate.link === element.link); + return members.some((member) => keyOf(member) === keyOf(element)) + ? members + : [element, ...members]; +} + +/** The one place link semantics live: `keys` plus every clip sharing a link with any of them. */ +export function expandToLinkedMembers( + keys: Iterable, + elements: readonly LinkedElement[], +): Set { + const expanded = new Set(keys); + const links = new Set( + elements.filter((el) => expanded.has(keyOf(el)) && isLinked(el)).map((el) => el.link), + ); + if (links.size === 0) return expanded; + for (const element of elements) { + if (isLinked(element) && links.has(element.link)) expanded.add(keyOf(element)); + } + return expanded; +} + +/** + * The clips a drag or trim of `grabbed` moves. Alt edits the grabbed clip alone; + * otherwise the selection (when it holds the grabbed clip) plus link partners. + */ +export function linkedGestureKeys( + selected: ReadonlySet, + grabbed: LinkedElement, + elements: readonly LinkedElement[], + altKey: boolean, +): Set { + const grabbedKey = keyOf(grabbed); + if (altKey) return new Set([grabbedKey]); + const base = selected.has(grabbedKey) ? selected : [grabbedKey]; + return expandToLinkedMembers(base, elements); } diff --git a/packages/studio/src/player/components/timelineClipDragTypes.ts b/packages/studio/src/player/components/timelineClipDragTypes.ts index 2f8ce3e6de..c80efc16d7 100644 --- a/packages/studio/src/player/components/timelineClipDragTypes.ts +++ b/packages/studio/src/player/components/timelineClipDragTypes.ts @@ -33,6 +33,8 @@ export interface DraggedClipState { snapTime: number | null; snapType: TimelineSnapType | null; started: boolean; + /** Alt held at pointerdown: edit this clip alone and unlink it from its partners. */ + altKey?: boolean; } export interface ResizingClipState { @@ -63,6 +65,8 @@ export interface ResizingClipState { playbackStart?: number; }[]; started: boolean; + /** Alt held at pointerdown: trim this clip alone and unlink it from its partners. */ + altKey?: boolean; } export interface BlockedClipState { diff --git a/packages/studio/src/player/components/timelineClipGestureHandlers.ts b/packages/studio/src/player/components/timelineClipGestureHandlers.ts index 4dc1671c95..02eb06ef8c 100644 --- a/packages/studio/src/player/components/timelineClipGestureHandlers.ts +++ b/packages/studio/src/player/components/timelineClipGestureHandlers.ts @@ -8,6 +8,7 @@ import { import type { TimelineEditCapabilities } from "./timelineEditCapabilities"; import type { TimelineEditCallbacks } from "./timelineCallbacks"; import { CLIP_HANDLE_W } from "./timelineLayout"; +import { selectLinkPartners } from "./timelineLinkSelection"; import { SPLIT_BOUNDARY_EPSILON_S } from "../../utils/timelineElementSplit"; export interface ClipGestureDeps { @@ -145,6 +146,7 @@ export function createClipGestureHandlers( previewDuration: el.duration, previewPlaybackStart: el.playbackStart, started: false, + altKey: e.altKey, }); }; @@ -191,6 +193,7 @@ export function createClipGestureHandlers( snapTime: null, snapType: null, started: false, + altKey: e.altKey, }); }; @@ -218,6 +221,7 @@ export function createClipGestureHandlers( } // Clip selection is idempotent; empty timeline space owns deselection. setSelectedElementId(elementKey); + if (!e.altKey) selectLinkPartners(elementKey); onSelectElement?.(el); }; diff --git a/packages/studio/src/player/components/timelineLinkSelection.ts b/packages/studio/src/player/components/timelineLinkSelection.ts new file mode 100644 index 0000000000..7cd22f9594 --- /dev/null +++ b/packages/studio/src/player/components/timelineLinkSelection.ts @@ -0,0 +1,10 @@ +import { usePlayerStore } from "../store/playerStore"; +import { expandToLinkedMembers } from "./audioClipLink"; + +/** After selecting `key`, widen the selection to its link partners (primary stays `key`). */ +export function selectLinkPartners(key: string): void { + const state = usePlayerStore.getState(); + const expanded = expandToLinkedMembers([key], state.elements); + if (expanded.size <= 1) return; + state.setSelection(expanded, key); +} diff --git a/packages/studio/src/player/components/useTimelineClipContextMenu.ts b/packages/studio/src/player/components/useTimelineClipContextMenu.ts index 25a47f369d..a02277e0be 100644 --- a/packages/studio/src/player/components/useTimelineClipContextMenu.ts +++ b/packages/studio/src/player/components/useTimelineClipContextMenu.ts @@ -1,6 +1,7 @@ import { useCallback } from "react"; import { usePlayerStore, type TimelineElement } from "../store/playerStore"; import type { ClipContextMenuState } from "./TimelineProvider"; +import { selectLinkPartners } from "./timelineLinkSelection"; /** A clicked member of an active multi-selection keeps the whole group * selected, so the context menu it opens acts on all of it. */ @@ -17,6 +18,7 @@ export function useClipContextMenu( const id = el.key ?? el.id; if (!(selectedElementIds.size > 1 && selectedElementIds.has(id))) { setSelectedElementId(id); + if (!e.altKey) selectLinkPartners(id); onSelectElement?.(el); } dismissGapMenu(); diff --git a/packages/studio/src/player/components/useTimelineClipDrag.ts b/packages/studio/src/player/components/useTimelineClipDrag.ts index 18e588d747..ddca889b7c 100644 --- a/packages/studio/src/player/components/useTimelineClipDrag.ts +++ b/packages/studio/src/player/components/useTimelineClipDrag.ts @@ -27,6 +27,7 @@ import type { BlockedClipState, } from "./timelineClipDragTypes"; import { getTimelineElementIndexes } from "../lib/timelineElementIndexes"; +import { linkedGestureKeys } from "./audioClipLink"; import type { TimelineRowGeometry } from "./timelineLayout"; import { mountTimelineClipDragGestureLifecycle, @@ -220,6 +221,12 @@ export function useTimelineClipDrag({ return; } beginGesture("drag", next.pointerId); + gestureSelectedKeysRef.current = linkedGestureKeys( + gestureSelectedKeysRef.current, + next.element, + elementsRef.current, + next.altKey === true, + ); publishDraggedClip(next); }, [beginGesture, publishDraggedClip], @@ -231,6 +238,12 @@ export function useTimelineClipDrag({ return; } beginGesture("resize", next.pointerId); + gestureSelectedKeysRef.current = linkedGestureKeys( + gestureSelectedKeysRef.current, + next.element, + elementsRef.current, + next.altKey === true, + ); publishResizingClip(next); }, [beginGesture, publishResizingClip], diff --git a/packages/studio/src/player/lib/timelineDOM.test.ts b/packages/studio/src/player/lib/timelineDOM.test.ts index 7143ae7521..f0b03876ef 100644 --- a/packages/studio/src/player/lib/timelineDOM.test.ts +++ b/packages/studio/src/player/lib/timelineDOM.test.ts @@ -281,6 +281,34 @@ describe("parseTimelineFromDOM — hfId from data-hf-id", () => { expect(element.hidden).toBe(true); }); + + it("reads data-link on both the DOM and manifest paths", () => { + const doc = makeDoc(` +
+ +
+ `); + expect(parseTimelineFromDOM(doc, 10).find((el) => el.domId === "talk")?.link).toBe("lk-1"); + const element = createTimelineElementFromManifestClip({ + clip: { + id: "talk", + label: "Talk", + kind: "element", + tagName: "video", + start: 0, + duration: 5, + track: 0, + compositionId: null, + parentCompositionId: null, + compositionSrc: null, + assetUrl: null, + }, + fallbackIndex: 0, + doc, + hostEl: doc.getElementById("talk"), + }); + expect(element.link).toBe("lk-1"); + }); }); describe("group info cache", () => { diff --git a/packages/studio/src/player/lib/timelineDOM.ts b/packages/studio/src/player/lib/timelineDOM.ts index c93108cb0b..6a3f821433 100644 --- a/packages/studio/src/player/lib/timelineDOM.ts +++ b/packages/studio/src/player/lib/timelineDOM.ts @@ -150,6 +150,8 @@ export function createTimelineElementFromManifestClip(params: { if (hostEl.hasAttribute("data-hidden")) entry.hidden = true; const timelineRole = hostEl.getAttribute("data-timeline-role"); if (timelineRole) entry.timelineRole = timelineRole; + const link = hostEl.getAttribute("data-link"); + if (link) entry.link = link; const audioGroup = hostEl.getAttribute("data-audio-group"); if (audioGroup) { entry.audioGroup = audioGroup; @@ -337,6 +339,8 @@ export function parseTimelineFromDOM( const timelineRole = el.getAttribute("data-timeline-role"); if (timelineRole) entry.timelineRole = timelineRole; + const domLink = el.getAttribute("data-link"); + if (domLink) entry.link = domLink; const domAudioGroup = el.getAttribute("data-audio-group"); if (domAudioGroup) { diff --git a/packages/studio/src/player/store/timelineElement.ts b/packages/studio/src/player/store/timelineElement.ts index 436391d791..91b06062bd 100644 --- a/packages/studio/src/player/store/timelineElement.ts +++ b/packages/studio/src/player/store/timelineElement.ts @@ -83,6 +83,8 @@ export interface TimelineElement { /** The owning group's serialized `data-fx-chain`, when set — resolved once per parse. */ audioGroupFxChain?: string; audioGroupAutomation?: string; + /** Verbatim `data-link`: clips sharing it are edited as one (move, trim, split, delete). */ + link?: string; /** * Master start of the composition this row runs in, which its tweens and its * `data-start` are local to; 0 at the root. Writes go through toAuthoredStart. From d0fea3b89c234f5af7e8347eb27542d0c07d22bf Mon Sep 17 00:00:00 2001 From: Vance Ingalls Date: Tue, 29 Sep 2026 23:07:46 -0700 Subject: [PATCH 04/21] feat(studio): detach, merge, link and unlink source transforms Co-Authored-By: Claude Opus 5.5 --- .../src/components/editor/mediaAudioEdits.ts | 14 +- .../components/editor/mediaLinkEdits.test.ts | 227 +++++++++++++++ .../src/components/editor/mediaLinkEdits.ts | 260 ++++++++++++++++++ 3 files changed, 494 insertions(+), 7 deletions(-) create mode 100644 packages/studio/src/components/editor/mediaLinkEdits.test.ts create mode 100644 packages/studio/src/components/editor/mediaLinkEdits.ts diff --git a/packages/studio/src/components/editor/mediaAudioEdits.ts b/packages/studio/src/components/editor/mediaAudioEdits.ts index 70f85b0cd0..c1571d21a8 100644 --- a/packages/studio/src/components/editor/mediaAudioEdits.ts +++ b/packages/studio/src/components/editor/mediaAudioEdits.ts @@ -29,7 +29,7 @@ const SOUND_KEPT_ON_LINKED_AUDIO_STAGE = "Background removed. Sound kept on a li const SOUND_NOT_KEPT_MESSAGE = "Background removed, but the sound could not be kept on a linked audio track. Undo to restore the original clip."; -const MOVED_SOUND_ATTRS = [ +export const MOVED_SOUND_ATTRS = [ "data-volume", HF_AUDIO_FADE_IN_ATTR, HF_AUDIO_FADE_OUT_ATTR, @@ -38,7 +38,7 @@ const MOVED_SOUND_ATTRS = [ HF_AUDIO_GROUP_ATTR, ]; -const COPIED_TIMING_ATTRS = [ +export const COPIED_TIMING_ATTRS = [ "start", "end", "duration", @@ -91,7 +91,7 @@ export function cutoutOps(input: { isVideo: boolean; cutoutSrc: string }): Patch return input.isVideo ? [src, htmlAttr("muted", "true"), dataAttr("has-audio", null)] : [src]; } -function firstFreeName(base: string, taken: Set): string { +export function firstFreeName(base: string, taken: Set): string { if (!taken.has(base)) return base; let n = 2; while (taken.has(`${base}-${n}`)) n += 1; @@ -117,7 +117,7 @@ function firstFreeTrackIndex(source: string): number { return track; } -function splitAutomation(raw: string): { videoKeeps: string | null; audioTakes: string } { +export function splitAutomation(raw: string): { videoKeeps: string | null; audioTakes: string } { let lanes: HfAutomationLane[]; try { lanes = parseAutomation(raw).lanes; @@ -132,11 +132,11 @@ function splitAutomation(raw: string): { videoKeeps: string | null; audioTakes: }; } -function formatAttrs(attrs: Array<[string, string]>): string { +export function formatAttrs(attrs: Array<[string, string]>): string { return attrs.map(([name, value]) => `${name}="${escapeHtmlAttribute(value)}"`).join(" "); } -function readAuthoredSrc(source: string, target: PatchTarget): string { +export function readAuthoredSrc(source: string, target: PatchTarget): string { const tag = findTagByTarget(source, target); if (!tag) return ""; const own = /\bsrc=(["'])([^"']*)\1/.exec(tag.tag); @@ -145,7 +145,7 @@ function readAuthoredSrc(source: string, target: PatchTarget): string { return firstSource?.[2] ? unescapeHtmlAttribute(firstSource[2]) : ""; } -function insertBeforeTarget(source: string, target: PatchTarget, markup: string): string { +export function insertBeforeTarget(source: string, target: PatchTarget, markup: string): string { const match = findTagByTarget(source, target); if (!match) return source; const lineStart = source.lastIndexOf("\n", match.start) + 1; diff --git a/packages/studio/src/components/editor/mediaLinkEdits.test.ts b/packages/studio/src/components/editor/mediaLinkEdits.test.ts new file mode 100644 index 0000000000..56cc8ed1b0 --- /dev/null +++ b/packages/studio/src/components/editor/mediaLinkEdits.test.ts @@ -0,0 +1,227 @@ +// @vitest-environment happy-dom +import { describe, expect, it } from "vitest"; +import { + canDetachAudio, + canLinkPair, + detachAudioInSource, + linkInSource, + findMergePair, + mergeAudioInSource, + pickDetachedAudioTrack, + removeElementInSource, + setLinkInSource, +} from "./mediaLinkEdits"; + +const automation = JSON.stringify({ + version: 1, + lanes: [ + { target: "volume", points: [{ t: 0, v: 1 }] }, + { target: "rate", points: [{ t: 0, v: 2 }] }, + ], +}).replaceAll('"', """); + +const VIDEO_TAG = + ''; + +const SOURCE = [ + '
', + ` ${VIDEO_TAG}`, + ' ', + "
", +].join("\n"); + +const video = { id: "talk" }; + +function attrsOf(source: string, selector: string): Record { + const el = new DOMParser().parseFromString(source, "text/html").querySelector(selector); + if (!el) throw new Error(`missing ${selector}`); + return Object.fromEntries(Array.from(el.attributes, (a) => [a.name, a.value])); +} + +describe("detachAudioInSource", () => { + const result = detachAudioInSource(SOURCE, { target: video, videoId: "talk", track: 2 }); + + it("inserts a linked audio with the video's file, window and sound", () => { + expect(result).not.toBeNull(); + const audio = attrsOf(result?.html ?? "", "audio#talk-audio"); + expect(audio).toMatchObject({ + src: "assets/talk.mp4", + "data-start": "2", + "data-duration": "6", + "data-media-start": "1", + "data-playback-rate": "1", + "data-track-index": "2", + "data-link": result?.linkId, + "data-volume": "0.5", + "data-fade-in": "0.3", + "data-fade-out": "0.2", + "data-fx-chain": "eq", + "data-fx-carve": "carve", + "data-audio-group": "voiceover", + }); + expect(audio["data-automation"]).toContain("volume"); + }); + + it("mutes the video, strips its sound attributes and links it", () => { + const after = attrsOf(result?.html ?? "", "video#talk"); + expect(after).toHaveProperty("muted"); + expect(after["data-link"]).toBe(result?.linkId); + for (const gone of [ + "data-has-audio", + "data-volume", + "data-fade-in", + "data-fade-out", + "data-fx-chain", + "data-fx-carve", + "data-audio-group", + ]) { + expect(after).not.toHaveProperty(gone); + } + expect(after["data-automation"]).toContain("rate"); + expect(after["data-automation"]).not.toContain("volume"); + }); + + it("suffixes the audio id when taken", () => { + const taken = SOURCE.replace('id="music"', 'id="talk-audio"'); + expect(detachAudioInSource(taken, { target: video, videoId: "talk", track: 2 })?.audioId).toBe( + "talk-audio-2", + ); + }); +}); + +describe("mergeAudioInSource", () => { + it("round-trips detach back to the original attributes", () => { + const detached = detachAudioInSource(SOURCE, { target: video, videoId: "talk", track: 2 }); + const merged = mergeAudioInSource(detached?.html ?? "", { + videoTarget: video, + audioTarget: { id: "talk-audio" }, + }); + const original = attrsOf(SOURCE, "video#talk"); + const roundTrip = attrsOf(merged ?? "", "video#talk"); + const lanes = (raw: string | undefined) => + JSON.parse(raw ?? "{}").lanes.map((lane: { target: string }) => lane.target); + expect(lanes(roundTrip["data-automation"]).sort()).toEqual( + lanes(original["data-automation"]).sort(), + ); + delete original["data-automation"]; + delete roundTrip["data-automation"]; + expect(roundTrip).toEqual(original); + expect(merged).not.toContain('