feat(studio): waveform strip, clip light, peak marks and video beat source (5/8)

packages/cli/src/commands/beats.ts

@@ -23,7 +23,8 @@ function resolveMusicTarget(project: ProjectDir): { rel: string; audioPath: stri
   const src = findMusicAudioSrc(readFileSync(project.indexPath, "utf-8"));
   if (!src) {
     fail(
-      'No music track found. Add data-timeline-role="music" to the <audio> element ' +
+      'No music track found. Add data-timeline-role="music" to the <audio> element, or to a ' +
+        '<video data-has-audio="true"> whose own sound is the music ' +
         "(or give it an id like music/bgm/soundtrack).",
     );
   }

packages/core/src/beats/beatFile.test.ts

@@ -0,0 +1,29 @@
+import { describe, expect, it } from "vitest";
+import { findMusicAudioSrc } from "./beatFile";
+
+describe("findMusicAudioSrc", () => {
+  it("finds a video with sound tagged as music", () => {
+    const html = `<video id="clip" src="band.mp4" data-has-audio="true" data-timeline-role="music"></video>`;
+    expect(findMusicAudioSrc(html)).toBe("band.mp4");
+  });
+
+  it("finds an audible video by a music id", () => {
+    expect(findMusicAudioSrc(`<video id="soundtrack" src="s.mp4" data-has-audio="true">`)).toBe(
+      "s.mp4",
+    );
+  });
+
+  it("skips a muted video and one without declared sound", () => {
+    const html = [
+      `<video id="music" src="muted.mp4" data-has-audio="true" muted>`,
+      `<video id="bgm" src="undeclared.mp4">`,
+      `<audio id="music-bed" src="bed.mp3">`,
+    ].join("");
+    expect(findMusicAudioSrc(html)).toBe("bed.mp3");
+  });
+
+  it("keeps document order across audio and video", () => {
+    const html = `<audio id="music" src="first.mp3"></audio><video id="bgm" src="v.mp4" data-has-audio="true">`;
+    expect(findMusicAudioSrc(html)).toBe("first.mp3");
+  });
+});

packages/core/src/beats/beatFile.ts

@@ -87,18 +87,25 @@ function attr(tag: string, name: string): string | null {
   return m ? m[1]! : null;
 }
 
+function isSoundingVideoTag(tag: string): boolean {
+  const unquoted = tag.replace(/"[^"]*"|'[^']*'/g, '""');
+  return attr(tag, "data-has-audio") === "true" && !/\smuted(?=[\s/>=])/i.test(unquoted);
+}
+
 /**
  * Find the music track's src in composition HTML, applying the SAME rules as the
- * Studio's `isMusicTrack` so the CLI and Studio agree on which `<audio>` is music:
- * the FIRST `<audio>` (in document order) where data-timeline-role="music", or —
+ * Studio's `isMusicTrack` so the CLI and Studio agree on which clip is music:
+ * the FIRST `<audio>`, or `<video data-has-audio="true">` without `muted` (in
+ * document order) where data-timeline-role="music", or —
  * when no role is set — whose id matches the music regex. An explicit non-music
  * role excludes the element. Returns the raw src attribute, or null.
  */
 export function findMusicAudioSrc(html: string): string | null {
   // `[^>]*` spans newlines (it's a negated class, not `.`), so multi-line opening
   // tags are handled. HyperFrames authors src as an attribute on <audio>.
-  const tags = html.match(/<audio\b[^>]*>/gi) ?? [];
+  const tags = html.match(/<(?:audio|video)\b[^>]*>/gi) ?? [];
   for (const tag of tags) {
+    if (/^<video/i.test(tag) && !isSoundingVideoTag(tag)) continue;
     const src = attr(tag, "src");
     if (!src) continue;
     const role = attr(tag, "data-timeline-role");

packages/studio-server/src/createStudioApi.ts

@@ -10,6 +10,7 @@ import { registerThumbnailRoutes } from "./routes/thumbnail.js";
 import { registerWaveformRoutes } from "./routes/waveform.js";
 import { registerFreezeFrameRoutes } from "./routes/freezeFrame.js";
 import { registerLoudnessRoutes } from "./routes/loudness.js";
+import { registerPeakRoutes } from "./routes/peaks.js";
 import { registerFontRoutes } from "./routes/fonts.js";
 import { registerRegistryRoutes } from "./routes/registry.js";
 import { registerSelectionRoutes } from "./routes/selection.js";
@@ -65,6 +66,7 @@ export function createStudioApi(adapter: StudioApiAdapter): Hono {
   registerWaveformRoutes(api, adapter);
   registerFreezeFrameRoutes(api, adapter);
   registerLoudnessRoutes(api, adapter);
+  registerPeakRoutes(api, adapter);
   registerFontRoutes(api);
   registerRegistryRoutes(api, adapter);
   registerGlobalAssetRoutes(api);

packages/studio-server/src/helpers/peakMap.decodeFailure.test.ts

@@ -0,0 +1,29 @@
+import { chmodSync, mkdtempSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { describe, expect, it, vi } from "vitest";
+
+// Stand-in binaries: ffprobe reports a stereo stream, ffmpeg writes samples and then fails.
+const fakeBin = vi.hoisted(() => ({ dir: "" }));
+vi.mock("@hyperframes/parsers/ff-binaries", () => ({
+  findFfBinary: (name: string) => join(fakeBin.dir, name),
+}));
+
+import { decodePeakMap } from "./peakMap";
+
+describe("decodePeakMap, decode failure", () => {
+  it.skipIf(process.platform === "win32")(
+    "rejects partial output when ffmpeg exits nonzero",
+    async () => {
+      fakeBin.dir = mkdtempSync(join(tmpdir(), "hf-fake-ff-"));
+      const script = (name: string, body: string) => {
+        const file = join(fakeBin.dir, name);
+        writeFileSync(file, `#!/bin/sh\n${body}\n`);
+        chmodSync(file, 0o755);
+      };
+      script("ffprobe", "echo 2");
+      script("ffmpeg", "head -c 38400 /dev/zero; exit 1");
+      await expect(decodePeakMap("truncated.mov")).rejects.toThrow(/exit 1/);
+    },
+  );
+});

packages/studio-server/src/helpers/peakMap.test.ts

@@ -0,0 +1,149 @@
+import { execFileSync } from "node:child_process";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { describe, expect, it } from "vitest";
+import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
+import {
+  accumulatePeaks,
+  buildPeakMapCacheKey,
+  createPeakAccumulator,
+  decodePeakMap,
+  finishPeaks,
+} from "./peakMap.js";
+
+function stereo(frames: Array<[number, number]>): Buffer {
+  const buf = Buffer.alloc(frames.length * 8);
+  frames.forEach(([l, r], i) => {
+    buf.writeFloatLE(l, i * 8);
+    buf.writeFloatLE(r, i * 8 + 4);
+  });
+  return buf;
+}
+
+describe("peak accumulation", () => {
+  it("keeps the absolute max of either channel per bin, un-normalized", () => {
+    const acc = createPeakAccumulator(4, 0.5, 2);
+    accumulatePeaks(
+      acc,
+      stereo([
+        [0.1, -0.3],
+        [0.2, 0],
+        [0, 0.05],
+        [-0.04, 0],
+      ]),
+    );
+    expect(finishPeaks(acc)).toEqual([0.3, 0.05]);
+  });
+
+  it("carries a frame split across chunks", () => {
+    const acc = createPeakAccumulator(2, 1, 2);
+    const whole = stereo([
+      [0.5, 0],
+      [0, -0.9],
+    ]);
+    accumulatePeaks(acc, whole.subarray(0, 5));
+    accumulatePeaks(acc, whole.subarray(5));
+    expect(finishPeaks(acc)).toEqual([0.9]);
+    expect(acc.frame).toBe(2);
+  });
+});
+
+describe("peak accumulation, mono", () => {
+  it("reads one channel per frame", () => {
+    const acc = createPeakAccumulator(2, 1, 1);
+    const buf = Buffer.alloc(8);
+    buf.writeFloatLE(-1, 0);
+    buf.writeFloatLE(0.5, 4);
+    accumulatePeaks(acc, buf);
+    expect(finishPeaks(acc)).toEqual([1]);
+  });
+});
+
+describe("buildPeakMapCacheKey", () => {
+  it("changes when the file is replaced and never collides with a waveform entry", () => {
+    const a = buildPeakMapCacheKey("a/talk.mp4", { size: 10, mtimeMs: 1 });
+    expect(a).toBe(buildPeakMapCacheKey("a/talk.mp4", { size: 10, mtimeMs: 1 }));
+    expect(a).not.toBe(buildPeakMapCacheKey("a/talk.mp4", { size: 11, mtimeMs: 1 }));
+    expect(a).not.toBe(buildPeakMapCacheKey("a/talk.mp4", { size: 10, mtimeMs: 2 }));
+    expect(a).toMatch(/^peaks-v2_a_talk\.mp4_10-1\.json$/);
+  });
+});
+
+describe("decodePeakMap", () => {
+  const ffmpeg = findFfBinary("ffmpeg");
+  it.skipIf(!ffmpeg)(
+    "reads a 0 dBFS video track near 1.0 and a -12 dBFS one near 0.25",
+    async () => {
+      const dir = mkdtempSync(join(tmpdir(), "hf-peaks-"));
+      try {
+        const make = (name: string, volume: string) => {
+          const file = join(dir, name);
+          execFileSync(ffmpeg ?? "ffmpeg", [
+            "-v",
+            "error",
+            "-f",
+            "lavfi",
+            "-i",
+            "color=c=black:s=32x32:d=1",
+            "-f",
+            "lavfi",
+            "-i",
+            `sine=frequency=440:duration=1,volume=${volume}`,
+            "-shortest",
+            "-c:v",
+            "libx264",
+            "-c:a",
+            "pcm_s16le",
+            file,
+          ]);
+          return file;
+        };
+        const loud = await decodePeakMap(make("loud.mov", "8"));
+        const quiet = await decodePeakMap(make("quiet.mov", "2"));
+        expect(Math.max(...loud)).toBeGreaterThan(0.95);
+        expect(Math.max(...quiet)).toBeLessThan(0.3);
+        expect(loud.length).toBeGreaterThanOrEqual(19);
+      } finally {
+        rmSync(dir, { recursive: true, force: true });
+      }
+    },
+  );
+
+  it.skipIf(!ffmpeg)(
+    "decodes the probed first audio stream when a louder stereo stream follows it",
+    async () => {
+      const dir = mkdtempSync(join(tmpdir(), "hf-peaks-"));
+      try {
+        const file = join(dir, "multi.mkv");
+        execFileSync(ffmpeg ?? "ffmpeg", [
+          "-v",
+          "error",
+          "-f",
+          "lavfi",
+          "-i",
+          "aevalsrc=0.1*sin(2*PI*440*t):s=48000:d=1",
+          "-f",
+          "lavfi",
+          "-i",
+          "aevalsrc=0.9*sin(2*PI*440*t)|0.9*sin(2*PI*440*t):s=48000:d=1",
+          "-map",
+          "0:a",
+          "-map",
+          "1:a",
+          "-c:a",
+          "pcm_f32le",
+          "-disposition:a",
+          "0",
+          file,
+        ]);
+        const peaks = await decodePeakMap(file);
+        expect(peaks.length).toBeGreaterThanOrEqual(19);
+        expect(peaks.length).toBeLessThanOrEqual(21);
+        expect(Math.max(...peaks)).toBeCloseTo(0.1, 2);
+      } finally {
+        rmSync(dir, { recursive: true, force: true });
+      }
+    },
+  );
+});

packages/studio-server/src/helpers/peakMap.ts

@@ -0,0 +1,131 @@
+import { execFile, spawn } from "node:child_process";
+import { promisify } from "node:util";
+import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
+
+export const PEAK_BIN_SECONDS = 0.05;
+const PEAK_SAMPLE_RATE = 48_000;
+const execFileAsync = promisify(execFile);
+const PEAK_MAP_CACHE_VERSION = "peaks-v2";
+
+/**
+ * Running max-abs per bin over interleaved f32le at the source's own channel
+ * count. Native channels, not `-ac 2`: ffmpeg's mono upmix pans at -3 dB,
+ * which would hide a clip that really does reach 0 dBFS.
+ */
+export interface PeakAccumulator {
+  bins: number[];
+  frame: number;
+  carry: Buffer;
+  framesPerBin: number;
+  channels: number;
+}
+
+export function createPeakAccumulator(
+  sampleRate = PEAK_SAMPLE_RATE,
+  binSeconds = PEAK_BIN_SECONDS,
+  channels = 2,
+): PeakAccumulator {
+  return {
+    bins: [],
+    frame: 0,
+    carry: Buffer.alloc(0),
+    framesPerBin: Math.max(1, Math.round(sampleRate * binSeconds)),
+    channels: Math.max(1, channels),
+  };
+}
+
+function framePeak(data: Buffer, offset: number, channels: number): number {
+  let peak = 0;
+  for (let ch = 0; ch < channels; ch++) {
+    peak = Math.max(peak, Math.abs(data.readFloatLE(offset + ch * 4)));
+  }
+  return peak;
+}
+
+export function accumulatePeaks(acc: PeakAccumulator, chunk: Buffer): void {
+  const data = acc.carry.length > 0 ? Buffer.concat([acc.carry, chunk]) : chunk;
+  const frameBytes = 4 * acc.channels;
+  const whole = data.length - (data.length % frameBytes);
+  for (let offset = 0; offset < whole; offset += frameBytes) {
+    const peak = framePeak(data, offset, acc.channels);
+    const bin = Math.floor(acc.frame / acc.framesPerBin);
+    if (acc.bins.length <= bin) acc.bins.push(0);
+    if (peak > (acc.bins[bin] ?? 0)) acc.bins[bin] = peak;
+    acc.frame += 1;
+  }
+  acc.carry = Buffer.from(data.subarray(whole));
+}
+
+/** Linear sample peak per bin, NOT normalized — the level is the point. */
+export function finishPeaks(acc: PeakAccumulator): number[] {
+  return acc.bins.map((peak) => Number(peak.toFixed(4)));
+}
+
+async function probeChannels(mediaPath: string): Promise<number> {
+  const { stdout } = await execFileAsync(
+    findFfBinary("ffprobe") ?? "ffprobe",
+    [
+      "-v",
+      "error",
+      "-select_streams",
+      "a:0",
+      "-show_entries",
+      "stream=channels",
+      "-of",
+      "csv=p=0",
+      "--",
+      mediaPath,
+    ],
+    { encoding: "utf8", timeout: 30_000, windowsHide: true },
+  );
+  const channels = Number.parseInt(stdout.trim(), 10);
+  if (!Number.isFinite(channels) || channels < 1) throw new Error("no audio stream");
+  return channels;
+}
+
+export async function decodePeakMap(mediaPath: string): Promise<number[]> {
+  const channels = await probeChannels(mediaPath);
+  return new Promise((resolvePromise, reject) => {
+    const proc = spawn(
+      findFfBinary("ffmpeg") ?? "ffmpeg",
+      [
+        "-v",
+        "error",
+        "-i",
+        mediaPath,
+        "-map",
+        "0:a:0",
+        "-vn",
+        "-ar",
+        String(PEAK_SAMPLE_RATE),
+        "-f",
+        "f32le",
+        "pipe:1",
+      ],
+      { stdio: ["ignore", "pipe", "ignore"], windowsHide: true },
+    );
+    const acc = createPeakAccumulator(PEAK_SAMPLE_RATE, PEAK_BIN_SECONDS, channels);
+    proc.stdout?.on("data", (chunk: Buffer) => accumulatePeaks(acc, chunk));
+    proc.on("close", (code) => {
+      if (code !== 0) {
+        reject(new Error(`ffmpeg failed decoding audio (exit ${code})`));
+        return;
+      }
+      if (acc.frame === 0) {
+        reject(new Error("ffmpeg produced no audio samples"));
+        return;
+      }
+      resolvePromise(finishPeaks(acc));
+    });
+    proc.on("error", reject);
+  });
+}
+
+/** Keyed on content (size + mtime) like the waveform cache, under its own version prefix. */
+export function buildPeakMapCacheKey(
+  assetPath: string,
+  fingerprint: { size: number; mtimeMs: number },
+): string {
+  const name = assetPath.replace(/[/\\]/g, "_");
+  return `${PEAK_MAP_CACHE_VERSION}_${name}_${fingerprint.size}-${Math.round(fingerprint.mtimeMs)}.json`;
+}