Skip to content

fix(cli): select containing source frames in snapshots #4809

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Dante-dan wants to merge 1 commit into
base: main
Choose a base branch
from
+225 −39
Open

fix(cli): select containing source frames in snapshots #4809

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
134 changes: 124 additions & 10 deletions packages/cli/src/commands/snapshot.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "nod
import { dirname, join } from "node:path";
import { tmpdir } from "node:os";
import { execFileSync } from "node:child_process";
import { findFFmpeg } from "../browser/ffmpeg.js";
import { findFFmpeg, findFFprobe } from "../browser/ffmpeg.js";
import { sourceTimeAt } from "@hyperframes/core";

const snapshotState = vi.hoisted(() => ({
Expand All @@ -28,6 +28,7 @@ vi.mock("../utils/staticProjectServer.js", () => ({

import snapshotCommand, {
extractVideoFrameToBuffer,
containingSourceFrameIndex,
computeSnapshotTimes,
formatSnapshotTimestamp,
parseZoomScale,
Expand Down Expand Up @@ -85,12 +86,8 @@ describe("transparent snapshot capture", () => {
it("pairs every frame with the frame-exact reference frame under --against", () => {
const source = readFileSync(new URL("./snapshot.ts", import.meta.url), "utf8");
expect(source).toContain("against: {");
expect(source).toContain("extractVideoFrameToBuffer(opts.against, time, false, true)");
expect(source).toContain("extractVideoFrameToBuffer(opts.against, time)");
expect(source).toContain('labels: ["render", "reference"]');
// accurate seek = `-ss` after `-i`, never the keyframe-snap fast path
expect(source).toContain(
'accurateSeek ? ["-i", videoPath, ...seek] : [...seek, "-i", videoPath]',
);
});

it("resolves and forwards the shared local browser GPU policy", () => {
Expand Down Expand Up @@ -304,10 +301,127 @@ describe("resolveSnapshotVideoFrameTime", () => {
});
});

describe("containingSourceFrameIndex", () => {
it.each([
[0.4, 9],
[22 / 30, 17],
[10 / 30, 8],
[4.1, 98],
])("selects frame %s on the source presentation grid", (time, expected) => {
expect(
containingSourceFrameIndex(
Array.from({ length: 120 }, (_, n) => n / 24),
time,
),
).toBe(expected);
});
it("keeps exact boundaries despite floating-point round-off", () => {
expect(containingSourceFrameIndex([0, 4.1], 4.1 - 4 * Number.EPSILON)).toBe(1);
expect(containingSourceFrameIndex([0, 4.1], 4.1 - 0.00001)).toBe(0);
});
it("uses irregular presentation intervals", () => {
expect(containingSourceFrameIndex([0, 0.02, 0.08, 0.15], 0.07)).toBe(1);
expect(containingSourceFrameIndex([0, 0.02, 0.08, 0.15], 0.08)).toBe(2);
});
});

describe("extractVideoFrameToBuffer", () => {
const ffmpeg = findFFmpeg();
const ffprobe = findFFprobe();

it.skipIf(!ffmpeg || !ffprobe)(
"selects containing frames for snapshots and reference pairs",
async () => {
const dir = mkdtempSync(join(tmpdir(), "hf-snapshot-containing-"));
try {
const clip = join(dir, "clip.mp4");
execFileSync(ffmpeg!, [
"-hide_banner",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
"testsrc=d=2:r=24:s=160x90",
"-pix_fmt",
"yuv420p",
clip,
]);
for (const [time, index] of [
[0.4, 9],
[22 / 30, 17],
[10 / 30, 8],
[9 / 24, 9],
] as const) {
const expected = join(dir, `frame-${index}.png`);
execFileSync(ffmpeg!, [
"-hide_banner",
"-loglevel",
"error",
"-i",
clip,
"-vf",
`select=eq(n\\,${index})`,
"-frames:v",
"1",
"-y",
expected,
]);
const actual = await extractVideoFrameToBuffer(clip, time);
expect(actual?.equals(readFileSync(expected))).toBe(true);
}
} finally {
rmSync(dir, { recursive: true, force: true });
}
},
);

it.skipIf(!ffmpeg || !ffprobe)(
"selects a variable-frame-rate presentation interval",
async () => {
const dir = mkdtempSync(join(tmpdir(), "hf-snapshot-vfr-"));
try {
const clip = join(dir, "clip.mp4");
execFileSync(ffmpeg!, [
"-hide_banner",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
"testsrc=d=1:r=10:s=160x90",
"-vf",
"select=eq(n\\,0)+eq(n\\,1)+eq(n\\,4)+eq(n\\,8)",
"-fps_mode",
"vfr",
"-pix_fmt",
"yuv420p",
clip,
]);
const expected = join(dir, "frame.png");
execFileSync(ffmpeg!, [
"-hide_banner",
"-loglevel",
"error",
"-i",
clip,
"-vf",
"select=eq(n\\,1)",
"-frames:v",
"1",
"-y",
expected,
]);
expect((await extractVideoFrameToBuffer(clip, 0.3))?.equals(readFileSync(expected))).toBe(
true,
);
} finally {
rmSync(dir, { recursive: true, force: true });
}
},
);

it.skipIf(!ffmpeg)(
it.skipIf(!ffmpeg || !ffprobe)(
"gives a 24 fps clip's real last frame for a held tail that lands past it",
async () => {
const dir = mkdtempSync(join(tmpdir(), "hf-snapshot-tail-"));
Expand All @@ -316,10 +430,10 @@ describe("extractVideoFrameToBuffer", () => {
const source = ["-f", "lavfi", "-i", "testsrc=d=1:r=24:s=160x90", "-pix_fmt", "yuv420p"];
execFileSync(ffmpeg!, ["-hide_banner", "-loglevel", "error", ...source, clip]);

const held = await extractVideoFrameToBuffer(clip, 1 - 1 / 30, false, false, true);
const lastFrame = await extractVideoFrameToBuffer(clip, 23 / 24, false, true);
const held = await extractVideoFrameToBuffer(clip, 1 - 1 / 30, false, true);
const lastFrame = await extractVideoFrameToBuffer(clip, 23 / 24);

expect(await extractVideoFrameToBuffer(clip, 1 - 1 / 30)).toBeNull();
expect((await extractVideoFrameToBuffer(clip, 1 - 1 / 30))?.equals(lastFrame!)).toBe(true);
expect(lastFrame).not.toBeNull();
expect(held?.equals(lastFrame!)).toBe(true);
} finally {
Expand Down
130 changes: 101 additions & 29 deletions packages/cli/src/commands/snapshot.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ import { formatLintFindings } from "../utils/lintFormat.js";
import { normalizeErrorMessage } from "../utils/errorMessage.js";
import { serveStaticProjectHtml } from "../utils/staticProjectServer.js";
import { c } from "../ui/colors.js";
import { findFFmpeg, getFFmpegInstallHint } from "../browser/ffmpeg.js";
import { runCancellableProcess } from "../utils/cancellableProcess.js";
import { findFFmpeg, findFFprobe, getFFmpegInstallHint } from "../browser/ffmpeg.js";
import { parseAngle, type Camera } from "./motionShotLayout.js";
import type { Example } from "./_examples.js";
import { loadOptionalPackage } from "../utils/optionalPackages.js";
Expand Down Expand Up @@ -139,51 +140,123 @@ export function requireSnapshotFfmpeg(ffmpegPath: string | undefined): string {
);
}

/**
* Extract a single frame from a video file at `timeSeconds` via FFmpeg.
* Used to work around Chrome-headless's inability to reliably seek
* <video> elements during snapshot capture.
*/
export function containingSourceFrameIndex(timestamps: readonly number[], time: number): number {
if (!Number.isFinite(time)) return -1;
const tolerance = 16 * Number.EPSILON * Math.max(1, Math.abs(time));
let index = -1;
for (const [i, timestamp] of timestamps.entries()) {
if (timestamp > time + tolerance) break;
index = i;
}
return index;
}

async function probeContainingSourceFrame(
videoPath: string,
time: number,
holdLastFrame: boolean,
): Promise<number> {
const ffprobe = findFFprobe();
if (!ffprobe)
throw new Error(`FFprobe is required for snapshot frame selection. ${getFFmpegInstallHint()}`);
const { stdout } = await runCancellableProcess(
ffprobe,
[
"-v",
"error",
"-select_streams",
"v:0",
"-read_intervals",
`%+${time + 1}`,
"-show_frames",
"-show_entries",
"frame=best_effort_timestamp:stream=time_base:format=start_time,duration",
"-of",
"json",
videoPath,
],
{ timeoutMs: FFMPEG_EXTRACT_TIMEOUT_MS, maxBufferBytes: 32 * 1024 * 1024 },
);
const data: unknown = JSON.parse(stdout);
if (
typeof data !== "object" ||
data === null ||
!("frames" in data) ||
!Array.isArray(data.frames) ||
!("streams" in data) ||
!Array.isArray(data.streams)
)
return -1;
const stream: unknown = data.streams[0];
if (
typeof stream !== "object" ||
stream === null ||
!("time_base" in stream) ||
typeof stream.time_base !== "string"
)
return -1;
const [numerator, denominator] = stream.time_base.split("/");
const timeBase = Number(numerator) / Number(denominator);
if (!Number.isFinite(timeBase) || timeBase <= 0) return -1;
const format = "format" in data ? data.format : null;
const start =
typeof format === "object" && format !== null && "start_time" in format
? Number(format.start_time)
: 0;
const duration =
typeof format === "object" && format !== null && "duration" in format
? Number(format.duration)
: NaN;
if (!holdLastFrame && Number.isFinite(duration) && time >= duration) return -1;
const timestamps: number[] = [];
for (const frame of data.frames) {
if (
typeof frame !== "object" ||
frame === null ||
!("best_effort_timestamp" in frame) ||
typeof frame.best_effort_timestamp !== "number"
)
return -1;
timestamps.push(frame.best_effort_timestamp * timeBase - (Number.isFinite(start) ? start : 0));
}
return containingSourceFrameIndex(timestamps, time);
}

export async function extractVideoFrameToBuffer(
videoPath: string,
timeSeconds: number,
useVp9AlphaDecoder = false,
accurateSeek = false,
holdLastFrame = false,
): Promise<Buffer | null> {
const tmp = mkdtempSync(join(tmpdir(), "hf-snapshot-frame-"));
const outPath = join(tmp, "frame.png");
try {
const ffmpegPath = requireSnapshotFfmpeg(findFFmpeg());
// `-ss` before `-i` performs a fast keyframe seek; adequate for snapshot accuracy
// (±1 frame) and orders of magnitude faster than the decode-and-scan alternative.
// `accurateSeek` puts `-ss` after `-i` (decode from the start) for frame-exact
// reference pairs, where ±1 frame would read as a real mismatch.
const frameIndex = await probeContainingSourceFrame(
videoPath,
Math.max(0, timeSeconds),
holdLastFrame,
);
if (frameIndex < 0) return null;
const args = ["-hide_banner", "-loglevel", "error"];
if (useVp9AlphaDecoder) {
args.push("-c:v", "libvpx-vp9");
}
const decoderArgs = [...args];
const seek = ["-ss", String(Math.max(0, timeSeconds))];
if (useVp9AlphaDecoder) args.push("-c:v", "libvpx-vp9");
args.push(
...(accurateSeek ? ["-i", videoPath, ...seek] : [...seek, "-i", videoPath]),
"-i",
videoPath,
"-map",
"0:v:0",
"-vf",
`select=eq(n\\,${frameIndex})`,
"-frames:v",
"1",
"-fps_mode",
"passthrough",
"-q:v",
"2",
"-y",
outPath,
);
let result = await runFfmpegOnce(ffmpegPath, args, FFMPEG_EXTRACT_TIMEOUT_MS);
if (holdLastFrame && result.code === 0 && !result.timedOut && !existsSync(outPath)) {
// Past the last frame's timestamp FFmpeg writes nothing; decode the final second and keep its last frame.
const tail = ["-sseof", "-1", "-i", videoPath, "-update", "1", "-q:v", "2", "-y", outPath];
result = await runFfmpegOnce(
ffmpegPath,
[...decoderArgs, ...tail],
FFMPEG_EXTRACT_TIMEOUT_MS,
);
}
const result = await runFfmpegOnce(ffmpegPath, args, FFMPEG_EXTRACT_TIMEOUT_MS);
if (result.code !== 0 || result.timedOut || !existsSync(outPath)) return null;
return readFileSync(outPath);
} finally {
Expand Down Expand Up @@ -562,7 +635,6 @@ async function captureSnapshots(
ffmpegInput,
Math.max(0, v.relTime),
useVp9AlphaDecoder,
false,
v.srcDuration > 0 && v.relTime >= v.srcDuration - 1,
);
if (!png) continue;
Expand Down Expand Up @@ -626,7 +698,7 @@ async function captureSnapshots(
if (opts.against) {
// Frame-exact reference frame beside the render, so a rebuild can be
// checked against its footage without hand-rolled ffmpeg + montage.
const refPng = await extractVideoFrameToBuffer(opts.against, time, false, true);
const refPng = await extractVideoFrameToBuffer(opts.against, time);
if (!refPng) {
console.error(
` ${c.warn("⚠")} --against has no frame at ${timeLabel} — reference pair skipped`,
Expand Down