From d1daa952d016bebf0e59c410782380602f3388c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miguel=20=C3=81ngel?= <miguel.sierra@heygen.com>
Date: Wed, 30 Sep 2026 05:35:55 -0400
Subject: [PATCH 01/12] test(studio): gate edit accuracy against the base
 branch baseline

Work in progress: the ratchet and its tests; the CI job follows.
---
 .../tests/e2e/edit-accuracy/ratchet.mjs       | 135 ++++++++++++++++++
 .../tests/e2e/edit-accuracy/ratchet.test.mjs  |  87 +++++++++++
 .../studio/tests/e2e/edit-accuracy/report.mjs |  46 +++---
 .../studio/tests/e2e/edit-accuracy/run.mjs    |   6 +-
 4 files changed, 250 insertions(+), 24 deletions(-)
 create mode 100644 packages/studio/tests/e2e/edit-accuracy/ratchet.mjs
 create mode 100644 packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs

diff --git a/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs b/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs
new file mode 100644
index 0000000000..e5a53a59a1
--- /dev/null
+++ b/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs
@@ -0,0 +1,135 @@
+#!/usr/bin/env node
+/**
+ * Edit accuracy gate against the base branch's baseline.json. Smoothness is reported, never gated.
+ *   node ratchet.mjs regressions <base baseline.json> <results.json>    ids that passed on base and failed here
+ *   node ratchet.mjs gate <base baseline.json> <head baseline.json> <out dir> <results.json>...
+ * A case fails only if it fails 2 of its 3 runs; one whose runs disagree is listed as unstable.
+ */
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { LIMIT_PX, entry, writeReport } from "./report.mjs";
+
+const GATED_PX = ["tracking", "pressJump", "drop", "reload", "render"];
+const LISTED = 30;
+
+/** Passes every gated metric; an unsettled preview fails the metrics it fed, all of them gated. */
+export const accurate = (e) =>
+  Boolean(e) &&
+  !e.error &&
+  !e.unsettled &&
+  e.undo === true &&
+  GATED_PX.every((m) => !(e[m] > LIMIT_PX));
+
+export const regressions = (base, results) =>
+  results.filter((r) => accurate(base.cases[r.id]) && !accurate(entry(r))).map((r) => r.id);
+
+const summary = (e) =>
+  e.error ? "error" : `${GATED_PX.map((m) => `${m} ${e[m] ?? "-"}`).join(", ")}, undo ${e.undo}`;
+
+/** Every run of every case: each shard's run plus the re-runs of its regressions. */
+export function gate(base, head, runs) {
+  const seen = new Map();
+  for (const r of runs) seen.set(r.id, [...(seen.get(r.id) ?? []), entry(r)]);
+  const cases = [...seen].map(([id, entries]) => {
+    const fails = entries.filter((e) => !accurate(e)).length;
+    return {
+      id,
+      entries,
+      passed: fails * 2 < entries.length,
+      basePassed: accurate(base.cases[id]),
+    };
+  });
+  const passing = cases.filter((c) => c.passed);
+  const result = {
+    basePassing: Object.values(base.cases).filter(accurate).length,
+    headPassing: passing.length,
+    regressed: cases.filter((c) => c.basePassed && !c.passed).map((c) => c.id),
+    unstable: cases
+      .filter((c) => new Set(c.entries.map(accurate)).size > 1)
+      .map((c) => ({ id: c.id, runs: c.entries.map(summary) })),
+    newlyPassing: passing.filter((c) => !c.basePassed).map((c) => c.id),
+    unbanked: passing.filter((c) => !accurate(head.cases[c.id])).map((c) => c.id),
+    overclaimed: cases.filter((c) => !c.passed && accurate(head.cases[c.id])).map((c) => c.id),
+    missing: Object.keys(base.cases).filter((id) => !seen.has(id)),
+  };
+  const reasons = [
+    result.regressed.length &&
+      `${result.regressed.length} case(s) that pass on the base branch fail here`,
+    result.headPassing < result.basePassing &&
+      `the passing count fell from ${result.basePassing} to ${result.headPassing}`,
+    result.unbanked.length &&
+      `${result.unbanked.length} newly passing case(s) are not banked in baseline.json`,
+    result.overclaimed.length &&
+      `baseline.json marks ${result.overclaimed.length} failing case(s) as passing`,
+  ].filter(Boolean);
+  return { ...result, ok: reasons.length === 0, reasons };
+}
+
+const list = (title, ids) =>
+  ids.length
+    ? [
+        `**${title}** (${ids.length})`,
+        ...ids.slice(0, LISTED).map((id) => `- ${id}`),
+        ids.length > LISTED ? `- ...` : "",
+        "",
+      ]
+    : [];
+
+export function comment(g) {
+  return [
+    "<!-- edit-accuracy -->",
+    `### Edit accuracy: ${g.headPassing} passing here, ${g.basePassing} on the base branch`,
+    "",
+    g.ok ? "The gate passes." : `The gate fails: ${g.reasons.join("; ")}.`,
+    "Smoothness is reported in the artifact, not gated. A case fails only if it fails 2 of 3 runs.",
+    "",
+    ...list("Regressed", g.regressed),
+    ...list("Newly passing", g.newlyPassing),
+    ...list("Not banked (commit the artifact's baseline.json)", g.unbanked),
+    ...list("Marked passing in baseline.json but failing", g.overclaimed),
+    ...list("In the base grid but not run", g.missing),
+    ...(g.unstable.length
+      ? [
+          `**Unstable** (${g.unstable.length})`,
+          ...g.unstable.slice(0, LISTED).map((u) => `- ${u.id}: ${u.runs.join(" / ")}`),
+          "",
+        ]
+      : []),
+  ].join("\n");
+}
+
+const readJson = (path) => JSON.parse(readFileSync(path, "utf8"));
+const readBaseline = (path) => {
+  try {
+    return readJson(path);
+  } catch {
+    return { cases: {} };
+  }
+};
+
+function main([command, basePath, ...rest]) {
+  const base = readBaseline(basePath);
+  if (command === "regressions") {
+    for (const id of regressions(base, readJson(rest[0]).cases)) console.log(id);
+    return 0;
+  }
+  const [headPath, out, ...resultPaths] = rest;
+  const runs = resultPaths.map((p) => readJson(p));
+  const g = gate(
+    base,
+    readBaseline(headPath),
+    runs.flatMap((r) => r.cases),
+  );
+  mkdirSync(out, { recursive: true });
+  writeFileSync(join(out, "comment.md"), comment(g));
+  writeFileSync(join(out, "gate.json"), JSON.stringify(g, null, 1));
+  // The first run of every shard, as a baseline.json to commit when cases newly pass.
+  const firsts = runs.filter((r) => !r.meta.rerun).flatMap((r) => r.cases);
+  writeReport(out, { ...runs[0].meta, grid: "full (CI)" }, firsts, 0);
+  console.log(comment(g));
+  return g.ok ? 0 : 1;
+}
+
+if (process.argv[1] === fileURLToPath(import.meta.url))
+  process.exitCode = main(process.argv.slice(2));
diff --git a/packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs b/packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs
new file mode 100644
index 0000000000..1e4bd1f7e8
--- /dev/null
+++ b/packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs
@@ -0,0 +1,87 @@
+import { describe, expect, it } from "vitest";
+import { accurate, gate, regressions } from "./ratchet.mjs";
+
+const good = {
+  tracking: 0.1,
+  pressJump: 0,
+  drop: 0,
+  reload: 0,
+  render: 0.02,
+  undo: true,
+  dropped: 0,
+  controlDropped: 0,
+  work: 3,
+  frameP95: 20,
+};
+const run = (id, drop = 0) => ({
+  id,
+  pass: false,
+  tracking: { max: 0.1 },
+  pressJump: 0,
+  drop,
+  reload: 0,
+  render: 0.02,
+  undo: { bytes: true, redoBytes: true, box: 0, redoBox: 0 },
+  checks: { undo: true },
+  smooth: { p95: 20, dropped: 0, workP95: 3, control: { dropped: 0 } },
+  unsettled: [],
+});
+const baseline = (cases) => ({ cases });
+
+describe("accurate", () => {
+  it("ignores smoothness and a metric the baseline does not hold", () => {
+    expect(accurate({ ...good, dropped: 4, work: 90 })).toBe(true);
+    expect(accurate({ tracking: 0.1, drop: 0, reload: 0, undo: true })).toBe(true);
+    expect(accurate({ ...good, pressJump: null })).toBe(true);
+  });
+
+  it("fails a px metric over the limit, a failed undo and an error", () => {
+    expect(accurate({ ...good, drop: 0.51 })).toBe(false);
+    expect(accurate({ ...good, pressJump: 0.51 })).toBe(false);
+    expect(accurate({ ...good, unsettled: ["committed"] })).toBe(false);
+    expect(accurate({ ...good, undo: false })).toBe(false);
+    expect(accurate({ pass: false, error: true })).toBe(false);
+    expect(accurate(undefined)).toBe(false);
+  });
+});
+
+describe("gate", () => {
+  const base = baseline({ a: good, b: good });
+
+  it("lists only cases that passed on base and fail here as regressions", () => {
+    expect(regressions(base, [run("a", 3), run("b"), run("c", 3)])).toEqual(["a"]);
+  });
+
+  it("fails a regression that fails 2 of 3 runs", () => {
+    const g = gate(base, base, [run("a", 3), run("a", 3), run("a"), run("b")]);
+    expect(g.regressed).toEqual(["a"]);
+    expect(g.ok).toBe(false);
+    expect(g.unstable.map((u) => u.id)).toEqual(["a"]);
+  });
+
+  it("passes a case that fails 1 of 3 runs but still lists it as unstable", () => {
+    const g = gate(base, base, [run("a", 3), run("a"), run("a"), run("b")]);
+    expect(g.regressed).toEqual([]);
+    expect(g.unstable).toHaveLength(1);
+    expect(g.unstable[0].runs[0]).toContain("drop 3");
+    expect(g.ok).toBe(true);
+  });
+
+  it("fails a newly passing case until baseline.json banks it", () => {
+    const before = baseline({ a: good, b: { ...good, drop: 9 } });
+    expect(gate(before, before, [run("a"), run("b")]).unbanked).toEqual(["b"]);
+    expect(gate(before, base, [run("a"), run("b")]).ok).toBe(true);
+  });
+
+  it("fails when baseline.json claims a pass the run does not reproduce", () => {
+    const g = gate(baseline({}), baseline({ a: good }), [run("a", 3)]);
+    expect(g.overclaimed).toEqual(["a"]);
+    expect(g.ok).toBe(false);
+  });
+
+  it("fails when the passing count falls, as when a passing case leaves the grid", () => {
+    const g = gate(base, base, [run("a")]);
+    expect(g.missing).toEqual(["b"]);
+    expect(g.reasons.join()).toContain("fell from 2 to 1");
+  });
+});
diff --git a/packages/studio/tests/e2e/edit-accuracy/report.mjs b/packages/studio/tests/e2e/edit-accuracy/report.mjs
index 07b94794a3..d7e0136541 100644
--- a/packages/studio/tests/e2e/edit-accuracy/report.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/report.mjs
@@ -3,7 +3,7 @@ import { writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { percentile } from "./geometry.mjs";
 
-const LIMIT_PX = 0.5;
+export const LIMIT_PX = 0.5;
 // A frame over 1.5 vsyncs is dropped; raw rAF p95 stays reported so a different rule re-scores without a re-run.
 const DROPPED_FRAME_MS = 25;
 const WORK_MS = 8;
@@ -157,32 +157,32 @@ function table(summary, meta, results) {
   return lines.join("\n") + "\n";
 }
 
+/** One case as baseline.json holds it; the gate reads the same projection. */
+export function entry(r) {
+  if (r.error) return { pass: false, error: true };
+  return {
+    pass: r.pass,
+    tracking: roundUp(r.tracking.max),
+    pressJump: roundUp(r.pressJump),
+    drop: roundUp(r.drop),
+    reload: roundUp(r.reload),
+    render: roundUp(r.render),
+    undo: r.checks.undo,
+    dropped: r.smooth.dropped,
+    controlDropped: r.smooth.control.dropped,
+    work: roundUp(r.smooth.workP95),
+    frameP95: roundUp(r.smooth.p95),
+    ...(r.unsettled.length && { unsettled: r.unsettled }),
+    ...(r.undoTimeout && { undoTimeout: r.undoTimeout }),
+    ...(r.renderError && { renderError: true }),
+  };
+}
+
 /** One line per case, so a baseline diff reads case by case. */
 function baseline(meta, results) {
   const entries = [...results]
     .sort((a, b) => a.id.localeCompare(b.id))
-    // fallow-ignore-next-line complexity
-    .map((r) => {
-      const v = r.error
-        ? { pass: false, error: true }
-        : {
-            pass: r.pass,
-            tracking: roundUp(r.tracking.max),
-            pressJump: roundUp(r.pressJump),
-            drop: roundUp(r.drop),
-            reload: roundUp(r.reload),
-            render: roundUp(r.render),
-            undo: r.checks.undo,
-            dropped: r.smooth.dropped,
-            controlDropped: r.smooth.control.dropped,
-            work: roundUp(r.smooth.workP95),
-            frameP95: roundUp(r.smooth.p95),
-            ...(r.unsettled.length && { unsettled: r.unsettled }),
-            ...(r.undoTimeout && { undoTimeout: r.undoTimeout }),
-            ...(r.renderError && { renderError: true }),
-          };
-      return `    ${JSON.stringify(r.id)}: ${JSON.stringify(v)}`;
-    });
+    .map((r) => `    ${JSON.stringify(r.id)}: ${JSON.stringify(entry(r))}`);
   return `{\n  "studio": ${JSON.stringify(meta.studio)},\n  "build": ${JSON.stringify(meta.build)},\n  "bench": ${JSON.stringify(meta.bench)},\n  "grid": ${JSON.stringify(meta.grid)},\n  "cases": {\n${entries.join(",\n")}\n  }\n}\n`;
 }
 
diff --git a/packages/studio/tests/e2e/edit-accuracy/run.mjs b/packages/studio/tests/e2e/edit-accuracy/run.mjs
index 6ff53308e8..4eb07ac1f0 100644
--- a/packages/studio/tests/e2e/edit-accuracy/run.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/run.mjs
@@ -1,6 +1,7 @@
 #!/usr/bin/env bun
 // Edit accuracy bench: real gestures in the built CLI Studio (build core, parsers, lint, studio-server first).
-// bun run --cwd packages/studio test:edit-accuracy -- --grid full|pr --jobs N [--shard i/n] [--filter re] [--lock path]
+// bun run --cwd packages/studio test:edit-accuracy -- --grid full|pr --jobs N [--shard i/n] [--filter re]
+//   [--lock path: the suite lock, taken per chunk] [--rerun: a confirmation run for the gate]
 import { createHash } from "node:crypto";
 import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { loadavg, tmpdir } from "node:os";
@@ -29,6 +30,8 @@ const { values: opt } = parseArgs({
     port: { type: "string", default: "5800" },
     cli: { type: "string", default: join(REPO, "packages/cli/dist/cli.js") },
     lock: { type: "string" },
+    // Marks a confirmation run of cases that regressed, so the gate keeps it out of the banked baseline.
+    rerun: { type: "boolean", default: false },
   },
 });
 const [shard, shards] = opt.shard.split("/").map(Number);
@@ -215,6 +218,7 @@ const meta = {
   studio,
   build,
   bench,
+  rerun: opt.rerun,
   grid:
     opt.grid +
     (opt.filter ? ` filter ${opt.filter}` : "") +

From 62167173cf30720b3aaed36e6ba617f21678b2ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miguel=20=C3=81ngel?= <miguel.sierra@heygen.com>
Date: Wed, 30 Sep 2026 05:45:12 -0400
Subject: [PATCH 02/12] test(studio): run the edit accuracy gate in CI across
 eight shards

Shares the built CLI from the Build job, re-runs twice each case that passes on the base branch and failed, and posts one sticky comment from the gate job.
---
 .github/workflows/ci.yml                      | 131 ++++++++++++++++++
 .../tests/e2e/edit-accuracy/ratchet.mjs       |   3 +
 2 files changed, 134 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 979886a970..35f14fb078 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -74,6 +74,7 @@ jobs:
       codex_plugin: ${{ steps.filter.outputs.codex_plugin }}
       gcp_beginframe: ${{ steps.filter.outputs.gcp_beginframe }}
       studio: ${{ steps.filter.outputs.studio }}
+      edit_accuracy: ${{ steps.filter.outputs.edit_accuracy }}
     steps:
       # Force git-based change detection instead of the pull_request REST API.
       # The API path can fail the whole workflow on transient listFiles
@@ -158,6 +159,19 @@ jobs:
               - "packages/studio-server/**"
               - "bun.lock"
               - ".github/workflows/ci.yml"
+            edit_accuracy:
+              - "packages/studio/src/components/editor/**"
+              - "packages/studio/src/components/nle/**"
+              - "packages/studio/src/hooks/**"
+              - "packages/studio/src/utils/**"
+              - "packages/studio/src/player/**"
+              - "packages/studio/src/webmcp/**"
+              - "packages/studio-server/src/**"
+              - "packages/core/src/**"
+              - "packages/cli/src/server/**"
+              - "packages/studio/tests/e2e/edit-accuracy/**"
+              - "bun.lock"
+              - ".github/workflows/ci.yml"
             gcp_beginframe:
               - "packages/gcp-cloud-run/Dockerfile"
               - "packages/aws-lambda/scripts/probe-beginframe.ts"
@@ -216,6 +230,14 @@ jobs:
       - run: bash scripts/ci/install-workspace-dependencies.sh
       - run: bun run build
       - run: bun run verify:packed-manifests
+      - name: Share the built CLI with the edit accuracy shards
+        if: github.event_name == 'pull_request' && needs.changes.outputs.edit_accuracy == 'true'
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: edit-accuracy-cli
+          path: packages/cli/dist
+          if-no-files-found: error
+          retention-days: 1
 
   lint:
     name: Lint
@@ -924,6 +946,115 @@ jobs:
             /tmp/studio-open-counts.json
           if-no-files-found: error
 
+  # Manual editing accuracy in the built Studio, gated against the base branch's baseline.json.
+  studio-edit-accuracy:
+    name: "Studio: edit accuracy (${{ matrix.shard }}/8)"
+    needs: [changes, build]
+    if: github.event_name == 'pull_request' && needs.changes.outputs.edit_accuracy == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2, 3, 4, 5, 6, 7, 8]
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        with:
+          node-version: 22
+      - uses: ./.github/actions/prepare-ffmpeg-bin
+      - run: bash scripts/ci/install-workspace-dependencies.sh
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          name: edit-accuracy-cli
+          path: packages/cli/dist
+      # Studio and the producer both run in the engine's pinned headless shell.
+      - run: node packages/cli/dist/cli.js browser ensure
+      - name: Read the base branch's edit accuracy baseline
+        env:
+          BASE_REF: ${{ github.base_ref }}
+        run: |
+          git fetch --no-tags --depth=1 origin "$BASE_REF"
+          if ! git show FETCH_HEAD:packages/studio/tests/e2e/edit-accuracy/baseline.json > /tmp/base-edit-accuracy.json; then
+            echo '{"cases":{}}' > /tmp/base-edit-accuracy.json
+            echo "The base branch has no edit accuracy baseline yet"
+          fi
+      - name: Run this shard, then re-run twice each case that passes on the base branch and failed
+        run: |
+          set -euo pipefail
+          bench() { bun run --cwd packages/studio test:edit-accuracy -- --grid full --jobs 2 "$@"; }
+          bench --shard "${{ matrix.shard }}/8" --out /tmp/edit-accuracy/run1
+          mapfile -t REGRESSED < <(node packages/studio/tests/e2e/edit-accuracy/ratchet.mjs regressions \
+            /tmp/base-edit-accuracy.json /tmp/edit-accuracy/run1/results.json)
+          if (( ${#REGRESSED[@]} > 0 )); then
+            FILTER="^($(IFS='|'; echo "${REGRESSED[*]}"))$"
+            for run in 2 3; do bench --filter "$FILTER" --rerun --out "/tmp/edit-accuracy/run${run}"; done
+          fi
+      - name: Upload shard evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: edit-accuracy-shard-${{ matrix.shard }}
+          path: /tmp/edit-accuracy
+          if-no-files-found: error
+
+  studio-edit-accuracy-gate:
+    name: "Studio: edit accuracy gate"
+    needs: [changes, studio-edit-accuracy]
+    if: ${{ !cancelled() && github.event_name == 'pull_request' && needs.changes.outputs.edit_accuracy == 'true' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+      pull-requests: write
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        with:
+          node-version: 22
+      - name: Read the base branch's edit accuracy baseline
+        env:
+          BASE_REF: ${{ github.base_ref }}
+        run: |
+          git fetch --no-tags --depth=1 origin "$BASE_REF"
+          if ! git show FETCH_HEAD:packages/studio/tests/e2e/edit-accuracy/baseline.json > /tmp/base-edit-accuracy.json; then
+            echo '{"cases":{}}' > /tmp/base-edit-accuracy.json
+            echo "The base branch has no edit accuracy baseline yet"
+          fi
+      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+        with:
+          pattern: edit-accuracy-shard-*
+          path: /tmp/edit-accuracy-shards
+      - name: Compare every run with the base branch
+        id: gate
+        # Smoothness is reported, not gated; a case fails only if it fails 2 of its 3 runs.
+        run: |
+          set +e
+          mapfile -t RESULTS < <(find /tmp/edit-accuracy-shards -name results.json | sort)
+          node packages/studio/tests/e2e/edit-accuracy/ratchet.mjs gate /tmp/base-edit-accuracy.json \
+            packages/studio/tests/e2e/edit-accuracy/baseline.json /tmp/edit-accuracy-gate "${RESULTS[@]}"
+          echo "exit_code=$?" >> "$GITHUB_OUTPUT"
+          cat /tmp/edit-accuracy-gate/comment.md >> "$GITHUB_STEP_SUMMARY" || true
+      - name: Post sticky comment
+        # Fork PRs get a read-only token; the gate below still fires.
+        continue-on-error: true
+        uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2.9.4
+        with:
+          header: edit-accuracy
+          path: /tmp/edit-accuracy-gate/comment.md
+      - name: Upload the gate report and a baseline.json to bank
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: edit-accuracy-gate
+          path: /tmp/edit-accuracy-gate
+      - name: Fail if the gate failed
+        if: steps.gate.outputs.exit_code != '0'
+        run: |
+          echo "::error::Edit accuracy fell against the base branch; see the PR comment."
+          exit 1
+
   smoke-global-install:
     name: "Smoke: global install"
     needs: [changes, build]
diff --git a/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs b/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs
index e5a53a59a1..705f1609d1 100644
--- a/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs
@@ -28,6 +28,7 @@ const summary = (e) =>
   e.error ? "error" : `${GATED_PX.map((m) => `${m} ${e[m] ?? "-"}`).join(", ")}, undo ${e.undo}`;
 
 /** Every run of every case: each shard's run plus the re-runs of its regressions. */
+// fallow-ignore-next-line complexity
 export function gate(base, head, runs) {
   const seen = new Map();
   for (const r of runs) seen.set(r.id, [...(seen.get(r.id) ?? []), entry(r)]);
@@ -108,6 +109,7 @@ const readBaseline = (path) => {
   }
 };
 
+// fallow-ignore-next-line complexity
 function main([command, basePath, ...rest]) {
   const base = readBaseline(basePath);
   if (command === "regressions") {
@@ -116,6 +118,7 @@ function main([command, basePath, ...rest]) {
   }
   const [headPath, out, ...resultPaths] = rest;
   const runs = resultPaths.map((p) => readJson(p));
+  if (!runs.length) throw new Error("no results.json from any shard");
   const g = gate(
     base,
     readBaseline(headPath),

From 8b394154230cd7b3b96a555222a7b58208cbd758 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miguel=20=C3=81ngel?= <miguel.sierra@heygen.com>
Date: Wed, 30 Sep 2026 08:59:44 -0400
Subject: [PATCH 03/12] test(studio): mark the ratchet accuracy check as
 intentionally branchy for fallow

---
 packages/studio/tests/e2e/edit-accuracy/ratchet.mjs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs b/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs
index 705f1609d1..a7854c6851 100644
--- a/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs
@@ -14,6 +14,7 @@ const GATED_PX = ["tracking", "pressJump", "drop", "reload", "render"];
 const LISTED = 30;
 
 /** Passes every gated metric; an unsettled preview fails the metrics it fed, all of them gated. */
+// fallow-ignore-next-line complexity
 export const accurate = (e) =>
   Boolean(e) &&
   !e.error &&

From 41d5be140c6c44195f0974cdd7fe81252fde78a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miguel=20=C3=81ngel?= <miguel.sierra@heygen.com>
Date: Wed, 30 Sep 2026 12:00:52 -0400
Subject: [PATCH 04/12] test(studio): confirm newly passing edit accuracy cases
 2 of 3 before banking them

The shards re-ran only regressions, so one lucky pass of a base-failing case failed the gate as unbanked and was never listed as unstable. Every case whose verdict differs from the base branch, either way, is now re-run twice and judged 2 of 3. A render error fails the gate like an unsettled snapshot, a malformed baseline.json fails the job instead of reading as empty, and the header is 3 lines.
---
 .github/workflows/ci.yml                      |  8 ++---
 .../tests/e2e/edit-accuracy/ratchet.mjs       | 33 ++++++++-----------
 .../tests/e2e/edit-accuracy/ratchet.test.mjs  | 20 +++++++++--
 .../studio/tests/e2e/edit-accuracy/report.mjs |  1 +
 4 files changed, 35 insertions(+), 27 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 35f14fb078..3ceeec73c3 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -980,15 +980,15 @@ jobs:
             echo '{"cases":{}}' > /tmp/base-edit-accuracy.json
             echo "The base branch has no edit accuracy baseline yet"
           fi
-      - name: Run this shard, then re-run twice each case that passes on the base branch and failed
+      - name: Run this shard, then re-run twice each case whose verdict differs from the base branch
         run: |
           set -euo pipefail
           bench() { bun run --cwd packages/studio test:edit-accuracy -- --grid full --jobs 2 "$@"; }
           bench --shard "${{ matrix.shard }}/8" --out /tmp/edit-accuracy/run1
-          mapfile -t REGRESSED < <(node packages/studio/tests/e2e/edit-accuracy/ratchet.mjs regressions \
+          mapfile -t FLIPPED < <(node packages/studio/tests/e2e/edit-accuracy/ratchet.mjs flipped \
             /tmp/base-edit-accuracy.json /tmp/edit-accuracy/run1/results.json)
-          if (( ${#REGRESSED[@]} > 0 )); then
-            FILTER="^($(IFS='|'; echo "${REGRESSED[*]}"))$"
+          if (( ${#FLIPPED[@]} > 0 )); then
+            FILTER="^($(IFS='|'; echo "${FLIPPED[*]}"))$"
             for run in 2 3; do bench --filter "$FILTER" --rerun --out "/tmp/edit-accuracy/run${run}"; done
           fi
       - name: Upload shard evidence
diff --git a/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs b/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs
index a7854c6851..88eeef9c22 100644
--- a/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs
@@ -1,10 +1,7 @@
 #!/usr/bin/env node
-/**
- * Edit accuracy gate against the base branch's baseline.json. Smoothness is reported, never gated.
- *   node ratchet.mjs regressions <base baseline.json> <results.json>    ids that passed on base and failed here
- *   node ratchet.mjs gate <base baseline.json> <head baseline.json> <out dir> <results.json>...
- * A case fails only if it fails 2 of its 3 runs; one whose runs disagree is listed as unstable.
- */
+// Edit accuracy gate against the base branch's baseline.json; smoothness is reported, never gated.
+// `flipped <base> <results>` lists cases to re-run twice; `gate <base> <head> <out> <results...>` judges 2 of 3.
+// A case whose runs disagree is listed as unstable.
 import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { fileURLToPath } from "node:url";
@@ -19,16 +16,18 @@ export const accurate = (e) =>
   Boolean(e) &&
   !e.error &&
   !e.unsettled &&
+  !e.renderError &&
   e.undo === true &&
   GATED_PX.every((m) => !(e[m] > LIMIT_PX));
 
-export const regressions = (base, results) =>
-  results.filter((r) => accurate(base.cases[r.id]) && !accurate(entry(r))).map((r) => r.id);
+/** Cases whose verdict here differs from the base branch, either way: each is re-run twice before the gate. */
+export const flipped = (base, results) =>
+  results.filter((r) => accurate(base.cases[r.id]) !== accurate(entry(r))).map((r) => r.id);
 
 const summary = (e) =>
   e.error ? "error" : `${GATED_PX.map((m) => `${m} ${e[m] ?? "-"}`).join(", ")}, undo ${e.undo}`;
 
-/** Every run of every case: each shard's run plus the re-runs of its regressions. */
+/** Every run of every case: each shard's run plus the re-runs of the cases it flipped. */
 // fallow-ignore-next-line complexity
 export function gate(base, head, runs) {
   const seen = new Map();
@@ -101,20 +100,14 @@ export function comment(g) {
   ].join("\n");
 }
 
+// Strict: CI writes {"cases":{}} for a missing base, so a file that does not parse is broken, not empty.
 const readJson = (path) => JSON.parse(readFileSync(path, "utf8"));
-const readBaseline = (path) => {
-  try {
-    return readJson(path);
-  } catch {
-    return { cases: {} };
-  }
-};
 
 // fallow-ignore-next-line complexity
 function main([command, basePath, ...rest]) {
-  const base = readBaseline(basePath);
-  if (command === "regressions") {
-    for (const id of regressions(base, readJson(rest[0]).cases)) console.log(id);
+  const base = readJson(basePath);
+  if (command === "flipped") {
+    for (const id of flipped(base, readJson(rest[0]).cases)) console.log(id);
     return 0;
   }
   const [headPath, out, ...resultPaths] = rest;
@@ -122,7 +115,7 @@ function main([command, basePath, ...rest]) {
   if (!runs.length) throw new Error("no results.json from any shard");
   const g = gate(
     base,
-    readBaseline(headPath),
+    readJson(headPath),
     runs.flatMap((r) => r.cases),
   );
   mkdirSync(out, { recursive: true });
diff --git a/packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs b/packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs
index 1e4bd1f7e8..e40490c9ac 100644
--- a/packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs
@@ -1,5 +1,5 @@
 import { describe, expect, it } from "vitest";
-import { accurate, gate, regressions } from "./ratchet.mjs";
+import { accurate, flipped, gate } from "./ratchet.mjs";
 
 const good = {
   tracking: 0.1,
@@ -39,6 +39,7 @@ describe("accurate", () => {
     expect(accurate({ ...good, drop: 0.51 })).toBe(false);
     expect(accurate({ ...good, pressJump: 0.51 })).toBe(false);
     expect(accurate({ ...good, unsettled: ["committed"] })).toBe(false);
+    expect(accurate({ ...good, render: null, renderError: true })).toBe(false);
     expect(accurate({ ...good, undo: false })).toBe(false);
     expect(accurate({ pass: false, error: true })).toBe(false);
     expect(accurate(undefined)).toBe(false);
@@ -48,8 +49,21 @@ describe("accurate", () => {
 describe("gate", () => {
   const base = baseline({ a: good, b: good });
 
-  it("lists only cases that passed on base and fail here as regressions", () => {
-    expect(regressions(base, [run("a", 3), run("b"), run("c", 3)])).toEqual(["a"]);
+  it("re-runs every case whose verdict differs from the base branch, in either direction", () => {
+    const mixed = baseline({ a: good, b: good, d: { ...good, drop: 9 } });
+    expect(flipped(mixed, [run("a", 3), run("b"), run("c", 3), run("d")])).toEqual(["a", "d"]);
+  });
+
+  it("banks a newly passing case only when it passes 2 of 3 runs, and lists a lucky pass as unstable", () => {
+    const before = baseline({ a: good, b: { ...good, drop: 9 } });
+    const lucky = gate(before, before, [run("a"), run("b"), run("b", 9), run("b", 9)]);
+    expect(lucky.newlyPassing).toEqual([]);
+    expect(lucky.unbanked).toEqual([]);
+    expect(lucky.unstable.map((u) => u.id)).toEqual(["b"]);
+    expect(lucky.ok).toBe(true);
+    const real = gate(before, before, [run("a"), run("b"), run("b"), run("b", 9)]);
+    expect(real.unbanked).toEqual(["b"]);
+    expect(real.ok).toBe(false);
   });
 
   it("fails a regression that fails 2 of 3 runs", () => {
diff --git a/packages/studio/tests/e2e/edit-accuracy/report.mjs b/packages/studio/tests/e2e/edit-accuracy/report.mjs
index d7e0136541..684aa0f077 100644
--- a/packages/studio/tests/e2e/edit-accuracy/report.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/report.mjs
@@ -158,6 +158,7 @@ function table(summary, meta, results) {
 }
 
 /** One case as baseline.json holds it; the gate reads the same projection. */
+// fallow-ignore-next-line complexity
 export function entry(r) {
   if (r.error) return { pass: false, error: true };
   return {

From d2ac8c11f664737dabd24c0f4aefbdf9f586a046 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miguel=20=C3=81ngel?= <miguel.sierra@heygen.com>
Date: Wed, 30 Sep 2026 17:04:32 -0400
Subject: [PATCH 05/12] test(studio): wip flash metric screencasts each edit
 window and counts off-state frames

---
 .../studio/tests/e2e/edit-accuracy/case.mjs   |  29 ++-
 .../studio/tests/e2e/edit-accuracy/flash.mjs  | 220 ++++++++++++++++++
 .../tests/e2e/edit-accuracy/ratchet.mjs       |   2 +
 .../tests/e2e/edit-accuracy/ratchet.test.mjs  |   4 +
 .../studio/tests/e2e/edit-accuracy/report.mjs |  20 +-
 .../studio/tests/e2e/edit-accuracy/run.mjs    |   6 +-
 6 files changed, 275 insertions(+), 6 deletions(-)
 create mode 100644 packages/studio/tests/e2e/edit-accuracy/flash.mjs

diff --git a/packages/studio/tests/e2e/edit-accuracy/case.mjs b/packages/studio/tests/e2e/edit-accuracy/case.mjs
index 3a62db0323..962f17e70a 100644
--- a/packages/studio/tests/e2e/edit-accuracy/case.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/case.mjs
@@ -3,6 +3,7 @@ import { spawn } from "node:child_process";
 import { readFileSync } from "node:fs";
 import { join } from "node:path";
 import { COMPOSITION, PLAYHEAD } from "./grid.mjs";
+import { panes, startCapture } from "./flash.mjs";
 import {
   angleOf,
   centre,
@@ -242,7 +243,7 @@ async function settled(ctx, timeout = 15_000) {
 
 /** Ready once Studio's own seek tool reports the composition and the playhead landed. */
 // fallow-ignore-next-line complexity
-async function openStudio(ctx) {
+async function openStudio(ctx, onReady) {
   ctx.handles = null;
   await ctx.page.waitForFunction(() => window.__editBench?.has("studio_seek"), { timeout: 90_000 });
   let seek = null;
@@ -255,6 +256,7 @@ async function openStudio(ctx) {
     seek = null;
   }
   if (!seek) throw new Error("studio never reported a seekable composition");
+  await onReady?.();
   await sleep(1000);
   return settled(ctx);
 }
@@ -546,6 +548,13 @@ async function controlDrag(browser, gesture) {
   }
 }
 
+/** The element's perimeter in screen px: how many pixels a 0.5 px shift of its edges touches. */
+const perimeterPx = (m) =>
+  Math.ceil(
+    m.visible.map(m.map.toScreen).reduce((sum, p, i, q) => sum + dist(p, q[(i + 1) % 4]), 0),
+  );
+
+/** Drives the drag and returns with the button still down; the flash window starts at pointer-up. */
 async function pointerGesture(ctx, gesture, pre) {
   const press = await handlePoint(ctx, pre, gesture);
   const pressComp = pre.map.toComp(press);
@@ -571,7 +580,6 @@ async function pointerGesture(ctx, gesture, pre) {
   }
   const rec = await recording(ctx.page, false);
   const smooth = smoothness(rec);
-  await ctx.page.mouse.up();
   const lastQuad = gesture === "crop" ? last.m.outline : last.m.visible;
   return {
     errors,
@@ -634,8 +642,13 @@ export async function runCase({ browser, spec, dir, files, url, evidence }) {
       spec.gesture === "nudge"
         ? await nudgeGesture(ctx, pre)
         : await pointerGesture(ctx, spec.gesture, pre);
+    // Flash windows: each action to its settle is screencast, untraced; the drag's trace has already stopped.
+    const flash = { regions: await panes(page), tolPx: perimeterPx(pre), windows: {} };
+    let capture = await startCapture(page);
+    if (spec.gesture !== "nudge") await page.mouse.up();
     await waitForFiles(ctx, { from: original, timeout: spec.gesture === "nudge" ? 6000 : 5000 });
-    await nextFrame(page, 2);
+    await settled(ctx);
+    flash.windows.release = await capture.stop();
     await blurPreview(page);
     await page.keyboard.press("Escape");
     const committed = await settled(ctx);
@@ -646,22 +659,29 @@ export async function runCase({ browser, spec, dir, files, url, evidence }) {
     // Undo and redo run before any reload. Each waits up to 15 s for its own write; redo waits for undo.
     const landed = (from) =>
       saved ? waitForFiles(ctx, { from, timeout: 15_000 }) : { reached: true, files: from };
+    capture = await startCapture(page);
     await chord(page, "Control+z");
     const undo = await landed(committedFiles);
     const undone = await settled(ctx);
+    flash.windows.undo = await capture.stop();
     await shoot("undone");
     let [redo, redone] = [{ reached: false }, null];
     if (undo.reached) {
       await blurPreview(page);
+      capture = await startCapture(page);
       await chord(page, "Control+Shift+z");
       redo = await landed(undo.files);
       redone = await settled(ctx);
+      flash.windows.redo = await capture.stop();
     }
     // A late write must not land under the reload.
     await waitForFiles(ctx, { timeout: 15_000 });
 
+    // The reload window starts once Studio can seek the composition; before that Studio itself is loading.
+    const lastSettled = (flash.windows.redo ?? flash.windows.undo).frames.at(-1);
     await page.reload();
-    const reloaded = await openStudio(ctx);
+    const reloaded = await openStudio(ctx, async () => (capture = await startCapture(page)));
+    flash.windows.reload = { ...(await capture.stop()), before: lastSettled };
     await shoot("reloaded");
     const quads = Object.fromEntries(
       Object.entries({ pre, committed, undone, redone, reloaded }).filter(([, m]) => m),
@@ -687,6 +707,7 @@ export async function runCase({ browser, spec, dir, files, url, evidence }) {
       // Which write never landed within 15 s; a redo that was never sent is untested, so undo fails.
       undoTimeout: saved && !undo.reached ? "undo" : saved && !redo.reached ? "redo" : null,
       smooth: { ...drive.smooth, control },
+      flash,
       unsettled: Object.keys(quads).filter((k) => quads[k].unsettled),
       reloaded,
       diag: {
diff --git a/packages/studio/tests/e2e/edit-accuracy/flash.mjs b/packages/studio/tests/e2e/edit-accuracy/flash.mjs
new file mode 100644
index 0000000000..b4aa5d7f8c
--- /dev/null
+++ b/packages/studio/tests/e2e/edit-accuracy/flash.mjs
@@ -0,0 +1,220 @@
+/** Preview flashes: every frame the screencast paints, from an action to its settle, against the frames around it. */
+
+// A 4x4 marker in the top-left corner, outside every region, repaints every rAF with the frame counter in its
+// colour, so the screencast (which only sends repainted frames) sends every frame and each one is numbered.
+function markerOn() {
+  const bench = window.__editBench;
+  if (bench.marker) return;
+  const el = Object.assign(document.createElement("div"), { id: "edit-bench-marker" });
+  el.style.cssText =
+    "position:fixed;left:0;top:0;width:4px;height:4px;z-index:2147483647;pointer-events:none";
+  document.documentElement.append(el);
+  bench.marker = { el, n: 0, on: true, masks: [] };
+  const tick = () => {
+    if (!bench.marker?.on) return;
+    const n = ++bench.marker.n;
+    el.style.background = `rgb(${n & 255},${(n >> 8) & 255},128)`;
+    // Toasts slide in and out by design; wherever one was during the window is left out of the comparison.
+    for (const t of document.querySelectorAll(".hf-toast-enter, .hf-toast-exit")) {
+      const r = t.getBoundingClientRect();
+      bench.marker.masks.push([r.left, r.top, r.right, r.bottom].map(Math.round));
+    }
+    requestAnimationFrame(tick);
+  };
+  requestAnimationFrame(tick);
+}
+
+function markerOff() {
+  const m = window.__editBench.marker;
+  if (!m) return;
+  m.on = false;
+  m.el.remove();
+  window.__editBench.marker = null;
+}
+
+/** The panes a flash can show in, as screen rects: preview (with its chrome), timeline and inspector. */
+function paneRects() {
+  const part = (el) => el?.closest(".dv-react-part");
+  const rect = (el) => {
+    const r = el.getBoundingClientRect();
+    return {
+      x: Math.round(r.x),
+      y: Math.round(r.y),
+      w: Math.round(r.width),
+      h: Math.round(r.height),
+    };
+  };
+  const preview = part(document.querySelector('[data-testid="preview-zoom-stage"]'));
+  const parts = [...document.querySelectorAll(".dv-react-part")].filter((e) => e.offsetWidth > 0);
+  const p = preview && rect(preview);
+  const timeline =
+    part(document.querySelector('[data-testid="timeline-clip"]')) ??
+    parts.reduce((a, b) => (!a || rect(b).y > rect(a).y ? b : a), null);
+  const area = (e) => rect(e).w * rect(e).h;
+  const inspector =
+    p &&
+    parts
+      .filter((e) => rect(e).x >= p.x + p.w - 1 && rect(e).y < p.y + p.h)
+      .reduce((a, b) => (!a || area(b) > area(a) ? b : a), null);
+  return Object.fromEntries(
+    Object.entries({ preview, timeline, inspector })
+      .filter(([, e]) => e)
+      .map(([k, e]) => [k, rect(e)]),
+  );
+}
+
+/** Starts a screencast of one action window; stop() returns its PNG frames and the marker range it spans. */
+export async function startCapture(page, { marker = true } = {}) {
+  const cdp = await page.createCDPSession();
+  const frames = [];
+  cdp.on("Page.screencastFrame", (f) => {
+    frames.push(f.data);
+    cdp.send("Page.screencastFrameAck", { sessionId: f.sessionId }).catch(() => undefined);
+  });
+  if (marker) await page.evaluate(markerOn);
+  const counter = () => page.evaluate(() => window.__editBench.marker?.n ?? null);
+  const masks = () => page.evaluate(() => window.__editBench.marker?.masks ?? []);
+  await cdp.send("Page.startScreencast", { format: "png", everyNthFrame: 1 });
+  // The first frame is the state before the action.
+  for (const deadline = Date.now() + 2000; !frames.length && Date.now() < deadline; )
+    await new Promise((r) => setTimeout(r, 10));
+  const from = await counter();
+  return {
+    async stop() {
+      const [to, mask] = [await counter(), await masks()];
+      await cdp.send("Page.stopScreencast").catch(() => undefined);
+      if (marker) await page.evaluate(markerOff).catch(() => undefined);
+      await cdp.detach().catch(() => undefined);
+      return { frames, from, to, masks: mask };
+    },
+  };
+}
+
+export const panes = (page) => page.evaluate(paneRects);
+
+/**
+ * In the decoder page: per frame, the marker counter and the pixels that differ from the before frame and from the
+ * after frame in each pane. A pixel differs when any channel moves more than `colourTol`.
+ */
+function compareFrames(frames, before, after, regions, masks, colourTol) {
+  // fallow-ignore-next-line complexity
+  return (async () => {
+    const decode = async (b64) => {
+      const bmp = await createImageBitmap(
+        await (await fetch(`data:image/png;base64,${b64}`)).blob(),
+      );
+      const g = new OffscreenCanvas(bmp.width, bmp.height).getContext("2d", {
+        willReadFrequently: true,
+      });
+      g.drawImage(bmp, 0, 0);
+      return g.getImageData(0, 0, bmp.width, bmp.height);
+    };
+    const masked = (x, y) => masks.some(([l, t, r, b]) => x >= l && x < r && y >= t && y < b);
+    // fallow-ignore-next-line complexity
+    const differing = (a, b, r) => {
+      let n = 0;
+      for (let y = r.y; y < r.y + r.h; y++)
+        for (let x = r.x; x < r.x + r.w; x++) {
+          if (masks.length && masked(x, y)) continue;
+          const i = (y * a.width + x) * 4;
+          if (
+            Math.abs(a.data[i] - b.data[i]) > colourTol ||
+            Math.abs(a.data[i + 1] - b.data[i + 1]) > colourTol ||
+            Math.abs(a.data[i + 2] - b.data[i + 2]) > colourTol
+          )
+            n++;
+        }
+      return n;
+    };
+    const [b, a] = [await decode(before), await decode(after)];
+    const out = [];
+    for (const f of frames) {
+      const img = await decode(f);
+      const counter = img.data[4 * (img.width + 1)] + 256 * img.data[4 * (img.width + 1) + 1];
+      const diffs = Object.fromEntries(
+        Object.entries(regions).map(([k, r]) => [k, [differing(img, b, r), differing(img, a, r)]]),
+      );
+      out.push({ counter, diffs });
+    }
+    return out;
+  })();
+}
+
+/**
+ * A frame is bad when some pane differs from the before frame and from the after frame by more pixels than a
+ * 0.5 px shift of the element's perimeter moves. Coverage is marker counters seen over counters in the window.
+ */
+// fallow-ignore-next-line complexity
+async function scoreWindow(
+  decoder,
+  { frames, from, to, before, after, masks = [] },
+  regions,
+  tolPx,
+) {
+  const b = before ?? frames[0];
+  const a = after ?? frames.at(-1);
+  if (!b || !a) return { frames: 0, coverage: 0, bad: [], longest: 0 };
+  const rows = await decoder.evaluate(
+    compareFrames,
+    frames,
+    b,
+    a,
+    regions,
+    dedupe(masks),
+    COLOUR_TOL,
+  );
+  const seen = new Set(rows.map((r) => r.counter).filter((c) => c >= from && c <= to));
+  const span = from === null || to === null ? null : to - from + 1;
+  const bad = [];
+  rows.forEach((r, i) => {
+    const panes = Object.entries(r.diffs)
+      .filter(([, [vsBefore, vsAfter]]) => vsBefore > tolPx && vsAfter > tolPx)
+      .map(([k, [vsBefore, vsAfter]]) => ({ pane: k, vsBefore, vsAfter }));
+    if (panes.length) bad.push({ frame: i, counter: r.counter, panes });
+  });
+  let [longest, run] = [0, 0];
+  rows.forEach((_, i) => {
+    run = bad.some((x) => x.frame === i) ? run + 1 : 0;
+    longest = Math.max(longest, run);
+  });
+  return {
+    frames: frames.length,
+    rafFrames: span,
+    coverage: span ? seen.size / span : null,
+    bad,
+    longest,
+  };
+}
+
+const dedupe = (rects) =>
+  [...new Set(rects.map((r) => r.join(",")))].map((k) => k.split(",").map(Number));
+
+// PNG is lossless; this absorbs only anti-aliasing and subpixel text noise between otherwise equal frames.
+const COLOUR_TOL = 24;
+const MIN_COVERAGE = 0.9;
+
+/** Scores every window of a case; the offending frames go to `evidence.flashFrames` as PNG. */
+export async function scoreFlash(decoder, { regions, tolPx, windows }, evidence) {
+  const out = {};
+  evidence.flashFrames = [];
+  for (const [name, win] of Object.entries(windows)) {
+    const scored = await scoreWindow(decoder, win, regions, tolPx);
+    for (const b of scored.bad)
+      evidence.flashFrames.push([
+        `${name}-${b.frame}-${b.panes.map((p) => p.pane).join("+")}`,
+        win.frames[b.frame],
+      ]);
+    out[name] = scored;
+  }
+  const all = Object.values(out);
+  const coverage = Math.min(...all.map((w) => w.coverage ?? 0));
+  return {
+    bad: all.reduce((n, w) => n + w.bad.length, 0),
+    longest: Math.max(0, ...all.map((w) => w.longest)),
+    coverage,
+    uncovered: !(coverage >= MIN_COVERAGE),
+    regions,
+    tolPx,
+    windows: out,
+  };
+}
diff --git a/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs b/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs
index 88eeef9c22..6b7035bd29 100644
--- a/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs
@@ -17,6 +17,8 @@ export const accurate = (e) =>
   !e.error &&
   !e.unsettled &&
   !e.renderError &&
+  !(e.flash > 0) &&
+  !e.flashUncovered &&
   e.undo === true &&
   GATED_PX.every((m) => !(e[m] > LIMIT_PX));
 
diff --git a/packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs b/packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs
index e40490c9ac..9f620779bb 100644
--- a/packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs
@@ -10,6 +10,7 @@ const good = {
   undo: true,
   dropped: 0,
   controlDropped: 0,
+  flash: 0,
   work: 3,
   frameP95: 20,
 };
@@ -25,6 +26,7 @@ const run = (id, drop = 0) => ({
   checks: { undo: true },
   smooth: { p95: 20, dropped: 0, workP95: 3, control: { dropped: 0 } },
   unsettled: [],
+  flash: { bad: 0, uncovered: false },
 });
 const baseline = (cases) => ({ cases });
 
@@ -40,6 +42,8 @@ describe("accurate", () => {
     expect(accurate({ ...good, pressJump: 0.51 })).toBe(false);
     expect(accurate({ ...good, unsettled: ["committed"] })).toBe(false);
     expect(accurate({ ...good, render: null, renderError: true })).toBe(false);
+    expect(accurate({ ...good, flash: 1 })).toBe(false);
+    expect(accurate({ ...good, flash: 0, flashUncovered: true })).toBe(false);
     expect(accurate({ ...good, undo: false })).toBe(false);
     expect(accurate({ pass: false, error: true })).toBe(false);
     expect(accurate(undefined)).toBe(false);
diff --git a/packages/studio/tests/e2e/edit-accuracy/report.mjs b/packages/studio/tests/e2e/edit-accuracy/report.mjs
index 684aa0f077..6746f375ac 100644
--- a/packages/studio/tests/e2e/edit-accuracy/report.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/report.mjs
@@ -7,7 +7,7 @@ export const LIMIT_PX = 0.5;
 // A frame over 1.5 vsyncs is dropped; raw rAF p95 stays reported so a different rule re-scores without a re-run.
 const DROPPED_FRAME_MS = 25;
 const WORK_MS = 8;
-export const METRICS = ["tracking", "press", "drop", "reload", "render", "undo", "smooth"];
+export const METRICS = ["tracking", "press", "drop", "reload", "render", "undo", "flash", "smooth"];
 
 /** Worst-first value per metric; undo ranks by box distance, and its byte failures are counted apart. */
 const worstValue = {
@@ -17,6 +17,7 @@ const worstValue = {
   reload: (r) => r.reload,
   render: (r) => r.render ?? 0,
   undo: (r) => Math.max(r.undo.box, r.undo.redoBox ?? 0),
+  flash: (r) => r.flash.bad,
   smooth: (r) => r.smooth.dropped - r.smooth.control.dropped,
 };
 
@@ -54,6 +55,8 @@ export function score(spec, r) {
     reload: r.reload <= LIMIT_PX,
     render: r.render !== null && r.render <= LIMIT_PX,
     undo: r.undo.bytes && r.undo.redoBytes && Math.max(r.undo.box, r.undo.redoBox) <= LIMIT_PX,
+    // A window the screencast covered under 90% is uncovered, never a pass.
+    flash: !r.flash.uncovered && r.flash.bad === 0,
     // Only drops beyond the blank page's, driven the same way in the same Chrome, are the edit's.
     smooth:
       smooth.dropped <= smooth.control.dropped &&
@@ -117,6 +120,18 @@ function summarize(results, seconds) {
     unsettled: measured.filter((r) => r.unsettled.length).length,
     undoTimeouts: measured.filter((r) => r.undoTimeout).length,
     renderErrors: measured.filter((r) => r.renderError).length,
+    flash: {
+      uncovered: measured.filter((r) => r.flash.uncovered).length,
+      badCases: measured.filter((r) => r.flash.bad > 0).length,
+      longest: Math.max(0, ...measured.map((r) => r.flash.longest)),
+      coverage: round(
+        percentile(
+          measured.map((r) => r.flash.coverage),
+          50,
+        ),
+      ),
+      lowest: round(Math.min(...measured.map((r) => r.flash.coverage))),
+    },
     smooth: smoothSummary(measured),
     seconds: Math.round(seconds),
   };
@@ -139,6 +154,7 @@ function table(summary, meta, results) {
     `The preview never held still for 1 s within 15 s in ${summary.unsettled} cases; the metrics that snapshot feeds fail.`,
     `An undo or redo write never landed within 15 s in ${summary.undoTimeouts} cases; undo fails there.`,
     `The producer failed to render ${summary.renderErrors} cases; render fails there.`,
+    `Flash: ${summary.flash.badCases} cases painted a frame matching neither the state before nor after (longest run ${summary.flash.longest} frames); ${summary.flash.uncovered} cases uncovered; screencast coverage median ${summary.flash.coverage}, lowest ${summary.flash.lowest}.`,
     `Smoothness: ${summary.smooth.unknown} cases with unknown work; dropped frames per case (median/max) ${summary.smooth.dropped}, blank-page control ${summary.smooth.control}; raw rAF p95 (median/max) ${summary.smooth.p95} ms, control ${summary.smooth.controlP95} ms.`,
     "",
     "| Metric | Pass | Worst | Worst case |",
@@ -176,6 +192,8 @@ export function entry(r) {
     ...(r.unsettled.length && { unsettled: r.unsettled }),
     ...(r.undoTimeout && { undoTimeout: r.undoTimeout }),
     ...(r.renderError && { renderError: true }),
+    flash: r.flash.bad,
+    ...(r.flash.uncovered && { flashUncovered: true }),
   };
 }
 
diff --git a/packages/studio/tests/e2e/edit-accuracy/run.mjs b/packages/studio/tests/e2e/edit-accuracy/run.mjs
index 4eb07ac1f0..73cb4e6a7d 100644
--- a/packages/studio/tests/e2e/edit-accuracy/run.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/run.mjs
@@ -15,6 +15,7 @@ import { buildGrid, writeFixture } from "./grid.mjs";
 import { killServers, runCase, startServer, stopServer } from "./case.mjs";
 import { METRICS, score, writeReport } from "./report.mjs";
 import { renderBox } from "./render.mjs";
+import { scoreFlash } from "./flash.mjs";
 import { aabb, boxDistance } from "./geometry.mjs";
 
 const HERE = dirname(fileURLToPath(import.meta.url));
@@ -54,6 +55,8 @@ function saveEvidence(id, evidence) {
     writeFileSync(join(caseDir, `saved-${name.replace("/", "-")}`), text);
   if (evidence.renderFailed && evidence.frame)
     writeFileSync(join(caseDir, "producer.jpg"), evidence.frame);
+  for (const [name, png] of evidence.flashFrames ?? [])
+    writeFileSync(join(caseDir, `flash-${name}.png`), Buffer.from(png, "base64"));
 }
 
 function verdict(r) {
@@ -115,7 +118,8 @@ async function runOne(spec, browser, decoder, port) {
     });
     await stopServer(server);
     server = null;
-    result = await withRender(dir, decoder, measured, evidence);
+    const flash = await scoreFlash(decoder, measured.flash, evidence);
+    result = await withRender(dir, decoder, { ...measured, flash }, evidence);
   } catch (error) {
     result = errorResult(error, log);
   } finally {

From ff1282752005e3871db43fdc104a3732f1a19e6e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miguel=20=C3=81ngel?= <miguel.sierra@heygen.com>
Date: Wed, 30 Sep 2026 17:52:22 -0400
Subject: [PATCH 06/12] test(studio): fail edit accuracy cases whose preview
 flashes or paints an edit late

Edit-to-paint: the first frame after a pointer-up, key, undo or redo shows the after-state.

A metric the base baseline never measured cannot regress the gate; it counts for banking.
---
 .../studio/tests/e2e/edit-accuracy/case.mjs   |  74 +++++++---
 .../studio/tests/e2e/edit-accuracy/flash.mjs  | 138 ++++++++++++++----
 .../tests/e2e/edit-accuracy/flash.test.mjs    |  63 ++++++++
 .../tests/e2e/edit-accuracy/ratchet.mjs       |  36 +++--
 .../tests/e2e/edit-accuracy/ratchet.test.mjs  |  24 ++-
 .../studio/tests/e2e/edit-accuracy/report.mjs |  25 +++-
 6 files changed, 292 insertions(+), 68 deletions(-)
 create mode 100644 packages/studio/tests/e2e/edit-accuracy/flash.test.mjs

diff --git a/packages/studio/tests/e2e/edit-accuracy/case.mjs b/packages/studio/tests/e2e/edit-accuracy/case.mjs
index 962f17e70a..20d4ccfa83 100644
--- a/packages/studio/tests/e2e/edit-accuracy/case.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/case.mjs
@@ -28,6 +28,10 @@ const ROTATE_BY = (25 * Math.PI) / 180;
 const CROP_BY = 40;
 const NUDGES = 5;
 const ZOOM_SENSITIVITY = 0.007; // previewZoom.ts: one wheel unit scales zoom by exp(0.007)
+// Flash controls, for proving the metric only: tap (release without a move), blink (hide the preview for one frame
+// after the release), reload (reload the preview frame mid-settle). EDIT_BENCH_MARKER=0 captures without the marker.
+const CONTROL = process.env.EDIT_BENCH_CONTROL;
+const CAPTURE = { marker: process.env.EDIT_BENCH_MARKER !== "0" };
 
 const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
 
@@ -523,7 +527,7 @@ async function controlDrag(browser, gesture) {
     const read = () => readQuads(ctx);
     if (gesture === "nudge") {
       await recording(page, true);
-      for (let i = 0; i < NUDGES; i++) {
+      for (let i = 1; i < NUDGES; i++) {
         await page.keyboard.press("ArrowRight");
         await nextFrame(page);
       }
@@ -554,7 +558,8 @@ const perimeterPx = (m) =>
     m.visible.map(m.map.toScreen).reduce((sum, p, i, q) => sum + dist(p, q[(i + 1) % 4]), 0),
   );
 
-/** Drives the drag and returns with the button still down; the flash window starts at pointer-up. */
+/** Drives the drag and returns with the button still down; `release` (pointer-up) runs in the flash window. */
+// fallow-ignore-next-line complexity
 async function pointerGesture(ctx, gesture, pre) {
   const press = await handlePoint(ctx, pre, gesture);
   const pressComp = pre.map.toComp(press);
@@ -572,7 +577,7 @@ async function pointerGesture(ctx, gesture, pre) {
   await recording(ctx.page, true);
   const errors = [];
   let last = s0;
-  for (const p of g.path) {
+  for (const p of CONTROL === "tap" ? g.path.map(() => press) : g.path) {
     await ctx.page.mouse.move(p[0], p[1]);
     await nextFrame(ctx.page);
     last = await sample(ctx, gesture, g.point, p);
@@ -582,8 +587,10 @@ async function pointerGesture(ctx, gesture, pre) {
   const smooth = smoothness(rec);
   const lastQuad = gesture === "crop" ? last.m.outline : last.m.visible;
   return {
-    errors,
-    lastQuad,
+    release: async () => {
+      await ctx.page.mouse.up();
+      return { errors, lastQuad };
+    },
     pressJump: quadDistance(s0.m.visible, pre.visible),
     smooth,
     diag: {
@@ -594,25 +601,50 @@ async function pointerGesture(ctx, gesture, pre) {
   };
 }
 
+/** Nudges all but the last step traced; `release` presses the last one in the flash window, so a key commits there. */
 async function nudgeGesture(ctx, pre) {
   await recording(ctx.page, true);
-  for (let i = 0; i < NUDGES; i++) {
+  for (let i = 1; i < NUDGES; i++) {
     await ctx.page.keyboard.press("ArrowRight");
     await nextFrame(ctx.page);
   }
   await nextFrame(ctx.page, 2);
   const smooth = smoothness(await recording(ctx.page, false));
-  const m = await measure(ctx);
-  const [a, b] = [centre(pre.visible), centre(m.visible)];
   return {
-    errors: [dist([b[0] - a[0], b[1] - a[1]], [NUDGES, 0])],
-    lastQuad: m.visible,
+    release: async () => {
+      await ctx.page.keyboard.press("ArrowRight");
+      await nextFrame(ctx.page, 2);
+      const m = await measure(ctx);
+      const [a, b] = [centre(pre.visible), centre(m.visible)];
+      return { errors: [dist([b[0] - a[0], b[1] - a[1]], [NUDGES, 0])], lastQuad: m.visible };
+    },
     pressJump: null,
     smooth,
     diag: {},
   };
 }
 
+/** The positive flash controls, started right after the committing input. */
+function injectControl(page) {
+  if (CONTROL === "blink")
+    return page.evaluate(() =>
+      requestAnimationFrame(() => {
+        const s = document.querySelector('[data-testid="preview-zoom-stage"]').style;
+        s.opacity = "0";
+        requestAnimationFrame(() => (s.opacity = ""));
+      }),
+    );
+  if (CONTROL === "reload")
+    return page.evaluate(async () => {
+      await new Promise((r) => setTimeout(r, 100));
+      const frames = [...document.querySelectorAll("iframe")].filter((f) =>
+        f.src.includes("/preview"),
+      );
+      const area = (f) => f.offsetWidth * f.offsetHeight;
+      frames.reduce((a, b) => (area(b) > area(a) ? b : a)).contentWindow.location.reload();
+    });
+}
+
 /** One case, end to end, in a fresh browser context against a Studio already serving `dir`. */
 // fallow-ignore-next-line complexity
 export async function runCase({ browser, spec, dir, files, url, evidence }) {
@@ -644,8 +676,9 @@ export async function runCase({ browser, spec, dir, files, url, evidence }) {
         : await pointerGesture(ctx, spec.gesture, pre);
     // Flash windows: each action to its settle is screencast, untraced; the drag's trace has already stopped.
     const flash = { regions: await panes(page), tolPx: perimeterPx(pre), windows: {} };
-    let capture = await startCapture(page);
-    if (spec.gesture !== "nudge") await page.mouse.up();
+    let capture = await startCapture(page, CAPTURE);
+    const { errors, lastQuad } = await drive.release();
+    await injectControl(page);
     await waitForFiles(ctx, { from: original, timeout: spec.gesture === "nudge" ? 6000 : 5000 });
     await settled(ctx);
     flash.windows.release = await capture.stop();
@@ -659,7 +692,7 @@ export async function runCase({ browser, spec, dir, files, url, evidence }) {
     // Undo and redo run before any reload. Each waits up to 15 s for its own write; redo waits for undo.
     const landed = (from) =>
       saved ? waitForFiles(ctx, { from, timeout: 15_000 }) : { reached: true, files: from };
-    capture = await startCapture(page);
+    capture = await startCapture(page, CAPTURE);
     await chord(page, "Control+z");
     const undo = await landed(committedFiles);
     const undone = await settled(ctx);
@@ -668,7 +701,7 @@ export async function runCase({ browser, spec, dir, files, url, evidence }) {
     let [redo, redone] = [{ reached: false }, null];
     if (undo.reached) {
       await blurPreview(page);
-      capture = await startCapture(page);
+      capture = await startCapture(page, CAPTURE);
       await chord(page, "Control+Shift+z");
       redo = await landed(undo.files);
       redone = await settled(ctx);
@@ -680,7 +713,10 @@ export async function runCase({ browser, spec, dir, files, url, evidence }) {
     // The reload window starts once Studio can seek the composition; before that Studio itself is loading.
     const lastSettled = (flash.windows.redo ?? flash.windows.undo).frames.at(-1);
     await page.reload();
-    const reloaded = await openStudio(ctx, async () => (capture = await startCapture(page)));
+    const reloaded = await openStudio(
+      ctx,
+      async () => (capture = await startCapture(page, CAPTURE)),
+    );
     flash.windows.reload = { ...(await capture.stop()), before: lastSettled };
     await shoot("reloaded");
     const quads = Object.fromEntries(
@@ -691,12 +727,12 @@ export async function runCase({ browser, spec, dir, files, url, evidence }) {
       zoom,
       saved,
       tracking: {
-        max: Math.max(...drive.errors),
-        p95: percentile(drive.errors, 95),
-        frames: drive.errors.length,
+        max: Math.max(...errors),
+        p95: percentile(errors, 95),
+        frames: errors.length,
       },
       pressJump: drive.pressJump,
-      drop: quadDistance(drive.lastQuad, committed.visible),
+      drop: quadDistance(lastQuad, committed.visible),
       reload: quadDistance(committed.visible, reloaded.visible),
       undo: {
         bytes: saved && undo.reached && sameFiles(undo.files, original),
diff --git a/packages/studio/tests/e2e/edit-accuracy/flash.mjs b/packages/studio/tests/e2e/edit-accuracy/flash.mjs
index b4aa5d7f8c..8f33362c8a 100644
--- a/packages/studio/tests/e2e/edit-accuracy/flash.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/flash.mjs
@@ -1,7 +1,11 @@
-/** Preview flashes: every frame the screencast paints, from an action to its settle, against the frames around it. */
+/**
+ * Preview flashes and edit-to-paint: every frame the screencast paints, from an action to its settle, against the
+ * frames around it.
+ */
 
 // A 4x4 marker in the top-left corner, outside every region, repaints every rAF with the frame counter in its
 // colour, so the screencast (which only sends repainted frames) sends every frame and each one is numbered.
+// It also logs the counter at each committing input (pointer-up, a non-modifier key), in any same-origin frame.
 function markerOn() {
   const bench = window.__editBench;
   if (bench.marker) return;
@@ -9,15 +13,36 @@ function markerOn() {
   el.style.cssText =
     "position:fixed;left:0;top:0;width:4px;height:4px;z-index:2147483647;pointer-events:none";
   document.documentElement.append(el);
-  bench.marker = { el, n: 0, on: true, masks: [] };
+  const marker = { el, n: 0, on: true, masks: [], times: [], inputs: [], unlisten: [] };
+  bench.marker = marker;
+  const MODIFIERS = new Set(["Control", "Shift", "Alt", "Meta"]);
+  const log = (e) => {
+    if (e.type === "pointerup" || !MODIFIERS.has(e.key))
+      marker.inputs.push({ type: e.type, key: e.key ?? null, n: marker.n, t: performance.now() });
+  };
+  const listen = (w) => {
+    for (const type of ["pointerup", "keydown"]) {
+      w.addEventListener(type, log, true);
+      marker.unlisten.push(() => w.removeEventListener(type, log, true));
+    }
+    for (const f of w.document.querySelectorAll("iframe")) {
+      try {
+        listen(f.contentWindow);
+      } catch {
+        // A cross-origin frame cannot take an input Studio handles.
+      }
+    }
+  };
+  listen(window);
   const tick = () => {
-    if (!bench.marker?.on) return;
-    const n = ++bench.marker.n;
+    if (!marker.on) return;
+    const n = ++marker.n;
+    marker.times[n] = performance.now();
     el.style.background = `rgb(${n & 255},${(n >> 8) & 255},128)`;
     // Toasts slide in and out by design; wherever one was during the window is left out of the comparison.
     for (const t of document.querySelectorAll(".hf-toast-enter, .hf-toast-exit")) {
       const r = t.getBoundingClientRect();
-      bench.marker.masks.push([r.left, r.top, r.right, r.bottom].map(Math.round));
+      marker.masks.push([r.left, r.top, r.right, r.bottom].map(Math.round));
     }
     requestAnimationFrame(tick);
   };
@@ -28,6 +53,7 @@ function markerOff() {
   const m = window.__editBench.marker;
   if (!m) return;
   m.on = false;
+  for (const off of m.unlisten) off();
   m.el.remove();
   window.__editBench.marker = null;
 }
@@ -63,7 +89,10 @@ function paneRects() {
   );
 }
 
-/** Starts a screencast of one action window; stop() returns its PNG frames and the marker range it spans. */
+/**
+ * Starts a screencast of one action window; stop() returns its PNG frames, the marker range it spans, the rAF time
+ * of each counter and the committing inputs logged in it.
+ */
 export async function startCapture(page, { marker = true } = {}) {
   const cdp = await page.createCDPSession();
   const frames = [];
@@ -72,20 +101,23 @@ export async function startCapture(page, { marker = true } = {}) {
     cdp.send("Page.screencastFrameAck", { sessionId: f.sessionId }).catch(() => undefined);
   });
   if (marker) await page.evaluate(markerOn);
-  const counter = () => page.evaluate(() => window.__editBench.marker?.n ?? null);
-  const masks = () => page.evaluate(() => window.__editBench.marker?.masks ?? []);
+  const state = () =>
+    page.evaluate(() => {
+      const m = window.__editBench.marker;
+      return m ? { n: m.n, masks: m.masks, times: m.times, inputs: m.inputs } : { n: null };
+    });
   await cdp.send("Page.startScreencast", { format: "png", everyNthFrame: 1 });
   // The first frame is the state before the action.
   for (const deadline = Date.now() + 2000; !frames.length && Date.now() < deadline; )
     await new Promise((r) => setTimeout(r, 10));
-  const from = await counter();
+  const from = (await state()).n;
   return {
     async stop() {
-      const [to, mask] = [await counter(), await masks()];
+      const { n: to, masks = [], times = [], inputs = [] } = await state();
       await cdp.send("Page.stopScreencast").catch(() => undefined);
       if (marker) await page.evaluate(markerOff).catch(() => undefined);
       await cdp.detach().catch(() => undefined);
-      return { frames, from, to, masks: mask };
+      return { frames, from, to, masks, times, inputs };
     },
   };
 }
@@ -140,29 +172,42 @@ function compareFrames(frames, before, after, regions, masks, colourTol) {
   })();
 }
 
+/** Marker counters the screencast never delivered, as [first, last] runs relative to the window start. */
+// fallow-ignore-next-line complexity
+function gaps(seen, from, to) {
+  const runs = [];
+  for (let c = from; c <= to; c++)
+    if (!seen.has(c)) {
+      const last = runs.at(-1);
+      if (last && last[1] === c - from - 1) last[1] = c - from;
+      else runs.push([c - from, c - from]);
+    }
+  return runs;
+}
+
+const showsAfter = (row, tolPx) =>
+  Object.values(row.diffs).every(([, vsAfter]) => vsAfter <= tolPx);
+
 /**
- * A frame is bad when some pane differs from the before frame and from the after frame by more pixels than a
- * 0.5 px shift of the element's perimeter moves. Coverage is marker counters seen over counters in the window.
+ * Edit-to-paint: frames from the committing input until the preview shows the after-state for good, and the ms from
+ * the input to that frame's rAF. One frame is the next paint; null when the window logged no input.
  */
 // fallow-ignore-next-line complexity
-async function scoreWindow(
-  decoder,
-  { frames, from, to, before, after, masks = [] },
-  regions,
-  tolPx,
-) {
-  const b = before ?? frames[0];
-  const a = after ?? frames.at(-1);
-  if (!b || !a) return { frames: 0, coverage: 0, bad: [], longest: 0 };
-  const rows = await decoder.evaluate(
-    compareFrames,
-    frames,
-    b,
-    a,
-    regions,
-    dedupe(masks),
-    COLOUR_TOL,
-  );
+function editToPaint(rows, input, times, tolPx) {
+  if (!input) return null;
+  let lastOff = null;
+  for (const r of rows) if (r.counter > input.n && !showsAfter(r, tolPx)) lastOff = r.counter;
+  const at = lastOff === null ? input.n + 1 : lastOff + 1;
+  return { frames: at - input.n, ms: times[at] == null ? null : times[at] - input.t };
+}
+
+/**
+ * A frame is bad (a flash) when some pane differs from the before frame and from the after frame by more pixels than
+ * a 0.5 px shift of the element's perimeter moves. Coverage is marker counters seen over counters in the window.
+ * `rows` holds each frame's marker counter and per-pane [vsBefore, vsAfter] differing pixel counts.
+ */
+// fallow-ignore-next-line complexity
+export function classify(rows, { from, to, times = [], inputs = [] }, tolPx) {
   const seen = new Set(rows.map((r) => r.counter).filter((c) => c >= from && c <= to));
   const span = from === null || to === null ? null : to - from + 1;
   const bad = [];
@@ -178,14 +223,34 @@ async function scoreWindow(
     longest = Math.max(longest, run);
   });
   return {
-    frames: frames.length,
+    frames: rows.length,
     rafFrames: span,
     coverage: span ? seen.size / span : null,
+    missing: span ? gaps(seen, from, to) : null,
     bad,
     longest,
+    input: inputs[0] ?? null,
+    paint: editToPaint(rows, inputs[0], times, tolPx),
   };
 }
 
+// fallow-ignore-next-line complexity
+async function scoreWindow(decoder, win, regions, tolPx) {
+  const b = win.before ?? win.frames[0];
+  const a = win.after ?? win.frames.at(-1);
+  if (!b || !a) return { frames: 0, coverage: 0, bad: [], longest: 0, paint: null };
+  const rows = await decoder.evaluate(
+    compareFrames,
+    win.frames,
+    b,
+    a,
+    regions,
+    dedupe(win.masks ?? []),
+    COLOUR_TOL,
+  );
+  return classify(rows, win, tolPx);
+}
+
 const dedupe = (rects) =>
   [...new Set(rects.map((r) => r.join(",")))].map((k) => k.split(",").map(Number));
 
@@ -193,7 +258,11 @@ const dedupe = (rects) =>
 const COLOUR_TOL = 24;
 const MIN_COVERAGE = 0.9;
 
+// The inputs that commit an edit; the reload window has none and scores flashes only.
+const COMMITTING = ["release", "undo", "redo"];
+
 /** Scores every window of a case; the offending frames go to `evidence.flashFrames` as PNG. */
+// fallow-ignore-next-line complexity
 export async function scoreFlash(decoder, { regions, tolPx, windows }, evidence) {
   const out = {};
   evidence.flashFrames = [];
@@ -208,11 +277,16 @@ export async function scoreFlash(decoder, { regions, tolPx, windows }, evidence)
   }
   const all = Object.values(out);
   const coverage = Math.min(...all.map((w) => w.coverage ?? 0));
+  const paints = COMMITTING.filter((k) => out[k]).map((k) => out[k].paint);
   return {
     bad: all.reduce((n, w) => n + w.bad.length, 0),
     longest: Math.max(0, ...all.map((w) => w.longest)),
     coverage,
     uncovered: !(coverage >= MIN_COVERAGE),
+    // The slowest committing input; unknown (null) when any of them logged no input.
+    paint: paints.some((p) => !p)
+      ? null
+      : paints.reduce((a, p) => (p.frames > a.frames ? p : a), { frames: 0, ms: 0 }),
     regions,
     tolPx,
     windows: out,
diff --git a/packages/studio/tests/e2e/edit-accuracy/flash.test.mjs b/packages/studio/tests/e2e/edit-accuracy/flash.test.mjs
new file mode 100644
index 0000000000..a79f612ad4
--- /dev/null
+++ b/packages/studio/tests/e2e/edit-accuracy/flash.test.mjs
@@ -0,0 +1,63 @@
+import { describe, expect, it } from "vitest";
+import { classify } from "./flash.mjs";
+
+const TOL = 10;
+// One row per painted frame: its marker counter and [vsBefore, vsAfter] differing pixels per pane.
+const before = (counter) => ({ counter, diffs: { preview: [0, 50] } });
+const after = (counter) => ({ counter, diffs: { preview: [50, 0] } });
+const neither = (counter) => ({ counter, diffs: { preview: [50, 50] } });
+const times = Array.from({ length: 20 }, (_, n) => n * 16);
+const win = (inputN) => ({
+  from: 1,
+  to: 8,
+  times,
+  inputs: [{ type: "pointerup", n: inputN, t: 40 }],
+});
+
+describe("classify", () => {
+  it("passes an input whose next frame already shows the after-state", () => {
+    const w = classify(
+      [before(1), before(2), after(3), after(4), after(5), after(6), after(7), after(8)],
+      win(2),
+      TOL,
+    );
+    expect(w.paint).toEqual({ frames: 1, ms: 8 });
+    expect(w.bad).toEqual([]);
+    expect(w.coverage).toBe(1);
+  });
+
+  it("counts frames that hold the old state after the input", () => {
+    const rows = [
+      before(1),
+      before(2),
+      before(3),
+      before(4),
+      after(5),
+      after(6),
+      after(7),
+      after(8),
+    ];
+    expect(classify(rows, win(2), TOL).paint).toEqual({ frames: 3, ms: 40 });
+  });
+
+  it("flags a frame that is neither state as a flash and as late paint", () => {
+    const w = classify(
+      [before(1), before(2), after(3), neither(4), after(5), after(6), after(7), after(8)],
+      win(2),
+      TOL,
+    );
+    expect(w.bad.map((b) => b.counter)).toEqual([4]);
+    expect(w.longest).toBe(1);
+    expect(w.paint.frames).toBe(3);
+  });
+
+  it("reports missing counters and never guesses paint without a logged input", () => {
+    const w = classify([before(1), after(4), after(8)], { from: 1, to: 8, times, inputs: [] }, TOL);
+    expect(w.coverage).toBe(3 / 8);
+    expect(w.missing).toEqual([
+      [1, 2],
+      [4, 6],
+    ]);
+    expect(w.paint).toBeNull();
+  });
+});
diff --git a/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs b/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs
index 6b7035bd29..b1da73ac2c 100644
--- a/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs
@@ -5,26 +5,31 @@
 import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { fileURLToPath } from "node:url";
-import { LIMIT_PX, entry, writeReport } from "./report.mjs";
+import { LIMIT_PX, PAINT_FRAMES, entry, writeReport } from "./report.mjs";
 
 const GATED_PX = ["tracking", "pressJump", "drop", "reload", "render"];
 const LISTED = 30;
 
-/** Passes every gated metric; an unsettled preview fails the metrics it fed, all of them gated. */
+/**
+ * Passes every gated metric; an unsettled preview fails the metrics it fed, all of them gated. Flash and
+ * edit-to-paint count only where `like` measured them, so a metric the base branch never measured cannot regress.
+ */
 // fallow-ignore-next-line complexity
-export const accurate = (e) =>
+export const accurate = (e, like = e) =>
   Boolean(e) &&
   !e.error &&
   !e.unsettled &&
   !e.renderError &&
-  !(e.flash > 0) &&
-  !e.flashUncovered &&
   e.undo === true &&
-  GATED_PX.every((m) => !(e[m] > LIMIT_PX));
+  GATED_PX.every((m) => !(e[m] > LIMIT_PX)) &&
+  (!like || !("flash" in like) || (e.flash === 0 && !e.flashUncovered)) &&
+  (!like || !("paint" in like) || (e.paint !== null && e.paint <= PAINT_FRAMES));
 
 /** Cases whose verdict here differs from the base branch, either way: each is re-run twice before the gate. */
 export const flipped = (base, results) =>
-  results.filter((r) => accurate(base.cases[r.id]) !== accurate(entry(r))).map((r) => r.id);
+  results
+    .filter((r) => accurate(base.cases[r.id]) !== accurate(entry(r), base.cases[r.id]))
+    .map((r) => r.id);
 
 const summary = (e) =>
   e.error ? "error" : `${GATED_PX.map((m) => `${m} ${e[m] ?? "-"}`).join(", ")}, undo ${e.undo}`;
@@ -35,21 +40,24 @@ export function gate(base, head, runs) {
   const seen = new Map();
   for (const r of runs) seen.set(r.id, [...(seen.get(r.id) ?? []), entry(r)]);
   const cases = [...seen].map(([id, entries]) => {
-    const fails = entries.filter((e) => !accurate(e)).length;
+    const like = base.cases[id];
+    const majority = (judge) => entries.filter((e) => !judge(e)).length * 2 < entries.length;
     return {
       id,
       entries,
-      passed: fails * 2 < entries.length,
-      basePassed: accurate(base.cases[id]),
+      passed: majority((e) => accurate(e)),
+      // Judged on the base entry's metrics, for regressions and the count.
+      passedLike: majority((e) => accurate(e, like)),
+      basePassed: accurate(like),
     };
   });
   const passing = cases.filter((c) => c.passed);
   const result = {
-    basePassing: Object.values(base.cases).filter(accurate).length,
-    headPassing: passing.length,
-    regressed: cases.filter((c) => c.basePassed && !c.passed).map((c) => c.id),
+    basePassing: Object.values(base.cases).filter((e) => accurate(e)).length,
+    headPassing: cases.filter((c) => c.passedLike).length,
+    regressed: cases.filter((c) => c.basePassed && !c.passedLike).map((c) => c.id),
     unstable: cases
-      .filter((c) => new Set(c.entries.map(accurate)).size > 1)
+      .filter((c) => new Set(c.entries.map((e) => accurate(e))).size > 1)
       .map((c) => ({ id: c.id, runs: c.entries.map(summary) })),
     newlyPassing: passing.filter((c) => !c.basePassed).map((c) => c.id),
     unbanked: passing.filter((c) => !accurate(head.cases[c.id])).map((c) => c.id),
diff --git a/packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs b/packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs
index 9f620779bb..9f94999453 100644
--- a/packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs
@@ -11,6 +11,7 @@ const good = {
   dropped: 0,
   controlDropped: 0,
   flash: 0,
+  paint: 1,
   work: 3,
   frameP95: 20,
 };
@@ -26,7 +27,7 @@ const run = (id, drop = 0) => ({
   checks: { undo: true },
   smooth: { p95: 20, dropped: 0, workP95: 3, control: { dropped: 0 } },
   unsettled: [],
-  flash: { bad: 0, uncovered: false },
+  flash: { bad: 0, uncovered: false, paint: { frames: 1, ms: 5 } },
 });
 const baseline = (cases) => ({ cases });
 
@@ -44,12 +45,33 @@ describe("accurate", () => {
     expect(accurate({ ...good, render: null, renderError: true })).toBe(false);
     expect(accurate({ ...good, flash: 1 })).toBe(false);
     expect(accurate({ ...good, flash: 0, flashUncovered: true })).toBe(false);
+    expect(accurate({ ...good, paint: 2 })).toBe(false);
+    expect(accurate({ ...good, paint: null })).toBe(false);
     expect(accurate({ ...good, undo: false })).toBe(false);
     expect(accurate({ pass: false, error: true })).toBe(false);
     expect(accurate(undefined)).toBe(false);
   });
 });
 
+describe("a metric the base branch never measured", () => {
+  const { flash: _f, paint: _p, ...older } = good;
+
+  it("is judged only where the like entry holds it", () => {
+    expect(accurate({ ...good, flash: 2, paint: 9 }, older)).toBe(true);
+    expect(accurate({ ...good, flash: 2 }, good)).toBe(false);
+  });
+
+  it("cannot regress a case or lower the count, and still needs banking to pass", () => {
+    const flashing = { ...run("a"), flash: { bad: 3, uncovered: false, paint: { frames: 4 } } };
+    const g = gate(baseline({ a: older }), baseline({ a: { ...good, flash: 3 } }), [flashing]);
+    expect(g.regressed).toEqual([]);
+    expect(g.headPassing).toBe(1);
+    expect(g.overclaimed).toEqual([]);
+    expect(g.ok).toBe(true);
+    expect(flipped(baseline({ a: older }), [flashing])).toEqual([]);
+  });
+});
+
 describe("gate", () => {
   const base = baseline({ a: good, b: good });
 
diff --git a/packages/studio/tests/e2e/edit-accuracy/report.mjs b/packages/studio/tests/e2e/edit-accuracy/report.mjs
index 6746f375ac..c58d4d048c 100644
--- a/packages/studio/tests/e2e/edit-accuracy/report.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/report.mjs
@@ -7,7 +7,19 @@ export const LIMIT_PX = 0.5;
 // A frame over 1.5 vsyncs is dropped; raw rAF p95 stays reported so a different rule re-scores without a re-run.
 const DROPPED_FRAME_MS = 25;
 const WORK_MS = 8;
-export const METRICS = ["tracking", "press", "drop", "reload", "render", "undo", "flash", "smooth"];
+export const METRICS = [
+  "tracking",
+  "press",
+  "drop",
+  "reload",
+  "render",
+  "undo",
+  "flash",
+  "paint",
+  "smooth",
+];
+// Edit-to-paint: the first frame painted after a committing input already shows the after-state.
+export const PAINT_FRAMES = 1;
 
 /** Worst-first value per metric; undo ranks by box distance, and its byte failures are counted apart. */
 const worstValue = {
@@ -18,6 +30,7 @@ const worstValue = {
   render: (r) => r.render ?? 0,
   undo: (r) => Math.max(r.undo.box, r.undo.redoBox ?? 0),
   flash: (r) => r.flash.bad,
+  paint: (r) => r.flash.paint?.frames ?? Infinity,
   smooth: (r) => r.smooth.dropped - r.smooth.control.dropped,
 };
 
@@ -57,6 +70,7 @@ export function score(spec, r) {
     undo: r.undo.bytes && r.undo.redoBytes && Math.max(r.undo.box, r.undo.redoBox) <= LIMIT_PX,
     // A window the screencast covered under 90% is uncovered, never a pass.
     flash: !r.flash.uncovered && r.flash.bad === 0,
+    paint: !r.flash.uncovered && r.flash.paint !== null && r.flash.paint.frames <= PAINT_FRAMES,
     // Only drops beyond the blank page's, driven the same way in the same Chrome, are the edit's.
     smooth:
       smooth.dropped <= smooth.control.dropped &&
@@ -132,6 +146,11 @@ function summarize(results, seconds) {
       ),
       lowest: round(Math.min(...measured.map((r) => r.flash.coverage))),
     },
+    paint: {
+      unknown: measured.filter((r) => !r.flash.paint).length,
+      frames: medianMax(measured.filter((r) => r.flash.paint).map((r) => r.flash.paint.frames)),
+      ms: medianMax(measured.filter((r) => r.flash.paint?.ms != null).map((r) => r.flash.paint.ms)),
+    },
     smooth: smoothSummary(measured),
     seconds: Math.round(seconds),
   };
@@ -148,13 +167,14 @@ function table(summary, meta, results) {
     `Every metric counts except smoothness, which is reported against the blank-page control: ${summary.perMetric.find((m) => m.metric === "smooth").pass}/${summary.total} pass it, and ${summary.passing}/${summary.total} pass everything including it.`,
     "",
     `Studio ${meta.studio} (build ${meta.build}), bench ${meta.bench}, grid \`${meta.grid}\`, ${meta.date}, ${summary.seconds}s with ${meta.jobs} jobs, ${summary.errors} harness errors, load ${meta.load}.`,
-    `Pass: tracking, press jump, drop, reload and render ≤ ${LIMIT_PX} px; undo and redo byte-identical with the box ≤ ${LIMIT_PX} px; no more frames over ${DROPPED_FRAME_MS} ms than the blank-page control, and main-thread work ≤ ${WORK_MS} ms per frame at p95.`,
+    `Pass: tracking, press jump, drop, reload and render ≤ ${LIMIT_PX} px; undo and redo byte-identical with the box ≤ ${LIMIT_PX} px; no flash frame and the after-state in the next frame; no more frames over ${DROPPED_FRAME_MS} ms than the blank-page control, and main-thread work ≤ ${WORK_MS} ms per frame at p95.`,
     "",
     `Undo or redo left different bytes in ${summary.bytesDiffer.undo} undo and ${summary.bytesDiffer.redo} redo cases.`,
     `The preview never held still for 1 s within 15 s in ${summary.unsettled} cases; the metrics that snapshot feeds fail.`,
     `An undo or redo write never landed within 15 s in ${summary.undoTimeouts} cases; undo fails there.`,
     `The producer failed to render ${summary.renderErrors} cases; render fails there.`,
     `Flash: ${summary.flash.badCases} cases painted a frame matching neither the state before nor after (longest run ${summary.flash.longest} frames); ${summary.flash.uncovered} cases uncovered; screencast coverage median ${summary.flash.coverage}, lowest ${summary.flash.lowest}.`,
+    `Edit-to-paint (pointer-up, key, undo, redo; pass at ${PAINT_FRAMES} frame): frames to the after-state (median/max) ${summary.paint.frames}, ms ${summary.paint.ms}; ${summary.paint.unknown} cases logged no input.`,
     `Smoothness: ${summary.smooth.unknown} cases with unknown work; dropped frames per case (median/max) ${summary.smooth.dropped}, blank-page control ${summary.smooth.control}; raw rAF p95 (median/max) ${summary.smooth.p95} ms, control ${summary.smooth.controlP95} ms.`,
     "",
     "| Metric | Pass | Worst | Worst case |",
@@ -194,6 +214,7 @@ export function entry(r) {
     ...(r.renderError && { renderError: true }),
     flash: r.flash.bad,
     ...(r.flash.uncovered && { flashUncovered: true }),
+    paint: r.flash.paint?.frames ?? null,
   };
 }
 

From d8dfb5cd0994a27fdbcab0e1d14d4b931d934bbf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miguel=20=C3=81ngel?= <miguel.sierra@heygen.com>
Date: Wed, 30 Sep 2026 18:14:45 -0400
Subject: [PATCH 07/12] test(studio): count a screencast frame delivered twice
 once in the flash metric

---
 .../studio/tests/e2e/edit-accuracy/flash.mjs  | 20 +++++++++++--------
 .../tests/e2e/edit-accuracy/flash.test.mjs    | 17 ++++++++++++++++
 2 files changed, 29 insertions(+), 8 deletions(-)

diff --git a/packages/studio/tests/e2e/edit-accuracy/flash.mjs b/packages/studio/tests/e2e/edit-accuracy/flash.mjs
index 8f33362c8a..24544b96cc 100644
--- a/packages/studio/tests/e2e/edit-accuracy/flash.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/flash.mjs
@@ -204,24 +204,28 @@ function editToPaint(rows, input, times, tolPx) {
 /**
  * A frame is bad (a flash) when some pane differs from the before frame and from the after frame by more pixels than
  * a 0.5 px shift of the element's perimeter moves. Coverage is marker counters seen over counters in the window.
- * `rows` holds each frame's marker counter and per-pane [vsBefore, vsAfter] differing pixel counts.
+ * `frames` holds each frame's marker counter and per-pane [vsBefore, vsAfter] differing pixel counts.
  */
 // fallow-ignore-next-line complexity
-export function classify(rows, { from, to, times = [], inputs = [] }, tolPx) {
+export function classify(frames, { from, to, times = [], inputs = [] }, tolPx) {
+  // The screencast can deliver one frame twice; with the marker on, each counter counts once.
+  const indexed = frames.map((r, frame) => ({ ...r, frame }));
+  const rows =
+    from === null ? indexed : indexed.filter((r, i) => r.counter !== frames[i - 1]?.counter);
   const seen = new Set(rows.map((r) => r.counter).filter((c) => c >= from && c <= to));
   const span = from === null || to === null ? null : to - from + 1;
   const bad = [];
-  rows.forEach((r, i) => {
+  for (const r of rows) {
     const panes = Object.entries(r.diffs)
       .filter(([, [vsBefore, vsAfter]]) => vsBefore > tolPx && vsAfter > tolPx)
       .map(([k, [vsBefore, vsAfter]]) => ({ pane: k, vsBefore, vsAfter }));
-    if (panes.length) bad.push({ frame: i, counter: r.counter, panes });
-  });
+    if (panes.length) bad.push({ frame: r.frame, counter: r.counter, panes });
+  }
   let [longest, run] = [0, 0];
-  rows.forEach((_, i) => {
-    run = bad.some((x) => x.frame === i) ? run + 1 : 0;
+  for (const r of rows) {
+    run = bad.some((x) => x.frame === r.frame) ? run + 1 : 0;
     longest = Math.max(longest, run);
-  });
+  }
   return {
     frames: rows.length,
     rafFrames: span,
diff --git a/packages/studio/tests/e2e/edit-accuracy/flash.test.mjs b/packages/studio/tests/e2e/edit-accuracy/flash.test.mjs
index a79f612ad4..c5190e0e55 100644
--- a/packages/studio/tests/e2e/edit-accuracy/flash.test.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/flash.test.mjs
@@ -51,6 +51,23 @@ describe("classify", () => {
     expect(w.paint.frames).toBe(3);
   });
 
+  it("counts a frame the screencast delivered twice once", () => {
+    const rows = [
+      before(1),
+      before(2),
+      neither(3),
+      neither(3),
+      after(4),
+      after(5),
+      after(6),
+      after(7),
+      after(8),
+    ];
+    const w = classify(rows, win(2), TOL);
+    expect(w.bad.map((b) => b.frame)).toEqual([2]);
+    expect(w.frames).toBe(8);
+  });
+
   it("reports missing counters and never guesses paint without a logged input", () => {
     const w = classify([before(1), after(4), after(8)], { from: 1, to: 8, times, inputs: [] }, TOL);
     expect(w.coverage).toBe(3 / 8);

From 67183ae39cbd3be81fe7b9120248a6359094002f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miguel=20=C3=81ngel?= <miguel.sierra@heygen.com>
Date: Wed, 30 Sep 2026 18:29:59 -0400
Subject: [PATCH 08/12] test(studio): capture flash windows as JPEG and judge
 coverage against a blank control

PNG capture misses 11-23% of frames even on a blank page, so every case read uncovered.
---
 .../studio/tests/e2e/edit-accuracy/case.mjs   | 64 ++++++++++++-------
 .../studio/tests/e2e/edit-accuracy/flash.mjs  | 45 +++++++++----
 .../tests/e2e/edit-accuracy/flash.test.mjs    | 11 +++-
 .../studio/tests/e2e/edit-accuracy/report.mjs | 19 +++++-
 4 files changed, 102 insertions(+), 37 deletions(-)

diff --git a/packages/studio/tests/e2e/edit-accuracy/case.mjs b/packages/studio/tests/e2e/edit-accuracy/case.mjs
index 20d4ccfa83..2f8b50a56b 100644
--- a/packages/studio/tests/e2e/edit-accuracy/case.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/case.mjs
@@ -28,7 +28,7 @@ const ROTATE_BY = (25 * Math.PI) / 180;
 const CROP_BY = 40;
 const NUDGES = 5;
 const ZOOM_SENSITIVITY = 0.007; // previewZoom.ts: one wheel unit scales zoom by exp(0.007)
-// Flash controls, for proving the metric only: tap (release without a move), blink (hide the preview for one frame
+// Flash controls, for proving the metric only: tap (release without a move), blinkN (hide the preview for N frames
 // after the release), reload (reload the preview frame mid-settle). EDIT_BENCH_MARKER=0 captures without the marker.
 const CONTROL = process.env.EDIT_BENCH_CONTROL;
 const CAPTURE = { marker: process.env.EDIT_BENCH_MARKER !== "0" };
@@ -514,7 +514,11 @@ function smoothness(rec) {
 const CONTROL_PAGE = `data:text/html,<body style="margin:0;background:%23202020"><div id="box"
   style="position:absolute;left:600px;top:300px;width:240px;height:160px;background:%23f0c020"></div>`;
 
-/** The case's drag schedule and per-frame reads on a blank page in the same Chrome: the machine's own frame drops. */
+/**
+ * The case's drag schedule and per-frame reads on a blank page in the same Chrome: the machine's own frame drops.
+ * Then 1 s of screencast there: the capture coverage the machine reaches right now.
+ */
+// fallow-ignore-next-line complexity
 async function controlDrag(browser, gesture) {
   const context = await browser.createBrowserContext();
   try {
@@ -525,6 +529,7 @@ async function controlDrag(browser, gesture) {
     const box = await page.$("#box");
     const ctx = { page, handles: { target: box, root: box } };
     const read = () => readQuads(ctx);
+    let smooth;
     if (gesture === "nudge") {
       await recording(page, true);
       for (let i = 1; i < NUDGES; i++) {
@@ -532,21 +537,24 @@ async function controlDrag(browser, gesture) {
         await nextFrame(page);
       }
       await nextFrame(page, 2);
-      return smoothness(await recording(page, false));
-    }
-    await page.mouse.move(700, 380);
-    await page.mouse.down();
-    await nextFrame(page);
-    await read();
-    await recording(page, true);
-    for (let i = 1; i <= STEPS; i++) {
-      await page.mouse.move(700 + (MOVE_BY[0] * i) / STEPS, 380 + (MOVE_BY[1] * i) / STEPS);
+      smooth = smoothness(await recording(page, false));
+    } else {
+      await page.mouse.move(700, 380);
+      await page.mouse.down();
       await nextFrame(page);
       await read();
+      await recording(page, true);
+      for (let i = 1; i <= STEPS; i++) {
+        await page.mouse.move(700 + (MOVE_BY[0] * i) / STEPS, 380 + (MOVE_BY[1] * i) / STEPS);
+        await nextFrame(page);
+        await read();
+      }
+      smooth = smoothness(await recording(page, false));
+      await page.mouse.up();
     }
-    const smooth = smoothness(await recording(page, false));
-    await page.mouse.up();
-    return smooth;
+    const capture = await startCapture(page, CAPTURE);
+    await sleep(1000);
+    return { smooth, capture: await capture.stop() };
   } finally {
     await context.close().catch(() => undefined);
   }
@@ -626,14 +634,17 @@ async function nudgeGesture(ctx, pre) {
 
 /** The positive flash controls, started right after the committing input. */
 function injectControl(page) {
-  if (CONTROL === "blink")
-    return page.evaluate(() =>
-      requestAnimationFrame(() => {
-        const s = document.querySelector('[data-testid="preview-zoom-stage"]').style;
-        s.opacity = "0";
-        requestAnimationFrame(() => (s.opacity = ""));
-      }),
-    );
+  const blink = /^blink(\d+)$/.exec(CONTROL ?? "");
+  if (blink)
+    return page.evaluate((frames) => {
+      const s = document.querySelector('[data-testid="preview-zoom-stage"]').style;
+      const hide = (left) =>
+        requestAnimationFrame(() => {
+          s.opacity = left ? "0" : "";
+          if (left) hide(left - 1);
+        });
+      hide(frames);
+    }, Number(blink[1]));
   if (CONTROL === "reload")
     return page.evaluate(async () => {
       await new Promise((r) => setTimeout(r, 100));
@@ -648,7 +659,7 @@ function injectControl(page) {
 /** One case, end to end, in a fresh browser context against a Studio already serving `dir`. */
 // fallow-ignore-next-line complexity
 export async function runCase({ browser, spec, dir, files, url, evidence }) {
-  const control = await controlDrag(browser, spec.gesture);
+  const { smooth: control, capture: controlCapture } = await controlDrag(browser, spec.gesture);
   const context = await browser.createBrowserContext();
   const page = await context.newPage();
   const ctx = { page, dir, files, handles: null };
@@ -675,7 +686,12 @@ export async function runCase({ browser, spec, dir, files, url, evidence }) {
         ? await nudgeGesture(ctx, pre)
         : await pointerGesture(ctx, spec.gesture, pre);
     // Flash windows: each action to its settle is screencast, untraced; the drag's trace has already stopped.
-    const flash = { regions: await panes(page), tolPx: perimeterPx(pre), windows: {} };
+    const flash = {
+      regions: await panes(page),
+      tolPx: perimeterPx(pre),
+      windows: {},
+      control: controlCapture,
+    };
     let capture = await startCapture(page, CAPTURE);
     const { errors, lastQuad } = await drive.release();
     await injectControl(page);
diff --git a/packages/studio/tests/e2e/edit-accuracy/flash.mjs b/packages/studio/tests/e2e/edit-accuracy/flash.mjs
index 24544b96cc..a3653cef9d 100644
--- a/packages/studio/tests/e2e/edit-accuracy/flash.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/flash.mjs
@@ -3,15 +3,16 @@
  * frames around it.
  */
 
-// A 4x4 marker in the top-left corner, outside every region, repaints every rAF with the frame counter in its
-// colour, so the screencast (which only sends repainted frames) sends every frame and each one is numbered.
+// A 16x16 marker in the top-left corner, outside every region, repaints every rAF with the frame counter in its
+// colour (5 bits per channel in steps of 8, which JPEG keeps), so the screencast (which only sends repainted frames)
+// sends every frame and each one is numbered.
 // It also logs the counter at each committing input (pointer-up, a non-modifier key), in any same-origin frame.
 function markerOn() {
   const bench = window.__editBench;
   if (bench.marker) return;
   const el = Object.assign(document.createElement("div"), { id: "edit-bench-marker" });
   el.style.cssText =
-    "position:fixed;left:0;top:0;width:4px;height:4px;z-index:2147483647;pointer-events:none";
+    "position:fixed;left:0;top:0;width:16px;height:16px;z-index:2147483647;pointer-events:none";
   document.documentElement.append(el);
   const marker = { el, n: 0, on: true, masks: [], times: [], inputs: [], unlisten: [] };
   bench.marker = marker;
@@ -38,7 +39,7 @@ function markerOn() {
     if (!marker.on) return;
     const n = ++marker.n;
     marker.times[n] = performance.now();
-    el.style.background = `rgb(${n & 255},${(n >> 8) & 255},128)`;
+    el.style.background = `rgb(${(n % 32) * 8},${((n >> 5) % 32) * 8},128)`;
     // Toasts slide in and out by design; wherever one was during the window is left out of the comparison.
     for (const t of document.querySelectorAll(".hf-toast-enter, .hf-toast-exit")) {
       const r = t.getBoundingClientRect();
@@ -106,7 +107,8 @@ export async function startCapture(page, { marker = true } = {}) {
       const m = window.__editBench.marker;
       return m ? { n: m.n, masks: m.masks, times: m.times, inputs: m.inputs } : { n: null };
     });
-  await cdp.send("Page.startScreencast", { format: "png", everyNthFrame: 1 });
+  // JPEG: PNG encoding cannot keep up with 60 fps even on a blank page; the colour tolerance absorbs JPEG noise.
+  await cdp.send("Page.startScreencast", { format: "jpeg", quality: 95, everyNthFrame: 1 });
   // The first frame is the state before the action.
   for (const deadline = Date.now() + 2000; !frames.length && Date.now() < deadline; )
     await new Promise((r) => setTimeout(r, 10));
@@ -133,7 +135,7 @@ function compareFrames(frames, before, after, regions, masks, colourTol) {
   return (async () => {
     const decode = async (b64) => {
       const bmp = await createImageBitmap(
-        await (await fetch(`data:image/png;base64,${b64}`)).blob(),
+        await (await fetch(`data:image/jpeg;base64,${b64}`)).blob(),
       );
       const g = new OffscreenCanvas(bmp.width, bmp.height).getContext("2d", {
         willReadFrequently: true,
@@ -162,7 +164,9 @@ function compareFrames(frames, before, after, regions, masks, colourTol) {
     const out = [];
     for (const f of frames) {
       const img = await decode(f);
-      const counter = img.data[4 * (img.width + 1)] + 256 * img.data[4 * (img.width + 1) + 1];
+      const px = 4 * (img.width * 8 + 8);
+      const level = (v) => Math.min(31, Math.round(v / 8));
+      const counter = level(img.data[px]) + 32 * level(img.data[px + 1]);
       const diffs = Object.fromEntries(
         Object.entries(regions).map(([k, r]) => [k, [differing(img, b, r), differing(img, a, r)]]),
       );
@@ -172,6 +176,19 @@ function compareFrames(frames, before, after, regions, masks, colourTol) {
   })();
 }
 
+const MARKER_STATES = 1024;
+
+/** The marker's counter wraps every 1024 frames; frames arrive in order, so each one unwraps from the one before. */
+export function unwrap(rows, from) {
+  let last = from;
+  return rows.map((r) => {
+    last +=
+      ((((r.counter - last) % MARKER_STATES) + MARKER_STATES * 1.5) % MARKER_STATES) -
+      MARKER_STATES / 2;
+    return { ...r, counter: last };
+  });
+}
+
 /** Marker counters the screencast never delivered, as [first, last] runs relative to the window start. */
 // fallow-ignore-next-line complexity
 function gaps(seen, from, to) {
@@ -252,22 +269,24 @@ async function scoreWindow(decoder, win, regions, tolPx) {
     dedupe(win.masks ?? []),
     COLOUR_TOL,
   );
-  return classify(rows, win, tolPx);
+  return classify(win.from === null ? rows : unwrap(rows, win.from), win, tolPx);
 }
 
 const dedupe = (rects) =>
   [...new Set(rects.map((r) => r.join(",")))].map((k) => k.split(",").map(Number));
 
-// PNG is lossless; this absorbs only anti-aliasing and subpixel text noise between otherwise equal frames.
+// Absorbs JPEG noise, anti-aliasing and subpixel text between otherwise equal frames.
 const COLOUR_TOL = 24;
+// A window is covered at 90% of its frames, or within 5 points of what the same run's blank control reached.
 const MIN_COVERAGE = 0.9;
+const CONTROL_SLACK = 0.05;
 
 // The inputs that commit an edit; the reload window has none and scores flashes only.
 const COMMITTING = ["release", "undo", "redo"];
 
 /** Scores every window of a case; the offending frames go to `evidence.flashFrames` as PNG. */
 // fallow-ignore-next-line complexity
-export async function scoreFlash(decoder, { regions, tolPx, windows }, evidence) {
+export async function scoreFlash(decoder, { regions, tolPx, windows, control }, evidence) {
   const out = {};
   evidence.flashFrames = [];
   for (const [name, win] of Object.entries(windows)) {
@@ -281,12 +300,16 @@ export async function scoreFlash(decoder, { regions, tolPx, windows }, evidence)
   }
   const all = Object.values(out);
   const coverage = Math.min(...all.map((w) => w.coverage ?? 0));
+  const controlCoverage = (await scoreWindow(decoder, control, {}, tolPx)).coverage ?? 0;
   const paints = COMMITTING.filter((k) => out[k]).map((k) => out[k].paint);
   return {
     bad: all.reduce((n, w) => n + w.bad.length, 0),
     longest: Math.max(0, ...all.map((w) => w.longest)),
     coverage,
-    uncovered: !(coverage >= MIN_COVERAGE),
+    controlCoverage,
+    // The chance a 1-frame flash fell in a frame the screencast never sent.
+    missChance: 1 - coverage,
+    uncovered: !(coverage >= Math.min(MIN_COVERAGE, controlCoverage - CONTROL_SLACK)),
     // The slowest committing input; unknown (null) when any of them logged no input.
     paint: paints.some((p) => !p)
       ? null
diff --git a/packages/studio/tests/e2e/edit-accuracy/flash.test.mjs b/packages/studio/tests/e2e/edit-accuracy/flash.test.mjs
index c5190e0e55..1ed213e459 100644
--- a/packages/studio/tests/e2e/edit-accuracy/flash.test.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/flash.test.mjs
@@ -1,5 +1,5 @@
 import { describe, expect, it } from "vitest";
-import { classify } from "./flash.mjs";
+import { classify, unwrap } from "./flash.mjs";
 
 const TOL = 10;
 // One row per painted frame: its marker counter and [vsBefore, vsAfter] differing pixels per pane.
@@ -78,3 +78,12 @@ describe("classify", () => {
     expect(w.paint).toBeNull();
   });
 });
+
+describe("unwrap", () => {
+  it("restores counters past the marker's 1024 states, including a frame from just before the window", () => {
+    const codes = [1028 % 1024, 1022, 1030 % 1024, 1034 % 1024, 1100 % 1024].map((counter) => ({
+      counter,
+    }));
+    expect(unwrap(codes, 1030).map((r) => r.counter)).toEqual([1028, 1022, 1030, 1034, 1100]);
+  });
+});
diff --git a/packages/studio/tests/e2e/edit-accuracy/report.mjs b/packages/studio/tests/e2e/edit-accuracy/report.mjs
index c58d4d048c..cad84ad360 100644
--- a/packages/studio/tests/e2e/edit-accuracy/report.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/report.mjs
@@ -145,6 +145,12 @@ function summarize(results, seconds) {
         ),
       ),
       lowest: round(Math.min(...measured.map((r) => r.flash.coverage))),
+      control: round(
+        percentile(
+          measured.map((r) => r.flash.controlCoverage),
+          50,
+        ),
+      ),
     },
     paint: {
       unknown: measured.filter((r) => !r.flash.paint).length,
@@ -173,7 +179,7 @@ function table(summary, meta, results) {
     `The preview never held still for 1 s within 15 s in ${summary.unsettled} cases; the metrics that snapshot feeds fail.`,
     `An undo or redo write never landed within 15 s in ${summary.undoTimeouts} cases; undo fails there.`,
     `The producer failed to render ${summary.renderErrors} cases; render fails there.`,
-    `Flash: ${summary.flash.badCases} cases painted a frame matching neither the state before nor after (longest run ${summary.flash.longest} frames); ${summary.flash.uncovered} cases uncovered; screencast coverage median ${summary.flash.coverage}, lowest ${summary.flash.lowest}.`,
+    `Flash: ${summary.flash.badCases} cases painted a frame matching neither the state before nor after (longest run ${summary.flash.longest} frames); ${summary.flash.uncovered} cases uncovered; screencast coverage median ${summary.flash.coverage}, lowest ${summary.flash.lowest}, blank control median ${summary.flash.control}.`,
     `Edit-to-paint (pointer-up, key, undo, redo; pass at ${PAINT_FRAMES} frame): frames to the after-state (median/max) ${summary.paint.frames}, ms ${summary.paint.ms}; ${summary.paint.unknown} cases logged no input.`,
     `Smoothness: ${summary.smooth.unknown} cases with unknown work; dropped frames per case (median/max) ${summary.smooth.dropped}, blank-page control ${summary.smooth.control}; raw rAF p95 (median/max) ${summary.smooth.p95} ms, control ${summary.smooth.controlP95} ms.`,
     "",
@@ -190,6 +196,17 @@ function table(summary, meta, results) {
       `| ${g} | ${rs.length} | ${rs.filter((r) => r.pass).length} | ${METRICS.map((m) => rs.filter((r) => r.checks[m]).length).join(" | ")} |`,
     );
   }
+  lines.push(
+    "",
+    "| Case | Flash frames | Paint frames | 1-frame flash miss chance | Blank control coverage |",
+    "|---|---|---|---|---|",
+    ...results
+      .filter((r) => !r.error)
+      .map(
+        (r) =>
+          `| ${r.id} | ${r.flash.bad} | ${r.flash.paint?.frames ?? "-"} | ${round(r.flash.missChance)} | ${round(r.flash.controlCoverage)} |`,
+      ),
+  );
   return lines.join("\n") + "\n";
 }
 

From cca3bb76dbbca2439f5277fcbd91ed6ee073665e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miguel=20=C3=81ngel?= <miguel.sierra@heygen.com>
Date: Wed, 30 Sep 2026 18:54:23 -0400
Subject: [PATCH 09/12] test(studio): fail edit-to-paint when the answering
 frame comes later than a dropped frame

A pointer-up whose next frame arrived 507 ms later passed on frame count alone.
---
 packages/studio/tests/e2e/edit-accuracy/flash.mjs  |  7 +++++--
 .../studio/tests/e2e/edit-accuracy/ratchet.mjs     |  4 ++--
 .../tests/e2e/edit-accuracy/ratchet.test.mjs       |  7 ++++++-
 packages/studio/tests/e2e/edit-accuracy/report.mjs | 14 +++++++++-----
 4 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/packages/studio/tests/e2e/edit-accuracy/flash.mjs b/packages/studio/tests/e2e/edit-accuracy/flash.mjs
index a3653cef9d..14af98cc0e 100644
--- a/packages/studio/tests/e2e/edit-accuracy/flash.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/flash.mjs
@@ -310,10 +310,13 @@ export async function scoreFlash(decoder, { regions, tolPx, windows, control },
     // The chance a 1-frame flash fell in a frame the screencast never sent.
     missChance: 1 - coverage,
     uncovered: !(coverage >= Math.min(MIN_COVERAGE, controlCoverage - CONTROL_SLACK)),
-    // The slowest committing input; unknown (null) when any of them logged no input.
+    // The slowest committing input, in frames and in ms; unknown (null) when any of them logged no input.
     paint: paints.some((p) => !p)
       ? null
-      : paints.reduce((a, p) => (p.frames > a.frames ? p : a), { frames: 0, ms: 0 }),
+      : {
+          frames: Math.max(...paints.map((p) => p.frames)),
+          ms: paints.some((p) => p.ms === null) ? null : Math.max(...paints.map((p) => p.ms)),
+        },
     regions,
     tolPx,
     windows: out,
diff --git a/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs b/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs
index b1da73ac2c..a31230e3ea 100644
--- a/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs
@@ -5,7 +5,7 @@
 import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { fileURLToPath } from "node:url";
-import { LIMIT_PX, PAINT_FRAMES, entry, writeReport } from "./report.mjs";
+import { LIMIT_PX, entry, paintOk, writeReport } from "./report.mjs";
 
 const GATED_PX = ["tracking", "pressJump", "drop", "reload", "render"];
 const LISTED = 30;
@@ -23,7 +23,7 @@ export const accurate = (e, like = e) =>
   e.undo === true &&
   GATED_PX.every((m) => !(e[m] > LIMIT_PX)) &&
   (!like || !("flash" in like) || (e.flash === 0 && !e.flashUncovered)) &&
-  (!like || !("paint" in like) || (e.paint !== null && e.paint <= PAINT_FRAMES));
+  (!like || !("paint" in like) || paintOk(e.paint, e.paintMs));
 
 /** Cases whose verdict here differs from the base branch, either way: each is re-run twice before the gate. */
 export const flipped = (base, results) =>
diff --git a/packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs b/packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs
index 9f94999453..77831efb23 100644
--- a/packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs
@@ -12,6 +12,7 @@ const good = {
   controlDropped: 0,
   flash: 0,
   paint: 1,
+  paintMs: 5,
   work: 3,
   frameP95: 20,
 };
@@ -47,6 +48,7 @@ describe("accurate", () => {
     expect(accurate({ ...good, flash: 0, flashUncovered: true })).toBe(false);
     expect(accurate({ ...good, paint: 2 })).toBe(false);
     expect(accurate({ ...good, paint: null })).toBe(false);
+    expect(accurate({ ...good, paintMs: 507 })).toBe(false);
     expect(accurate({ ...good, undo: false })).toBe(false);
     expect(accurate({ pass: false, error: true })).toBe(false);
     expect(accurate(undefined)).toBe(false);
@@ -62,7 +64,10 @@ describe("a metric the base branch never measured", () => {
   });
 
   it("cannot regress a case or lower the count, and still needs banking to pass", () => {
-    const flashing = { ...run("a"), flash: { bad: 3, uncovered: false, paint: { frames: 4 } } };
+    const flashing = {
+      ...run("a"),
+      flash: { bad: 3, uncovered: false, paint: { frames: 4, ms: 60 } },
+    };
     const g = gate(baseline({ a: older }), baseline({ a: { ...good, flash: 3 } }), [flashing]);
     expect(g.regressed).toEqual([]);
     expect(g.headPassing).toBe(1);
diff --git a/packages/studio/tests/e2e/edit-accuracy/report.mjs b/packages/studio/tests/e2e/edit-accuracy/report.mjs
index cad84ad360..f85d3dd033 100644
--- a/packages/studio/tests/e2e/edit-accuracy/report.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/report.mjs
@@ -18,8 +18,11 @@ export const METRICS = [
   "paint",
   "smooth",
 ];
-// Edit-to-paint: the first frame painted after a committing input already shows the after-state.
-export const PAINT_FRAMES = 1;
+// Edit-to-paint: the first frame painted after a committing input already shows the after-state, and that frame
+// comes no later than a frame counts as dropped.
+const PAINT_FRAMES = 1;
+export const paintOk = (frames, ms) =>
+  frames != null && frames <= PAINT_FRAMES && ms != null && ms <= DROPPED_FRAME_MS;
 
 /** Worst-first value per metric; undo ranks by box distance, and its byte failures are counted apart. */
 const worstValue = {
@@ -70,7 +73,7 @@ export function score(spec, r) {
     undo: r.undo.bytes && r.undo.redoBytes && Math.max(r.undo.box, r.undo.redoBox) <= LIMIT_PX,
     // A window the screencast covered under 90% is uncovered, never a pass.
     flash: !r.flash.uncovered && r.flash.bad === 0,
-    paint: !r.flash.uncovered && r.flash.paint !== null && r.flash.paint.frames <= PAINT_FRAMES,
+    paint: !r.flash.uncovered && paintOk(r.flash.paint?.frames, r.flash.paint?.ms),
     // Only drops beyond the blank page's, driven the same way in the same Chrome, are the edit's.
     smooth:
       smooth.dropped <= smooth.control.dropped &&
@@ -180,7 +183,7 @@ function table(summary, meta, results) {
     `An undo or redo write never landed within 15 s in ${summary.undoTimeouts} cases; undo fails there.`,
     `The producer failed to render ${summary.renderErrors} cases; render fails there.`,
     `Flash: ${summary.flash.badCases} cases painted a frame matching neither the state before nor after (longest run ${summary.flash.longest} frames); ${summary.flash.uncovered} cases uncovered; screencast coverage median ${summary.flash.coverage}, lowest ${summary.flash.lowest}, blank control median ${summary.flash.control}.`,
-    `Edit-to-paint (pointer-up, key, undo, redo; pass at ${PAINT_FRAMES} frame): frames to the after-state (median/max) ${summary.paint.frames}, ms ${summary.paint.ms}; ${summary.paint.unknown} cases logged no input.`,
+    `Edit-to-paint (pointer-up, key, undo, redo; pass at ${PAINT_FRAMES} frame within ${DROPPED_FRAME_MS} ms): frames to the after-state (median/max) ${summary.paint.frames}, ms ${summary.paint.ms}; ${summary.paint.unknown} cases logged no input.`,
     `Smoothness: ${summary.smooth.unknown} cases with unknown work; dropped frames per case (median/max) ${summary.smooth.dropped}, blank-page control ${summary.smooth.control}; raw rAF p95 (median/max) ${summary.smooth.p95} ms, control ${summary.smooth.controlP95} ms.`,
     "",
     "| Metric | Pass | Worst | Worst case |",
@@ -204,7 +207,7 @@ function table(summary, meta, results) {
       .filter((r) => !r.error)
       .map(
         (r) =>
-          `| ${r.id} | ${r.flash.bad} | ${r.flash.paint?.frames ?? "-"} | ${round(r.flash.missChance)} | ${round(r.flash.controlCoverage)} |`,
+          `| ${r.id} | ${r.flash.bad} | ${r.flash.paint ? `${r.flash.paint.frames} (${round(r.flash.paint.ms)} ms)` : "-"} | ${round(r.flash.missChance)} | ${round(r.flash.controlCoverage)} |`,
       ),
   );
   return lines.join("\n") + "\n";
@@ -232,6 +235,7 @@ export function entry(r) {
     flash: r.flash.bad,
     ...(r.flash.uncovered && { flashUncovered: true }),
     paint: r.flash.paint?.frames ?? null,
+    paintMs: roundUp(r.flash.paint?.ms ?? null),
   };
 }
 

From 7495af85381af6077a09ae91a472ce36ac0058fc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miguel=20=C3=81ngel?= <miguel.sierra@heygen.com>
Date: Wed, 30 Sep 2026 18:59:36 -0400
Subject: [PATCH 10/12] test(studio): reach the preview iframe inside the
 player shadow root in flash capture

---
 packages/studio/tests/e2e/edit-accuracy/case.mjs | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/packages/studio/tests/e2e/edit-accuracy/case.mjs b/packages/studio/tests/e2e/edit-accuracy/case.mjs
index 2f8b50a56b..82f6aea17f 100644
--- a/packages/studio/tests/e2e/edit-accuracy/case.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/case.mjs
@@ -648,8 +648,15 @@ function injectControl(page) {
   if (CONTROL === "reload")
     return page.evaluate(async () => {
       await new Promise((r) => setTimeout(r, 100));
+      const url = (f) => {
+        try {
+          return f.contentWindow.location.href;
+        } catch {
+          return "";
+        }
+      };
       const frames = [...document.querySelectorAll("iframe")].filter((f) =>
-        f.src.includes("/preview"),
+        url(f).includes("/preview"),
       );
       const area = (f) => f.offsetWidth * f.offsetHeight;
       frames.reduce((a, b) => (area(b) > area(a) ? b : a)).contentWindow.location.reload();

From 07b30439197fee54a54d5dc523638f8ac36bb91d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miguel=20=C3=81ngel?= <miguel.sierra@heygen.com>
Date: Wed, 30 Sep 2026 19:34:20 -0400
Subject: [PATCH 11/12] test(studio): run the edit accuracy Chrome with
 software compositing so flash capture keeps up

Under swiftshader GL the screencast saw 40-70% of Studio's frames; software compositing sees ~95%.
---
 packages/studio/tests/e2e/edit-accuracy/run.mjs | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/packages/studio/tests/e2e/edit-accuracy/run.mjs b/packages/studio/tests/e2e/edit-accuracy/run.mjs
index 73cb4e6a7d..689377e59b 100644
--- a/packages/studio/tests/e2e/edit-accuracy/run.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/run.mjs
@@ -207,7 +207,14 @@ const browsers = await Promise.all(
     puppeteer.launch({
       executablePath: chrome,
       headless: true,
-      args: ["--no-sandbox", "--use-angle=swiftshader", "--enable-unsafe-swiftshader"],
+      // Software compositing: the screencast reads each frame back cheaply, so flash windows see ~95% of frames
+      // instead of 40-70% under swiftshader GL.
+      args: [
+        "--no-sandbox",
+        "--use-angle=swiftshader",
+        "--enable-unsafe-swiftshader",
+        "--disable-gpu-compositing",
+      ],
     }),
   ),
 );

From 2e83db5a4ba62deb7ec9cc977b790c1d49914ba7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Miguel=20=C3=81ngel?= <miguel.sierra@heygen.com>
Date: Wed, 30 Sep 2026 19:54:28 -0400
Subject: [PATCH 12/12] test(studio): walk open shadow roots to the preview
 iframe in the flash controls and input log

The previous commit only switched the frame check to the live URL; the iframe walk did not land.
---
 packages/studio/tests/e2e/edit-accuracy/case.mjs  | 8 +++++---
 packages/studio/tests/e2e/edit-accuracy/flash.mjs | 7 ++++++-
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/packages/studio/tests/e2e/edit-accuracy/case.mjs b/packages/studio/tests/e2e/edit-accuracy/case.mjs
index 82f6aea17f..1ac22fb666 100644
--- a/packages/studio/tests/e2e/edit-accuracy/case.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/case.mjs
@@ -655,9 +655,11 @@ function injectControl(page) {
           return "";
         }
       };
-      const frames = [...document.querySelectorAll("iframe")].filter((f) =>
-        url(f).includes("/preview"),
-      );
+      const iframes = (root) =>
+        [...root.querySelectorAll("*")].flatMap((el) =>
+          el.tagName === "IFRAME" ? [el] : el.shadowRoot ? iframes(el.shadowRoot) : [],
+        );
+      const frames = iframes(document).filter((f) => url(f).includes("/preview"));
       const area = (f) => f.offsetWidth * f.offsetHeight;
       frames.reduce((a, b) => (area(b) > area(a) ? b : a)).contentWindow.location.reload();
     });
diff --git a/packages/studio/tests/e2e/edit-accuracy/flash.mjs b/packages/studio/tests/e2e/edit-accuracy/flash.mjs
index 14af98cc0e..48b898a184 100644
--- a/packages/studio/tests/e2e/edit-accuracy/flash.mjs
+++ b/packages/studio/tests/e2e/edit-accuracy/flash.mjs
@@ -21,12 +21,17 @@ function markerOn() {
     if (e.type === "pointerup" || !MODIFIERS.has(e.key))
       marker.inputs.push({ type: e.type, key: e.key ?? null, n: marker.n, t: performance.now() });
   };
+  // Studio's preview iframe sits in the player's shadow root, out of querySelectorAll's reach.
+  const iframes = (root) =>
+    [...root.querySelectorAll("*")].flatMap((el) =>
+      el.tagName === "IFRAME" ? [el] : el.shadowRoot ? iframes(el.shadowRoot) : [],
+    );
   const listen = (w) => {
     for (const type of ["pointerup", "keydown"]) {
       w.addEventListener(type, log, true);
       marker.unlisten.push(() => w.removeEventListener(type, log, true));
     }
-    for (const f of w.document.querySelectorAll("iframe")) {
+    for (const f of iframes(w.document)) {
       try {
         listen(f.contentWindow);
       } catch {