diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 979886a970..3ceeec73c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,6 +74,7 @@ jobs: codex_plugin: ${{ steps.filter.outputs.codex_plugin }} gcp_beginframe: ${{ steps.filter.outputs.gcp_beginframe }} studio: ${{ steps.filter.outputs.studio }} + edit_accuracy: ${{ steps.filter.outputs.edit_accuracy }} steps: # Force git-based change detection instead of the pull_request REST API. # The API path can fail the whole workflow on transient listFiles @@ -158,6 +159,19 @@ jobs: - "packages/studio-server/**" - "bun.lock" - ".github/workflows/ci.yml" + edit_accuracy: + - "packages/studio/src/components/editor/**" + - "packages/studio/src/components/nle/**" + - "packages/studio/src/hooks/**" + - "packages/studio/src/utils/**" + - "packages/studio/src/player/**" + - "packages/studio/src/webmcp/**" + - "packages/studio-server/src/**" + - "packages/core/src/**" + - "packages/cli/src/server/**" + - "packages/studio/tests/e2e/edit-accuracy/**" + - "bun.lock" + - ".github/workflows/ci.yml" gcp_beginframe: - "packages/gcp-cloud-run/Dockerfile" - "packages/aws-lambda/scripts/probe-beginframe.ts" @@ -216,6 +230,14 @@ jobs: - run: bash scripts/ci/install-workspace-dependencies.sh - run: bun run build - run: bun run verify:packed-manifests + - name: Share the built CLI with the edit accuracy shards + if: github.event_name == 'pull_request' && needs.changes.outputs.edit_accuracy == 'true' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: edit-accuracy-cli + path: packages/cli/dist + if-no-files-found: error + retention-days: 1 lint: name: Lint @@ -924,6 +946,115 @@ jobs: /tmp/studio-open-counts.json if-no-files-found: error + # Manual editing accuracy in the built Studio, gated against the base branch's baseline.json. + studio-edit-accuracy: + name: "Studio: edit accuracy (${{ matrix.shard }}/8)" + needs: [changes, build] + if: github.event_name == 'pull_request' && needs.changes.outputs.edit_accuracy == 'true' + runs-on: ubuntu-latest + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3, 4, 5, 6, 7, 8] + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 22 + - uses: ./.github/actions/prepare-ffmpeg-bin + - run: bash scripts/ci/install-workspace-dependencies.sh + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: edit-accuracy-cli + path: packages/cli/dist + # Studio and the producer both run in the engine's pinned headless shell. + - run: node packages/cli/dist/cli.js browser ensure + - name: Read the base branch's edit accuracy baseline + env: + BASE_REF: ${{ github.base_ref }} + run: | + git fetch --no-tags --depth=1 origin "$BASE_REF" + if ! git show FETCH_HEAD:packages/studio/tests/e2e/edit-accuracy/baseline.json > /tmp/base-edit-accuracy.json; then + echo '{"cases":{}}' > /tmp/base-edit-accuracy.json + echo "The base branch has no edit accuracy baseline yet" + fi + - name: Run this shard, then re-run twice each case whose verdict differs from the base branch + run: | + set -euo pipefail + bench() { bun run --cwd packages/studio test:edit-accuracy -- --grid full --jobs 2 "$@"; } + bench --shard "${{ matrix.shard }}/8" --out /tmp/edit-accuracy/run1 + mapfile -t FLIPPED < <(node packages/studio/tests/e2e/edit-accuracy/ratchet.mjs flipped \ + /tmp/base-edit-accuracy.json /tmp/edit-accuracy/run1/results.json) + if (( ${#FLIPPED[@]} > 0 )); then + FILTER="^($(IFS='|'; echo "${FLIPPED[*]}"))$" + for run in 2 3; do bench --filter "$FILTER" --rerun --out "/tmp/edit-accuracy/run${run}"; done + fi + - name: Upload shard evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: edit-accuracy-shard-${{ matrix.shard }} + path: /tmp/edit-accuracy + if-no-files-found: error + + studio-edit-accuracy-gate: + name: "Studio: edit accuracy gate" + needs: [changes, studio-edit-accuracy] + if: ${{ !cancelled() && github.event_name == 'pull_request' && needs.changes.outputs.edit_accuracy == 'true' }} + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 22 + - name: Read the base branch's edit accuracy baseline + env: + BASE_REF: ${{ github.base_ref }} + run: | + git fetch --no-tags --depth=1 origin "$BASE_REF" + if ! git show FETCH_HEAD:packages/studio/tests/e2e/edit-accuracy/baseline.json > /tmp/base-edit-accuracy.json; then + echo '{"cases":{}}' > /tmp/base-edit-accuracy.json + echo "The base branch has no edit accuracy baseline yet" + fi + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: edit-accuracy-shard-* + path: /tmp/edit-accuracy-shards + - name: Compare every run with the base branch + id: gate + # Smoothness is reported, not gated; a case fails only if it fails 2 of its 3 runs. + run: | + set +e + mapfile -t RESULTS < <(find /tmp/edit-accuracy-shards -name results.json | sort) + node packages/studio/tests/e2e/edit-accuracy/ratchet.mjs gate /tmp/base-edit-accuracy.json \ + packages/studio/tests/e2e/edit-accuracy/baseline.json /tmp/edit-accuracy-gate "${RESULTS[@]}" + echo "exit_code=$?" >> "$GITHUB_OUTPUT" + cat /tmp/edit-accuracy-gate/comment.md >> "$GITHUB_STEP_SUMMARY" || true + - name: Post sticky comment + # Fork PRs get a read-only token; the gate below still fires. + continue-on-error: true + uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2.9.4 + with: + header: edit-accuracy + path: /tmp/edit-accuracy-gate/comment.md + - name: Upload the gate report and a baseline.json to bank + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: edit-accuracy-gate + path: /tmp/edit-accuracy-gate + - name: Fail if the gate failed + if: steps.gate.outputs.exit_code != '0' + run: | + echo "::error::Edit accuracy fell against the base branch; see the PR comment." + exit 1 + smoke-global-install: name: "Smoke: global install" needs: [changes, build] diff --git a/packages/studio/tests/e2e/edit-accuracy/case.mjs b/packages/studio/tests/e2e/edit-accuracy/case.mjs index 3a62db0323..1ac22fb666 100644 --- a/packages/studio/tests/e2e/edit-accuracy/case.mjs +++ b/packages/studio/tests/e2e/edit-accuracy/case.mjs @@ -3,6 +3,7 @@ import { spawn } from "node:child_process"; import { readFileSync } from "node:fs"; import { join } from "node:path"; import { COMPOSITION, PLAYHEAD } from "./grid.mjs"; +import { panes, startCapture } from "./flash.mjs"; import { angleOf, centre, @@ -27,6 +28,10 @@ const ROTATE_BY = (25 * Math.PI) / 180; const CROP_BY = 40; const NUDGES = 5; const ZOOM_SENSITIVITY = 0.007; // previewZoom.ts: one wheel unit scales zoom by exp(0.007) +// Flash controls, for proving the metric only: tap (release without a move), blinkN (hide the preview for N frames +// after the release), reload (reload the preview frame mid-settle). EDIT_BENCH_MARKER=0 captures without the marker. +const CONTROL = process.env.EDIT_BENCH_CONTROL; +const CAPTURE = { marker: process.env.EDIT_BENCH_MARKER !== "0" }; const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); @@ -242,7 +247,7 @@ async function settled(ctx, timeout = 15_000) { /** Ready once Studio's own seek tool reports the composition and the playhead landed. */ // fallow-ignore-next-line complexity -async function openStudio(ctx) { +async function openStudio(ctx, onReady) { ctx.handles = null; await ctx.page.waitForFunction(() => window.__editBench?.has("studio_seek"), { timeout: 90_000 }); let seek = null; @@ -255,6 +260,7 @@ async function openStudio(ctx) { seek = null; } if (!seek) throw new Error("studio never reported a seekable composition"); + await onReady?.(); await sleep(1000); return settled(ctx); } @@ -508,7 +514,11 @@ function smoothness(rec) { const CONTROL_PAGE = `data:text/html,
`; -/** The case's drag schedule and per-frame reads on a blank page in the same Chrome: the machine's own frame drops. */ +/** + * The case's drag schedule and per-frame reads on a blank page in the same Chrome: the machine's own frame drops. + * Then 1 s of screencast there: the capture coverage the machine reaches right now. + */ +// fallow-ignore-next-line complexity async function controlDrag(browser, gesture) { const context = await browser.createBrowserContext(); try { @@ -519,33 +529,45 @@ async function controlDrag(browser, gesture) { const box = await page.$("#box"); const ctx = { page, handles: { target: box, root: box } }; const read = () => readQuads(ctx); + let smooth; if (gesture === "nudge") { await recording(page, true); - for (let i = 0; i < NUDGES; i++) { + for (let i = 1; i < NUDGES; i++) { await page.keyboard.press("ArrowRight"); await nextFrame(page); } await nextFrame(page, 2); - return smoothness(await recording(page, false)); - } - await page.mouse.move(700, 380); - await page.mouse.down(); - await nextFrame(page); - await read(); - await recording(page, true); - for (let i = 1; i <= STEPS; i++) { - await page.mouse.move(700 + (MOVE_BY[0] * i) / STEPS, 380 + (MOVE_BY[1] * i) / STEPS); + smooth = smoothness(await recording(page, false)); + } else { + await page.mouse.move(700, 380); + await page.mouse.down(); await nextFrame(page); await read(); + await recording(page, true); + for (let i = 1; i <= STEPS; i++) { + await page.mouse.move(700 + (MOVE_BY[0] * i) / STEPS, 380 + (MOVE_BY[1] * i) / STEPS); + await nextFrame(page); + await read(); + } + smooth = smoothness(await recording(page, false)); + await page.mouse.up(); } - const smooth = smoothness(await recording(page, false)); - await page.mouse.up(); - return smooth; + const capture = await startCapture(page, CAPTURE); + await sleep(1000); + return { smooth, capture: await capture.stop() }; } finally { await context.close().catch(() => undefined); } } +/** The element's perimeter in screen px: how many pixels a 0.5 px shift of its edges touches. */ +const perimeterPx = (m) => + Math.ceil( + m.visible.map(m.map.toScreen).reduce((sum, p, i, q) => sum + dist(p, q[(i + 1) % 4]), 0), + ); + +/** Drives the drag and returns with the button still down; `release` (pointer-up) runs in the flash window. */ +// fallow-ignore-next-line complexity async function pointerGesture(ctx, gesture, pre) { const press = await handlePoint(ctx, pre, gesture); const pressComp = pre.map.toComp(press); @@ -563,7 +585,7 @@ async function pointerGesture(ctx, gesture, pre) { await recording(ctx.page, true); const errors = []; let last = s0; - for (const p of g.path) { + for (const p of CONTROL === "tap" ? g.path.map(() => press) : g.path) { await ctx.page.mouse.move(p[0], p[1]); await nextFrame(ctx.page); last = await sample(ctx, gesture, g.point, p); @@ -571,11 +593,12 @@ async function pointerGesture(ctx, gesture, pre) { } const rec = await recording(ctx.page, false); const smooth = smoothness(rec); - await ctx.page.mouse.up(); const lastQuad = gesture === "crop" ? last.m.outline : last.m.visible; return { - errors, - lastQuad, + release: async () => { + await ctx.page.mouse.up(); + return { errors, lastQuad }; + }, pressJump: quadDistance(s0.m.visible, pre.visible), smooth, diag: { @@ -586,29 +609,66 @@ async function pointerGesture(ctx, gesture, pre) { }; } +/** Nudges all but the last step traced; `release` presses the last one in the flash window, so a key commits there. */ async function nudgeGesture(ctx, pre) { await recording(ctx.page, true); - for (let i = 0; i < NUDGES; i++) { + for (let i = 1; i < NUDGES; i++) { await ctx.page.keyboard.press("ArrowRight"); await nextFrame(ctx.page); } await nextFrame(ctx.page, 2); const smooth = smoothness(await recording(ctx.page, false)); - const m = await measure(ctx); - const [a, b] = [centre(pre.visible), centre(m.visible)]; return { - errors: [dist([b[0] - a[0], b[1] - a[1]], [NUDGES, 0])], - lastQuad: m.visible, + release: async () => { + await ctx.page.keyboard.press("ArrowRight"); + await nextFrame(ctx.page, 2); + const m = await measure(ctx); + const [a, b] = [centre(pre.visible), centre(m.visible)]; + return { errors: [dist([b[0] - a[0], b[1] - a[1]], [NUDGES, 0])], lastQuad: m.visible }; + }, pressJump: null, smooth, diag: {}, }; } +/** The positive flash controls, started right after the committing input. */ +function injectControl(page) { + const blink = /^blink(\d+)$/.exec(CONTROL ?? ""); + if (blink) + return page.evaluate((frames) => { + const s = document.querySelector('[data-testid="preview-zoom-stage"]').style; + const hide = (left) => + requestAnimationFrame(() => { + s.opacity = left ? "0" : ""; + if (left) hide(left - 1); + }); + hide(frames); + }, Number(blink[1])); + if (CONTROL === "reload") + return page.evaluate(async () => { + await new Promise((r) => setTimeout(r, 100)); + const url = (f) => { + try { + return f.contentWindow.location.href; + } catch { + return ""; + } + }; + const iframes = (root) => + [...root.querySelectorAll("*")].flatMap((el) => + el.tagName === "IFRAME" ? [el] : el.shadowRoot ? iframes(el.shadowRoot) : [], + ); + const frames = iframes(document).filter((f) => url(f).includes("/preview")); + const area = (f) => f.offsetWidth * f.offsetHeight; + frames.reduce((a, b) => (area(b) > area(a) ? b : a)).contentWindow.location.reload(); + }); +} + /** One case, end to end, in a fresh browser context against a Studio already serving `dir`. */ // fallow-ignore-next-line complexity export async function runCase({ browser, spec, dir, files, url, evidence }) { - const control = await controlDrag(browser, spec.gesture); + const { smooth: control, capture: controlCapture } = await controlDrag(browser, spec.gesture); const context = await browser.createBrowserContext(); const page = await context.newPage(); const ctx = { page, dir, files, handles: null }; @@ -634,8 +694,19 @@ export async function runCase({ browser, spec, dir, files, url, evidence }) { spec.gesture === "nudge" ? await nudgeGesture(ctx, pre) : await pointerGesture(ctx, spec.gesture, pre); + // Flash windows: each action to its settle is screencast, untraced; the drag's trace has already stopped. + const flash = { + regions: await panes(page), + tolPx: perimeterPx(pre), + windows: {}, + control: controlCapture, + }; + let capture = await startCapture(page, CAPTURE); + const { errors, lastQuad } = await drive.release(); + await injectControl(page); await waitForFiles(ctx, { from: original, timeout: spec.gesture === "nudge" ? 6000 : 5000 }); - await nextFrame(page, 2); + await settled(ctx); + flash.windows.release = await capture.stop(); await blurPreview(page); await page.keyboard.press("Escape"); const committed = await settled(ctx); @@ -646,22 +717,32 @@ export async function runCase({ browser, spec, dir, files, url, evidence }) { // Undo and redo run before any reload. Each waits up to 15 s for its own write; redo waits for undo. const landed = (from) => saved ? waitForFiles(ctx, { from, timeout: 15_000 }) : { reached: true, files: from }; + capture = await startCapture(page, CAPTURE); await chord(page, "Control+z"); const undo = await landed(committedFiles); const undone = await settled(ctx); + flash.windows.undo = await capture.stop(); await shoot("undone"); let [redo, redone] = [{ reached: false }, null]; if (undo.reached) { await blurPreview(page); + capture = await startCapture(page, CAPTURE); await chord(page, "Control+Shift+z"); redo = await landed(undo.files); redone = await settled(ctx); + flash.windows.redo = await capture.stop(); } // A late write must not land under the reload. await waitForFiles(ctx, { timeout: 15_000 }); + // The reload window starts once Studio can seek the composition; before that Studio itself is loading. + const lastSettled = (flash.windows.redo ?? flash.windows.undo).frames.at(-1); await page.reload(); - const reloaded = await openStudio(ctx); + const reloaded = await openStudio( + ctx, + async () => (capture = await startCapture(page, CAPTURE)), + ); + flash.windows.reload = { ...(await capture.stop()), before: lastSettled }; await shoot("reloaded"); const quads = Object.fromEntries( Object.entries({ pre, committed, undone, redone, reloaded }).filter(([, m]) => m), @@ -671,12 +752,12 @@ export async function runCase({ browser, spec, dir, files, url, evidence }) { zoom, saved, tracking: { - max: Math.max(...drive.errors), - p95: percentile(drive.errors, 95), - frames: drive.errors.length, + max: Math.max(...errors), + p95: percentile(errors, 95), + frames: errors.length, }, pressJump: drive.pressJump, - drop: quadDistance(drive.lastQuad, committed.visible), + drop: quadDistance(lastQuad, committed.visible), reload: quadDistance(committed.visible, reloaded.visible), undo: { bytes: saved && undo.reached && sameFiles(undo.files, original), @@ -687,6 +768,7 @@ export async function runCase({ browser, spec, dir, files, url, evidence }) { // Which write never landed within 15 s; a redo that was never sent is untested, so undo fails. undoTimeout: saved && !undo.reached ? "undo" : saved && !redo.reached ? "redo" : null, smooth: { ...drive.smooth, control }, + flash, unsettled: Object.keys(quads).filter((k) => quads[k].unsettled), reloaded, diag: { diff --git a/packages/studio/tests/e2e/edit-accuracy/flash.mjs b/packages/studio/tests/e2e/edit-accuracy/flash.mjs new file mode 100644 index 0000000000..48b898a184 --- /dev/null +++ b/packages/studio/tests/e2e/edit-accuracy/flash.mjs @@ -0,0 +1,329 @@ +/** + * Preview flashes and edit-to-paint: every frame the screencast paints, from an action to its settle, against the + * frames around it. + */ + +// A 16x16 marker in the top-left corner, outside every region, repaints every rAF with the frame counter in its +// colour (5 bits per channel in steps of 8, which JPEG keeps), so the screencast (which only sends repainted frames) +// sends every frame and each one is numbered. +// It also logs the counter at each committing input (pointer-up, a non-modifier key), in any same-origin frame. +function markerOn() { + const bench = window.__editBench; + if (bench.marker) return; + const el = Object.assign(document.createElement("div"), { id: "edit-bench-marker" }); + el.style.cssText = + "position:fixed;left:0;top:0;width:16px;height:16px;z-index:2147483647;pointer-events:none"; + document.documentElement.append(el); + const marker = { el, n: 0, on: true, masks: [], times: [], inputs: [], unlisten: [] }; + bench.marker = marker; + const MODIFIERS = new Set(["Control", "Shift", "Alt", "Meta"]); + const log = (e) => { + if (e.type === "pointerup" || !MODIFIERS.has(e.key)) + marker.inputs.push({ type: e.type, key: e.key ?? null, n: marker.n, t: performance.now() }); + }; + // Studio's preview iframe sits in the player's shadow root, out of querySelectorAll's reach. + const iframes = (root) => + [...root.querySelectorAll("*")].flatMap((el) => + el.tagName === "IFRAME" ? [el] : el.shadowRoot ? iframes(el.shadowRoot) : [], + ); + const listen = (w) => { + for (const type of ["pointerup", "keydown"]) { + w.addEventListener(type, log, true); + marker.unlisten.push(() => w.removeEventListener(type, log, true)); + } + for (const f of iframes(w.document)) { + try { + listen(f.contentWindow); + } catch { + // A cross-origin frame cannot take an input Studio handles. + } + } + }; + listen(window); + const tick = () => { + if (!marker.on) return; + const n = ++marker.n; + marker.times[n] = performance.now(); + el.style.background = `rgb(${(n % 32) * 8},${((n >> 5) % 32) * 8},128)`; + // Toasts slide in and out by design; wherever one was during the window is left out of the comparison. + for (const t of document.querySelectorAll(".hf-toast-enter, .hf-toast-exit")) { + const r = t.getBoundingClientRect(); + marker.masks.push([r.left, r.top, r.right, r.bottom].map(Math.round)); + } + requestAnimationFrame(tick); + }; + requestAnimationFrame(tick); +} + +function markerOff() { + const m = window.__editBench.marker; + if (!m) return; + m.on = false; + for (const off of m.unlisten) off(); + m.el.remove(); + window.__editBench.marker = null; +} + +/** The panes a flash can show in, as screen rects: preview (with its chrome), timeline and inspector. */ +function paneRects() { + const part = (el) => el?.closest(".dv-react-part"); + const rect = (el) => { + const r = el.getBoundingClientRect(); + return { + x: Math.round(r.x), + y: Math.round(r.y), + w: Math.round(r.width), + h: Math.round(r.height), + }; + }; + const preview = part(document.querySelector('[data-testid="preview-zoom-stage"]')); + const parts = [...document.querySelectorAll(".dv-react-part")].filter((e) => e.offsetWidth > 0); + const p = preview && rect(preview); + const timeline = + part(document.querySelector('[data-testid="timeline-clip"]')) ?? + parts.reduce((a, b) => (!a || rect(b).y > rect(a).y ? b : a), null); + const area = (e) => rect(e).w * rect(e).h; + const inspector = + p && + parts + .filter((e) => rect(e).x >= p.x + p.w - 1 && rect(e).y < p.y + p.h) + .reduce((a, b) => (!a || area(b) > area(a) ? b : a), null); + return Object.fromEntries( + Object.entries({ preview, timeline, inspector }) + .filter(([, e]) => e) + .map(([k, e]) => [k, rect(e)]), + ); +} + +/** + * Starts a screencast of one action window; stop() returns its PNG frames, the marker range it spans, the rAF time + * of each counter and the committing inputs logged in it. + */ +export async function startCapture(page, { marker = true } = {}) { + const cdp = await page.createCDPSession(); + const frames = []; + cdp.on("Page.screencastFrame", (f) => { + frames.push(f.data); + cdp.send("Page.screencastFrameAck", { sessionId: f.sessionId }).catch(() => undefined); + }); + if (marker) await page.evaluate(markerOn); + const state = () => + page.evaluate(() => { + const m = window.__editBench.marker; + return m ? { n: m.n, masks: m.masks, times: m.times, inputs: m.inputs } : { n: null }; + }); + // JPEG: PNG encoding cannot keep up with 60 fps even on a blank page; the colour tolerance absorbs JPEG noise. + await cdp.send("Page.startScreencast", { format: "jpeg", quality: 95, everyNthFrame: 1 }); + // The first frame is the state before the action. + for (const deadline = Date.now() + 2000; !frames.length && Date.now() < deadline; ) + await new Promise((r) => setTimeout(r, 10)); + const from = (await state()).n; + return { + async stop() { + const { n: to, masks = [], times = [], inputs = [] } = await state(); + await cdp.send("Page.stopScreencast").catch(() => undefined); + if (marker) await page.evaluate(markerOff).catch(() => undefined); + await cdp.detach().catch(() => undefined); + return { frames, from, to, masks, times, inputs }; + }, + }; +} + +export const panes = (page) => page.evaluate(paneRects); + +/** + * In the decoder page: per frame, the marker counter and the pixels that differ from the before frame and from the + * after frame in each pane. A pixel differs when any channel moves more than `colourTol`. + */ +function compareFrames(frames, before, after, regions, masks, colourTol) { + // fallow-ignore-next-line complexity + return (async () => { + const decode = async (b64) => { + const bmp = await createImageBitmap( + await (await fetch(`data:image/jpeg;base64,${b64}`)).blob(), + ); + const g = new OffscreenCanvas(bmp.width, bmp.height).getContext("2d", { + willReadFrequently: true, + }); + g.drawImage(bmp, 0, 0); + return g.getImageData(0, 0, bmp.width, bmp.height); + }; + const masked = (x, y) => masks.some(([l, t, r, b]) => x >= l && x < r && y >= t && y < b); + // fallow-ignore-next-line complexity + const differing = (a, b, r) => { + let n = 0; + for (let y = r.y; y < r.y + r.h; y++) + for (let x = r.x; x < r.x + r.w; x++) { + if (masks.length && masked(x, y)) continue; + const i = (y * a.width + x) * 4; + if ( + Math.abs(a.data[i] - b.data[i]) > colourTol || + Math.abs(a.data[i + 1] - b.data[i + 1]) > colourTol || + Math.abs(a.data[i + 2] - b.data[i + 2]) > colourTol + ) + n++; + } + return n; + }; + const [b, a] = [await decode(before), await decode(after)]; + const out = []; + for (const f of frames) { + const img = await decode(f); + const px = 4 * (img.width * 8 + 8); + const level = (v) => Math.min(31, Math.round(v / 8)); + const counter = level(img.data[px]) + 32 * level(img.data[px + 1]); + const diffs = Object.fromEntries( + Object.entries(regions).map(([k, r]) => [k, [differing(img, b, r), differing(img, a, r)]]), + ); + out.push({ counter, diffs }); + } + return out; + })(); +} + +const MARKER_STATES = 1024; + +/** The marker's counter wraps every 1024 frames; frames arrive in order, so each one unwraps from the one before. */ +export function unwrap(rows, from) { + let last = from; + return rows.map((r) => { + last += + ((((r.counter - last) % MARKER_STATES) + MARKER_STATES * 1.5) % MARKER_STATES) - + MARKER_STATES / 2; + return { ...r, counter: last }; + }); +} + +/** Marker counters the screencast never delivered, as [first, last] runs relative to the window start. */ +// fallow-ignore-next-line complexity +function gaps(seen, from, to) { + const runs = []; + for (let c = from; c <= to; c++) + if (!seen.has(c)) { + const last = runs.at(-1); + if (last && last[1] === c - from - 1) last[1] = c - from; + else runs.push([c - from, c - from]); + } + return runs; +} + +const showsAfter = (row, tolPx) => + Object.values(row.diffs).every(([, vsAfter]) => vsAfter <= tolPx); + +/** + * Edit-to-paint: frames from the committing input until the preview shows the after-state for good, and the ms from + * the input to that frame's rAF. One frame is the next paint; null when the window logged no input. + */ +// fallow-ignore-next-line complexity +function editToPaint(rows, input, times, tolPx) { + if (!input) return null; + let lastOff = null; + for (const r of rows) if (r.counter > input.n && !showsAfter(r, tolPx)) lastOff = r.counter; + const at = lastOff === null ? input.n + 1 : lastOff + 1; + return { frames: at - input.n, ms: times[at] == null ? null : times[at] - input.t }; +} + +/** + * A frame is bad (a flash) when some pane differs from the before frame and from the after frame by more pixels than + * a 0.5 px shift of the element's perimeter moves. Coverage is marker counters seen over counters in the window. + * `frames` holds each frame's marker counter and per-pane [vsBefore, vsAfter] differing pixel counts. + */ +// fallow-ignore-next-line complexity +export function classify(frames, { from, to, times = [], inputs = [] }, tolPx) { + // The screencast can deliver one frame twice; with the marker on, each counter counts once. + const indexed = frames.map((r, frame) => ({ ...r, frame })); + const rows = + from === null ? indexed : indexed.filter((r, i) => r.counter !== frames[i - 1]?.counter); + const seen = new Set(rows.map((r) => r.counter).filter((c) => c >= from && c <= to)); + const span = from === null || to === null ? null : to - from + 1; + const bad = []; + for (const r of rows) { + const panes = Object.entries(r.diffs) + .filter(([, [vsBefore, vsAfter]]) => vsBefore > tolPx && vsAfter > tolPx) + .map(([k, [vsBefore, vsAfter]]) => ({ pane: k, vsBefore, vsAfter })); + if (panes.length) bad.push({ frame: r.frame, counter: r.counter, panes }); + } + let [longest, run] = [0, 0]; + for (const r of rows) { + run = bad.some((x) => x.frame === r.frame) ? run + 1 : 0; + longest = Math.max(longest, run); + } + return { + frames: rows.length, + rafFrames: span, + coverage: span ? seen.size / span : null, + missing: span ? gaps(seen, from, to) : null, + bad, + longest, + input: inputs[0] ?? null, + paint: editToPaint(rows, inputs[0], times, tolPx), + }; +} + +// fallow-ignore-next-line complexity +async function scoreWindow(decoder, win, regions, tolPx) { + const b = win.before ?? win.frames[0]; + const a = win.after ?? win.frames.at(-1); + if (!b || !a) return { frames: 0, coverage: 0, bad: [], longest: 0, paint: null }; + const rows = await decoder.evaluate( + compareFrames, + win.frames, + b, + a, + regions, + dedupe(win.masks ?? []), + COLOUR_TOL, + ); + return classify(win.from === null ? rows : unwrap(rows, win.from), win, tolPx); +} + +const dedupe = (rects) => + [...new Set(rects.map((r) => r.join(",")))].map((k) => k.split(",").map(Number)); + +// Absorbs JPEG noise, anti-aliasing and subpixel text between otherwise equal frames. +const COLOUR_TOL = 24; +// A window is covered at 90% of its frames, or within 5 points of what the same run's blank control reached. +const MIN_COVERAGE = 0.9; +const CONTROL_SLACK = 0.05; + +// The inputs that commit an edit; the reload window has none and scores flashes only. +const COMMITTING = ["release", "undo", "redo"]; + +/** Scores every window of a case; the offending frames go to `evidence.flashFrames` as PNG. */ +// fallow-ignore-next-line complexity +export async function scoreFlash(decoder, { regions, tolPx, windows, control }, evidence) { + const out = {}; + evidence.flashFrames = []; + for (const [name, win] of Object.entries(windows)) { + const scored = await scoreWindow(decoder, win, regions, tolPx); + for (const b of scored.bad) + evidence.flashFrames.push([ + `${name}-${b.frame}-${b.panes.map((p) => p.pane).join("+")}`, + win.frames[b.frame], + ]); + out[name] = scored; + } + const all = Object.values(out); + const coverage = Math.min(...all.map((w) => w.coverage ?? 0)); + const controlCoverage = (await scoreWindow(decoder, control, {}, tolPx)).coverage ?? 0; + const paints = COMMITTING.filter((k) => out[k]).map((k) => out[k].paint); + return { + bad: all.reduce((n, w) => n + w.bad.length, 0), + longest: Math.max(0, ...all.map((w) => w.longest)), + coverage, + controlCoverage, + // The chance a 1-frame flash fell in a frame the screencast never sent. + missChance: 1 - coverage, + uncovered: !(coverage >= Math.min(MIN_COVERAGE, controlCoverage - CONTROL_SLACK)), + // The slowest committing input, in frames and in ms; unknown (null) when any of them logged no input. + paint: paints.some((p) => !p) + ? null + : { + frames: Math.max(...paints.map((p) => p.frames)), + ms: paints.some((p) => p.ms === null) ? null : Math.max(...paints.map((p) => p.ms)), + }, + regions, + tolPx, + windows: out, + }; +} diff --git a/packages/studio/tests/e2e/edit-accuracy/flash.test.mjs b/packages/studio/tests/e2e/edit-accuracy/flash.test.mjs new file mode 100644 index 0000000000..1ed213e459 --- /dev/null +++ b/packages/studio/tests/e2e/edit-accuracy/flash.test.mjs @@ -0,0 +1,89 @@ +import { describe, expect, it } from "vitest"; +import { classify, unwrap } from "./flash.mjs"; + +const TOL = 10; +// One row per painted frame: its marker counter and [vsBefore, vsAfter] differing pixels per pane. +const before = (counter) => ({ counter, diffs: { preview: [0, 50] } }); +const after = (counter) => ({ counter, diffs: { preview: [50, 0] } }); +const neither = (counter) => ({ counter, diffs: { preview: [50, 50] } }); +const times = Array.from({ length: 20 }, (_, n) => n * 16); +const win = (inputN) => ({ + from: 1, + to: 8, + times, + inputs: [{ type: "pointerup", n: inputN, t: 40 }], +}); + +describe("classify", () => { + it("passes an input whose next frame already shows the after-state", () => { + const w = classify( + [before(1), before(2), after(3), after(4), after(5), after(6), after(7), after(8)], + win(2), + TOL, + ); + expect(w.paint).toEqual({ frames: 1, ms: 8 }); + expect(w.bad).toEqual([]); + expect(w.coverage).toBe(1); + }); + + it("counts frames that hold the old state after the input", () => { + const rows = [ + before(1), + before(2), + before(3), + before(4), + after(5), + after(6), + after(7), + after(8), + ]; + expect(classify(rows, win(2), TOL).paint).toEqual({ frames: 3, ms: 40 }); + }); + + it("flags a frame that is neither state as a flash and as late paint", () => { + const w = classify( + [before(1), before(2), after(3), neither(4), after(5), after(6), after(7), after(8)], + win(2), + TOL, + ); + expect(w.bad.map((b) => b.counter)).toEqual([4]); + expect(w.longest).toBe(1); + expect(w.paint.frames).toBe(3); + }); + + it("counts a frame the screencast delivered twice once", () => { + const rows = [ + before(1), + before(2), + neither(3), + neither(3), + after(4), + after(5), + after(6), + after(7), + after(8), + ]; + const w = classify(rows, win(2), TOL); + expect(w.bad.map((b) => b.frame)).toEqual([2]); + expect(w.frames).toBe(8); + }); + + it("reports missing counters and never guesses paint without a logged input", () => { + const w = classify([before(1), after(4), after(8)], { from: 1, to: 8, times, inputs: [] }, TOL); + expect(w.coverage).toBe(3 / 8); + expect(w.missing).toEqual([ + [1, 2], + [4, 6], + ]); + expect(w.paint).toBeNull(); + }); +}); + +describe("unwrap", () => { + it("restores counters past the marker's 1024 states, including a frame from just before the window", () => { + const codes = [1028 % 1024, 1022, 1030 % 1024, 1034 % 1024, 1100 % 1024].map((counter) => ({ + counter, + })); + expect(unwrap(codes, 1030).map((r) => r.counter)).toEqual([1028, 1022, 1030, 1034, 1100]); + }); +}); diff --git a/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs b/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs new file mode 100644 index 0000000000..a31230e3ea --- /dev/null +++ b/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs @@ -0,0 +1,142 @@ +#!/usr/bin/env node +// Edit accuracy gate against the base branch's baseline.json; smoothness is reported, never gated. +// `flipped