diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 979886a970..3ceeec73c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,6 +74,7 @@ jobs: codex_plugin: ${{ steps.filter.outputs.codex_plugin }} gcp_beginframe: ${{ steps.filter.outputs.gcp_beginframe }} studio: ${{ steps.filter.outputs.studio }} + edit_accuracy: ${{ steps.filter.outputs.edit_accuracy }} steps: # Force git-based change detection instead of the pull_request REST API. # The API path can fail the whole workflow on transient listFiles @@ -158,6 +159,19 @@ jobs: - "packages/studio-server/**" - "bun.lock" - ".github/workflows/ci.yml" + edit_accuracy: + - "packages/studio/src/components/editor/**" + - "packages/studio/src/components/nle/**" + - "packages/studio/src/hooks/**" + - "packages/studio/src/utils/**" + - "packages/studio/src/player/**" + - "packages/studio/src/webmcp/**" + - "packages/studio-server/src/**" + - "packages/core/src/**" + - "packages/cli/src/server/**" + - "packages/studio/tests/e2e/edit-accuracy/**" + - "bun.lock" + - ".github/workflows/ci.yml" gcp_beginframe: - "packages/gcp-cloud-run/Dockerfile" - "packages/aws-lambda/scripts/probe-beginframe.ts" @@ -216,6 +230,14 @@ jobs: - run: bash scripts/ci/install-workspace-dependencies.sh - run: bun run build - run: bun run verify:packed-manifests + - name: Share the built CLI with the edit accuracy shards + if: github.event_name == 'pull_request' && needs.changes.outputs.edit_accuracy == 'true' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: edit-accuracy-cli + path: packages/cli/dist + if-no-files-found: error + retention-days: 1 lint: name: Lint @@ -924,6 +946,115 @@ jobs: /tmp/studio-open-counts.json if-no-files-found: error + # Manual editing accuracy in the built Studio, gated against the base branch's baseline.json. + studio-edit-accuracy: + name: "Studio: edit accuracy (${{ matrix.shard }}/8)" + needs: [changes, build] + if: github.event_name == 'pull_request' && needs.changes.outputs.edit_accuracy == 'true' + runs-on: ubuntu-latest + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3, 4, 5, 6, 7, 8] + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 22 + - uses: ./.github/actions/prepare-ffmpeg-bin + - run: bash scripts/ci/install-workspace-dependencies.sh + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: edit-accuracy-cli + path: packages/cli/dist + # Studio and the producer both run in the engine's pinned headless shell. + - run: node packages/cli/dist/cli.js browser ensure + - name: Read the base branch's edit accuracy baseline + env: + BASE_REF: ${{ github.base_ref }} + run: | + git fetch --no-tags --depth=1 origin "$BASE_REF" + if ! git show FETCH_HEAD:packages/studio/tests/e2e/edit-accuracy/baseline.json > /tmp/base-edit-accuracy.json; then + echo '{"cases":{}}' > /tmp/base-edit-accuracy.json + echo "The base branch has no edit accuracy baseline yet" + fi + - name: Run this shard, then re-run twice each case whose verdict differs from the base branch + run: | + set -euo pipefail + bench() { bun run --cwd packages/studio test:edit-accuracy -- --grid full --jobs 2 "$@"; } + bench --shard "${{ matrix.shard }}/8" --out /tmp/edit-accuracy/run1 + mapfile -t FLIPPED < <(node packages/studio/tests/e2e/edit-accuracy/ratchet.mjs flipped \ + /tmp/base-edit-accuracy.json /tmp/edit-accuracy/run1/results.json) + if (( ${#FLIPPED[@]} > 0 )); then + FILTER="^($(IFS='|'; echo "${FLIPPED[*]}"))$" + for run in 2 3; do bench --filter "$FILTER" --rerun --out "/tmp/edit-accuracy/run${run}"; done + fi + - name: Upload shard evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: edit-accuracy-shard-${{ matrix.shard }} + path: /tmp/edit-accuracy + if-no-files-found: error + + studio-edit-accuracy-gate: + name: "Studio: edit accuracy gate" + needs: [changes, studio-edit-accuracy] + if: ${{ !cancelled() && github.event_name == 'pull_request' && needs.changes.outputs.edit_accuracy == 'true' }} + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 22 + - name: Read the base branch's edit accuracy baseline + env: + BASE_REF: ${{ github.base_ref }} + run: | + git fetch --no-tags --depth=1 origin "$BASE_REF" + if ! git show FETCH_HEAD:packages/studio/tests/e2e/edit-accuracy/baseline.json > /tmp/base-edit-accuracy.json; then + echo '{"cases":{}}' > /tmp/base-edit-accuracy.json + echo "The base branch has no edit accuracy baseline yet" + fi + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: edit-accuracy-shard-* + path: /tmp/edit-accuracy-shards + - name: Compare every run with the base branch + id: gate + # Smoothness is reported, not gated; a case fails only if it fails 2 of its 3 runs. + run: | + set +e + mapfile -t RESULTS < <(find /tmp/edit-accuracy-shards -name results.json | sort) + node packages/studio/tests/e2e/edit-accuracy/ratchet.mjs gate /tmp/base-edit-accuracy.json \ + packages/studio/tests/e2e/edit-accuracy/baseline.json /tmp/edit-accuracy-gate "${RESULTS[@]}" + echo "exit_code=$?" >> "$GITHUB_OUTPUT" + cat /tmp/edit-accuracy-gate/comment.md >> "$GITHUB_STEP_SUMMARY" || true + - name: Post sticky comment + # Fork PRs get a read-only token; the gate below still fires. + continue-on-error: true + uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2.9.4 + with: + header: edit-accuracy + path: /tmp/edit-accuracy-gate/comment.md + - name: Upload the gate report and a baseline.json to bank + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: edit-accuracy-gate + path: /tmp/edit-accuracy-gate + - name: Fail if the gate failed + if: steps.gate.outputs.exit_code != '0' + run: | + echo "::error::Edit accuracy fell against the base branch; see the PR comment." + exit 1 + smoke-global-install: name: "Smoke: global install" needs: [changes, build] diff --git a/packages/studio/tests/e2e/edit-accuracy/case.mjs b/packages/studio/tests/e2e/edit-accuracy/case.mjs index 3a62db0323..1ac22fb666 100644 --- a/packages/studio/tests/e2e/edit-accuracy/case.mjs +++ b/packages/studio/tests/e2e/edit-accuracy/case.mjs @@ -3,6 +3,7 @@ import { spawn } from "node:child_process"; import { readFileSync } from "node:fs"; import { join } from "node:path"; import { COMPOSITION, PLAYHEAD } from "./grid.mjs"; +import { panes, startCapture } from "./flash.mjs"; import { angleOf, centre, @@ -27,6 +28,10 @@ const ROTATE_BY = (25 * Math.PI) / 180; const CROP_BY = 40; const NUDGES = 5; const ZOOM_SENSITIVITY = 0.007; // previewZoom.ts: one wheel unit scales zoom by exp(0.007) +// Flash controls, for proving the metric only: tap (release without a move), blinkN (hide the preview for N frames +// after the release), reload (reload the preview frame mid-settle). EDIT_BENCH_MARKER=0 captures without the marker. +const CONTROL = process.env.EDIT_BENCH_CONTROL; +const CAPTURE = { marker: process.env.EDIT_BENCH_MARKER !== "0" }; const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); @@ -242,7 +247,7 @@ async function settled(ctx, timeout = 15_000) { /** Ready once Studio's own seek tool reports the composition and the playhead landed. */ // fallow-ignore-next-line complexity -async function openStudio(ctx) { +async function openStudio(ctx, onReady) { ctx.handles = null; await ctx.page.waitForFunction(() => window.__editBench?.has("studio_seek"), { timeout: 90_000 }); let seek = null; @@ -255,6 +260,7 @@ async function openStudio(ctx) { seek = null; } if (!seek) throw new Error("studio never reported a seekable composition"); + await onReady?.(); await sleep(1000); return settled(ctx); } @@ -508,7 +514,11 @@ function smoothness(rec) { const CONTROL_PAGE = `data:text/html,
`; -/** The case's drag schedule and per-frame reads on a blank page in the same Chrome: the machine's own frame drops. */ +/** + * The case's drag schedule and per-frame reads on a blank page in the same Chrome: the machine's own frame drops. + * Then 1 s of screencast there: the capture coverage the machine reaches right now. + */ +// fallow-ignore-next-line complexity async function controlDrag(browser, gesture) { const context = await browser.createBrowserContext(); try { @@ -519,33 +529,45 @@ async function controlDrag(browser, gesture) { const box = await page.$("#box"); const ctx = { page, handles: { target: box, root: box } }; const read = () => readQuads(ctx); + let smooth; if (gesture === "nudge") { await recording(page, true); - for (let i = 0; i < NUDGES; i++) { + for (let i = 1; i < NUDGES; i++) { await page.keyboard.press("ArrowRight"); await nextFrame(page); } await nextFrame(page, 2); - return smoothness(await recording(page, false)); - } - await page.mouse.move(700, 380); - await page.mouse.down(); - await nextFrame(page); - await read(); - await recording(page, true); - for (let i = 1; i <= STEPS; i++) { - await page.mouse.move(700 + (MOVE_BY[0] * i) / STEPS, 380 + (MOVE_BY[1] * i) / STEPS); + smooth = smoothness(await recording(page, false)); + } else { + await page.mouse.move(700, 380); + await page.mouse.down(); await nextFrame(page); await read(); + await recording(page, true); + for (let i = 1; i <= STEPS; i++) { + await page.mouse.move(700 + (MOVE_BY[0] * i) / STEPS, 380 + (MOVE_BY[1] * i) / STEPS); + await nextFrame(page); + await read(); + } + smooth = smoothness(await recording(page, false)); + await page.mouse.up(); } - const smooth = smoothness(await recording(page, false)); - await page.mouse.up(); - return smooth; + const capture = await startCapture(page, CAPTURE); + await sleep(1000); + return { smooth, capture: await capture.stop() }; } finally { await context.close().catch(() => undefined); } } +/** The element's perimeter in screen px: how many pixels a 0.5 px shift of its edges touches. */ +const perimeterPx = (m) => + Math.ceil( + m.visible.map(m.map.toScreen).reduce((sum, p, i, q) => sum + dist(p, q[(i + 1) % 4]), 0), + ); + +/** Drives the drag and returns with the button still down; `release` (pointer-up) runs in the flash window. */ +// fallow-ignore-next-line complexity async function pointerGesture(ctx, gesture, pre) { const press = await handlePoint(ctx, pre, gesture); const pressComp = pre.map.toComp(press); @@ -563,7 +585,7 @@ async function pointerGesture(ctx, gesture, pre) { await recording(ctx.page, true); const errors = []; let last = s0; - for (const p of g.path) { + for (const p of CONTROL === "tap" ? g.path.map(() => press) : g.path) { await ctx.page.mouse.move(p[0], p[1]); await nextFrame(ctx.page); last = await sample(ctx, gesture, g.point, p); @@ -571,11 +593,12 @@ async function pointerGesture(ctx, gesture, pre) { } const rec = await recording(ctx.page, false); const smooth = smoothness(rec); - await ctx.page.mouse.up(); const lastQuad = gesture === "crop" ? last.m.outline : last.m.visible; return { - errors, - lastQuad, + release: async () => { + await ctx.page.mouse.up(); + return { errors, lastQuad }; + }, pressJump: quadDistance(s0.m.visible, pre.visible), smooth, diag: { @@ -586,29 +609,66 @@ async function pointerGesture(ctx, gesture, pre) { }; } +/** Nudges all but the last step traced; `release` presses the last one in the flash window, so a key commits there. */ async function nudgeGesture(ctx, pre) { await recording(ctx.page, true); - for (let i = 0; i < NUDGES; i++) { + for (let i = 1; i < NUDGES; i++) { await ctx.page.keyboard.press("ArrowRight"); await nextFrame(ctx.page); } await nextFrame(ctx.page, 2); const smooth = smoothness(await recording(ctx.page, false)); - const m = await measure(ctx); - const [a, b] = [centre(pre.visible), centre(m.visible)]; return { - errors: [dist([b[0] - a[0], b[1] - a[1]], [NUDGES, 0])], - lastQuad: m.visible, + release: async () => { + await ctx.page.keyboard.press("ArrowRight"); + await nextFrame(ctx.page, 2); + const m = await measure(ctx); + const [a, b] = [centre(pre.visible), centre(m.visible)]; + return { errors: [dist([b[0] - a[0], b[1] - a[1]], [NUDGES, 0])], lastQuad: m.visible }; + }, pressJump: null, smooth, diag: {}, }; } +/** The positive flash controls, started right after the committing input. */ +function injectControl(page) { + const blink = /^blink(\d+)$/.exec(CONTROL ?? ""); + if (blink) + return page.evaluate((frames) => { + const s = document.querySelector('[data-testid="preview-zoom-stage"]').style; + const hide = (left) => + requestAnimationFrame(() => { + s.opacity = left ? "0" : ""; + if (left) hide(left - 1); + }); + hide(frames); + }, Number(blink[1])); + if (CONTROL === "reload") + return page.evaluate(async () => { + await new Promise((r) => setTimeout(r, 100)); + const url = (f) => { + try { + return f.contentWindow.location.href; + } catch { + return ""; + } + }; + const iframes = (root) => + [...root.querySelectorAll("*")].flatMap((el) => + el.tagName === "IFRAME" ? [el] : el.shadowRoot ? iframes(el.shadowRoot) : [], + ); + const frames = iframes(document).filter((f) => url(f).includes("/preview")); + const area = (f) => f.offsetWidth * f.offsetHeight; + frames.reduce((a, b) => (area(b) > area(a) ? b : a)).contentWindow.location.reload(); + }); +} + /** One case, end to end, in a fresh browser context against a Studio already serving `dir`. */ // fallow-ignore-next-line complexity export async function runCase({ browser, spec, dir, files, url, evidence }) { - const control = await controlDrag(browser, spec.gesture); + const { smooth: control, capture: controlCapture } = await controlDrag(browser, spec.gesture); const context = await browser.createBrowserContext(); const page = await context.newPage(); const ctx = { page, dir, files, handles: null }; @@ -634,8 +694,19 @@ export async function runCase({ browser, spec, dir, files, url, evidence }) { spec.gesture === "nudge" ? await nudgeGesture(ctx, pre) : await pointerGesture(ctx, spec.gesture, pre); + // Flash windows: each action to its settle is screencast, untraced; the drag's trace has already stopped. + const flash = { + regions: await panes(page), + tolPx: perimeterPx(pre), + windows: {}, + control: controlCapture, + }; + let capture = await startCapture(page, CAPTURE); + const { errors, lastQuad } = await drive.release(); + await injectControl(page); await waitForFiles(ctx, { from: original, timeout: spec.gesture === "nudge" ? 6000 : 5000 }); - await nextFrame(page, 2); + await settled(ctx); + flash.windows.release = await capture.stop(); await blurPreview(page); await page.keyboard.press("Escape"); const committed = await settled(ctx); @@ -646,22 +717,32 @@ export async function runCase({ browser, spec, dir, files, url, evidence }) { // Undo and redo run before any reload. Each waits up to 15 s for its own write; redo waits for undo. const landed = (from) => saved ? waitForFiles(ctx, { from, timeout: 15_000 }) : { reached: true, files: from }; + capture = await startCapture(page, CAPTURE); await chord(page, "Control+z"); const undo = await landed(committedFiles); const undone = await settled(ctx); + flash.windows.undo = await capture.stop(); await shoot("undone"); let [redo, redone] = [{ reached: false }, null]; if (undo.reached) { await blurPreview(page); + capture = await startCapture(page, CAPTURE); await chord(page, "Control+Shift+z"); redo = await landed(undo.files); redone = await settled(ctx); + flash.windows.redo = await capture.stop(); } // A late write must not land under the reload. await waitForFiles(ctx, { timeout: 15_000 }); + // The reload window starts once Studio can seek the composition; before that Studio itself is loading. + const lastSettled = (flash.windows.redo ?? flash.windows.undo).frames.at(-1); await page.reload(); - const reloaded = await openStudio(ctx); + const reloaded = await openStudio( + ctx, + async () => (capture = await startCapture(page, CAPTURE)), + ); + flash.windows.reload = { ...(await capture.stop()), before: lastSettled }; await shoot("reloaded"); const quads = Object.fromEntries( Object.entries({ pre, committed, undone, redone, reloaded }).filter(([, m]) => m), @@ -671,12 +752,12 @@ export async function runCase({ browser, spec, dir, files, url, evidence }) { zoom, saved, tracking: { - max: Math.max(...drive.errors), - p95: percentile(drive.errors, 95), - frames: drive.errors.length, + max: Math.max(...errors), + p95: percentile(errors, 95), + frames: errors.length, }, pressJump: drive.pressJump, - drop: quadDistance(drive.lastQuad, committed.visible), + drop: quadDistance(lastQuad, committed.visible), reload: quadDistance(committed.visible, reloaded.visible), undo: { bytes: saved && undo.reached && sameFiles(undo.files, original), @@ -687,6 +768,7 @@ export async function runCase({ browser, spec, dir, files, url, evidence }) { // Which write never landed within 15 s; a redo that was never sent is untested, so undo fails. undoTimeout: saved && !undo.reached ? "undo" : saved && !redo.reached ? "redo" : null, smooth: { ...drive.smooth, control }, + flash, unsettled: Object.keys(quads).filter((k) => quads[k].unsettled), reloaded, diag: { diff --git a/packages/studio/tests/e2e/edit-accuracy/flash.mjs b/packages/studio/tests/e2e/edit-accuracy/flash.mjs new file mode 100644 index 0000000000..48b898a184 --- /dev/null +++ b/packages/studio/tests/e2e/edit-accuracy/flash.mjs @@ -0,0 +1,329 @@ +/** + * Preview flashes and edit-to-paint: every frame the screencast paints, from an action to its settle, against the + * frames around it. + */ + +// A 16x16 marker in the top-left corner, outside every region, repaints every rAF with the frame counter in its +// colour (5 bits per channel in steps of 8, which JPEG keeps), so the screencast (which only sends repainted frames) +// sends every frame and each one is numbered. +// It also logs the counter at each committing input (pointer-up, a non-modifier key), in any same-origin frame. +function markerOn() { + const bench = window.__editBench; + if (bench.marker) return; + const el = Object.assign(document.createElement("div"), { id: "edit-bench-marker" }); + el.style.cssText = + "position:fixed;left:0;top:0;width:16px;height:16px;z-index:2147483647;pointer-events:none"; + document.documentElement.append(el); + const marker = { el, n: 0, on: true, masks: [], times: [], inputs: [], unlisten: [] }; + bench.marker = marker; + const MODIFIERS = new Set(["Control", "Shift", "Alt", "Meta"]); + const log = (e) => { + if (e.type === "pointerup" || !MODIFIERS.has(e.key)) + marker.inputs.push({ type: e.type, key: e.key ?? null, n: marker.n, t: performance.now() }); + }; + // Studio's preview iframe sits in the player's shadow root, out of querySelectorAll's reach. + const iframes = (root) => + [...root.querySelectorAll("*")].flatMap((el) => + el.tagName === "IFRAME" ? [el] : el.shadowRoot ? iframes(el.shadowRoot) : [], + ); + const listen = (w) => { + for (const type of ["pointerup", "keydown"]) { + w.addEventListener(type, log, true); + marker.unlisten.push(() => w.removeEventListener(type, log, true)); + } + for (const f of iframes(w.document)) { + try { + listen(f.contentWindow); + } catch { + // A cross-origin frame cannot take an input Studio handles. + } + } + }; + listen(window); + const tick = () => { + if (!marker.on) return; + const n = ++marker.n; + marker.times[n] = performance.now(); + el.style.background = `rgb(${(n % 32) * 8},${((n >> 5) % 32) * 8},128)`; + // Toasts slide in and out by design; wherever one was during the window is left out of the comparison. + for (const t of document.querySelectorAll(".hf-toast-enter, .hf-toast-exit")) { + const r = t.getBoundingClientRect(); + marker.masks.push([r.left, r.top, r.right, r.bottom].map(Math.round)); + } + requestAnimationFrame(tick); + }; + requestAnimationFrame(tick); +} + +function markerOff() { + const m = window.__editBench.marker; + if (!m) return; + m.on = false; + for (const off of m.unlisten) off(); + m.el.remove(); + window.__editBench.marker = null; +} + +/** The panes a flash can show in, as screen rects: preview (with its chrome), timeline and inspector. */ +function paneRects() { + const part = (el) => el?.closest(".dv-react-part"); + const rect = (el) => { + const r = el.getBoundingClientRect(); + return { + x: Math.round(r.x), + y: Math.round(r.y), + w: Math.round(r.width), + h: Math.round(r.height), + }; + }; + const preview = part(document.querySelector('[data-testid="preview-zoom-stage"]')); + const parts = [...document.querySelectorAll(".dv-react-part")].filter((e) => e.offsetWidth > 0); + const p = preview && rect(preview); + const timeline = + part(document.querySelector('[data-testid="timeline-clip"]')) ?? + parts.reduce((a, b) => (!a || rect(b).y > rect(a).y ? b : a), null); + const area = (e) => rect(e).w * rect(e).h; + const inspector = + p && + parts + .filter((e) => rect(e).x >= p.x + p.w - 1 && rect(e).y < p.y + p.h) + .reduce((a, b) => (!a || area(b) > area(a) ? b : a), null); + return Object.fromEntries( + Object.entries({ preview, timeline, inspector }) + .filter(([, e]) => e) + .map(([k, e]) => [k, rect(e)]), + ); +} + +/** + * Starts a screencast of one action window; stop() returns its PNG frames, the marker range it spans, the rAF time + * of each counter and the committing inputs logged in it. + */ +export async function startCapture(page, { marker = true } = {}) { + const cdp = await page.createCDPSession(); + const frames = []; + cdp.on("Page.screencastFrame", (f) => { + frames.push(f.data); + cdp.send("Page.screencastFrameAck", { sessionId: f.sessionId }).catch(() => undefined); + }); + if (marker) await page.evaluate(markerOn); + const state = () => + page.evaluate(() => { + const m = window.__editBench.marker; + return m ? { n: m.n, masks: m.masks, times: m.times, inputs: m.inputs } : { n: null }; + }); + // JPEG: PNG encoding cannot keep up with 60 fps even on a blank page; the colour tolerance absorbs JPEG noise. + await cdp.send("Page.startScreencast", { format: "jpeg", quality: 95, everyNthFrame: 1 }); + // The first frame is the state before the action. + for (const deadline = Date.now() + 2000; !frames.length && Date.now() < deadline; ) + await new Promise((r) => setTimeout(r, 10)); + const from = (await state()).n; + return { + async stop() { + const { n: to, masks = [], times = [], inputs = [] } = await state(); + await cdp.send("Page.stopScreencast").catch(() => undefined); + if (marker) await page.evaluate(markerOff).catch(() => undefined); + await cdp.detach().catch(() => undefined); + return { frames, from, to, masks, times, inputs }; + }, + }; +} + +export const panes = (page) => page.evaluate(paneRects); + +/** + * In the decoder page: per frame, the marker counter and the pixels that differ from the before frame and from the + * after frame in each pane. A pixel differs when any channel moves more than `colourTol`. + */ +function compareFrames(frames, before, after, regions, masks, colourTol) { + // fallow-ignore-next-line complexity + return (async () => { + const decode = async (b64) => { + const bmp = await createImageBitmap( + await (await fetch(`data:image/jpeg;base64,${b64}`)).blob(), + ); + const g = new OffscreenCanvas(bmp.width, bmp.height).getContext("2d", { + willReadFrequently: true, + }); + g.drawImage(bmp, 0, 0); + return g.getImageData(0, 0, bmp.width, bmp.height); + }; + const masked = (x, y) => masks.some(([l, t, r, b]) => x >= l && x < r && y >= t && y < b); + // fallow-ignore-next-line complexity + const differing = (a, b, r) => { + let n = 0; + for (let y = r.y; y < r.y + r.h; y++) + for (let x = r.x; x < r.x + r.w; x++) { + if (masks.length && masked(x, y)) continue; + const i = (y * a.width + x) * 4; + if ( + Math.abs(a.data[i] - b.data[i]) > colourTol || + Math.abs(a.data[i + 1] - b.data[i + 1]) > colourTol || + Math.abs(a.data[i + 2] - b.data[i + 2]) > colourTol + ) + n++; + } + return n; + }; + const [b, a] = [await decode(before), await decode(after)]; + const out = []; + for (const f of frames) { + const img = await decode(f); + const px = 4 * (img.width * 8 + 8); + const level = (v) => Math.min(31, Math.round(v / 8)); + const counter = level(img.data[px]) + 32 * level(img.data[px + 1]); + const diffs = Object.fromEntries( + Object.entries(regions).map(([k, r]) => [k, [differing(img, b, r), differing(img, a, r)]]), + ); + out.push({ counter, diffs }); + } + return out; + })(); +} + +const MARKER_STATES = 1024; + +/** The marker's counter wraps every 1024 frames; frames arrive in order, so each one unwraps from the one before. */ +export function unwrap(rows, from) { + let last = from; + return rows.map((r) => { + last += + ((((r.counter - last) % MARKER_STATES) + MARKER_STATES * 1.5) % MARKER_STATES) - + MARKER_STATES / 2; + return { ...r, counter: last }; + }); +} + +/** Marker counters the screencast never delivered, as [first, last] runs relative to the window start. */ +// fallow-ignore-next-line complexity +function gaps(seen, from, to) { + const runs = []; + for (let c = from; c <= to; c++) + if (!seen.has(c)) { + const last = runs.at(-1); + if (last && last[1] === c - from - 1) last[1] = c - from; + else runs.push([c - from, c - from]); + } + return runs; +} + +const showsAfter = (row, tolPx) => + Object.values(row.diffs).every(([, vsAfter]) => vsAfter <= tolPx); + +/** + * Edit-to-paint: frames from the committing input until the preview shows the after-state for good, and the ms from + * the input to that frame's rAF. One frame is the next paint; null when the window logged no input. + */ +// fallow-ignore-next-line complexity +function editToPaint(rows, input, times, tolPx) { + if (!input) return null; + let lastOff = null; + for (const r of rows) if (r.counter > input.n && !showsAfter(r, tolPx)) lastOff = r.counter; + const at = lastOff === null ? input.n + 1 : lastOff + 1; + return { frames: at - input.n, ms: times[at] == null ? null : times[at] - input.t }; +} + +/** + * A frame is bad (a flash) when some pane differs from the before frame and from the after frame by more pixels than + * a 0.5 px shift of the element's perimeter moves. Coverage is marker counters seen over counters in the window. + * `frames` holds each frame's marker counter and per-pane [vsBefore, vsAfter] differing pixel counts. + */ +// fallow-ignore-next-line complexity +export function classify(frames, { from, to, times = [], inputs = [] }, tolPx) { + // The screencast can deliver one frame twice; with the marker on, each counter counts once. + const indexed = frames.map((r, frame) => ({ ...r, frame })); + const rows = + from === null ? indexed : indexed.filter((r, i) => r.counter !== frames[i - 1]?.counter); + const seen = new Set(rows.map((r) => r.counter).filter((c) => c >= from && c <= to)); + const span = from === null || to === null ? null : to - from + 1; + const bad = []; + for (const r of rows) { + const panes = Object.entries(r.diffs) + .filter(([, [vsBefore, vsAfter]]) => vsBefore > tolPx && vsAfter > tolPx) + .map(([k, [vsBefore, vsAfter]]) => ({ pane: k, vsBefore, vsAfter })); + if (panes.length) bad.push({ frame: r.frame, counter: r.counter, panes }); + } + let [longest, run] = [0, 0]; + for (const r of rows) { + run = bad.some((x) => x.frame === r.frame) ? run + 1 : 0; + longest = Math.max(longest, run); + } + return { + frames: rows.length, + rafFrames: span, + coverage: span ? seen.size / span : null, + missing: span ? gaps(seen, from, to) : null, + bad, + longest, + input: inputs[0] ?? null, + paint: editToPaint(rows, inputs[0], times, tolPx), + }; +} + +// fallow-ignore-next-line complexity +async function scoreWindow(decoder, win, regions, tolPx) { + const b = win.before ?? win.frames[0]; + const a = win.after ?? win.frames.at(-1); + if (!b || !a) return { frames: 0, coverage: 0, bad: [], longest: 0, paint: null }; + const rows = await decoder.evaluate( + compareFrames, + win.frames, + b, + a, + regions, + dedupe(win.masks ?? []), + COLOUR_TOL, + ); + return classify(win.from === null ? rows : unwrap(rows, win.from), win, tolPx); +} + +const dedupe = (rects) => + [...new Set(rects.map((r) => r.join(",")))].map((k) => k.split(",").map(Number)); + +// Absorbs JPEG noise, anti-aliasing and subpixel text between otherwise equal frames. +const COLOUR_TOL = 24; +// A window is covered at 90% of its frames, or within 5 points of what the same run's blank control reached. +const MIN_COVERAGE = 0.9; +const CONTROL_SLACK = 0.05; + +// The inputs that commit an edit; the reload window has none and scores flashes only. +const COMMITTING = ["release", "undo", "redo"]; + +/** Scores every window of a case; the offending frames go to `evidence.flashFrames` as PNG. */ +// fallow-ignore-next-line complexity +export async function scoreFlash(decoder, { regions, tolPx, windows, control }, evidence) { + const out = {}; + evidence.flashFrames = []; + for (const [name, win] of Object.entries(windows)) { + const scored = await scoreWindow(decoder, win, regions, tolPx); + for (const b of scored.bad) + evidence.flashFrames.push([ + `${name}-${b.frame}-${b.panes.map((p) => p.pane).join("+")}`, + win.frames[b.frame], + ]); + out[name] = scored; + } + const all = Object.values(out); + const coverage = Math.min(...all.map((w) => w.coverage ?? 0)); + const controlCoverage = (await scoreWindow(decoder, control, {}, tolPx)).coverage ?? 0; + const paints = COMMITTING.filter((k) => out[k]).map((k) => out[k].paint); + return { + bad: all.reduce((n, w) => n + w.bad.length, 0), + longest: Math.max(0, ...all.map((w) => w.longest)), + coverage, + controlCoverage, + // The chance a 1-frame flash fell in a frame the screencast never sent. + missChance: 1 - coverage, + uncovered: !(coverage >= Math.min(MIN_COVERAGE, controlCoverage - CONTROL_SLACK)), + // The slowest committing input, in frames and in ms; unknown (null) when any of them logged no input. + paint: paints.some((p) => !p) + ? null + : { + frames: Math.max(...paints.map((p) => p.frames)), + ms: paints.some((p) => p.ms === null) ? null : Math.max(...paints.map((p) => p.ms)), + }, + regions, + tolPx, + windows: out, + }; +} diff --git a/packages/studio/tests/e2e/edit-accuracy/flash.test.mjs b/packages/studio/tests/e2e/edit-accuracy/flash.test.mjs new file mode 100644 index 0000000000..1ed213e459 --- /dev/null +++ b/packages/studio/tests/e2e/edit-accuracy/flash.test.mjs @@ -0,0 +1,89 @@ +import { describe, expect, it } from "vitest"; +import { classify, unwrap } from "./flash.mjs"; + +const TOL = 10; +// One row per painted frame: its marker counter and [vsBefore, vsAfter] differing pixels per pane. +const before = (counter) => ({ counter, diffs: { preview: [0, 50] } }); +const after = (counter) => ({ counter, diffs: { preview: [50, 0] } }); +const neither = (counter) => ({ counter, diffs: { preview: [50, 50] } }); +const times = Array.from({ length: 20 }, (_, n) => n * 16); +const win = (inputN) => ({ + from: 1, + to: 8, + times, + inputs: [{ type: "pointerup", n: inputN, t: 40 }], +}); + +describe("classify", () => { + it("passes an input whose next frame already shows the after-state", () => { + const w = classify( + [before(1), before(2), after(3), after(4), after(5), after(6), after(7), after(8)], + win(2), + TOL, + ); + expect(w.paint).toEqual({ frames: 1, ms: 8 }); + expect(w.bad).toEqual([]); + expect(w.coverage).toBe(1); + }); + + it("counts frames that hold the old state after the input", () => { + const rows = [ + before(1), + before(2), + before(3), + before(4), + after(5), + after(6), + after(7), + after(8), + ]; + expect(classify(rows, win(2), TOL).paint).toEqual({ frames: 3, ms: 40 }); + }); + + it("flags a frame that is neither state as a flash and as late paint", () => { + const w = classify( + [before(1), before(2), after(3), neither(4), after(5), after(6), after(7), after(8)], + win(2), + TOL, + ); + expect(w.bad.map((b) => b.counter)).toEqual([4]); + expect(w.longest).toBe(1); + expect(w.paint.frames).toBe(3); + }); + + it("counts a frame the screencast delivered twice once", () => { + const rows = [ + before(1), + before(2), + neither(3), + neither(3), + after(4), + after(5), + after(6), + after(7), + after(8), + ]; + const w = classify(rows, win(2), TOL); + expect(w.bad.map((b) => b.frame)).toEqual([2]); + expect(w.frames).toBe(8); + }); + + it("reports missing counters and never guesses paint without a logged input", () => { + const w = classify([before(1), after(4), after(8)], { from: 1, to: 8, times, inputs: [] }, TOL); + expect(w.coverage).toBe(3 / 8); + expect(w.missing).toEqual([ + [1, 2], + [4, 6], + ]); + expect(w.paint).toBeNull(); + }); +}); + +describe("unwrap", () => { + it("restores counters past the marker's 1024 states, including a frame from just before the window", () => { + const codes = [1028 % 1024, 1022, 1030 % 1024, 1034 % 1024, 1100 % 1024].map((counter) => ({ + counter, + })); + expect(unwrap(codes, 1030).map((r) => r.counter)).toEqual([1028, 1022, 1030, 1034, 1100]); + }); +}); diff --git a/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs b/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs new file mode 100644 index 0000000000..a31230e3ea --- /dev/null +++ b/packages/studio/tests/e2e/edit-accuracy/ratchet.mjs @@ -0,0 +1,142 @@ +#!/usr/bin/env node +// Edit accuracy gate against the base branch's baseline.json; smoothness is reported, never gated. +// `flipped ` lists cases to re-run twice; `gate ` judges 2 of 3. +// A case whose runs disagree is listed as unstable. +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { LIMIT_PX, entry, paintOk, writeReport } from "./report.mjs"; + +const GATED_PX = ["tracking", "pressJump", "drop", "reload", "render"]; +const LISTED = 30; + +/** + * Passes every gated metric; an unsettled preview fails the metrics it fed, all of them gated. Flash and + * edit-to-paint count only where `like` measured them, so a metric the base branch never measured cannot regress. + */ +// fallow-ignore-next-line complexity +export const accurate = (e, like = e) => + Boolean(e) && + !e.error && + !e.unsettled && + !e.renderError && + e.undo === true && + GATED_PX.every((m) => !(e[m] > LIMIT_PX)) && + (!like || !("flash" in like) || (e.flash === 0 && !e.flashUncovered)) && + (!like || !("paint" in like) || paintOk(e.paint, e.paintMs)); + +/** Cases whose verdict here differs from the base branch, either way: each is re-run twice before the gate. */ +export const flipped = (base, results) => + results + .filter((r) => accurate(base.cases[r.id]) !== accurate(entry(r), base.cases[r.id])) + .map((r) => r.id); + +const summary = (e) => + e.error ? "error" : `${GATED_PX.map((m) => `${m} ${e[m] ?? "-"}`).join(", ")}, undo ${e.undo}`; + +/** Every run of every case: each shard's run plus the re-runs of the cases it flipped. */ +// fallow-ignore-next-line complexity +export function gate(base, head, runs) { + const seen = new Map(); + for (const r of runs) seen.set(r.id, [...(seen.get(r.id) ?? []), entry(r)]); + const cases = [...seen].map(([id, entries]) => { + const like = base.cases[id]; + const majority = (judge) => entries.filter((e) => !judge(e)).length * 2 < entries.length; + return { + id, + entries, + passed: majority((e) => accurate(e)), + // Judged on the base entry's metrics, for regressions and the count. + passedLike: majority((e) => accurate(e, like)), + basePassed: accurate(like), + }; + }); + const passing = cases.filter((c) => c.passed); + const result = { + basePassing: Object.values(base.cases).filter((e) => accurate(e)).length, + headPassing: cases.filter((c) => c.passedLike).length, + regressed: cases.filter((c) => c.basePassed && !c.passedLike).map((c) => c.id), + unstable: cases + .filter((c) => new Set(c.entries.map((e) => accurate(e))).size > 1) + .map((c) => ({ id: c.id, runs: c.entries.map(summary) })), + newlyPassing: passing.filter((c) => !c.basePassed).map((c) => c.id), + unbanked: passing.filter((c) => !accurate(head.cases[c.id])).map((c) => c.id), + overclaimed: cases.filter((c) => !c.passed && accurate(head.cases[c.id])).map((c) => c.id), + missing: Object.keys(base.cases).filter((id) => !seen.has(id)), + }; + const reasons = [ + result.regressed.length && + `${result.regressed.length} case(s) that pass on the base branch fail here`, + result.headPassing < result.basePassing && + `the passing count fell from ${result.basePassing} to ${result.headPassing}`, + result.unbanked.length && + `${result.unbanked.length} newly passing case(s) are not banked in baseline.json`, + result.overclaimed.length && + `baseline.json marks ${result.overclaimed.length} failing case(s) as passing`, + ].filter(Boolean); + return { ...result, ok: reasons.length === 0, reasons }; +} + +const list = (title, ids) => + ids.length + ? [ + `**${title}** (${ids.length})`, + ...ids.slice(0, LISTED).map((id) => `- ${id}`), + ids.length > LISTED ? `- ...` : "", + "", + ] + : []; + +export function comment(g) { + return [ + "", + `### Edit accuracy: ${g.headPassing} passing here, ${g.basePassing} on the base branch`, + "", + g.ok ? "The gate passes." : `The gate fails: ${g.reasons.join("; ")}.`, + "Smoothness is reported in the artifact, not gated. A case fails only if it fails 2 of 3 runs.", + "", + ...list("Regressed", g.regressed), + ...list("Newly passing", g.newlyPassing), + ...list("Not banked (commit the artifact's baseline.json)", g.unbanked), + ...list("Marked passing in baseline.json but failing", g.overclaimed), + ...list("In the base grid but not run", g.missing), + ...(g.unstable.length + ? [ + `**Unstable** (${g.unstable.length})`, + ...g.unstable.slice(0, LISTED).map((u) => `- ${u.id}: ${u.runs.join(" / ")}`), + "", + ] + : []), + ].join("\n"); +} + +// Strict: CI writes {"cases":{}} for a missing base, so a file that does not parse is broken, not empty. +const readJson = (path) => JSON.parse(readFileSync(path, "utf8")); + +// fallow-ignore-next-line complexity +function main([command, basePath, ...rest]) { + const base = readJson(basePath); + if (command === "flipped") { + for (const id of flipped(base, readJson(rest[0]).cases)) console.log(id); + return 0; + } + const [headPath, out, ...resultPaths] = rest; + const runs = resultPaths.map((p) => readJson(p)); + if (!runs.length) throw new Error("no results.json from any shard"); + const g = gate( + base, + readJson(headPath), + runs.flatMap((r) => r.cases), + ); + mkdirSync(out, { recursive: true }); + writeFileSync(join(out, "comment.md"), comment(g)); + writeFileSync(join(out, "gate.json"), JSON.stringify(g, null, 1)); + // The first run of every shard, as a baseline.json to commit when cases newly pass. + const firsts = runs.filter((r) => !r.meta.rerun).flatMap((r) => r.cases); + writeReport(out, { ...runs[0].meta, grid: "full (CI)" }, firsts, 0); + console.log(comment(g)); + return g.ok ? 0 : 1; +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) + process.exitCode = main(process.argv.slice(2)); diff --git a/packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs b/packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs new file mode 100644 index 0000000000..77831efb23 --- /dev/null +++ b/packages/studio/tests/e2e/edit-accuracy/ratchet.test.mjs @@ -0,0 +1,132 @@ +import { describe, expect, it } from "vitest"; +import { accurate, flipped, gate } from "./ratchet.mjs"; + +const good = { + tracking: 0.1, + pressJump: 0, + drop: 0, + reload: 0, + render: 0.02, + undo: true, + dropped: 0, + controlDropped: 0, + flash: 0, + paint: 1, + paintMs: 5, + work: 3, + frameP95: 20, +}; +const run = (id, drop = 0) => ({ + id, + pass: false, + tracking: { max: 0.1 }, + pressJump: 0, + drop, + reload: 0, + render: 0.02, + undo: { bytes: true, redoBytes: true, box: 0, redoBox: 0 }, + checks: { undo: true }, + smooth: { p95: 20, dropped: 0, workP95: 3, control: { dropped: 0 } }, + unsettled: [], + flash: { bad: 0, uncovered: false, paint: { frames: 1, ms: 5 } }, +}); +const baseline = (cases) => ({ cases }); + +describe("accurate", () => { + it("ignores smoothness and a metric the baseline does not hold", () => { + expect(accurate({ ...good, dropped: 4, work: 90 })).toBe(true); + expect(accurate({ tracking: 0.1, drop: 0, reload: 0, undo: true })).toBe(true); + expect(accurate({ ...good, pressJump: null })).toBe(true); + }); + + it("fails a px metric over the limit, a failed undo and an error", () => { + expect(accurate({ ...good, drop: 0.51 })).toBe(false); + expect(accurate({ ...good, pressJump: 0.51 })).toBe(false); + expect(accurate({ ...good, unsettled: ["committed"] })).toBe(false); + expect(accurate({ ...good, render: null, renderError: true })).toBe(false); + expect(accurate({ ...good, flash: 1 })).toBe(false); + expect(accurate({ ...good, flash: 0, flashUncovered: true })).toBe(false); + expect(accurate({ ...good, paint: 2 })).toBe(false); + expect(accurate({ ...good, paint: null })).toBe(false); + expect(accurate({ ...good, paintMs: 507 })).toBe(false); + expect(accurate({ ...good, undo: false })).toBe(false); + expect(accurate({ pass: false, error: true })).toBe(false); + expect(accurate(undefined)).toBe(false); + }); +}); + +describe("a metric the base branch never measured", () => { + const { flash: _f, paint: _p, ...older } = good; + + it("is judged only where the like entry holds it", () => { + expect(accurate({ ...good, flash: 2, paint: 9 }, older)).toBe(true); + expect(accurate({ ...good, flash: 2 }, good)).toBe(false); + }); + + it("cannot regress a case or lower the count, and still needs banking to pass", () => { + const flashing = { + ...run("a"), + flash: { bad: 3, uncovered: false, paint: { frames: 4, ms: 60 } }, + }; + const g = gate(baseline({ a: older }), baseline({ a: { ...good, flash: 3 } }), [flashing]); + expect(g.regressed).toEqual([]); + expect(g.headPassing).toBe(1); + expect(g.overclaimed).toEqual([]); + expect(g.ok).toBe(true); + expect(flipped(baseline({ a: older }), [flashing])).toEqual([]); + }); +}); + +describe("gate", () => { + const base = baseline({ a: good, b: good }); + + it("re-runs every case whose verdict differs from the base branch, in either direction", () => { + const mixed = baseline({ a: good, b: good, d: { ...good, drop: 9 } }); + expect(flipped(mixed, [run("a", 3), run("b"), run("c", 3), run("d")])).toEqual(["a", "d"]); + }); + + it("banks a newly passing case only when it passes 2 of 3 runs, and lists a lucky pass as unstable", () => { + const before = baseline({ a: good, b: { ...good, drop: 9 } }); + const lucky = gate(before, before, [run("a"), run("b"), run("b", 9), run("b", 9)]); + expect(lucky.newlyPassing).toEqual([]); + expect(lucky.unbanked).toEqual([]); + expect(lucky.unstable.map((u) => u.id)).toEqual(["b"]); + expect(lucky.ok).toBe(true); + const real = gate(before, before, [run("a"), run("b"), run("b"), run("b", 9)]); + expect(real.unbanked).toEqual(["b"]); + expect(real.ok).toBe(false); + }); + + it("fails a regression that fails 2 of 3 runs", () => { + const g = gate(base, base, [run("a", 3), run("a", 3), run("a"), run("b")]); + expect(g.regressed).toEqual(["a"]); + expect(g.ok).toBe(false); + expect(g.unstable.map((u) => u.id)).toEqual(["a"]); + }); + + it("passes a case that fails 1 of 3 runs but still lists it as unstable", () => { + const g = gate(base, base, [run("a", 3), run("a"), run("a"), run("b")]); + expect(g.regressed).toEqual([]); + expect(g.unstable).toHaveLength(1); + expect(g.unstable[0].runs[0]).toContain("drop 3"); + expect(g.ok).toBe(true); + }); + + it("fails a newly passing case until baseline.json banks it", () => { + const before = baseline({ a: good, b: { ...good, drop: 9 } }); + expect(gate(before, before, [run("a"), run("b")]).unbanked).toEqual(["b"]); + expect(gate(before, base, [run("a"), run("b")]).ok).toBe(true); + }); + + it("fails when baseline.json claims a pass the run does not reproduce", () => { + const g = gate(baseline({}), baseline({ a: good }), [run("a", 3)]); + expect(g.overclaimed).toEqual(["a"]); + expect(g.ok).toBe(false); + }); + + it("fails when the passing count falls, as when a passing case leaves the grid", () => { + const g = gate(base, base, [run("a")]); + expect(g.missing).toEqual(["b"]); + expect(g.reasons.join()).toContain("fell from 2 to 1"); + }); +}); diff --git a/packages/studio/tests/e2e/edit-accuracy/report.mjs b/packages/studio/tests/e2e/edit-accuracy/report.mjs index 07b94794a3..f85d3dd033 100644 --- a/packages/studio/tests/e2e/edit-accuracy/report.mjs +++ b/packages/studio/tests/e2e/edit-accuracy/report.mjs @@ -3,11 +3,26 @@ import { writeFileSync } from "node:fs"; import { join } from "node:path"; import { percentile } from "./geometry.mjs"; -const LIMIT_PX = 0.5; +export const LIMIT_PX = 0.5; // A frame over 1.5 vsyncs is dropped; raw rAF p95 stays reported so a different rule re-scores without a re-run. const DROPPED_FRAME_MS = 25; const WORK_MS = 8; -export const METRICS = ["tracking", "press", "drop", "reload", "render", "undo", "smooth"]; +export const METRICS = [ + "tracking", + "press", + "drop", + "reload", + "render", + "undo", + "flash", + "paint", + "smooth", +]; +// Edit-to-paint: the first frame painted after a committing input already shows the after-state, and that frame +// comes no later than a frame counts as dropped. +const PAINT_FRAMES = 1; +export const paintOk = (frames, ms) => + frames != null && frames <= PAINT_FRAMES && ms != null && ms <= DROPPED_FRAME_MS; /** Worst-first value per metric; undo ranks by box distance, and its byte failures are counted apart. */ const worstValue = { @@ -17,6 +32,8 @@ const worstValue = { reload: (r) => r.reload, render: (r) => r.render ?? 0, undo: (r) => Math.max(r.undo.box, r.undo.redoBox ?? 0), + flash: (r) => r.flash.bad, + paint: (r) => r.flash.paint?.frames ?? Infinity, smooth: (r) => r.smooth.dropped - r.smooth.control.dropped, }; @@ -54,6 +71,9 @@ export function score(spec, r) { reload: r.reload <= LIMIT_PX, render: r.render !== null && r.render <= LIMIT_PX, undo: r.undo.bytes && r.undo.redoBytes && Math.max(r.undo.box, r.undo.redoBox) <= LIMIT_PX, + // A window the screencast covered under 90% is uncovered, never a pass. + flash: !r.flash.uncovered && r.flash.bad === 0, + paint: !r.flash.uncovered && paintOk(r.flash.paint?.frames, r.flash.paint?.ms), // Only drops beyond the blank page's, driven the same way in the same Chrome, are the edit's. smooth: smooth.dropped <= smooth.control.dropped && @@ -117,6 +137,29 @@ function summarize(results, seconds) { unsettled: measured.filter((r) => r.unsettled.length).length, undoTimeouts: measured.filter((r) => r.undoTimeout).length, renderErrors: measured.filter((r) => r.renderError).length, + flash: { + uncovered: measured.filter((r) => r.flash.uncovered).length, + badCases: measured.filter((r) => r.flash.bad > 0).length, + longest: Math.max(0, ...measured.map((r) => r.flash.longest)), + coverage: round( + percentile( + measured.map((r) => r.flash.coverage), + 50, + ), + ), + lowest: round(Math.min(...measured.map((r) => r.flash.coverage))), + control: round( + percentile( + measured.map((r) => r.flash.controlCoverage), + 50, + ), + ), + }, + paint: { + unknown: measured.filter((r) => !r.flash.paint).length, + frames: medianMax(measured.filter((r) => r.flash.paint).map((r) => r.flash.paint.frames)), + ms: medianMax(measured.filter((r) => r.flash.paint?.ms != null).map((r) => r.flash.paint.ms)), + }, smooth: smoothSummary(measured), seconds: Math.round(seconds), }; @@ -133,12 +176,14 @@ function table(summary, meta, results) { `Every metric counts except smoothness, which is reported against the blank-page control: ${summary.perMetric.find((m) => m.metric === "smooth").pass}/${summary.total} pass it, and ${summary.passing}/${summary.total} pass everything including it.`, "", `Studio ${meta.studio} (build ${meta.build}), bench ${meta.bench}, grid \`${meta.grid}\`, ${meta.date}, ${summary.seconds}s with ${meta.jobs} jobs, ${summary.errors} harness errors, load ${meta.load}.`, - `Pass: tracking, press jump, drop, reload and render ≤ ${LIMIT_PX} px; undo and redo byte-identical with the box ≤ ${LIMIT_PX} px; no more frames over ${DROPPED_FRAME_MS} ms than the blank-page control, and main-thread work ≤ ${WORK_MS} ms per frame at p95.`, + `Pass: tracking, press jump, drop, reload and render ≤ ${LIMIT_PX} px; undo and redo byte-identical with the box ≤ ${LIMIT_PX} px; no flash frame and the after-state in the next frame; no more frames over ${DROPPED_FRAME_MS} ms than the blank-page control, and main-thread work ≤ ${WORK_MS} ms per frame at p95.`, "", `Undo or redo left different bytes in ${summary.bytesDiffer.undo} undo and ${summary.bytesDiffer.redo} redo cases.`, `The preview never held still for 1 s within 15 s in ${summary.unsettled} cases; the metrics that snapshot feeds fail.`, `An undo or redo write never landed within 15 s in ${summary.undoTimeouts} cases; undo fails there.`, `The producer failed to render ${summary.renderErrors} cases; render fails there.`, + `Flash: ${summary.flash.badCases} cases painted a frame matching neither the state before nor after (longest run ${summary.flash.longest} frames); ${summary.flash.uncovered} cases uncovered; screencast coverage median ${summary.flash.coverage}, lowest ${summary.flash.lowest}, blank control median ${summary.flash.control}.`, + `Edit-to-paint (pointer-up, key, undo, redo; pass at ${PAINT_FRAMES} frame within ${DROPPED_FRAME_MS} ms): frames to the after-state (median/max) ${summary.paint.frames}, ms ${summary.paint.ms}; ${summary.paint.unknown} cases logged no input.`, `Smoothness: ${summary.smooth.unknown} cases with unknown work; dropped frames per case (median/max) ${summary.smooth.dropped}, blank-page control ${summary.smooth.control}; raw rAF p95 (median/max) ${summary.smooth.p95} ms, control ${summary.smooth.controlP95} ms.`, "", "| Metric | Pass | Worst | Worst case |", @@ -154,35 +199,51 @@ function table(summary, meta, results) { `| ${g} | ${rs.length} | ${rs.filter((r) => r.pass).length} | ${METRICS.map((m) => rs.filter((r) => r.checks[m]).length).join(" | ")} |`, ); } + lines.push( + "", + "| Case | Flash frames | Paint frames | 1-frame flash miss chance | Blank control coverage |", + "|---|---|---|---|---|", + ...results + .filter((r) => !r.error) + .map( + (r) => + `| ${r.id} | ${r.flash.bad} | ${r.flash.paint ? `${r.flash.paint.frames} (${round(r.flash.paint.ms)} ms)` : "-"} | ${round(r.flash.missChance)} | ${round(r.flash.controlCoverage)} |`, + ), + ); return lines.join("\n") + "\n"; } +/** One case as baseline.json holds it; the gate reads the same projection. */ +// fallow-ignore-next-line complexity +export function entry(r) { + if (r.error) return { pass: false, error: true }; + return { + pass: r.pass, + tracking: roundUp(r.tracking.max), + pressJump: roundUp(r.pressJump), + drop: roundUp(r.drop), + reload: roundUp(r.reload), + render: roundUp(r.render), + undo: r.checks.undo, + dropped: r.smooth.dropped, + controlDropped: r.smooth.control.dropped, + work: roundUp(r.smooth.workP95), + frameP95: roundUp(r.smooth.p95), + ...(r.unsettled.length && { unsettled: r.unsettled }), + ...(r.undoTimeout && { undoTimeout: r.undoTimeout }), + ...(r.renderError && { renderError: true }), + flash: r.flash.bad, + ...(r.flash.uncovered && { flashUncovered: true }), + paint: r.flash.paint?.frames ?? null, + paintMs: roundUp(r.flash.paint?.ms ?? null), + }; +} + /** One line per case, so a baseline diff reads case by case. */ function baseline(meta, results) { const entries = [...results] .sort((a, b) => a.id.localeCompare(b.id)) - // fallow-ignore-next-line complexity - .map((r) => { - const v = r.error - ? { pass: false, error: true } - : { - pass: r.pass, - tracking: roundUp(r.tracking.max), - pressJump: roundUp(r.pressJump), - drop: roundUp(r.drop), - reload: roundUp(r.reload), - render: roundUp(r.render), - undo: r.checks.undo, - dropped: r.smooth.dropped, - controlDropped: r.smooth.control.dropped, - work: roundUp(r.smooth.workP95), - frameP95: roundUp(r.smooth.p95), - ...(r.unsettled.length && { unsettled: r.unsettled }), - ...(r.undoTimeout && { undoTimeout: r.undoTimeout }), - ...(r.renderError && { renderError: true }), - }; - return ` ${JSON.stringify(r.id)}: ${JSON.stringify(v)}`; - }); + .map((r) => ` ${JSON.stringify(r.id)}: ${JSON.stringify(entry(r))}`); return `{\n "studio": ${JSON.stringify(meta.studio)},\n "build": ${JSON.stringify(meta.build)},\n "bench": ${JSON.stringify(meta.bench)},\n "grid": ${JSON.stringify(meta.grid)},\n "cases": {\n${entries.join(",\n")}\n }\n}\n`; } diff --git a/packages/studio/tests/e2e/edit-accuracy/run.mjs b/packages/studio/tests/e2e/edit-accuracy/run.mjs index 6ff53308e8..689377e59b 100644 --- a/packages/studio/tests/e2e/edit-accuracy/run.mjs +++ b/packages/studio/tests/e2e/edit-accuracy/run.mjs @@ -1,6 +1,7 @@ #!/usr/bin/env bun // Edit accuracy bench: real gestures in the built CLI Studio (build core, parsers, lint, studio-server first). -// bun run --cwd packages/studio test:edit-accuracy -- --grid full|pr --jobs N [--shard i/n] [--filter re] [--lock path] +// bun run --cwd packages/studio test:edit-accuracy -- --grid full|pr --jobs N [--shard i/n] [--filter re] +// [--lock path: the suite lock, taken per chunk] [--rerun: a confirmation run for the gate] import { createHash } from "node:crypto"; import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { loadavg, tmpdir } from "node:os"; @@ -14,6 +15,7 @@ import { buildGrid, writeFixture } from "./grid.mjs"; import { killServers, runCase, startServer, stopServer } from "./case.mjs"; import { METRICS, score, writeReport } from "./report.mjs"; import { renderBox } from "./render.mjs"; +import { scoreFlash } from "./flash.mjs"; import { aabb, boxDistance } from "./geometry.mjs"; const HERE = dirname(fileURLToPath(import.meta.url)); @@ -29,6 +31,8 @@ const { values: opt } = parseArgs({ port: { type: "string", default: "5800" }, cli: { type: "string", default: join(REPO, "packages/cli/dist/cli.js") }, lock: { type: "string" }, + // Marks a confirmation run of cases that regressed, so the gate keeps it out of the banked baseline. + rerun: { type: "boolean", default: false }, }, }); const [shard, shards] = opt.shard.split("/").map(Number); @@ -51,6 +55,8 @@ function saveEvidence(id, evidence) { writeFileSync(join(caseDir, `saved-${name.replace("/", "-")}`), text); if (evidence.renderFailed && evidence.frame) writeFileSync(join(caseDir, "producer.jpg"), evidence.frame); + for (const [name, png] of evidence.flashFrames ?? []) + writeFileSync(join(caseDir, `flash-${name}.png`), Buffer.from(png, "base64")); } function verdict(r) { @@ -112,7 +118,8 @@ async function runOne(spec, browser, decoder, port) { }); await stopServer(server); server = null; - result = await withRender(dir, decoder, measured, evidence); + const flash = await scoreFlash(decoder, measured.flash, evidence); + result = await withRender(dir, decoder, { ...measured, flash }, evidence); } catch (error) { result = errorResult(error, log); } finally { @@ -200,7 +207,14 @@ const browsers = await Promise.all( puppeteer.launch({ executablePath: chrome, headless: true, - args: ["--no-sandbox", "--use-angle=swiftshader", "--enable-unsafe-swiftshader"], + // Software compositing: the screencast reads each frame back cheaply, so flash windows see ~95% of frames + // instead of 40-70% under swiftshader GL. + args: [ + "--no-sandbox", + "--use-angle=swiftshader", + "--enable-unsafe-swiftshader", + "--disable-gpu-compositing", + ], }), ), ); @@ -215,6 +229,7 @@ const meta = { studio, build, bench, + rerun: opt.rerun, grid: opt.grid + (opt.filter ? ` filter ${opt.filter}` : "") +