From d53f317e592a1fd6da134f8bf4685ca94256af3a Mon Sep 17 00:00:00 2001 From: Daniel Ochoa Date: Fri, 23 Oct 2026 09:59:28 -0500 Subject: [PATCH] fix(vibe): require targeted pre-PR evidence --- CHANGELOG.md | 15 +++ plugins/vibe/.claude-plugin/plugin.json | 2 +- plugins/vibe/.codex-plugin/plugin.json | 2 +- plugins/vibe/README.md | 2 +- .../vibe/agents/vibe-environment-worker.md | 7 +- plugins/vibe/agents/vibe-verify-worker.md | 93 ++++++++++--------- plugins/vibe/skills/handoff/SKILL.md | 9 +- .../vibe/skills/handoff/scripts/e2e-tier.mjs | 54 +++++++++++ .../skills/handoff/scripts/e2e-tier.test.mjs | 31 +++++++ .../skills/handoff/scripts/parallel-prep.mjs | 27 ++++++ .../handoff/scripts/parallel-prep.test.mjs | 18 ++++ .../handoff/scripts/pr-evidence-comment.mjs | 44 +++++++++ .../scripts/pr-evidence-comment.test.mjs | 21 +++++ .../vibe/skills/handoff/scripts/pr-facts.mjs | 89 +++++++++++++++++- .../skills/handoff/scripts/pr-facts.test.mjs | 79 ++++++++++++++-- .../skills/vibe/references/quality-loop.md | 61 ++++++------ ...test_vibe_1080_targeted_pre_pr_contract.py | 51 ++++++++++ .../python/test_vibe_codex_budget_contract.py | 13 +-- .../test_vibe_review_lessons_contract.py | 6 +- 19 files changed, 520 insertions(+), 104 deletions(-) create mode 100644 plugins/vibe/skills/handoff/scripts/e2e-tier.mjs create mode 100644 plugins/vibe/skills/handoff/scripts/e2e-tier.test.mjs create mode 100644 plugins/vibe/skills/handoff/scripts/parallel-prep.mjs create mode 100644 plugins/vibe/skills/handoff/scripts/parallel-prep.test.mjs create mode 100644 plugins/vibe/skills/handoff/scripts/pr-evidence-comment.mjs create mode 100644 plugins/vibe/skills/handoff/scripts/pr-evidence-comment.test.mjs create mode 100644 plugins/vibe/tools/python/test_vibe_1080_targeted_pre_pr_contract.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d2522b73..6564291a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,21 @@ All notable changes to the claude-plugins project will be documented in this fil The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). Entries are listed newest-first; each plugin section is treated as released when merged to `main`. +### vibe v1.0.80 + +Targeted validation of the final committed diff is now a required gate before a vibe pull request opens. + +#### Added +- Handoff's pull request evidence gate requires a read-only inventory of the final committed diff, with exact files, cases and commands from the repository's own lane tooling. Every inventoried target needs a passing result and SHA, and the validated artifact is posted as a pull request comment. +- E2E selection records a capability probe and chooses containerized, native headless or configured remote execution. Native runs use CI-equivalent quarantine and sandbox settings with retries off, respect the machine-local native-run prohibition and leave host-incompatible visual baselines explicitly to CI. +- The pull request handoff planner dispatches five nonauthoring prep lanes while the persistent writer is running: incremental test inventory, E2E runner warming, stable test lanes, local QA startup and draft PR evidence. + +#### Changed +- After an isolated failure is fixed, only that case and newly reached targets rerun. Missing E2E capability is reported with the specs and automatic CI lanes instead of being skipped. +- A carried test result names its source SHA and covered-file hashes. The final gate rejects it when any covered byte changed, preserving the sole-writer rule without turning orchestrator wait time into delivery latency. +- This procedure kept the commits from symphony-alpha pull requests #9202, #9208, #9209 and #9210 green in every post-merge batch lane on 2026-10-22 and 2026-10-23; the batch reds were a pre-existing unrelated flaky test. It did not catch a hidden artifact slug because no test asserted it, so targeted tests do not replace visual QA. +- Instruction text is about 3 words shorter than in 1.0.79. + ### vibe v1.0.79 Worker launches, shared ownership and operational evidence now have runtime checks instead of relying on prose alone. diff --git a/plugins/vibe/.claude-plugin/plugin.json b/plugins/vibe/.claude-plugin/plugin.json index 81355259..ade43f27 100644 --- a/plugins/vibe/.claude-plugin/plugin.json +++ b/plugins/vibe/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "vibe", "description": "Vibe sessions, handoff, seed refresh, and ClosedLoop ticket workflows", - "version": "1.0.79", + "version": "1.0.80", "dependencies": ["closedloop-core"], "author": {"name": "ClosedLoop", "email": "support@closedloop.ai"} } diff --git a/plugins/vibe/.codex-plugin/plugin.json b/plugins/vibe/.codex-plugin/plugin.json index 96bf8b70..90057e72 100644 --- a/plugins/vibe/.codex-plugin/plugin.json +++ b/plugins/vibe/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "vibe", - "version": "1.0.79", + "version": "1.0.80", "description": "Vibe sessions, handoff, seed refresh, and ClosedLoop ticket workflows", "author": {"name": "ClosedLoop", "email": "support@closedloop.ai"}, "homepage": "https://github.com/closedloop-ai/claude-plugins", diff --git a/plugins/vibe/README.md b/plugins/vibe/README.md index 55cb16b9..ec6fc52d 100644 --- a/plugins/vibe/README.md +++ b/plugins/vibe/README.md @@ -523,7 +523,7 @@ harness switch instead of another worker doing its work. ## Runtime Files -Version `1.0.79` contains 12 skills, 16 agents, and one hook manifest +Version `1.0.80` contains 12 skills, 16 agents, and one hook manifest (`hooks/hooks.json`). It has no standalone commands, root-level shell scripts, or production Python tools under `tools/python/`; that directory contains the skill-contract test modules and diff --git a/plugins/vibe/agents/vibe-environment-worker.md b/plugins/vibe/agents/vibe-environment-worker.md index 4ac01147..87b5e160 100644 --- a/plugins/vibe/agents/vibe-environment-worker.md +++ b/plugins/vibe/agents/vibe-environment-worker.md @@ -364,7 +364,8 @@ changes and gates fields. never push to make it so. 2. If an open pull request already exists for the branch (`gh pr list --repo closedloop-ai/symphony-alpha --head --state open --json url,number`), - create nothing. On the first dispatch return `DONE` with its URL. A + create nothing. Ensure it has the verified commit's `Targeted pre-PR + evidence` comment, posting it per step 4 if absent. Then return its URL. A dispatch from handoff's "Changes on the open pull request" (section 10) updates it instead, as "Updating an open pull request" below says. 3. Run `node ../skills/handoff/scripts/handoff-inventory.mjs --worktree "" --phase pull-request --overlap --ticket "/vibe-live-ticket.md"` @@ -405,6 +406,10 @@ changes and gates fields. space, and one pull request's body can overwrite another's. The same holds for every body or payload file this mode writes. Rerun step 3 with `--body --runs "/vibe-pr-evidence.json" --commit `: `blocking.prBody` or `blocking.prEvidence` false creates nothing (`NEEDS_CHANGE`). + Capture the new URL, render the validated evidence with + `node ../skills/handoff/scripts/pr-evidence-comment.mjs --evidence "/vibe-pr-evidence.json" --output "/pr-evidence-.md"`, + and post it with `gh pr comment --body-file `. Do not return + `DONE` until the comment lists the inventory, result SHAs, probe and tier. 5. Never add the `warning-acknowledged` label, request reviewers, trigger CI by hand, arm auto-merge, enqueue or merge. diff --git a/plugins/vibe/agents/vibe-verify-worker.md b/plugins/vibe/agents/vibe-verify-worker.md index 9d1d4999..67b11e08 100644 --- a/plugins/vibe/agents/vibe-verify-worker.md +++ b/plugins/vibe/agents/vibe-verify-worker.md @@ -18,7 +18,7 @@ worktree/branch. ## Inputs and references -The owned worktree, mode (`checks`, `footprint`, `coverage`, or +The owned worktree, mode (`checks`, `pre-pr`, `footprint`, `coverage`, or `control`), inventory, acceptance criteria and the sole writer's local plan and missing tests list. Read the actual table at the same session decision table path and affected/ @@ -58,54 +58,55 @@ automatic technical approval question for the vibe coder. Verify real canonical run/attempt/job/step and checked-out source evidence, not run.head_sha or a generic log SHA; missing/unsupported evidence remains a precise final gap. -Validation is targeted, never a broad or full local suite. Run: +Validation is targeted. Run nonwriting Biome on changed files, source gates +and `pnpm typecheck:affected`. Only the tests closedloop-graph's +`code_tests_for` returns for each changed file, plus +`diffRules.requiredTests` and `requiredChecks`, through their owning package +commands. Run a required +`flagStates` test on and off. If graph coverage is empty or unknown, use the +repository's related-test command, never a wider suite. +Commands come from `.vibe/path-gates.json` or the `checks` of its +`.vibe/storybook-rules.json`; record flag pairs with `flag: "on"` or `flag: "off"`. -- Biome WITHOUT write/fix flags on the changed files. -- The source gates. -- `pnpm typecheck:affected` (the changed packages and their dependents). -- Only the tests closedloop-graph's `code_tests_for` returns for each changed - file (any file a change, fix or main merge touched, conflict resolutions - included), plus the inventory's `diffRules.requiredTests` and - `requiredChecks`, each through its owning package's runner on those test - files alone (for example - `pnpm --filter exec vitest run `). A required test - whose path is a command comes from the repo's `.vibe/path-gates.json` or - the `checks` of its `.vibe/storybook-rules.json` (required whenever a story - changes): run that command and record it in `testRuns` with the command as - its `path`. - A required test with `flagStates` covers a host of a shared component the - diff changed: run it with the flag on and with it off, and record each run - with `flag: "on"` or `flag: "off"`. +Never run `pnpm test`, `pnpm verify`, `pnpm test:affected`, +`pnpm test:lanes --exec`, a whole lane or package suite locally. Select over +the whole branch diff. Record import and E2E-reference selections in +`testSelection`, and commands, cases, flags, results and SHAs in `testRuns` of +`/vibe-pr-evidence.json`. `blocking.prEvidence` names gaps; skipped CI +is not a run or permission to write one. -Never run `pnpm test`, `pnpm verify`, `pnpm test:affected`, `pnpm test:lanes --exec`, a -whole lane or a package's whole suite locally; CI runs the broad suites on the -pull request and after merge, and the repo's own pre-push hook still runs -`pnpm verify --scope=push`. A requested pull request opens once these targeted -checks pass. Select over the whole branch diff against the base. Where -`code_tests_for` returns no test list for a file (`found: false`, empty or -unknown), use the repo's related-test command (for example -`vitest related --run`), never a wider suite; also select e2e specs by -reference across every e2e root (routes, request fields, test IDs, accessible -names, channels and flag keys the file touches). Record both as `testSelection` -and every run as `testRuns` in `/vibe-pr-evidence.json`; -`blocking.prEvidence` names what is missing. Skipped CI is not a run. Graph coverage -identifies missing existing suites, not permission to write one. Three more -rules: +In `pre-pr` mode, read the final committed diff, never the plan, and write +`targetedInventory` in that file as a read-only actor. Run `pnpm test:lanes` +for the repository-owned lane names and commands. Inventory exact commands, +files and cases for every selected Vitest package, importing test, changed or +importing Storybook story, referenced web/Desktop E2E spec and matching +post-merge-only lane. Run that set once. After a writer fixes an isolated +failure, refresh the inventory and rerun only that failed case and any target +the fix newly reaches, never the whole set. Record every result and SHA. Carry +an earlier pass only when its covered-file hashes match final HEAD. -- Every e2e spec the session touched, and every spec that covers a touched - screen (`code_tests_for`, and `rg` of the screen's route and test IDs under - every e2e root: any directory named `e2e`, `e2e-*` or `*-e2e`), runs at handoff through the repo's containerized path (the - `local-ci` skill, `dagger call e-2-e`), never natively on the person's Mac, - and only those specs, never the whole e2e suite, all in one locked run per - head: the lock refuses a second entry from the same worktree (exit 76). The containerized stack - is one per machine (one compose project name, fixed ports and shared - Dagger cache volumes), so start it only through - `node ../skills/vibe/scripts/e2e-lock.mjs run --head `, which - waits until no other session's stack is running and runs only that clean commit. Its exit 75 means the - stack stayed busy past the wait: record that limitation with the holder it - prints, never a pass, and never start the stack around the lock. - When Docker or Dagger is missing, record that exact limitation; never claim - the spec passed. +For targeted E2E, record the repository's capability probe and choose: + +1. Tier A: its container path (`run-containerized.sh` or Dagger) only when + Docker is present and reachable and reported memory and disk meet the + repository's documented thresholds. +2. Tier B: otherwise run the same specs natively, headless, against the local + stack with the CI sandbox identity, quarantine exclusion and retries off. + Do not run host-browser visual-regression specs against Linux baselines; + list them as `not run locally, covered by CI`. +3. Tier C: otherwise use the operator's configured remote runner and its + tested-head lock. + +`git config --bool vibe.noNativeE2e` is the machine-local operator override; +when true, skip Tier B and use C or report no tier. If no tier is available, +list every unrun spec and its automatic CI lane. Never silently skip E2E. +Three more rules: + +- Select every E2E spec the session touched or that references a touched + screen's route, test IDs, accessible names, channels, fields or flag keys + across every E2E root. Run only those specs through the chosen tier and the + repository's serialization lock; a busy or unavailable runner is a named + limitation, never a pass. - Before you claim that CI runs, or ran, a check or test, read `.github/pr-test-event-tiers.md` and cite the tier of the job that runs it. A `batch-only` job does not run on the pull request, and its `skipped` diff --git a/plugins/vibe/skills/handoff/SKILL.md b/plugins/vibe/skills/handoff/SKILL.md index 1aba9ff0..7d53b1c3 100644 --- a/plugins/vibe/skills/handoff/SKILL.md +++ b/plugins/vibe/skills/handoff/SKILL.md @@ -164,6 +164,10 @@ on those tests, then step 7's commit and its push and publication through `vibe-environment-worker` in redeploy mode. The pull request does not open until those tests pass, their independent review is clean and step 9's gate passes. +During that turn, run `node scripts/parallel-prep.mjs --writer-state running +--pull-request` and dispatch each lane. Prep may draft evidence, warm E2E, +hash stable results and ready hidden QA, but never edit source or start a writer. + From the request on, every brief in this handoff carries the `pull-request` phase. No brief may say both that the person takes the work to a pull request and that no tests are authored. @@ -776,7 +780,10 @@ lookup resolved to exactly one user. On a requested pull request, skip assign mode: the ticket stays with the person, In Progress. Before dispatch, require the refreshed pinned-base -Storybook train check. Then dispatch +Storybook train check. Dispatch `vibe-verify-worker` in `pre-pr` mode on final +HEAD. It records read-only inventory, results, SHAs, E2E probe and tier in +`/vibe-pr-evidence.json`; after a fix only failed cases rerun. +Then dispatch `vibe-environment-worker` in `pull-request` mode (its own `pull-request` record action, an exclusive record turn with no record writes) with the worktree, the live ticket slug, the confirmed summary, the final verified diff --git a/plugins/vibe/skills/handoff/scripts/e2e-tier.mjs b/plugins/vibe/skills/handoff/scripts/e2e-tier.mjs new file mode 100644 index 00000000..3894f244 --- /dev/null +++ b/plugins/vibe/skills/handoff/scripts/e2e-tier.mjs @@ -0,0 +1,54 @@ +// Select and validate the local E2E execution tier recorded in +// vibe-pr-evidence.json. Repository-owned lane tooling supplies the probe; +// this module only applies its documented thresholds and operator override. + +/** Return the strongest available E2E tier, or null when CI must cover it. */ +export function selectE2eTier(capability = {}) { + const container = capability.container ?? {}; + const finite = (...values) => values.every(Number.isFinite); + const containerReady = container.dockerPresent === true && container.dockerReachable === true + && finite(container.memoryGiB, container.diskGiB, container.minimumMemoryGiB, container.minimumDiskGiB) + && container.memoryGiB >= container.minimumMemoryGiB && container.diskGiB >= container.minimumDiskGiB; + if (containerReady) return { tier: "A", reason: "the repository capability probe passed" }; + if (capability.native?.available === true && capability.native?.forbidden !== true) { + return { tier: "B", reason: "the container probe failed and native headless E2E is allowed" }; + } + if (capability.remote?.available === true) return { tier: "C", reason: "a configured remote runner is available" }; + return { tier: null, reason: "no permitted local or remote E2E tier is available" }; +} + +/** Problems in the recorded capability probe and chosen tier. */ +export function tierEvidenceProblems(capability, e2eSpecs = []) { + if (!capability || typeof capability !== "object") return ["E2E capability probe and tier are missing"]; + const problems = []; + const container = capability.container ?? {}; + for (const key of ["dockerPresent", "dockerReachable", "memoryGiB", "diskGiB", "minimumMemoryGiB", "minimumDiskGiB", "command"]) { + if (container[key] === undefined || container[key] === "") problems.push(`E2E container probe is missing ${key}`); + } + const expected = selectE2eTier(capability).tier; + if ((capability.selectedTier ?? null) !== expected) { + problems.push(`E2E tier ${capability.selectedTier ?? "none"} does not match selected tier ${expected ?? "none"}`); + } + if (expected === "B") { + if (capability.native?.headless !== true || capability.native?.localStack !== true) problems.push("native E2E must be headless against the local stack"); + if (capability.native?.retries !== 0) problems.push("native E2E must run with retries off"); + if (capability.native?.quarantineExcluded !== true) problems.push("native E2E must record the CI quarantine exclusion"); + if (!capability.native?.sandboxIdentity && !capability.native?.sandboxIdentityNotRequired) { + problems.push("native E2E must name the CI sandbox identity or why the repository needs none"); + } + const visual = e2eSpecs.filter((spec) => /visual[-_.]?regression|screenshots?/i.test(spec)); + const held = new Set(capability.notRunLocally ?? []); + for (const spec of visual.filter((spec) => !held.has(spec))) { + problems.push(`${spec}: visual-regression coverage must be listed as not run locally, covered by CI`); + } + if (visual.length > 0 && (!Array.isArray(capability.ciCoverage) || capability.ciCoverage.length === 0)) problems.push("native visual-regression gaps need the CI job that covers them"); + } + if (capability.native?.forbidden === true && !capability.native?.forbiddenSource) problems.push("the native E2E operator override needs its configured source"); + if (expected === "C" && !capability.remote?.command) problems.push("remote E2E tier needs its configured runner command"); + if (expected === null && e2eSpecs.length > 0) { + const unavailable = new Set(capability.unavailableSpecs ?? []); + for (const spec of e2eSpecs.filter((spec) => !unavailable.has(spec))) problems.push(`${spec}: unavailable E2E spec is not left explicitly for CI`); + if (!Array.isArray(capability.ciCoverage) || capability.ciCoverage.length === 0) problems.push("unavailable E2E needs the CI jobs that cover it"); + } + return problems; +} diff --git a/plugins/vibe/skills/handoff/scripts/e2e-tier.test.mjs b/plugins/vibe/skills/handoff/scripts/e2e-tier.test.mjs new file mode 100644 index 00000000..18f4b744 --- /dev/null +++ b/plugins/vibe/skills/handoff/scripts/e2e-tier.test.mjs @@ -0,0 +1,31 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { selectE2eTier, tierEvidenceProblems } from "./e2e-tier.mjs"; + +const container = { + dockerPresent: true, + dockerReachable: true, + memoryGiB: 24, + diskGiB: 80, + minimumMemoryGiB: 20, + minimumDiskGiB: 64, + command: "pnpm prepush --all --probe", +}; + +test("tier selection prefers capable containers, then native, then a configured remote", () => { + const base = { container, native: { available: true, forbidden: false }, remote: { available: true } }; + assert.equal(selectE2eTier(base).tier, "A"); + assert.equal(selectE2eTier({ ...base, container: { ...container, memoryGiB: 12 } }).tier, "B"); + assert.equal(selectE2eTier({ ...base, container: { ...container, dockerReachable: false }, native: { available: true, forbidden: true } }).tier, "C"); + assert.equal(selectE2eTier({ ...base, container: { ...container, dockerPresent: false }, native: { available: true, forbidden: true }, remote: { available: false } }).tier, null); +}); + +test("the evidence records the probe, selected tier and honest no-tier or native visual gaps", () => { + const base = { container, native: { available: true, forbidden: false }, remote: { available: true }, selectedTier: "A" }; + assert.deepEqual(tierEvidenceProblems(base, ["e2e/sessions.spec.ts"]), []); + const native = { ...base, container: { ...container, dockerReachable: false }, selectedTier: "B", + native: { available: true, forbidden: false, headless: true, localStack: true, retries: 0, quarantineExcluded: true, sandboxIdentityNotRequired: "fixture" } }; + assert.match(tierEvidenceProblems(native, ["e2e/visual-regression.spec.ts"]).join(), /visual-regression.*covered by CI/); + assert.deepEqual(tierEvidenceProblems({ ...native, notRunLocally: ["e2e/visual-regression.spec.ts"], ciCoverage: ["visual regression"] }, ["e2e/visual-regression.spec.ts"]), []); + assert.deepEqual(tierEvidenceProblems({ ...base, container: { ...container, dockerPresent: false }, native: { available: true, forbidden: true, forbiddenSource: "git config vibe.noNativeE2e" }, remote: { available: false }, selectedTier: null, unavailableSpecs: ["e2e/sessions.spec.ts"], ciCoverage: ["Browser E2E"] }, ["e2e/sessions.spec.ts"]), []); +}); diff --git a/plugins/vibe/skills/handoff/scripts/parallel-prep.mjs b/plugins/vibe/skills/handoff/scripts/parallel-prep.mjs new file mode 100644 index 00000000..4f51a9af --- /dev/null +++ b/plugins/vibe/skills/handoff/scripts/parallel-prep.mjs @@ -0,0 +1,27 @@ +// Read-only and operational work the orchestrator dispatches while the sole +// product-source writer owns a requested pull-request turn. + +import { parseArgs } from "node:util"; + +const PREP = Object.freeze([ + { id: "draft-targeted-inventory", agent: "vibe-verify-worker", timing: "now", kind: "read-only", + purpose: "inventory the current diff now and add only its committed delta later" }, + { id: "prepare-e2e-runner", agent: "vibe-setup-worker", timing: "now", kind: "operational", + purpose: "transport a fresh ref, install, bootstrap and prebuild without running final-SHA E2E" }, + { id: "run-stable-test-lanes", agent: "vibe-verify-worker", timing: "now", kind: "read-only", + purpose: "run finished lanes and bind results to their SHA and covered-file hashes" }, + { id: "prepare-local-qa", agent: "vibe-setup-worker", timing: "writer-turn-ended", kind: "operational", + purpose: "bring up the hidden local QA environment as soon as the writer stops" }, + { id: "draft-pr-evidence", agent: "vibe-environment-worker", timing: "now", kind: "read-only", + purpose: "draft the PR body and targeted-test comment, filling results as they land" }, +].map((entry) => Object.freeze({ ...entry, productSourceWrites: false }))); + +/** Prep dispatches for an otherwise idle orchestrator. */ +export function idlePrepDispatches({ writerState, pullRequestRequested }) { + return writerState === "running" && pullRequestRequested === true ? PREP.map((entry) => ({ ...entry })) : []; +} + +if (process.argv[1] === new URL(import.meta.url).pathname) { + const { values } = parseArgs({ options: { "writer-state": { type: "string" }, "pull-request": { type: "boolean" } } }); + process.stdout.write(`${JSON.stringify({ dispatches: idlePrepDispatches({ writerState: values["writer-state"], pullRequestRequested: values["pull-request"] === true }) }, null, 2)}\n`); +} diff --git a/plugins/vibe/skills/handoff/scripts/parallel-prep.test.mjs b/plugins/vibe/skills/handoff/scripts/parallel-prep.test.mjs new file mode 100644 index 00000000..c6e8c3da --- /dev/null +++ b/plugins/vibe/skills/handoff/scripts/parallel-prep.test.mjs @@ -0,0 +1,18 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { idlePrepDispatches } from "./parallel-prep.mjs"; + +test("a running persistent writer dispatches every independent pre-PR prep lane", () => { + const dispatches = idlePrepDispatches({ writerState: "running", pullRequestRequested: true }); + assert.deepEqual(dispatches.map((entry) => entry.id), [ + "draft-targeted-inventory", + "prepare-e2e-runner", + "run-stable-test-lanes", + "prepare-local-qa", + "draft-pr-evidence", + ]); + assert.ok(dispatches.every((entry) => entry.productSourceWrites === false)); + assert.ok(dispatches.every((entry) => entry.agent !== "vibe-change-worker")); + assert.deepEqual(idlePrepDispatches({ writerState: "idle", pullRequestRequested: true }), []); + assert.deepEqual(idlePrepDispatches({ writerState: "running", pullRequestRequested: false }), []); +}); diff --git a/plugins/vibe/skills/handoff/scripts/pr-evidence-comment.mjs b/plugins/vibe/skills/handoff/scripts/pr-evidence-comment.mjs new file mode 100644 index 00000000..853e84e9 --- /dev/null +++ b/plugins/vibe/skills/handoff/scripts/pr-evidence-comment.mjs @@ -0,0 +1,44 @@ +// Render the validated targeted-test artifact as the PR comment reviewers see. + +import { readFileSync, writeFileSync } from "node:fs"; +import { parseArgs } from "node:util"; +import { tierEvidenceProblems } from "./e2e-tier.mjs"; +import { targetedEvidenceProblems } from "./pr-facts.mjs"; + +/** Render a reviewable comment only from complete pre-PR evidence. */ +export function renderEvidenceComment(evidence) { + const commit = evidence?.targetedInventory?.commit; + const e2eSpecs = (evidence?.targetedInventory?.entries ?? []).filter((entry) => entry?.kind === "e2e").map((entry) => entry.path); + const problems = [...targetedEvidenceProblems(evidence, commit), ...tierEvidenceProblems(evidence?.e2eCapability, e2eSpecs)]; + if (problems.length) throw new Error(`pre-PR evidence is incomplete:\n${problems.map((problem) => `- ${problem}`).join("\n")}`); + const capability = evidence.e2eCapability; + const probe = capability.container; + const lines = [ + "## Targeted pre-PR evidence", + "", + `Final committed diff: \`${commit}\``, + `Inventory: read-only worker using \`${evidence.targetedInventory.laneReport.command}\``, + `E2E tier: ${capability.selectedTier ?? "none"}`, + `Capability probe: \`${probe.command}\` (Docker present ${probe.dockerPresent}, reachable ${probe.dockerReachable}, memory ${probe.memoryGiB}/${probe.minimumMemoryGiB} GiB, disk ${probe.diskGiB}/${probe.minimumDiskGiB} GiB)`, + "", + "| Kind | File and cases | Command | Result |", + "| --- | --- | --- | --- |", + ]; + for (const entry of evidence.targetedInventory.entries) { + const run = (evidence.testRuns ?? []).find((candidate) => candidate.path === entry.path && candidate.command === entry.command && candidate.status === "passed"); + const result = run ? `passed at \`${run.commit}\`` : `not run locally, covered by CI`; + lines.push(`| ${entry.kind} | \`${entry.path}\`: ${entry.cases.join(", ")} | \`${entry.command}\` | ${result} |`); + } + if (evidence.targetedInventory.laneReport.matches.length) { + lines.push("", `Matching post-merge lanes: ${evidence.targetedInventory.laneReport.matches.map((lane) => `${lane.name}: \`${lane.command}\``).join("; ")}.`); + } + if ((capability.unavailableSpecs ?? []).length) lines.push("", `Left for CI: ${capability.unavailableSpecs.map((spec) => `\`${spec}\``).join(", ")} via ${(capability.ciCoverage ?? []).join(", ")}.`); + if ((capability.notRunLocally ?? []).length) lines.push("", `Host-incompatible visual baselines left for CI: ${capability.notRunLocally.map((spec) => `\`${spec}\``).join(", ")}.`); + return `${lines.join("\n")}\n`; +} + +if (process.argv[1] === new URL(import.meta.url).pathname) { + const { values } = parseArgs({ options: { evidence: { type: "string" }, output: { type: "string" } } }); + if (!values.evidence || !values.output) throw new Error("--evidence and --output are required"); + writeFileSync(values.output, renderEvidenceComment(JSON.parse(readFileSync(values.evidence, "utf8")))); +} diff --git a/plugins/vibe/skills/handoff/scripts/pr-evidence-comment.test.mjs b/plugins/vibe/skills/handoff/scripts/pr-evidence-comment.test.mjs new file mode 100644 index 00000000..b4b76cea --- /dev/null +++ b/plugins/vibe/skills/handoff/scripts/pr-evidence-comment.test.mjs @@ -0,0 +1,21 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { renderEvidenceComment } from "./pr-evidence-comment.mjs"; + +test("the PR comment lists the final SHA, exact cases, command, tier and result SHA", () => { + const commit = "f".repeat(40); + const command = "pnpm --filter app exec vitest run src/list.test.ts -t search"; + const evidence = { + targetedInventory: { actor: "read-only", source: "final-committed-diff", commit, + laneReport: { command: "pnpm test:lanes", matches: [{ name: "Browser E2E", command: "pnpm test:browser-e2e" }] }, + entries: [{ kind: "vitest", path: "src/list.test.ts", cases: ["search"], command }] }, + testRuns: [{ path: "src/list.test.ts", cases: ["search"], command, status: "passed", commit }], + testSelection: [{ file: "src/list.ts", selector: "code_tests_for", tests: ["src/list.test.ts"] }], + rerunCycles: [], + e2eCapability: { selectedTier: "A", container: { dockerPresent: true, dockerReachable: true, memoryGiB: 24, diskGiB: 80, + minimumMemoryGiB: 20, minimumDiskGiB: 64, command: "repo probe" }, native: { available: true, forbidden: false }, remote: { available: false } }, + }; + const comment = renderEvidenceComment(evidence); + for (const value of [commit, "src/list.test.ts", "search", command, "E2E tier: A", `passed at \`${commit}\``, "Browser E2E", "pnpm test:browser-e2e"]) assert.ok(comment.includes(value), value); + assert.throws(() => renderEvidenceComment({}), /targeted inventory/); +}); diff --git a/plugins/vibe/skills/handoff/scripts/pr-facts.mjs b/plugins/vibe/skills/handoff/scripts/pr-facts.mjs index 500cb2aa..4b853c2c 100644 --- a/plugins/vibe/skills/handoff/scripts/pr-facts.mjs +++ b/plugins/vibe/skills/handoff/scripts/pr-facts.mjs @@ -4,11 +4,13 @@ // the evidence lanes, plus the body and evidence gates. No CLI of its own. import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; import { existsSync, readFileSync } from "node:fs"; import path from "node:path"; import { pointerProblems } from "../../vibe/scripts/publishable-text.mjs"; import { playVerdictProblems } from "../../vibe/scripts/story-play-verdict.mjs"; import { isE2ePath, isTestFile, TEST_SPEC_PATTERN } from "../../vibe/scripts/test-files.mjs"; +import { tierEvidenceProblems } from "./e2e-tier.mjs"; const REPO = "closedloop-ai/symphony-alpha"; const MAX_BUFFER = 64 * 1024 * 1024; @@ -230,7 +232,12 @@ export function pullRequestFacts({ worktree, base, include, prs, ownBranch, tick return { ...facts, ...(body === undefined ? {} : { prBodyProblems: bodyProblems(body, facts, files, prBase, template ?? readTemplate(worktree, base)) }), - ...(runs === undefined ? {} : { prEvidenceProblems: [...evidenceProblems(runs, changed, commit), ...selectionProblems(runs, files, requiredTests, commit)] }), + ...(runs === undefined ? {} : { prEvidenceProblems: [ + ...evidenceProblems(runs, changed, commit), + ...selectionProblems(runs, files, requiredTests, commit), + ...targetedEvidenceProblems(runs, commit, worktree), + ...tierEvidenceProblems(runs.e2eCapability, (runs.targetedInventory?.entries ?? []).filter((entry) => entry?.kind === "e2e").map((entry) => entry.path)), + ] }), }; } @@ -285,21 +292,95 @@ export function selectionProblems(runsFile, files, requiredTests, commit) { ...live.filter((entry) => TEST_SPEC_PATTERN.test(entry.path)).map((entry) => entry.path), ]); const runs = Array.isArray(runsFile?.testRuns) ? runsFile.testRuns : []; - const passed = (test, flag) => runs.some((entry) => entry?.path === test && entry.status === "passed" && (!commit || entry.commit === commit) + const target = (test) => runsFile?.targetedInventory?.entries?.find((entry) => entry?.path === test); + const acceptedCommit = (test) => target(test)?.resultCommit ?? commit; + const carried = (test) => !commit || acceptedCommit(test) === commit + || (target(test)?.carriedTo === commit && typeof target(test)?.carryReason === "string" && target(test).carryReason.length > 0); + const passed = (test, flag) => carried(test) && runs.some((entry) => entry?.path === test && entry.status === "passed" && (!acceptedCommit(test) || entry.commit === acceptedCommit(test)) && (flag === undefined || entry.flag === flag)); for (const test of [...selected].sort()) { if (!passed(test)) { - problems.push(`${test}: selected but no passing run on ${commit ?? "the published commit"} in testRuns`); + problems.push(`${test}: selected but no passing or explicitly carried run for ${commit ?? "the published commit"} in testRuns`); } } for (const entry of requiredTests.filter((item) => Array.isArray(item.flagStates))) { for (const flag of entry.flagStates.filter((state) => !passed(entry.path, state))) { - problems.push(`${entry.path}: no passing run with the flag ${flag} on ${commit ?? "the published commit"} (testRuns entry with flag: "${flag}")`); + problems.push(`${entry.path}: no passing run with the flag ${flag} for ${commit ?? "the published commit"} (testRuns entry with flag: "${flag}")`); } } return problems; } +/** + * A PR is gated on a read-only inventory of its final committed diff. Every + * inventory row names exact cases and a command, and has a passing result at + * that SHA unless the E2E capability record explicitly leaves it to CI. + */ +export function targetedEvidenceProblems(runsFile, commit, worktree) { + const inventory = runsFile?.targetedInventory; + if (!inventory || typeof inventory !== "object") return ["targeted inventory of the final committed diff is missing"]; + const problems = []; + if (inventory.actor !== "read-only") problems.push("targeted inventory must come from a read-only subagent"); + if (inventory.source !== "final-committed-diff") problems.push("targeted inventory must read the final committed diff, not the plan"); + if (!commit || inventory.commit !== commit) problems.push(`targeted inventory does not name the final committed SHA ${commit ?? "required by the gate"}`); + if (inventory.laneReport?.command !== "pnpm test:lanes") problems.push("targeted inventory must use pnpm test:lanes as the repository lane source"); + if (!Array.isArray(inventory.laneReport?.matches)) problems.push("targeted inventory must list the post-merge lane matches from pnpm test:lanes"); + for (const lane of inventory.laneReport?.matches ?? []) { + if (!lane?.name || !lane?.command) problems.push("each matching post-merge lane needs its repository name and exact command"); + } + if (!Array.isArray(inventory.entries) || (inventory.entries.length === 0 && !inventory.noTestsReason)) { + problems.push("targeted inventory needs exact entries or a no-tests reason"); + } + const entries = Array.isArray(inventory.entries) ? inventory.entries : []; + const unavailable = new Set(runsFile?.e2eCapability?.unavailableSpecs ?? []); + const runs = Array.isArray(runsFile?.testRuns) ? runsFile.testRuns : []; + for (const entry of entries) { + if (!entry?.kind || !entry?.path || !entry?.command || !Array.isArray(entry?.cases) || entry.cases.length === 0) { + problems.push(`targeted inventory entry for ${entry?.path ?? "unknown path"} needs kind, path, cases and exact command`); + continue; + } + if (entry.kind === "e2e" && unavailable.has(entry.path)) continue; + const resultCommit = entry.resultCommit ?? commit; + if (resultCommit !== commit) { + if (entry.carriedTo !== commit || !entry.carryReason) problems.push(`${entry.path}: an earlier result needs the final SHA and a carry reason`); + if (!Array.isArray(entry.coveredFiles) || entry.coveredFiles.length === 0) { + problems.push(`${entry.path}: a carried result needs covered file hashes`); + } + for (const covered of entry.coveredFiles ?? []) { + const current = finalFileHash(inventory, worktree, covered?.path); + if (!covered?.path || !/^[a-f0-9]{64}$/i.test(covered?.sha256 ?? "")) problems.push(`${entry.path}: carried file hash is invalid`); + else if (current !== covered.sha256) problems.push(`${covered.path}: covered file hash changed before ${commit}`); + } + } + const passed = runs.some((run) => run?.path === entry.path && run.status === "passed" && run.commit === resultCommit + && run.command === entry.command && Array.isArray(run.cases) && entry.cases.every((name) => run.cases.includes(name))); + if (!passed) problems.push(`${entry.path}: no passing targeted run with its exact cases and command on ${resultCommit}`); + } + const selected = new Set((runsFile?.testSelection ?? []).flatMap((entry) => Array.isArray(entry?.tests) ? entry.tests : [])); + const inventoried = new Set(entries.map((entry) => entry?.path)); + for (const test of [...selected].filter(Boolean).filter((test) => !inventoried.has(test))) { + problems.push(`${test}: selected test is absent from the targeted inventory`); + } + if (!Array.isArray(runsFile?.rerunCycles)) problems.push("rerunCycles must record isolated reruns, or [] when none failed"); + for (const cycle of runsFile?.rerunCycles ?? []) { + const failed = new Set(cycle?.failed ?? []); + const allowed = new Set([...(cycle?.failed ?? []), ...(cycle?.newlyReached ?? [])]); + const rerun = new Set(cycle?.rerun ?? []); + if (!cycle?.fromCommit || !cycle?.toCommit || failed.size === 0) problems.push("each rerun cycle needs its source SHA, fixed SHA and failed cases"); + if (rerun.size !== allowed.size || [...rerun].some((item) => !allowed.has(item))) problems.push("a rerun cycle may contain only its failed cases and targets newly reached by the fix"); + } + return problems; +} + +function finalFileHash(inventory, worktree, file) { + if (!file) return undefined; + if (worktree) { + try { return createHash("sha256").update(readFileSync(path.join(worktree, file))).digest("hex"); } + catch { return undefined; } + } + return inventory.fileHashes?.[file]; +} + function readTemplate(worktree, base) { for (const file of TEMPLATE_PATHS) { try { diff --git a/plugins/vibe/skills/handoff/scripts/pr-facts.test.mjs b/plugins/vibe/skills/handoff/scripts/pr-facts.test.mjs index 2f48d602..acbbe09c 100644 --- a/plugins/vibe/skills/handoff/scripts/pr-facts.test.mjs +++ b/plugins/vibe/skills/handoff/scripts/pr-facts.test.mjs @@ -3,7 +3,7 @@ import { unlinkSync, writeFileSync } from "node:fs"; import path from "node:path"; import test from "node:test"; import { fileURLToPath } from "node:url"; -import { evidenceProblems, pullRequestFacts, requiredLanes, selectionProblems, STACKED_BASE_STATEMENT, templateHeadingProblems } from "./pr-facts.mjs"; +import { evidenceProblems, pullRequestFacts, requiredLanes, selectionProblems, STACKED_BASE_STATEMENT, targetedEvidenceProblems, templateHeadingProblems } from "./pr-facts.mjs"; import { git, makeCheckout, makeHome, runNode } from "../../vibe/scripts/test-fixtures.mjs"; const HERE = path.dirname(fileURLToPath(import.meta.url)); @@ -34,7 +34,17 @@ const COMPLETE_BODY = [ const SOURCES = [ROUTE, "packages/api/src/types/links.ts", "packages/api/src/service.ts"]; const SELECTED = { testSelection: SOURCES.flatMap((file) => [{ file, selector: "code_tests_for", tests: [TEST] }, { file, selector: "e2e-reference", tests: [] }]), - testRuns: [{ path: TEST, status: "passed", commit: "a".repeat(40), command: "vitest run" }], + targetedInventory: { + actor: "read-only", source: "final-committed-diff", commit: "a".repeat(40), laneReport: { command: "pnpm test:lanes", matches: [] }, + entries: [{ kind: "vitest", path: TEST, cases: ["limits"], command: "vitest run" }], + }, + rerunCycles: [], + e2eCapability: { + selectedTier: "A", + container: { dockerPresent: true, dockerReachable: true, memoryGiB: 24, diskGiB: 80, minimumMemoryGiB: 20, minimumDiskGiB: 64, command: "repo probe" }, + native: { available: true, forbidden: false }, remote: { available: false }, + }, + testRuns: [{ path: TEST, cases: ["limits"], status: "passed", commit: "a".repeat(40), command: "vitest run" }], }; function session(t) { @@ -121,8 +131,8 @@ test("the inventory reports the facts and gates only the pull-request phase", (t assert.equal(handoff.json.serverOnly, true); assert.equal(handoff.json.overlap, null); assert.equal("prBody" in handoff.json.blocking, false); - const run = (body) => runNode(INVENTORY, ["--worktree", s.wt, "--phase", "pull-request", "--ticket", ticket, "--prs-json", prs, - "--body", s.file("body.md", body), "--runs", s.file("runs.json", { touchFlows: [], touchFlowsReason: "server only", runs: [], ...SELECTED }), + const run = (body, evidence = SELECTED) => runNode(INVENTORY, ["--worktree", s.wt, "--phase", "pull-request", "--ticket", ticket, "--prs-json", prs, + "--body", s.file("body.md", body), "--runs", s.file("runs.json", { touchFlows: [], touchFlowsReason: "server only", runs: [], ...evidence }), "--commit", "a".repeat(40)], s.home); const failing = run("Summary only."); assert.equal(failing.status, 1); @@ -130,6 +140,9 @@ test("the inventory reports the facts and gates only the pull-request phase", (t const passing = run(COMPLETE_BODY); assert.equal(passing.json.blocking.prBody, true, JSON.stringify(passing.json.prBodyProblems)); assert.equal(passing.json.blocking.prEvidence, true); + const noTargetedArtifact = run(COMPLETE_BODY, { testSelection: SELECTED.testSelection, testRuns: SELECTED.testRuns }); + assert.equal(noTargetedArtifact.json.blocking.prEvidence, false); + assert.match(noTargetedArtifact.json.prEvidenceProblems.join(), /targeted inventory.*missing/); assert.match(runNode(INVENTORY, ["--worktree", s.wt, "--phase", "handoff", "--body", ticket], s.home).json.error, /--phase pull-request/); }); @@ -152,15 +165,15 @@ test("test selection covers the whole branch diff, and every selected, required assert.ok(empty.some((problem) => problem.startsWith(`${file}: no import selection`)), file); assert.ok(empty.some((problem) => problem.startsWith(`${file}: no e2e by-reference selection`)), file); } - assert.ok(empty.some((problem) => problem.startsWith(`${TEST}: selected but no passing run`)), "an edited spec is always selected"); + assert.ok(empty.some((problem) => problem.startsWith(`${TEST}: selected but no passing`)), "an edited spec is always selected"); const unindexed = SELECTED.testSelection.map((entry) => (entry.file === "packages/api/src/service.ts" && entry.selector === "code_tests_for" ? { ...entry, tests: null } : entry)); assert.match(gate({ ...SELECTED, testSelection: unindexed }).join(), /service\.ts: no import selection/); assert.deepEqual(gate({ ...SELECTED, testSelection: [...unindexed, { file: "packages/api/src/service.ts", selector: "related", tests: [] }] }), []); const emptyGraph = SELECTED.testSelection.map((entry) => (entry.selector === "code_tests_for" && entry.file === ROUTE ? { ...entry, tests: [] } : entry)); assert.match(gate({ ...SELECTED, testSelection: emptyGraph }).join(), /route\.ts: no import selection/); - assert.match(gate(SELECTED, [{ path: "e2e/links.spec.ts", reason: "names the endpoint" }]).join(), /e2e\/links\.spec\.ts: selected but no passing run/); - assert.match(gate({ ...SELECTED, testRuns: [{ ...SELECTED.testRuns[0], commit: "b".repeat(40) }] }).join(), /links\.test\.ts: selected but no passing run/); - assert.match(gate({ ...SELECTED, testRuns: [{ ...SELECTED.testRuns[0], status: "skipped" }] }).join(), /links\.test\.ts: selected but no passing run/); + assert.match(gate(SELECTED, [{ path: "e2e/links.spec.ts", reason: "names the endpoint" }]).join(), /e2e\/links\.spec\.ts: selected but no passing/); + assert.match(gate({ ...SELECTED, testRuns: [{ ...SELECTED.testRuns[0], commit: "b".repeat(40) }] }).join(), /links\.test\.ts: selected but no passing/); + assert.match(gate({ ...SELECTED, testRuns: [{ ...SELECTED.testRuns[0], status: "skipped" }] }).join(), /links\.test\.ts: selected but no passing/); assert.match(selectionProblems({ testSelection: [{ file: ROUTE, selector: "guess", tests: [] }] }, [], [], commit).join(), /selector "guess"/); }); @@ -169,7 +182,55 @@ test("a required test with flag states needs a passing run with the flag on and const required = [{ path: "apps/a/sessions.test.tsx", reason: "shared", flagStates: ["on", "off"] }]; const run = (flag) => ({ path: "apps/a/sessions.test.tsx", status: "passed", commit, ...(flag ? { flag } : {}) }); const problems = (testRuns) => selectionProblems({ testSelection: [], testRuns }, [], required, commit); - assert.deepEqual(problems([run("on")]), [`apps/a/sessions.test.tsx: no passing run with the flag off on ${commit} (testRuns entry with flag: "off")`]); + assert.deepEqual(problems([run("on")]), [`apps/a/sessions.test.tsx: no passing run with the flag off for ${commit} (testRuns entry with flag: "off")`]); assert.equal(problems([run()]).length, 2, "a run that names no flag state counts for neither"); assert.deepEqual(problems([run("on"), run("off")]), []); }); + +test("a pull request needs a read-only final-diff inventory and exact targeted results at its commit", () => { + const commit = "d".repeat(40); + const entry = { kind: "vitest", path: TEST, cases: ["limits"], command: `pnpm --filter api exec vitest run ${TEST} -t limits` }; + const complete = { + targetedInventory: { + actor: "read-only", + source: "final-committed-diff", + commit, + laneReport: { command: "pnpm test:lanes", matches: [] }, + entries: [entry], + }, + testSelection: [{ file: ROUTE, selector: "code_tests_for", tests: [TEST] }], + testRuns: [{ path: TEST, cases: ["limits"], command: entry.command, status: "passed", commit }], + rerunCycles: [], + }; + assert.deepEqual(targetedEvidenceProblems(complete, commit), []); + assert.match(targetedEvidenceProblems({}, commit).join(), /targeted inventory.*missing/); + assert.match(targetedEvidenceProblems({ ...complete, targetedInventory: { ...complete.targetedInventory, actor: "writer" } }, commit).join(), /read-only/); + assert.match(targetedEvidenceProblems({ ...complete, targetedInventory: { ...complete.targetedInventory, commit: "e".repeat(40) } }, commit).join(), /final committed SHA/); + assert.match(targetedEvidenceProblems({ ...complete, testRuns: [] }, commit).join(), /no passing targeted run/); +}); + +test("after a fix only failed and newly reached targets may be rerun", () => { + const before = "1".repeat(40); + const after = "2".repeat(40); + const sameHash = "a".repeat(64); + const entry = (path, extra = {}) => ({ kind: "vitest", path, cases: [path], command: `vitest ${path}`, ...extra }); + const evidence = { + targetedInventory: { actor: "read-only", source: "final-committed-diff", commit: after, + fileHashes: { "src/passed.ts": sameHash }, + laneReport: { command: "pnpm test:lanes", matches: [] }, entries: [ + entry("passed.test.ts", { resultCommit: before, carriedTo: after, carryReason: "the isolated fix cannot reach this module", + coveredFiles: [{ path: "src/passed.ts", sha256: sameHash }] }), + entry("failed.test.ts"), + ] }, + testRuns: [ + { path: "passed.test.ts", cases: ["passed.test.ts"], command: "vitest passed.test.ts", status: "passed", commit: before }, + { path: "failed.test.ts", cases: ["failed.test.ts"], command: "vitest failed.test.ts", status: "passed", commit: after }, + ], + rerunCycles: [{ fromCommit: before, toCommit: after, failed: ["failed.test.ts"], newlyReached: [], rerun: ["failed.test.ts"] }], + }; + assert.deepEqual(targetedEvidenceProblems(evidence, after), []); + const broad = { ...evidence, rerunCycles: [{ ...evidence.rerunCycles[0], rerun: ["failed.test.ts", "passed.test.ts"] }] }; + assert.match(targetedEvidenceProblems(broad, after).join(), /only its failed cases/); + const changed = { ...evidence, targetedInventory: { ...evidence.targetedInventory, fileHashes: { "src/passed.ts": "b".repeat(64) } } }; + assert.match(targetedEvidenceProblems(changed, after).join(), /src\/passed\.ts.*hash changed/); +}); diff --git a/plugins/vibe/skills/vibe/references/quality-loop.md b/plugins/vibe/skills/vibe/references/quality-loop.md index 074370df..ca6f9592 100644 --- a/plugins/vibe/skills/vibe/references/quality-loop.md +++ b/plugins/vibe/skills/vibe/references/quality-loop.md @@ -110,37 +110,34 @@ requirement there wins. ## One persistent writer and a serial queue -The orchestrator queues incoming requests/requirements and forwards them to the -same writer in order. Use the owned `scripts/dist/writer-state.mjs` ledger: -register the actual native Codex worker ID once, or the actual Claude session -ID managed by `scripts/dist/claude-worker.mjs`. Both use the session's existing -private Git metadata and verify the same worktree and branch. No new hidden -environment variable or generic worker platform is involved. - -Keep one active request. A plan, review finding, product answer or commit wait -continues that request in the same context; it does not authorize starting the -next queued request or a fresh writer. Claim/finish the queue through the owned -state API. If identity, branch or resume cannot be verified, block internally -rather than silently fork. Preserve pending inputs when interrupted. - -Codex: spawn the canonical writer once with a read-only hold brief: no source, -test or record mutation until the coordinator registers its returned actual ID -and claims the queued request. Then use native follow-up/resume for that same -ID and the claimed input. Claude Code: use the owned launcher -with the existing scoped name `vibe:vibe-change-worker`, canonical own-plugin -definition and exact parent-discovered capabilities. The launcher binds at a -supported fresh CLI launch and explicitly resumes the recorded session ID; -native Agent per-call tool overriding is not assumed. Briefs and requests go -through stdin, not sensitive argv. Preserve the same model, role and binding. - -For an already registered session, retain its actual recorded writer ID and -original recorded binding: `binding.agentRoot`, role and exact -`binding.capabilities`. Recover these only from the original registration -evidence or a read-only helper's bounded private-ledger inspection; return -binding/identity facts, never private queued inputs or credentials. Verify the -original definition and recorded digest before new code. If it is present and -matching, resume that writer with its original binding. Root supplies current -policy through its continuation without changing test-authoring authority. +Queue requests to the same writer in order through `scripts/dist/writer-state.mjs`. Register one actual +Codex ID or launcher-managed Claude ID, bound to private Git metadata, +worktree and branch. + +Keep one active request. Plans, findings, answers and commit waits continue it; +this does not authorize starting the next queued request or a fresh writer. Use the owned state API and block on +unverifiable identity, branch or resume. + +During a requested pull-request writer turn, run +`node ../handoff/scripts/parallel-prep.mjs --writer-state running --pull-request` +and dispatch its lanes. Draft the targeted inventory from the live +diff and later inspect only its committed delta. Configured E2E runner: fresh +ref, install, bootstrap, prebuild app images; never force. +Rebuild only when final build-input hashes differ. Run stable lanes for finished +files; bind results to SHA and covered-file hashes. At turn end, start hidden +local QA so visual QA overlaps tests. Draft the PR body and evidence comment as +results land. Lanes are read-only or operational: no product-source/test edits +or second writer. Carry a result only when its recorded hashes +match final HEAD. + +Codex: hold, register and resume one canonical writer. Claude Code: launch and +resume the canonical `vibe:vibe-change-worker` with parent-discovered +capabilities. Briefs use stdin. Preserve model, role and binding. + +Retain the recorded writer ID, `binding.agentRoot`, role, exact +`binding.capabilities` and original definition. Recover from registration or +bounded read-only ledger inspection. If present and matching, verify its digest +and resume it. Never return queued inputs or credentials. ### Recorded definition migration @@ -156,7 +153,7 @@ Never use `register` to replace a writer or copy old agent definitions. 1. Read `status` and the bounded binding facts from the private ledger. Wait for source and record turns to finish with actual stopped-turn evidence; a short-lived CLI PID exiting does not release an owned turn. Resolve any - writer refusal in the original request first. Migration rejects `BLOCKED` + writer refusal in the original request first. Never migrate around a refusal. Migration rejects `BLOCKED` (except a removed definition, below), running leases, matching definitions, unproven upgrades and stale identity. 2. Codex: spawn one current canonical `vibe-change-worker` with a read-only diff --git a/plugins/vibe/tools/python/test_vibe_1080_targeted_pre_pr_contract.py b/plugins/vibe/tools/python/test_vibe_1080_targeted_pre_pr_contract.py new file mode 100644 index 00000000..afebd582 --- /dev/null +++ b/plugins/vibe/tools/python/test_vibe_1080_targeted_pre_pr_contract.py @@ -0,0 +1,51 @@ +"""Contracts for the vibe 1.0.80 targeted pre-PR evidence gate.""" + +from __future__ import annotations + +import json +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] + + +def read(path: str) -> str: + return (ROOT / path).read_text() + + +def test_release_and_both_manifests_are_1080() -> None: + for manifest in (".claude-plugin/plugin.json", ".codex-plugin/plugin.json"): + assert json.loads(read(manifest))["version"] == "1.0.80" + + +def test_handoff_requires_final_diff_inventory_before_pr_creation() -> None: + handoff = read("skills/handoff/SKILL.md") + verify = read("agents/vibe-verify-worker.md") + environment = read("agents/vibe-environment-worker.md") + assert "`pre-pr` mode on final\nHEAD" in handoff + assert "read the final committed diff, never the plan" in verify + assert "Run `pnpm test:lanes`" in verify + assert "post-merge-only lane" in verify + assert "rerun only that failed case" in verify + assert "pr-evidence-comment.mjs" in environment + assert "blocking.prEvidence" in environment + + +def test_e2e_tiers_and_operator_override_are_owned_rules() -> None: + verify = read("agents/vibe-verify-worker.md") + for text in ("Tier A:", "Tier B:", "Tier C:", "git config --bool vibe.noNativeE2e"): + assert text in verify + assert "not run locally, covered by CI" in verify + assert "Never silently skip E2E" in verify + + +def test_running_writer_triggers_nonauthoring_parallel_prep() -> None: + quality = read("skills/vibe/references/quality-loop.md") + handoff = read("skills/handoff/SKILL.md") + assert "parallel-prep.mjs --writer-state running --pull-request" in quality + assert "Draft the targeted inventory from the live\ndiff" in quality + assert "prebuild app images" in quality + assert "covered-file hashes" in quality + assert "visual QA overlaps tests" in quality + assert "never edit source or start a writer" in handoff + assert (ROOT / "skills/handoff/scripts/parallel-prep.mjs").exists() diff --git a/plugins/vibe/tools/python/test_vibe_codex_budget_contract.py b/plugins/vibe/tools/python/test_vibe_codex_budget_contract.py index 6db97898..1ddad349 100644 --- a/plugins/vibe/tools/python/test_vibe_codex_budget_contract.py +++ b/plugins/vibe/tools/python/test_vibe_codex_budget_contract.py @@ -7,8 +7,7 @@ `skills/vibe/scripts/worker-model.mjs`); Claude Code explorers on sonnet in agent frontmatter. 3. Targeted validation: the tests `code_tests_for` returns per changed file plus typecheck and lint, never a broad or full local suite. -4. One containerized e2e stack per machine, through `skills/vibe/scripts/e2e-lock.mjs`, running only the - specs the change needs. +4. Targeted E2E uses the strongest capable container, native or remote tier and the repository's lock. 5. Usage awareness: stop and tell the person when a usage limit is reached, unless they allowed credits. """ @@ -90,11 +89,13 @@ def test_handoff_recipe_runs_no_test_suite() -> None: assert '"typecheck:affected"' in recipe -def test_e2e_runs_one_stack_per_machine_and_only_needed_specs() -> None: +def test_e2e_uses_a_capability_tier_and_only_needed_specs() -> None: checks = section(agent("vibe-verify-worker"), "## Existing checks") - assert "node ../skills/vibe/scripts/e2e-lock.mjs run --head " in checks - assert "only those specs, never the whole e2e suite" in checks - assert "never start the stack around the lock" in checks + assert "Tier A:" in checks + assert "Tier B:" in checks + assert "Tier C:" in checks + assert "Run only those specs through the chosen tier" in checks + assert "git config --bool vibe.noNativeE2e" in checks handoff = flat(HANDOFF_SKILL.read_text()) assert "and no other spec" in handoff assert "one stack per machine through `../vibe/scripts/e2e-lock.mjs`" in handoff diff --git a/plugins/vibe/tools/python/test_vibe_review_lessons_contract.py b/plugins/vibe/tools/python/test_vibe_review_lessons_contract.py index 326bd110..412d7070 100644 --- a/plugins/vibe/tools/python/test_vibe_review_lessons_contract.py +++ b/plugins/vibe/tools/python/test_vibe_review_lessons_contract.py @@ -209,12 +209,14 @@ def test_pull_request_tests_follow_the_review_rules() -> None: )) -def test_verify_runs_sort_order_tests_containerized_e2e_and_reads_ci_tiers() -> None: +def test_verify_runs_sort_order_tests_by_capability_and_reads_ci_tiers() -> None: verify = flat(agent("vibe-verify-worker")) assert "apps/storybook/__tests__/preview-story-sort-order.test.ts" in DIFF_RULES.read_text() assert_all(verify, ( "`diffRules.requiredTests`", - "runs at handoff through the repo's containerized path (the `local-ci` skill, `dagger call e-2-e`)", + "Tier A: its container path (`run-containerized.sh` or Dagger)", + "Tier B: otherwise run the same specs natively, headless", + "Tier C: otherwise use the operator's configured remote runner", "read `.github/pr-test-event-tiers.md` and cite the tier", )) handoff = flat(HANDOFF_SKILL.read_text())