From 68a3a822a77ce72cc68f2c0a8c1020ac06d74644 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 12 Oct 2026 22:47:16 -0400 Subject: [PATCH 01/18] feat(source-control): add a promotion-evidence publisher in report mode publish-promotion-evidence.mjs reads GitHub through read-only gh api calls and emits the contrary-events array in the checker's {cell, event, at} shape plus a C3 promotion-evidence report: each promotable cell's predicate terms with counts, window and pass or fail. Report mode writes nothing. --out writes evidence.json and an evidence.meta.json sidecar by temp file and rename, and refuses a set that drops a previously published event. Deployment (host, identity, schedule) and the resolver's sidecar check wait on the open merge-authority decision. Merge authority is unchanged. Refs: #4588 Co-Authored-By: Claude Opus 5.5 --- ...8-promotion-evidence-publisher-3634507d.md | 7 + .../reference/promotion-evidence-bootstrap.md | 23 ++ .../publish-promotion-evidence/recorded.json | 57 +++ .../scripts/publish-promotion-evidence.mjs | 337 ++++++++++++++++++ .../publish-promotion-evidence.test.mjs | 183 ++++++++++ .../publish-promotion-evidence.test.sh | 13 + 6 files changed, 620 insertions(+) create mode 100644 .changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md create mode 100644 plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json create mode 100644 plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs create mode 100644 plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs create mode 100755 plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.sh diff --git a/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md b/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md new file mode 100644 index 0000000000..c121c26a19 --- /dev/null +++ b/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md @@ -0,0 +1,7 @@ +--- +bump: minor +--- + +### Added + +- `babysit-loop` ships `scripts/publish-promotion-evidence.mjs`, an operator-side publisher for the promotion-evidence events file (#4588). It reads GitHub only, through read-only `gh api` calls, and emits the `{cell, event, at}` contrary events the security-binding checker reads (`reverted-merge`, `gate-failure`, and `verification-divergence` for a missed-blocking finding a human confirmed) plus a C3 promotion-evidence report: each promotable cell's predicate terms with counts, window and pass or fail. Report mode, the default, writes nothing. `--out ` writes `evidence.json` and an `evidence.meta.json` sidecar (`generated_at`, `through`, `sha256`, `prev_sha256`) by temp file and rename, and refuses a set that drops a previously published event. Where it runs and under which identity is not decided yet, and the resolver does not read the sidecar; merge authority is unchanged. diff --git a/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md b/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md index 02bd114f5b..e15d74f95e 100644 --- a/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md +++ b/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md @@ -144,6 +144,29 @@ Agent-unwritable, because an agent that can write the file can delete a `reverte keep its cell promoted. Outside the target repository's blast radius, for the same reason as the binding. +#### The publisher + +[`scripts/publish-promotion-evidence.mjs`](../scripts/publish-promotion-evidence.mjs) builds the +events file from GitHub alone, through read-only `gh api` calls: merged pull requests with a +`work-class:` label, which of them the `--lane-login` identities merged, reverts of those merges, +check runs on their merge commits, and the missed-blocking label a human applied. Its header +comment states what counts as each event, including the judgment mappings. + +- **Report mode** (no `--out`) prints the events array and the C3 promotion-evidence report to + stdout and writes nothing. The report gives each promotable cell's predicate terms from + [work-classes.md](https://raw.githubusercontent.com/melodic-software/claude-code-plugins/main/plugins/autonomy/reference/guardrails/work-classes.md) + "Suggested default predicates" with counts, window and pass or fail, and lists merged reverts + it could not match to a merge. +- **`--out `** writes `evidence.json` and the sidecar `evidence.meta.json` + (`generated_at`, `through`, `sha256` of the events file, `prev_sha256`), each by a temp file + and a rename. It refuses to publish when the new events drop one the previous file held. + +Run it as the operator, never from the lane, and point `--out` at a directory the lane cannot +write. Which host runs it, under which identity and on what schedule is not decided yet: it waits +on the open decision about the single merge authority +([#4588](https://github.com/melodic-software/claude-code-plugins/issues/4588)). Until then the +resolver does not read the sidecar. + ### `promotion_evidence_checker` The `check-security-binding.mjs` the helper runs (the autonomy plugin's diff --git a/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json b/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json new file mode 100644 index 0000000000..3fe39d58eb --- /dev/null +++ b/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json @@ -0,0 +1,57 @@ +{ + "pullListItem": { + "body": "Refs: #7339", + "labels": [], + "merge_commit_sha": "aad0b2b69bc22154c61b0857307a183d0c7821a8", + "merged_at": "2026-10-13T02:42:17Z", + "number": 7352, + "title": "fix(source-control): settle unattended merge residuals before Phase 15", + "updated_at": "2026-10-13T02:42:19Z", + "user": { + "login": "kyle-sexton", + "type": "User" + } + }, + "pull": { + "labels": [], + "merge_commit_sha": "aad0b2b69bc22154c61b0857307a183d0c7821a8", + "merged_at": "2026-10-13T02:42:17Z", + "merged_by": { + "login": "kyle-sexton", + "type": "User" + }, + "number": 7352, + "title": "fix(source-control): settle unattended merge residuals before Phase 15" + }, + "checkRun": { + "completed_at": "2026-10-13T02:38:54Z", + "conclusion": "success", + "name": "lint-repo", + "status": "completed" + }, + "commit": { + "commit": { + "committer": { + "date": "2026-10-13T02:35:38Z" + }, + "message": "fix(source-control): settle unattended merge residuals before Phase 15 (#7352)" + }, + "sha": "aad0b2b69bc22154c61b0857307a183d0c7821a8" + }, + "issueEvent": { + "actor": { + "login": "kyle-sexton", + "type": "User" + }, + "created_at": "2026-10-13T01:37:37Z", + "event": "ready_for_review", + "label": null + }, + "review": { + "state": "COMMENTED", + "user": { + "login": "chatgpt-codex-connector[bot]", + "type": "Bot" + } + } +} diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs new file mode 100644 index 0000000000..5b3d3895bd --- /dev/null +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs @@ -0,0 +1,337 @@ +#!/usr/bin/env node + +// Operator-side publisher for the promotion-evidence events file +// (promotion_evidence_source in reference/promotion-evidence-bootstrap.md) and the C3 +// promotion-evidence report. It reads GitHub only, through GET `gh api` calls, and never +// runs inside the lane: the lane must not be able to write its output. +// +// Usage: node publish-promotion-evidence.mjs --repo --since --lane-login [--lane-login ...] +// [--through ] [--ai-reviewer ...] [--missed-blocking-label ] [--out ] +// Exit 0 = report printed (and published with --out); 1 = publish refused; 2 = usage or GitHub read error. +// +// Report mode (no --out) prints {repo, since, through, events, unresolvedReverts, cells} to +// stdout and writes nothing. With --out it also writes /evidence.json (the events array) +// and /evidence.meta.json ({generated_at, through, sha256, prev_sha256}), each through a +// temp file and a rename, and refuses when evidence.json already holds an event the new array +// lacks or cannot be read as an array. +// +// What counts (judgment where the contract names no GitHub source): +// - A merged PR's work class is its highest `work-class:` label, the label set +// babysit-prs/scripts/babysit_merge_policy.py WORK_CLASS_LABELS maps. A PR with no class +// label is not counted. +// - Autonomous means merged_by is a --lane-login identity. +// - reverted-merge: an autonomous C2/C3 merge named by a merged PR body or a default-branch +// commit as `Reverts #` (GitHub's revert button) or `This reverts commit +// ` (git revert). A merged PR titled revert... with neither form is listed in +// unresolvedReverts for a human to read. +// - gate-failure: a check run on an autonomous merge's merge commit concluding failure, +// timed_out or startup_failure. A run still in progress, cancelled or action_required is +// unsettled: no event, and the gate-pass term fails until it settles. +// - verification-divergence: a human (a User actor outside --lane-login) applied +// --missed-blocking-label to a merged C3 PR (cell C3-ai-review-blocking) or an autonomous C2 +// PR (cell C2-auto-merge). +// +// Event shape and vocabulary: plugins/autonomy/skills/setup/scripts/check-security-binding.mjs, +// CONTRARY_EVIDENCE_EVENTS and resolveEffectivePromotion; restated here, not imported, because +// no plugin reads another plugin's files. Predicate terms and suggested default thresholds: +// plugins/autonomy/reference/guardrails/work-classes.md "Suggested default predicates". + +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; + +const CLASS_LABELS = { + "work-class: read-only": "C1", + "work-class: mechanical": "C2", + "work-class: scoped": "C3", + "work-class: structural": "C4", + "work-class: untrusted-provenance": "C5", +}; +const GATE_FAILED = new Set(["failure", "timed_out", "startup_failure"]); +const GATE_OK = new Set(["success", "neutral", "skipped"]); +const PAGE = 100; +const DAY_MS = 86_400_000; +const ISO = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:\d{2})$/; + +export class PublishError extends Error {} + +function instant(value, name) { + const ms = typeof value === "string" && ISO.test(value) ? Date.parse(value) : Number.NaN; + if (Number.isNaN(ms)) throw new PublishError(`${name}: not an ISO 8601 instant with an offset: ${JSON.stringify(value)}`); + return ms; +} + +function realGh(path) { + const run = spawnSync("gh", ["api", "--method", "GET", path], { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 }); + if (run.status !== 0) throw new PublishError(`gh api ${path} failed: ${(run.stderr || run.error?.message || "").trim()}`); + return JSON.parse(run.stdout); +} + +function pages(gh, path, key, stop = () => false) { + const out = []; + for (let page = 1; ; page += 1) { + const body = gh(`${path}${path.includes("?") ? "&" : "?"}per_page=${PAGE}&page=${page}`); + const items = key ? body?.[key] : body; + if (!Array.isArray(items)) throw new PublishError(`gh api ${path}: expected an array${key ? ` at .${key}` : ""}`); + out.push(...items); + if (items.length < PAGE || stop(items)) return out; + } +} + +function workClass(labels) { + return (labels ?? []).map((label) => CLASS_LABELS[label?.name]).filter(Boolean).sort().at(-1) ?? null; +} + +function eventKey(event) { + return JSON.stringify([event.cell, event.event, event.at]); +} + +export function collect({ repo, since, through, laneLogins, aiReviewers = [], missedBlockingLabel, gh = realGh }) { + const sinceMs = instant(since, "--since"); + const throughMs = instant(through, "--through"); + const lane = new Set(laneLogins); + const inWindow = (at) => typeof at === "string" && Date.parse(at) >= sinceMs && Date.parse(at) <= throughMs; + const events = new Map(); + const addEvent = (cell, event, at) => { + const record = { cell, event, at }; + events.set(eventKey(record), record); + }; + + const merged = pages(gh, `repos/${repo}/pulls?state=closed&sort=updated&direction=desc`, null, (items) => + Date.parse(items.at(-1).updated_at) < sinceMs, + ).filter((pr) => inWindow(pr.merged_at)); + + const details = new Map(); + const record = (number) => { + if (!details.has(number)) { + const pr = gh(`repos/${repo}/pulls/${number}`); + details.set(number, { + number, + cls: workClass(pr.labels), + autonomous: lane.has(pr.merged_by?.login), + mergedAt: pr.merged_at, + sha: pr.merge_commit_sha, + labels: (pr.labels ?? []).map((label) => label?.name), + reverted: false, + }); + } + return details.get(number); + }; + const counted = merged.filter((pr) => ["C2", "C3"].includes(workClass(pr.labels))).map((pr) => record(pr.number)); + const bySha = new Map(counted.map((r) => [r.sha, r])); + + // Reverts, from merged PR bodies and default-branch commit messages. + const own = repo.toLowerCase(); + const revertsOf = (text) => { + const targets = []; + for (const m of (text ?? "").matchAll(/\bReverts\s+(?:([\w.-]+\/[\w.-]+))?#(\d+)/g)) { + if (!m[1] || m[1].toLowerCase() === own) targets.push({ number: Number(m[2]) }); + } + for (const m of (text ?? "").matchAll(/This reverts commit ([0-9a-f]{40})/g)) targets.push({ sha: m[1] }); + return targets; + }; + const reverts = []; + const unresolvedReverts = []; + for (const pr of merged) { + const targets = revertsOf(pr.body); + for (const target of targets) reverts.push({ ...target, at: pr.merged_at }); + if (targets.length === 0 && /^revert\b/i.test(pr.title ?? "")) unresolvedReverts.push({ number: pr.number, title: pr.title }); + } + const commits = pages(gh, `repos/${repo}/commits?since=${new Date(sinceMs).toISOString()}&until=${new Date(throughMs).toISOString()}`); + for (const commit of commits) { + for (const target of revertsOf(commit.commit?.message)) reverts.push({ ...target, at: commit.commit?.committer?.date }); + } + for (const revert of reverts) { + if (!inWindow(revert.at)) continue; + let number = revert.number; + if (number === undefined) { + const known = bySha.get(revert.sha); + number = known ? known.number : gh(`repos/${repo}/commits/${revert.sha}/pulls`).find((pr) => pr.merged_at)?.number; + if (number === undefined) continue; + } + const target = record(number); + if (target.autonomous && ["C2", "C3"].includes(target.cls)) { + target.reverted = true; + addEvent(`${target.cls}-auto-merge`, "reverted-merge", revert.at); + } + } + + // Post-merge gate on each autonomous merge commit. + for (const r of counted.filter((c) => c.autonomous)) { + const runs = pages(gh, `repos/${repo}/commits/${r.sha}/check-runs`, "check_runs"); + const failed = runs.filter((run) => run.status === "completed" && GATE_FAILED.has(run.conclusion) && inWindow(run.completed_at)); + r.gate = failed.length > 0 ? "failed" : runs.length > 0 && runs.every((run) => run.status === "completed" && GATE_OK.has(run.conclusion)) ? "passed" : "unsettled"; + for (const run of failed) addEvent(`${r.cls}-auto-merge`, "gate-failure", run.completed_at); + } + + // Human-confirmed missed-blocking AI-review findings, and advisory AI reviews on C3. + for (const r of counted.filter((c) => c.cls === "C3" || c.autonomous)) { + r.missedBlocking = 0; + if (!r.labels.includes(missedBlockingLabel)) continue; + const confirmed = pages(gh, `repos/${repo}/issues/${r.number}/events`).filter( + (e) => e.event === "labeled" && e.label?.name === missedBlockingLabel && e.actor?.type === "User" && !lane.has(e.actor.login) && inWindow(e.created_at), + ); + r.missedBlocking = confirmed.length > 0 ? 1 : 0; + for (const e of confirmed) addEvent(r.cls === "C3" ? "C3-ai-review-blocking" : "C2-auto-merge", "verification-divergence", e.created_at); + } + const reviewers = new Set(aiReviewers); + if (reviewers.size > 0) { + for (const r of counted.filter((c) => c.cls === "C3")) { + r.aiReviewed = pages(gh, `repos/${repo}/pulls/${r.number}/reviews`).some((review) => reviewers.has(review.user?.login)); + } + } + + const sorted = [...events.values()].sort((a, b) => Date.parse(a.at) - Date.parse(b.at) || eventKey(a).localeCompare(eventKey(b))); + return { + repo, + since, + through, + events: sorted, + unresolvedReverts, + cells: report(counted, sorted, throughMs, reviewers.size > 0), + }; +} + +function term(name, value, threshold, pass, detail) { + return { term: name, value, threshold, pass, ...(detail ? { detail } : {}) }; +} + +function windowOf(records, throughMs) { + if (records.length === 0) return { from: null, through: new Date(throughMs).toISOString(), days: 0 }; + const from = Math.min(...records.map((r) => Date.parse(r.mergedAt))); + return { from: new Date(from).toISOString(), through: new Date(throughMs).toISOString(), days: Math.floor(((throughMs - from) / DAY_MS) * 100) / 100 }; +} + +function gateTerm(name, records) { + const detail = { passed: 0, failed: 0, unsettled: 0, total: records.length }; + for (const r of records) detail[r.gate] += 1; + const value = records.length === 0 ? null : Math.floor((detail.passed / records.length) * 10000) / 100; + return term(name, value, "100%", records.length > 0 && detail.passed === records.length, detail); +} + +function cell(window, terms) { + return { window, pass: terms.every((t) => t.pass), terms }; +} + +// Thresholds are work-classes.md's suggested defaults. +function report(counted, events, throughMs, reviewsMeasured) { + const c2 = counted.filter((r) => r.cls === "C2" && r.autonomous); + const c3 = counted.filter((r) => r.cls === "C3" && r.autonomous); + const c3All = counted.filter((r) => r.cls === "C3"); + const c2Window = windowOf(c2, throughMs); + const reverted = (records) => records.filter((r) => r.reverted).length; + const missed = (records) => records.reduce((sum, r) => sum + (r.missedBlocking ?? 0), 0); + const c2Demotions = events.filter((e) => e.cell === "C2-auto-merge").length; + const reviewed = c3All.filter((r) => r.aiReviewed); + return { + "C2-auto-merge": cell(c2Window, [ + term("autonomous C2 completions", c2.length, ">= 20", c2.length >= 20), + term("window days", c2Window.days, ">= 14", c2Window.days >= 14), + gateTerm("deterministic-gate pass", c2), + term("human-reverted merges", reverted(c2), "0", reverted(c2) === 0), + ]), + "C3-auto-merge": cell(windowOf([...c2, ...c3], throughMs), [ + term("autonomous C2 merges", c2.length, ">= 20", c2.length >= 20), + term("C2 window days", c2Window.days, ">= 14", c2Window.days >= 14), + term("C2 demotion events", c2Demotions, "0", c2Demotions === 0), + term("autonomous C3 completions", c3.length, ">= 10", c3.length >= 10), + gateTerm("C3 deterministic-gate pass", c3), + term("C3 human-reverted merges", reverted(c3), "0", reverted(c3) === 0), + term("C3 human-confirmed missed-blocking findings", missed(c3), "0", missed(c3) === 0), + ]), + "C3-ai-review-blocking": cell(windowOf(c3All, throughMs), [ + reviewsMeasured + ? term("advisory AI reviews on C3", reviewed.length, ">= 30", reviewed.length >= 30) + : term("advisory AI reviews on C3", null, ">= 30", false, { unmeasured: "no --ai-reviewer given" }), + term("human-confirmed missed-blocking findings", missed(c3All), "0", missed(c3All) === 0), + ]), + }; +} + +function sha256(text) { + return createHash("sha256").update(text).digest("hex"); +} + +function writeAtomic(path, text) { + const temp = `${path}.${process.pid}.tmp`; + try { + writeFileSync(temp, text, { flag: "wx" }); + renameSync(temp, path); + } catch (error) { + rmSync(temp, { force: true }); + throw error; + } +} + +export function publish(outDir, events, through, now = new Date()) { + const file = join(outDir, "evidence.json"); + let previous = null; + try { + previous = readFileSync(file, "utf8"); + } catch (error) { + if (error.code !== "ENOENT") throw new PublishError(`cannot read ${file}: ${error.message}`); + } + if (previous !== null) { + let prior; + try { + prior = JSON.parse(previous); + } catch (error) { + throw new PublishError(`refusing to publish: ${file} is not JSON (${error.message}), so the superset check cannot run`); + } + if (!Array.isArray(prior)) throw new PublishError(`refusing to publish: ${file} is not a JSON array, so the superset check cannot run`); + const keys = new Set(events.map(eventKey)); + const dropped = prior.filter((event) => !keys.has(eventKey(event))); + if (dropped.length > 0) { + throw new PublishError(`refusing to publish: the new events drop ${dropped.length} previously published event(s): ${dropped.map(eventKey).join(", ")}`); + } + } + const text = `${JSON.stringify(events, null, 2)}\n`; + const meta = { generated_at: now.toISOString(), through, sha256: sha256(text), prev_sha256: previous === null ? null : sha256(previous) }; + writeAtomic(file, text); + writeAtomic(join(outDir, "evidence.meta.json"), `${JSON.stringify(meta, null, 2)}\n`); + return meta; +} + +function main() { + let values; + try { + ({ values } = parseArgs({ + options: { + repo: { type: "string" }, + since: { type: "string" }, + through: { type: "string" }, + "lane-login": { type: "string", multiple: true }, + "ai-reviewer": { type: "string", multiple: true }, + "missed-blocking-label": { type: "string", default: "ai-review: missed-blocking" }, + out: { type: "string" }, + }, + })); + if (!/^[\w.-]+\/[\w.-]+$/.test(values.repo ?? "") || !values.since || !values["lane-login"]?.length) { + throw new PublishError("--repo , --since and at least one --lane-login are required"); + } + } catch (error) { + process.stderr.write(`usage: ${error.message}\n`); + process.exit(2); + } + try { + const result = collect({ + repo: values.repo, + since: values.since, + through: values.through ?? new Date().toISOString(), + laneLogins: values["lane-login"], + aiReviewers: values["ai-reviewer"] ?? [], + missedBlockingLabel: values["missed-blocking-label"], + }); + if (values.out) publish(values.out, result.events, result.through); + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); + } catch (error) { + process.stderr.write(`publish-promotion-evidence: ${error.message}\n`); + process.exit(error instanceof PublishError && error.message.startsWith("refusing") ? 1 : 2); + } +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) main(); diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs new file mode 100644 index 0000000000..aff9ab60b4 --- /dev/null +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs @@ -0,0 +1,183 @@ +#!/usr/bin/env node +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { after, test } from "node:test"; +import { fileURLToPath } from "node:url"; +import { collect, PublishError, publish } from "./publish-promotion-evidence.mjs"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +// Real `gh api` responses from this repository (PR #7352, its merge commit, a check run, an +// issue event and a review), recorded 2026-10-12 and trimmed to the fields the publisher reads. +// Each case copies a record and varies number, labels, merger and timestamps. +const REC = JSON.parse(readFileSync(join(HERE, "fixtures", "publish-promotion-evidence", "recorded.json"), "utf8")); +const REPO = "melodic-software/claude-code-plugins"; +const LANE = "lane-bot[bot]"; +const SINCE = "2026-08-01T00:00:00Z"; +const THROUGH = "2026-10-05T00:00:00Z"; +const MECHANICAL = "work-class: mechanical"; +const SCOPED = "work-class: scoped"; +const MISSED = "ai-review: missed-blocking"; + +const base = mkdtempSync(join(tmpdir(), "publish-promotion-evidence-")); +after(() => rmSync(base, { recursive: true, force: true })); + +const shaOf = (number) => number.toString(16).padStart(40, "0"); +const later = (at, minutes) => new Date(Date.parse(at) + minutes * 60_000).toISOString().replace(".000Z", "Z"); + +function pr(number, mergedAt, { labels = [], merger = LANE, title = REC.pullListItem.title, body = REC.pullListItem.body } = {}) { + const names = labels.map((name) => ({ name })); + return { + list: { ...REC.pullListItem, number, title, body, labels: names, merged_at: mergedAt, updated_at: later(mergedAt, 1), merge_commit_sha: shaOf(number) }, + detail: { ...REC.pull, number, title, labels: names, merged_at: mergedAt, merge_commit_sha: shaOf(number), merged_by: { login: merger, type: merger === LANE ? "Bot" : "User" } }, + }; +} + +// A recorded GitHub: answers only the GET paths the publisher is expected to call and throws on +// any other, so an unexpected or mutating call fails the case. +function github({ pulls = [], commits = [], checkRuns = {}, issueEvents = {}, reviews = {} }) { + const page = (items, path) => { + const n = Number(/[?&]page=(\d+)/.exec(path)[1]); + return items.slice((n - 1) * 100, n * 100); + }; + const runsFor = (number) => checkRuns[number] ?? [{ ...REC.checkRun, completed_at: later(pulls.find((p) => p.detail.number === number).detail.merged_at, 5) }]; + return (path) => { + const prefix = `repos/${REPO}/`; + assert.ok(path.startsWith(prefix), path); + const rest = path.slice(prefix.length); + let m; + if (rest.startsWith("pulls?state=closed&sort=updated&direction=desc&")) { + return page(pulls.map((p) => p.list).sort((a, b) => b.updated_at.localeCompare(a.updated_at)), rest); + } + if ((m = /^pulls\/(\d+)$/.exec(rest))) return pulls.find((p) => p.detail.number === Number(m[1])).detail; + if ((m = /^pulls\/(\d+)\/reviews\?/.exec(rest))) return page(reviews[m[1]] ?? [], rest); + if (/^commits\?since=[^&]+&until=[^&]+&/.test(rest)) return page(commits, rest); + if ((m = /^commits\/([0-9a-f]{40})\/check-runs\?/.exec(rest))) { + const number = parseInt(m[1], 16); + return { total_count: runsFor(number).length, check_runs: page(runsFor(number), rest) }; + } + if ((m = /^issues\/(\d+)\/events\?/.exec(rest))) return page(issueEvents[m[1]] ?? [], rest); + throw new Error(`unrecorded gh api call: ${path}`); + }; +} + +function run(fixture, extra = {}) { + return collect({ repo: REPO, since: SINCE, through: THROUGH, laneLogins: [LANE], missedBlockingLabel: MISSED, gh: github(fixture), ...extra }); +} + +const termOf = (result, cell, name) => result.cells[cell].terms.find((t) => t.term === name); + +test("a revert-button PR of an autonomous C2 merge yields reverted-merge at the revert's merge time", () => { + const result = run({ + pulls: [ + pr(101, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] }), + pr(102, "2026-09-02T00:00:00Z", { title: 'Revert "chore: bump"', body: `Reverts ${REPO}#101`, merger: "kyle-sexton" }), + ], + }); + assert.deepEqual(result.events, [{ cell: "C2-auto-merge", event: "reverted-merge", at: "2026-09-02T00:00:00Z" }]); + assert.deepEqual(termOf(result, "C2-auto-merge", "human-reverted merges"), { term: "human-reverted merges", value: 1, threshold: "0", pass: false }); +}); + +test("a git-revert commit of an autonomous C3 merge yields reverted-merge for C3 auto-merge", () => { + const result = run({ + pulls: [pr(201, "2026-09-01T00:00:00Z", { labels: [SCOPED] })], + commits: [{ ...REC.commit, sha: shaOf(999), commit: { message: `Revert "x"\n\nThis reverts commit ${shaOf(201)}.`, committer: { date: "2026-09-03T00:00:00Z" } } }], + }); + assert.deepEqual(result.events, [{ cell: "C3-auto-merge", event: "reverted-merge", at: "2026-09-03T00:00:00Z" }]); +}); + +test("a revert of a human merge, and a revert written only in prose, emit no event; prose is listed unresolved", () => { + const result = run({ + pulls: [ + pr(301, "2026-09-01T00:00:00Z", { labels: [MECHANICAL], merger: "kyle-sexton" }), + pr(302, "2026-09-02T00:00:00Z", { body: `Reverts ${REPO}#301` }), + pr(303, "2026-09-03T00:00:00Z", { title: "revert(x): drop the K3 check", body: "Reverts the bullet #301 added." }), + ], + }); + assert.deepEqual(result.events, []); + assert.deepEqual(result.unresolvedReverts, [{ number: 303, title: "revert(x): drop the K3 check" }]); +}); + +test("a failing check run on an autonomous merge commit yields gate-failure at its completion", () => { + const failing = { ...REC.checkRun, conclusion: "failure", completed_at: "2026-09-01T00:10:00Z" }; + const result = run({ pulls: [pr(401, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] })], checkRuns: { 401: [REC.checkRun, failing] } }); + assert.deepEqual(result.events, [{ cell: "C2-auto-merge", event: "gate-failure", at: "2026-09-01T00:10:00Z" }]); + assert.deepEqual(termOf(result, "C2-auto-merge", "deterministic-gate pass").detail, { passed: 0, failed: 1, unsettled: 0, total: 1 }); +}); + +test("a missed-blocking label a human applied to a C3 PR yields verification-divergence; one a bot applied does not", () => { + const labeled = (login, type, at) => ({ ...REC.issueEvent, event: "labeled", label: { name: MISSED }, actor: { login, type }, created_at: at }); + const result = run({ + pulls: [ + pr(501, "2026-09-01T00:00:00Z", { labels: [SCOPED, MISSED], merger: "kyle-sexton" }), + pr(502, "2026-09-01T00:00:00Z", { labels: [SCOPED, MISSED] }), + pr(503, "2026-09-01T00:00:00Z", { labels: [SCOPED, MISSED], merger: "kyle-sexton" }), + ], + issueEvents: { + 501: [labeled("kyle-sexton", "User", "2026-09-04T00:00:00Z")], + 502: [labeled(LANE, "Bot", "2026-09-05T00:00:00Z")], + 503: [labeled("github-actions[bot]", "Bot", "2026-09-06T00:00:00Z")], + }, + }); + assert.deepEqual(result.events, [{ cell: "C3-ai-review-blocking", event: "verification-divergence", at: "2026-09-04T00:00:00Z" }]); +}); + +test("20 autonomous C2 merges over 14+ days pass the C2 term and 19 fail it", () => { + const merges = (count) => Array.from({ length: count }, (_, i) => pr(600 + i, `2026-09-${String(i + 1).padStart(2, "0")}T00:00:00Z`, { labels: [MECHANICAL] })); + const twenty = run({ pulls: merges(20) }); + assert.equal(termOf(twenty, "C2-auto-merge", "autonomous C2 completions").pass, true); + assert.equal(twenty.cells["C2-auto-merge"].pass, true); + const nineteen = run({ pulls: merges(19) }); + assert.deepEqual(termOf(nineteen, "C2-auto-merge", "autonomous C2 completions"), { term: "autonomous C2 completions", value: 19, threshold: ">= 20", pass: false }); + assert.equal(nineteen.cells["C2-auto-merge"].pass, false); +}); + +test("advisory reviews are counted from the named AI reviewers and reported unmeasured without one", () => { + const fixture = { pulls: [pr(701, "2026-09-01T00:00:00Z", { labels: [SCOPED], merger: "kyle-sexton" })], reviews: { 701: [REC.review] } }; + assert.equal(termOf(run(fixture, { aiReviewers: [REC.review.user.login] }), "C3-ai-review-blocking", "advisory AI reviews on C3").value, 1); + assert.equal(termOf(run(fixture), "C3-ai-review-blocking", "advisory AI reviews on C3").value, null); +}); + +const A = { cell: "C2-auto-merge", event: "reverted-merge", at: "2026-09-02T00:00:00Z" }; +const B = { cell: "C3-auto-merge", event: "gate-failure", at: "2026-09-03T00:00:00Z" }; +const hash = (text) => createHash("sha256").update(text).digest("hex"); + +test("the sidecar hash matches the published file, and prev_sha256 names the file it replaced", () => { + const out = mkdtempSync(join(base, "out-")); + const first = publish(out, [A], THROUGH, new Date("2026-10-05T00:00:01Z")); + const firstBytes = readFileSync(join(out, "evidence.json"), "utf8"); + assert.deepEqual(JSON.parse(firstBytes), [A]); + assert.deepEqual(JSON.parse(readFileSync(join(out, "evidence.meta.json"), "utf8")), { + generated_at: "2026-10-05T00:00:01.000Z", + through: THROUGH, + sha256: hash(firstBytes), + prev_sha256: null, + }); + assert.equal(first.sha256, hash(firstBytes)); + const second = publish(out, [A, B], THROUGH); + assert.equal(second.sha256, hash(readFileSync(join(out, "evidence.json"), "utf8"))); + assert.equal(second.prev_sha256, hash(firstBytes)); +}); + +test("publishing a set that drops a previously published event is refused and leaves both files unchanged", () => { + const out = mkdtempSync(join(base, "out-")); + publish(out, [A, B], THROUGH); + const before = [readFileSync(join(out, "evidence.json"), "utf8"), readFileSync(join(out, "evidence.meta.json"), "utf8")]; + assert.throws(() => publish(out, [B], THROUGH), (error) => error instanceof PublishError && /refusing.*drop 1/.test(error.message)); + assert.deepEqual([readFileSync(join(out, "evidence.json"), "utf8"), readFileSync(join(out, "evidence.meta.json"), "utf8")], before); +}); + +test("an existing events file that is not a JSON array is refused", () => { + const out = mkdtempSync(join(base, "out-")); + writeFileSync(join(out, "evidence.json"), "{}"); + assert.throws(() => publish(out, [A], THROUGH), /refusing.*not a JSON array/); +}); + +test("the CLI exits 2 on missing required arguments", () => { + const cli = spawnSync(process.execPath, [join(HERE, "publish-promotion-evidence.mjs"), "--repo", REPO], { encoding: "utf8" }); + assert.equal(cli.status, 2); + assert.match(cli.stderr, /--lane-login/); +}); diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.sh b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.sh new file mode 100755 index 0000000000..1e99296c52 --- /dev/null +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Discovery wrapper: scripts/run-plugin-tests.sh finds plugins/**/*.test.sh, so +# this hands off to the Node suite. SKIPs (exit 0) when Node is unavailable. +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +if ! command -v node >/dev/null 2>&1; then + echo "SKIP: node not installed" >&2 + exit 0 +fi + +exec node --test "$SCRIPT_DIR/publish-promotion-evidence.test.mjs" From 82d4562391e1ceb500891b2db55c899d66e1fecb Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 12 Oct 2026 22:51:56 -0400 Subject: [PATCH 02/18] fix(source-control): class merged PRs by their linked issues' labels The publisher now reads each merged PR's work class the way babysit_merge_policy.py does: from the work-class label on its linked issues (closingIssuesReferences, one read-only GraphQL query per merged PR), with the PR's own labels as the fallback, the highest class winning. The temp-file write no longer removes a temp file it did not create: an existing one fails the publish with EEXIST and stays as it was. Refs: #4588 Co-Authored-By: Claude Opus 5.5 --- .../reference/promotion-evidence-bootstrap.md | 5 ++- .../publish-promotion-evidence/recorded.json | 31 +++++++++++++++ .../scripts/publish-promotion-evidence.mjs | 39 +++++++++++++++---- .../publish-promotion-evidence.test.mjs | 29 +++++++++++++- 4 files changed, 92 insertions(+), 12 deletions(-) diff --git a/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md b/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md index e15d74f95e..1088595a2f 100644 --- a/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md +++ b/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md @@ -147,8 +147,9 @@ binding. #### The publisher [`scripts/publish-promotion-evidence.mjs`](../scripts/publish-promotion-evidence.mjs) builds the -events file from GitHub alone, through read-only `gh api` calls: merged pull requests with a -`work-class:` label, which of them the `--lane-login` identities merged, reverts of those merges, +events file from GitHub alone, through read-only `gh api` calls: merged pull requests classed by the +`work-class:` label on their linked issues (or on the pull request when no linked issue has one), +which of them the `--lane-login` identities merged, reverts of those merges, check runs on their merge commits, and the missed-blocking label a human applied. Its header comment states what counts as each event, including the judgment mappings. diff --git a/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json b/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json index 3fe39d58eb..51d4679abb 100644 --- a/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json +++ b/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json @@ -53,5 +53,36 @@ "login": "chatgpt-codex-connector[bot]", "type": "Bot" } + }, + "linkedIssues": { + "data": { + "repository": { + "pullRequest": { + "closingIssuesReferences": { + "nodes": [ + { + "number": 6007, + "labels": { + "nodes": [ + { + "name": "priority: medium" + }, + { + "name": "status: ready" + }, + { + "name": "needs-human" + }, + { + "name": "work-class: scoped" + } + ] + } + } + ] + } + } + } + } } } diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs index 5b3d3895bd..76e9e456e5 100644 --- a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs @@ -16,9 +16,10 @@ // lacks or cannot be read as an array. // // What counts (judgment where the contract names no GitHub source): -// - A merged PR's work class is its highest `work-class:` label, the label set -// babysit-prs/scripts/babysit_merge_policy.py WORK_CLASS_LABELS maps. A PR with no class -// label is not counted. +// - A merged PR's work class is the highest `work-class:` label (the label set +// babysit-prs/scripts/babysit_merge_policy.py WORK_CLASS_LABELS maps) on its linked issues +// (closingIssuesReferences, one read-only GraphQL query per merged PR) and its own labels, +// which are the fallback when no linked issue carries one. A PR with no class is not counted. // - Autonomous means merged_by is a --lane-login identity. // - reverted-merge: an autonomous C2/C3 merge named by a merged PR body or a default-branch // commit as `Reverts #` (GitHub's revert button) or `This reverts commit @@ -65,12 +66,19 @@ function instant(value, name) { return ms; } -function realGh(path) { - const run = spawnSync("gh", ["api", "--method", "GET", path], { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 }); +// A REST path is read with GET; "graphql" runs a query (read-only: LINKED_ISSUES is the only one). +function realGh(path, graphql) { + const args = graphql + ? ["api", "graphql", "-f", `query=${graphql.query}`, ...Object.entries(graphql.variables).flatMap(([key, value]) => ["-F", `${key}=${value}`])] + : ["api", "--method", "GET", path]; + const run = spawnSync("gh", args, { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 }); if (run.status !== 0) throw new PublishError(`gh api ${path} failed: ${(run.stderr || run.error?.message || "").trim()}`); return JSON.parse(run.stdout); } +const LINKED_ISSUES = + "query($owner:String!,$name:String!,$number:Int!){repository(owner:$owner,name:$name){pullRequest(number:$number){closingIssuesReferences(first:20){nodes{number labels(first:50){nodes{name}}}}}}}"; + function pages(gh, path, key, stop = () => false) { const out = []; for (let page = 1; ; page += 1) { @@ -105,13 +113,27 @@ export function collect({ repo, since, through, laneLogins, aiReviewers = [], mi Date.parse(items.at(-1).updated_at) < sinceMs, ).filter((pr) => inWindow(pr.merged_at)); + // Class labels live on the linked issues (closingIssuesReferences); a PR's own labels are the + // fallback. The highest class across both wins, as in babysit_merge_policy.py. + const [owner, name] = repo.split("/"); + const classes = new Map(); + const classOf = (number, prLabels) => { + if (!classes.has(number)) { + const nodes = gh("graphql", { query: LINKED_ISSUES, variables: { owner, name, number } })?.data?.repository?.pullRequest + ?.closingIssuesReferences?.nodes; + if (!Array.isArray(nodes)) throw new PublishError(`gh api graphql: no closingIssuesReferences for #${number}`); + classes.set(number, workClass([...(prLabels ?? []), ...nodes.flatMap((issue) => issue?.labels?.nodes ?? [])])); + } + return classes.get(number); + }; + const details = new Map(); const record = (number) => { if (!details.has(number)) { const pr = gh(`repos/${repo}/pulls/${number}`); details.set(number, { number, - cls: workClass(pr.labels), + cls: classOf(number, pr.labels), autonomous: lane.has(pr.merged_by?.login), mergedAt: pr.merged_at, sha: pr.merge_commit_sha, @@ -121,7 +143,7 @@ export function collect({ repo, since, through, laneLogins, aiReviewers = [], mi } return details.get(number); }; - const counted = merged.filter((pr) => ["C2", "C3"].includes(workClass(pr.labels))).map((pr) => record(pr.number)); + const counted = merged.filter((pr) => ["C2", "C3"].includes(classOf(pr.number, pr.labels))).map((pr) => record(pr.number)); const bySha = new Map(counted.map((r) => [r.sha, r])); // Reverts, from merged PR bodies and default-branch commit messages. @@ -258,8 +280,9 @@ function sha256(text) { function writeAtomic(path, text) { const temp = `${path}.${process.pid}.tmp`; + // "wx" refuses a temp file that already exists rather than overwrite or remove it. + writeFileSync(temp, text, { flag: "wx" }); try { - writeFileSync(temp, text, { flag: "wx" }); renameSync(temp, path); } catch (error) { rmSync(temp, { force: true }); diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs index aff9ab60b4..5085273fd7 100644 --- a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs @@ -38,13 +38,19 @@ function pr(number, mergedAt, { labels = [], merger = LANE, title = REC.pullList // A recorded GitHub: answers only the GET paths the publisher is expected to call and throws on // any other, so an unexpected or mutating call fails the case. -function github({ pulls = [], commits = [], checkRuns = {}, issueEvents = {}, reviews = {} }) { +function github({ pulls = [], commits = [], checkRuns = {}, issueEvents = {}, reviews = {}, linked = {} }) { const page = (items, path) => { const n = Number(/[?&]page=(\d+)/.exec(path)[1]); return items.slice((n - 1) * 100, n * 100); }; const runsFor = (number) => checkRuns[number] ?? [{ ...REC.checkRun, completed_at: later(pulls.find((p) => p.detail.number === number).detail.merged_at, 5) }]; - return (path) => { + return (path, graphql) => { + if (path === "graphql") { + assert.deepEqual([graphql.variables.owner, graphql.variables.name], REPO.split("/")); + assert.match(graphql.query, /^query\(/); + const recorded = linked[graphql.variables.number]; + return recorded ?? { data: { repository: { pullRequest: { closingIssuesReferences: { nodes: [] } } } } }; + } const prefix = `repos/${REPO}/`; assert.ok(path.startsWith(prefix), path); const rest = path.slice(prefix.length); @@ -135,6 +141,16 @@ test("20 autonomous C2 merges over 14+ days pass the C2 term and 19 fail it", () assert.equal(nineteen.cells["C2-auto-merge"].pass, false); }); +test("a class carried only by the linked issue counts, and the higher of issue and PR class wins", () => { + // REC.linkedIssues: PR #7063 closes issue #6007, labeled `work-class: scoped`. + const result = run({ + pulls: [pr(801, "2026-09-01T00:00:00Z"), pr(802, "2026-09-02T00:00:00Z", { labels: [MECHANICAL] })], + linked: { 801: REC.linkedIssues, 802: REC.linkedIssues }, + }); + assert.equal(termOf(result, "C3-auto-merge", "autonomous C3 completions").value, 2); + assert.equal(termOf(result, "C2-auto-merge", "autonomous C2 completions").value, 0); +}); + test("advisory reviews are counted from the named AI reviewers and reported unmeasured without one", () => { const fixture = { pulls: [pr(701, "2026-09-01T00:00:00Z", { labels: [SCOPED], merger: "kyle-sexton" })], reviews: { 701: [REC.review] } }; assert.equal(termOf(run(fixture, { aiReviewers: [REC.review.user.login] }), "C3-ai-review-blocking", "advisory AI reviews on C3").value, 1); @@ -170,6 +186,15 @@ test("publishing a set that drops a previously published event is refused and le assert.deepEqual([readFileSync(join(out, "evidence.json"), "utf8"), readFileSync(join(out, "evidence.meta.json"), "utf8")], before); }); +test("an existing temp file is neither overwritten nor removed, and nothing is published", () => { + const out = mkdtempSync(join(base, "out-")); + const temp = join(out, `evidence.json.${process.pid}.tmp`); + writeFileSync(temp, "planted"); + assert.throws(() => publish(out, [A], THROUGH), { code: "EEXIST" }); + assert.equal(readFileSync(temp, "utf8"), "planted"); + assert.throws(() => readFileSync(join(out, "evidence.json")), { code: "ENOENT" }); +}); + test("an existing events file that is not a JSON array is refused", () => { const out = mkdtempSync(join(base, "out-")); writeFileSync(join(out, "evidence.json"), "{}"); From a550d87a7bed9d55c9384dbf69534beca619330e Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 12 Oct 2026 23:15:49 -0400 Subject: [PATCH 03/18] fix(source-control): close the publisher's evidence gaps from the security gate - Read check runs with filter=all and the commit statuses list, so a failed run later re-run green, or a failure or error status, still emits gate-failure, including one completed after --through. - Read missed-blocking findings from the label timeline, so removing the label later does not undo a human's confirmation. - Rebuild each PR's and linked issue's labels as of merged_at, so relabeling after the merge neither adds nor drops a counted merge. - End a cell's window at its last counted merge and clamp --through to now, so 20 merges in one day fail the 14-day term. - Request pageInfo on linked issues and their labels; another page fails closed. - One reverted-merge per reverted merge, at the earliest revert seen. - Verify the existing sidecar's sha256 before chaining prev_sha256, and write both temp files before renaming the events file. - List closed PRs by creation date ascending, so an update while paging cannot skip a merge. - Document how to set --lane-login: merged_by is the merge-queue enqueuer. Refs: #4588 Co-Authored-By: Claude Opus 5.5 --- ...8-promotion-evidence-publisher-3634507d.md | 2 +- .../reference/promotion-evidence-bootstrap.md | 12 +- .../publish-promotion-evidence/recorded.json | 13 ++ .../scripts/publish-promotion-evidence.mjs | 203 ++++++++++++------ .../publish-promotion-evidence.test.mjs | 149 ++++++++++++- 5 files changed, 296 insertions(+), 83 deletions(-) diff --git a/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md b/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md index c121c26a19..60aee5c89d 100644 --- a/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md +++ b/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md @@ -4,4 +4,4 @@ bump: minor ### Added -- `babysit-loop` ships `scripts/publish-promotion-evidence.mjs`, an operator-side publisher for the promotion-evidence events file (#4588). It reads GitHub only, through read-only `gh api` calls, and emits the `{cell, event, at}` contrary events the security-binding checker reads (`reverted-merge`, `gate-failure`, and `verification-divergence` for a missed-blocking finding a human confirmed) plus a C3 promotion-evidence report: each promotable cell's predicate terms with counts, window and pass or fail. Report mode, the default, writes nothing. `--out ` writes `evidence.json` and an `evidence.meta.json` sidecar (`generated_at`, `through`, `sha256`, `prev_sha256`) by temp file and rename, and refuses a set that drops a previously published event. Where it runs and under which identity is not decided yet, and the resolver does not read the sidecar; merge authority is unchanged. +- `babysit-loop` ships `scripts/publish-promotion-evidence.mjs`, an operator-side publisher for the promotion-evidence events file (#4588). It reads GitHub only, through read-only `gh api` calls, and emits the `{cell, event, at}` contrary events the security-binding checker reads (`reverted-merge`, `gate-failure`, and `verification-divergence` for a missed-blocking finding a human confirmed) plus a C3 promotion-evidence report: each promotable cell's predicate terms with counts, window and pass or fail. Report mode, the default, writes nothing. `--out ` writes `evidence.json` and an `evidence.meta.json` sidecar (`generated_at`, `through`, `sha256`, `prev_sha256`) by temp file and rename, and refuses a set that drops a previously published event or an existing file that does not match its sidecar. Work classes are read from the linked issues' and the PR's labels as of the merge, and every check run (re-runs included) and commit status on a merge commit counts toward `gate-failure`. Where it runs and under which identity is not decided yet, and the resolver does not read the sidecar; merge authority is unchanged. diff --git a/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md b/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md index 1088595a2f..40db9ac329 100644 --- a/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md +++ b/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md @@ -148,10 +148,11 @@ binding. [`scripts/publish-promotion-evidence.mjs`](../scripts/publish-promotion-evidence.mjs) builds the events file from GitHub alone, through read-only `gh api` calls: merged pull requests classed by the -`work-class:` label on their linked issues (or on the pull request when no linked issue has one), -which of them the `--lane-login` identities merged, reverts of those merges, -check runs on their merge commits, and the missed-blocking label a human applied. Its header -comment states what counts as each event, including the judgment mappings. +`work-class:` labels their linked issues and the pull request carried at merge time, which of them +the `--lane-login` identities merged, reverts of those merges, every check run and commit status +on their merge commits, and the missed-blocking label a human applied. Its header comment states +what counts as each event, including the judgment mappings and how to set `--lane-login` for +merge-queue merges. - **Report mode** (no `--out`) prints the events array and the C3 promotion-evidence report to stdout and writes nothing. The report gives each promotable cell's predicate terms from @@ -160,7 +161,8 @@ comment states what counts as each event, including the judgment mappings. it could not match to a merge. - **`--out `** writes `evidence.json` and the sidecar `evidence.meta.json` (`generated_at`, `through`, `sha256` of the events file, `prev_sha256`), each by a temp file - and a rename. It refuses to publish when the new events drop one the previous file held. + and a rename. It refuses to publish when the previous events file does not match its sidecar's + `sha256`, or when the new events drop one the previous file held. Run it as the operator, never from the lane, and point `--out` at a directory the lane cannot write. Which host runs it, under which identity and on what schedule is not decided yet: it waits diff --git a/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json b/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json index 51d4679abb..3ccad36a6e 100644 --- a/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json +++ b/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json @@ -2,6 +2,7 @@ "pullListItem": { "body": "Refs: #7339", "labels": [], + "created_at": "2026-10-13T00:54:35Z", "merge_commit_sha": "aad0b2b69bc22154c61b0857307a183d0c7821a8", "merged_at": "2026-10-13T02:42:17Z", "number": 7352, @@ -29,6 +30,12 @@ "name": "lint-repo", "status": "completed" }, + "status": { + "context": "ci-lanes", + "state": "success", + "created_at": "2026-10-13T02:42:00Z", + "updated_at": "2026-10-13T02:42:00Z" + }, "commit": { "commit": { "committer": { @@ -59,10 +66,16 @@ "repository": { "pullRequest": { "closingIssuesReferences": { + "pageInfo": { + "hasNextPage": false + }, "nodes": [ { "number": 6007, "labels": { + "pageInfo": { + "hasNextPage": false + }, "nodes": [ { "name": "priority: medium" diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs index 76e9e456e5..954c3d5988 100644 --- a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs @@ -2,35 +2,47 @@ // Operator-side publisher for the promotion-evidence events file // (promotion_evidence_source in reference/promotion-evidence-bootstrap.md) and the C3 -// promotion-evidence report. It reads GitHub only, through GET `gh api` calls, and never -// runs inside the lane: the lane must not be able to write its output. +// promotion-evidence report. It reads GitHub only, through `gh api` GET calls and one read-only +// GraphQL query, and never runs inside the lane: the lane must not be able to write its output. // // Usage: node publish-promotion-evidence.mjs --repo --since --lane-login [--lane-login ...] // [--through ] [--ai-reviewer ...] [--missed-blocking-label ] [--out ] // Exit 0 = report printed (and published with --out); 1 = publish refused; 2 = usage or GitHub read error. // // Report mode (no --out) prints {repo, since, through, events, unresolvedReverts, cells} to -// stdout and writes nothing. With --out it also writes /evidence.json (the events array) -// and /evidence.meta.json ({generated_at, through, sha256, prev_sha256}), each through a -// temp file and a rename, and refuses when evidence.json already holds an event the new array -// lacks or cannot be read as an array. +// stdout and writes nothing. --through defaults to, and is clamped to, now. With --out it also +// writes /evidence.json (the events array) and /evidence.meta.json ({generated_at, +// through, sha256, prev_sha256}). Both temp files are written before either rename, the events +// file renamed first. It refuses when an existing evidence.json is not a JSON array, has no +// sidecar or does not match the sidecar's sha256, or holds an event the new array lacks. +// +// --lane-login: GitHub records the merge-queue enqueuer as merged_by. Probed 2026-10-12 on PR +// #7352: merged_by.login was the login of the AddedToMergeQueueEvent enqueuer, not a queue bot. +// So pass the identity the lane enqueues or merges with (its App bot login, `[bot]`); a +// lane PR a human enqueued counts as a human merge. Recheck if merged_by stops matching the +// enqueuer. // // What counts (judgment where the contract names no GitHub source): // - A merged PR's work class is the highest `work-class:` label (the label set // babysit-prs/scripts/babysit_merge_policy.py WORK_CLASS_LABELS maps) on its linked issues -// (closingIssuesReferences, one read-only GraphQL query per merged PR) and its own labels, -// which are the fallback when no linked issue carries one. A PR with no class is not counted. +// (closingIssuesReferences) and on the PR, each label set rebuilt as of merged_at by undoing +// later labeled/unlabeled events, so relabeling after the merge changes nothing. A linked-issue +// or label list with another page fails closed. A PR with no class is not counted. // - Autonomous means merged_by is a --lane-login identity. // - reverted-merge: an autonomous C2/C3 merge named by a merged PR body or a default-branch // commit as `Reverts #` (GitHub's revert button) or `This reverts commit -// ` (git revert). A merged PR titled revert... with neither form is listed in -// unresolvedReverts for a human to read. -// - gate-failure: a check run on an autonomous merge's merge commit concluding failure, -// timed_out or startup_failure. A run still in progress, cancelled or action_required is -// unsettled: no event, and the gate-pass term fails until it settles. +// ` (git revert); one event per reverted merge, at the earliest revert seen. A merged PR +// titled revert... with neither form is listed in unresolvedReverts for a human to read. +// - gate-failure: every check run on an autonomous merge's merge commit (filter=all, so a run +// later re-run green still counts) concluding failure, timed_out or startup_failure, and every +// commit status in state failure or error, whenever it completed. The gate-pass term also +// fails while a run is not completed with success, neutral or skipped, or a context's latest +// status is not success. // - verification-divergence: a human (a User actor outside --lane-login) applied // --missed-blocking-label to a merged C3 PR (cell C3-ai-review-blocking) or an autonomous C2 -// PR (cell C2-auto-merge). +// PR (cell C2-auto-merge), read from the label timeline, so removing the label later does not +// undo it. +// - A cell's window runs from its first counted merge to its last. // // Event shape and vocabulary: plugins/autonomy/skills/setup/scripts/check-security-binding.mjs, // CONTRARY_EVIDENCE_EVENTS and resolveEffectivePromotion; restated here, not imported, because @@ -54,6 +66,7 @@ const CLASS_LABELS = { }; const GATE_FAILED = new Set(["failure", "timed_out", "startup_failure"]); const GATE_OK = new Set(["success", "neutral", "skipped"]); +const STATUS_FAILED = new Set(["failure", "error"]); const PAGE = 100; const DAY_MS = 86_400_000; const ISO = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:\d{2})$/; @@ -77,21 +90,31 @@ function realGh(path, graphql) { } const LINKED_ISSUES = - "query($owner:String!,$name:String!,$number:Int!){repository(owner:$owner,name:$name){pullRequest(number:$number){closingIssuesReferences(first:20){nodes{number labels(first:50){nodes{name}}}}}}}"; + "query($owner:String!,$name:String!,$number:Int!){repository(owner:$owner,name:$name){pullRequest(number:$number){closingIssuesReferences(first:50){pageInfo{hasNextPage} nodes{number labels(first:100){pageInfo{hasNextPage} nodes{name}}}}}}}"; -function pages(gh, path, key, stop = () => false) { +function pages(gh, path, key) { const out = []; for (let page = 1; ; page += 1) { const body = gh(`${path}${path.includes("?") ? "&" : "?"}per_page=${PAGE}&page=${page}`); const items = key ? body?.[key] : body; if (!Array.isArray(items)) throw new PublishError(`gh api ${path}: expected an array${key ? ` at .${key}` : ""}`); out.push(...items); - if (items.length < PAGE || stop(items)) return out; + if (items.length < PAGE) return out; } } -function workClass(labels) { - return (labels ?? []).map((label) => CLASS_LABELS[label?.name]).filter(Boolean).sort().at(-1) ?? null; +function workClass(names) { + return names.map((name) => CLASS_LABELS[name]).filter(Boolean).sort().at(-1) ?? null; +} + +// The label set as of `atMs`: the current set with every later labeled/unlabeled event undone. +function labelsAsOf(current, events, atMs) { + const set = new Set(current); + const later = events + .filter((e) => (e.event === "labeled" || e.event === "unlabeled") && Date.parse(e.created_at) > atMs) + .sort((a, b) => Date.parse(b.created_at) - Date.parse(a.created_at)); + for (const e of later) e.event === "labeled" ? set.delete(e.label?.name) : set.add(e.label?.name); + return [...set]; } function eventKey(event) { @@ -100,7 +123,7 @@ function eventKey(event) { export function collect({ repo, since, through, laneLogins, aiReviewers = [], missedBlockingLabel, gh = realGh }) { const sinceMs = instant(since, "--since"); - const throughMs = instant(through, "--through"); + const throughMs = Math.min(instant(through, "--through"), Date.now()); const lane = new Set(laneLogins); const inWindow = (at) => typeof at === "string" && Date.parse(at) >= sinceMs && Date.parse(at) <= throughMs; const events = new Map(); @@ -109,20 +132,34 @@ export function collect({ repo, since, through, laneLogins, aiReviewers = [], mi events.set(eventKey(record), record); }; - const merged = pages(gh, `repos/${repo}/pulls?state=closed&sort=updated&direction=desc`, null, (items) => - Date.parse(items.at(-1).updated_at) < sinceMs, - ).filter((pr) => inWindow(pr.merged_at)); + // Created-ascending order is stable while paging: an update moves no PR between pages. + const seen = new Set(); + const merged = pages(gh, `repos/${repo}/pulls?state=closed&sort=created&direction=asc`).filter( + (pr) => inWindow(pr.merged_at) && !seen.has(pr.number) && seen.add(pr.number), + ); - // Class labels live on the linked issues (closingIssuesReferences); a PR's own labels are the - // fallback. The highest class across both wins, as in babysit_merge_policy.py. + const issueEvents = new Map(); + const eventsOf = (number) => { + if (!issueEvents.has(number)) issueEvents.set(number, pages(gh, `repos/${repo}/issues/${number}/events`)); + return issueEvents.get(number); + }; const [owner, name] = repo.split("/"); const classes = new Map(); - const classOf = (number, prLabels) => { + const classOf = (number, prLabels, mergedAt) => { if (!classes.has(number)) { - const nodes = gh("graphql", { query: LINKED_ISSUES, variables: { owner, name, number } })?.data?.repository?.pullRequest - ?.closingIssuesReferences?.nodes; - if (!Array.isArray(nodes)) throw new PublishError(`gh api graphql: no closingIssuesReferences for #${number}`); - classes.set(number, workClass([...(prLabels ?? []), ...nodes.flatMap((issue) => issue?.labels?.nodes ?? [])])); + const refs = gh("graphql", { query: LINKED_ISSUES, variables: { owner, name, number } })?.data?.repository?.pullRequest?.closingIssuesReferences; + if (!Array.isArray(refs?.nodes) || refs.pageInfo?.hasNextPage !== false) { + throw new PublishError(`gh api graphql: closingIssuesReferences for #${number} unreadable or hasNextPage`); + } + const atMs = Date.parse(mergedAt); + const names = labelsAsOf((prLabels ?? []).map((label) => label?.name), eventsOf(number), atMs); + for (const issue of refs.nodes) { + if (!Array.isArray(issue?.labels?.nodes) || issue.labels.pageInfo?.hasNextPage !== false) { + throw new PublishError(`gh api graphql: labels of issue #${issue?.number} unreadable or hasNextPage`); + } + names.push(...labelsAsOf(issue.labels.nodes.map((label) => label?.name), eventsOf(issue.number), atMs)); + } + classes.set(number, workClass(names)); } return classes.get(number); }; @@ -133,20 +170,19 @@ export function collect({ repo, since, through, laneLogins, aiReviewers = [], mi const pr = gh(`repos/${repo}/pulls/${number}`); details.set(number, { number, - cls: classOf(number, pr.labels), + cls: classOf(number, pr.labels, pr.merged_at), autonomous: lane.has(pr.merged_by?.login), mergedAt: pr.merged_at, sha: pr.merge_commit_sha, - labels: (pr.labels ?? []).map((label) => label?.name), reverted: false, }); } return details.get(number); }; - const counted = merged.filter((pr) => ["C2", "C3"].includes(classOf(pr.number, pr.labels))).map((pr) => record(pr.number)); + const counted = merged.filter((pr) => ["C2", "C3"].includes(classOf(pr.number, pr.labels, pr.merged_at))).map((pr) => record(pr.number)); const bySha = new Map(counted.map((r) => [r.sha, r])); - // Reverts, from merged PR bodies and default-branch commit messages. + // Reverts, from merged PR bodies and default-branch commit messages; earliest per target. const own = repo.toLowerCase(); const revertsOf = (text) => { const targets = []; @@ -167,6 +203,7 @@ export function collect({ repo, since, through, laneLogins, aiReviewers = [], mi for (const commit of commits) { for (const target of revertsOf(commit.commit?.message)) reverts.push({ ...target, at: commit.commit?.committer?.date }); } + const firstRevert = new Map(); for (const revert of reverts) { if (!inWindow(revert.at)) continue; let number = revert.number; @@ -175,26 +212,38 @@ export function collect({ repo, since, through, laneLogins, aiReviewers = [], mi number = known ? known.number : gh(`repos/${repo}/commits/${revert.sha}/pulls`).find((pr) => pr.merged_at)?.number; if (number === undefined) continue; } + if (!firstRevert.has(number) || Date.parse(revert.at) < Date.parse(firstRevert.get(number))) firstRevert.set(number, revert.at); + } + for (const [number, at] of firstRevert) { const target = record(number); if (target.autonomous && ["C2", "C3"].includes(target.cls)) { target.reverted = true; - addEvent(`${target.cls}-auto-merge`, "reverted-merge", revert.at); + addEvent(`${target.cls}-auto-merge`, "reverted-merge", at); } } - // Post-merge gate on each autonomous merge commit. + // Post-merge gate on each autonomous merge commit: every check run and every commit status. for (const r of counted.filter((c) => c.autonomous)) { - const runs = pages(gh, `repos/${repo}/commits/${r.sha}/check-runs`, "check_runs"); - const failed = runs.filter((run) => run.status === "completed" && GATE_FAILED.has(run.conclusion) && inWindow(run.completed_at)); - r.gate = failed.length > 0 ? "failed" : runs.length > 0 && runs.every((run) => run.status === "completed" && GATE_OK.has(run.conclusion)) ? "passed" : "unsettled"; - for (const run of failed) addEvent(`${r.cls}-auto-merge`, "gate-failure", run.completed_at); + const runs = pages(gh, `repos/${repo}/commits/${r.sha}/check-runs?filter=all`, "check_runs"); + const statuses = pages(gh, `repos/${repo}/commits/${r.sha}/statuses`); + const failedRuns = runs.filter((run) => run.status === "completed" && GATE_FAILED.has(run.conclusion)); + const failedStatuses = statuses.filter((status) => STATUS_FAILED.has(status.state)); + for (const run of failedRuns) addEvent(`${r.cls}-auto-merge`, "gate-failure", run.completed_at); + for (const status of failedStatuses) addEvent(`${r.cls}-auto-merge`, "gate-failure", status.created_at); + const latest = new Map(); + for (const status of [...statuses].sort((a, b) => Date.parse(b.created_at) - Date.parse(a.created_at))) { + if (!latest.has(status.context)) latest.set(status.context, status.state); + } + const settled = + runs.length + latest.size > 0 && + runs.every((run) => run.status === "completed" && GATE_OK.has(run.conclusion)) && + [...latest.values()].every((state) => state === "success"); + r.gate = failedRuns.length + failedStatuses.length > 0 ? "failed" : settled ? "passed" : "unsettled"; } - // Human-confirmed missed-blocking AI-review findings, and advisory AI reviews on C3. + // Human-confirmed missed-blocking AI-review findings, from the label timeline. for (const r of counted.filter((c) => c.cls === "C3" || c.autonomous)) { - r.missedBlocking = 0; - if (!r.labels.includes(missedBlockingLabel)) continue; - const confirmed = pages(gh, `repos/${repo}/issues/${r.number}/events`).filter( + const confirmed = eventsOf(r.number).filter( (e) => e.event === "labeled" && e.label?.name === missedBlockingLabel && e.actor?.type === "User" && !lane.has(e.actor.login) && inWindow(e.created_at), ); r.missedBlocking = confirmed.length > 0 ? 1 : 0; @@ -211,10 +260,10 @@ export function collect({ repo, since, through, laneLogins, aiReviewers = [], mi return { repo, since, - through, + through: new Date(throughMs).toISOString(), events: sorted, unresolvedReverts, - cells: report(counted, sorted, throughMs, reviewers.size > 0), + cells: report(counted, sorted, reviewers.size > 0), }; } @@ -222,10 +271,11 @@ function term(name, value, threshold, pass, detail) { return { term: name, value, threshold, pass, ...(detail ? { detail } : {}) }; } -function windowOf(records, throughMs) { - if (records.length === 0) return { from: null, through: new Date(throughMs).toISOString(), days: 0 }; - const from = Math.min(...records.map((r) => Date.parse(r.mergedAt))); - return { from: new Date(from).toISOString(), through: new Date(throughMs).toISOString(), days: Math.floor(((throughMs - from) / DAY_MS) * 100) / 100 }; +function windowOf(records) { + if (records.length === 0) return { from: null, to: null, days: 0 }; + const times = records.map((r) => Date.parse(r.mergedAt)); + const [from, to] = [Math.min(...times), Math.max(...times)]; + return { from: new Date(from).toISOString(), to: new Date(to).toISOString(), days: Math.floor(((to - from) / DAY_MS) * 100) / 100 }; } function gateTerm(name, records) { @@ -240,11 +290,11 @@ function cell(window, terms) { } // Thresholds are work-classes.md's suggested defaults. -function report(counted, events, throughMs, reviewsMeasured) { +function report(counted, events, reviewsMeasured) { const c2 = counted.filter((r) => r.cls === "C2" && r.autonomous); const c3 = counted.filter((r) => r.cls === "C3" && r.autonomous); const c3All = counted.filter((r) => r.cls === "C3"); - const c2Window = windowOf(c2, throughMs); + const c2Window = windowOf(c2); const reverted = (records) => records.filter((r) => r.reverted).length; const missed = (records) => records.reduce((sum, r) => sum + (r.missedBlocking ?? 0), 0); const c2Demotions = events.filter((e) => e.cell === "C2-auto-merge").length; @@ -256,7 +306,7 @@ function report(counted, events, throughMs, reviewsMeasured) { gateTerm("deterministic-gate pass", c2), term("human-reverted merges", reverted(c2), "0", reverted(c2) === 0), ]), - "C3-auto-merge": cell(windowOf([...c2, ...c3], throughMs), [ + "C3-auto-merge": cell(windowOf([...c2, ...c3]), [ term("autonomous C2 merges", c2.length, ">= 20", c2.length >= 20), term("C2 window days", c2Window.days, ">= 14", c2Window.days >= 14), term("C2 demotion events", c2Demotions, "0", c2Demotions === 0), @@ -265,7 +315,7 @@ function report(counted, events, throughMs, reviewsMeasured) { term("C3 human-reverted merges", reverted(c3), "0", reverted(c3) === 0), term("C3 human-confirmed missed-blocking findings", missed(c3), "0", missed(c3) === 0), ]), - "C3-ai-review-blocking": cell(windowOf(c3All, throughMs), [ + "C3-ai-review-blocking": cell(windowOf(c3All), [ reviewsMeasured ? term("advisory AI reviews on C3", reviewed.length, ">= 30", reviewed.length >= 30) : term("advisory AI reviews on C3", null, ">= 30", false, { unmeasured: "no --ai-reviewer given" }), @@ -278,26 +328,26 @@ function sha256(text) { return createHash("sha256").update(text).digest("hex"); } -function writeAtomic(path, text) { +// "wx" refuses a temp file that already exists rather than overwrite or remove it. +function writeTemp(path, text) { const temp = `${path}.${process.pid}.tmp`; - // "wx" refuses a temp file that already exists rather than overwrite or remove it. writeFileSync(temp, text, { flag: "wx" }); + return temp; +} + +function readIfPresent(path) { try { - renameSync(temp, path); + return readFileSync(path, "utf8"); } catch (error) { - rmSync(temp, { force: true }); - throw error; + if (error.code === "ENOENT") return null; + throw new PublishError(`cannot read ${path}: ${error.message}`); } } export function publish(outDir, events, through, now = new Date()) { const file = join(outDir, "evidence.json"); - let previous = null; - try { - previous = readFileSync(file, "utf8"); - } catch (error) { - if (error.code !== "ENOENT") throw new PublishError(`cannot read ${file}: ${error.message}`); - } + const metaFile = join(outDir, "evidence.meta.json"); + const previous = readIfPresent(file); if (previous !== null) { let prior; try { @@ -306,6 +356,15 @@ export function publish(outDir, events, through, now = new Date()) { throw new PublishError(`refusing to publish: ${file} is not JSON (${error.message}), so the superset check cannot run`); } if (!Array.isArray(prior)) throw new PublishError(`refusing to publish: ${file} is not a JSON array, so the superset check cannot run`); + let priorMeta; + try { + priorMeta = JSON.parse(readIfPresent(metaFile)); + } catch { + priorMeta = null; + } + if (priorMeta?.sha256 !== sha256(previous)) { + throw new PublishError(`refusing to publish: ${file} does not match the sha256 in its sidecar ${metaFile} (missing, unreadable or different)`); + } const keys = new Set(events.map(eventKey)); const dropped = prior.filter((event) => !keys.has(eventKey(event))); if (dropped.length > 0) { @@ -314,8 +373,18 @@ export function publish(outDir, events, through, now = new Date()) { } const text = `${JSON.stringify(events, null, 2)}\n`; const meta = { generated_at: now.toISOString(), through, sha256: sha256(text), prev_sha256: previous === null ? null : sha256(previous) }; - writeAtomic(file, text); - writeAtomic(join(outDir, "evidence.meta.json"), `${JSON.stringify(meta, null, 2)}\n`); + const eventsTemp = writeTemp(file, text); + let metaTemp; + try { + metaTemp = writeTemp(metaFile, `${JSON.stringify(meta, null, 2)}\n`); + renameSync(eventsTemp, file); + } catch (error) { + rmSync(eventsTemp, { force: true }); + if (metaTemp) rmSync(metaTemp, { force: true }); + throw error; + } + // A failure here leaves a pair the next run refuses on the sha256 check: fail closed. + renameSync(metaTemp, metaFile); return meta; } diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs index 5085273fd7..73a8affecc 100644 --- a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs @@ -31,14 +31,30 @@ const later = (at, minutes) => new Date(Date.parse(at) + minutes * 60_000).toISO function pr(number, mergedAt, { labels = [], merger = LANE, title = REC.pullListItem.title, body = REC.pullListItem.body } = {}) { const names = labels.map((name) => ({ name })); return { - list: { ...REC.pullListItem, number, title, body, labels: names, merged_at: mergedAt, updated_at: later(mergedAt, 1), merge_commit_sha: shaOf(number) }, + list: { + ...REC.pullListItem, + number, + title, + body, + labels: names, + created_at: later(mergedAt, -60), + merged_at: mergedAt, + updated_at: later(mergedAt, 1), + merge_commit_sha: shaOf(number), + }, detail: { ...REC.pull, number, title, labels: names, merged_at: mergedAt, merge_commit_sha: shaOf(number), merged_by: { login: merger, type: merger === LANE ? "Bot" : "User" } }, }; } -// A recorded GitHub: answers only the GET paths the publisher is expected to call and throws on -// any other, so an unexpected or mutating call fails the case. -function github({ pulls = [], commits = [], checkRuns = {}, issueEvents = {}, reviews = {}, linked = {} }) { +const linkedTo = (nodes, hasNextPage = false) => ({ data: { repository: { pullRequest: { closingIssuesReferences: { pageInfo: { hasNextPage }, nodes } } } } }); +const labelEvent = (event, name, at, login = "kyle-sexton", type = "User") => ({ ...REC.issueEvent, event, label: { name }, actor: { login, type }, created_at: at }); + +// A recorded GitHub: answers only the GET paths the publisher is expected to call, with the +// documented semantics of each (list sort order, check-runs `filter=latest` by default, statuses +// newest first), and throws on any other, so an unexpected or mutating call fails the case. +// A run marked `superseded` was re-run: only `filter=all` returns it. +// `bumpOnPage2` updates that PR (a comment, a label) between the listing's first and second page. +function github({ pulls = [], commits = [], checkRuns = {}, statuses = {}, issueEvents = {}, reviews = {}, linked = {}, bumpOnPage2 }) { const page = (items, path) => { const n = Number(/[?&]page=(\d+)/.exec(path)[1]); return items.slice((n - 1) * 100, n * 100); @@ -48,23 +64,26 @@ function github({ pulls = [], commits = [], checkRuns = {}, issueEvents = {}, re if (path === "graphql") { assert.deepEqual([graphql.variables.owner, graphql.variables.name], REPO.split("/")); assert.match(graphql.query, /^query\(/); - const recorded = linked[graphql.variables.number]; - return recorded ?? { data: { repository: { pullRequest: { closingIssuesReferences: { nodes: [] } } } } }; + return linked[graphql.variables.number] ?? linkedTo([]); } const prefix = `repos/${REPO}/`; assert.ok(path.startsWith(prefix), path); const rest = path.slice(prefix.length); let m; - if (rest.startsWith("pulls?state=closed&sort=updated&direction=desc&")) { - return page(pulls.map((p) => p.list).sort((a, b) => b.updated_at.localeCompare(a.updated_at)), rest); + if ((m = /^pulls\?state=closed&sort=(created|updated)&direction=(asc|desc)&/.exec(rest))) { + if (bumpOnPage2 && /[?&]page=2\b/.test(rest)) pulls.find((p) => p.list.number === bumpOnPage2).list.updated_at = "2026-10-04T00:00:00Z"; + const key = m[1] === "created" ? "created_at" : "updated_at"; + const sign = m[2] === "asc" ? 1 : -1; + return page(pulls.map((p) => p.list).sort((a, b) => sign * a[key].localeCompare(b[key])), rest); } if ((m = /^pulls\/(\d+)$/.exec(rest))) return pulls.find((p) => p.detail.number === Number(m[1])).detail; if ((m = /^pulls\/(\d+)\/reviews\?/.exec(rest))) return page(reviews[m[1]] ?? [], rest); if (/^commits\?since=[^&]+&until=[^&]+&/.test(rest)) return page(commits, rest); if ((m = /^commits\/([0-9a-f]{40})\/check-runs\?/.exec(rest))) { - const number = parseInt(m[1], 16); - return { total_count: runsFor(number).length, check_runs: page(runsFor(number), rest) }; + const runs = runsFor(parseInt(m[1], 16)).filter((r) => /[?&]filter=all\b/.test(rest) || !r.superseded); + return { total_count: runs.length, check_runs: page(runs, rest) }; } + if ((m = /^commits\/([0-9a-f]{40})\/statuses\?/.exec(rest))) return page(statuses[parseInt(m[1], 16)] ?? [], rest); if ((m = /^issues\/(\d+)\/events\?/.exec(rest))) return page(issueEvents[m[1]] ?? [], rest); throw new Error(`unrecorded gh api call: ${path}`); }; @@ -157,6 +176,98 @@ test("advisory reviews are counted from the named AI reviewers and reported unme assert.equal(termOf(run(fixture), "C3-ai-review-blocking", "advisory AI reviews on C3").value, null); }); +test("a failed check run that was re-run green still yields gate-failure (filter=all)", () => { + const failed = { ...REC.checkRun, conclusion: "failure", completed_at: "2026-09-01T00:10:00Z", superseded: true }; + const result = run({ pulls: [pr(901, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] })], checkRuns: { 901: [failed, REC.checkRun] } }); + assert.deepEqual(result.events, [{ cell: "C2-auto-merge", event: "gate-failure", at: "2026-09-01T00:10:00Z" }]); + assert.equal(termOf(result, "C2-auto-merge", "deterministic-gate pass").pass, false); +}); + +test("a failure or error commit status, even one later superseded by success, yields gate-failure", () => { + const statuses = [ + { ...REC.status, state: "success", created_at: "2026-09-01T00:30:00Z" }, + { ...REC.status, state: "error", created_at: "2026-09-01T00:20:00Z" }, + { ...REC.status, context: "other", state: "failure", created_at: "2026-09-01T00:15:00Z" }, + ]; + const result = run({ pulls: [pr(902, "2026-09-01T00:00:00Z", { labels: [SCOPED] })], statuses: { 902: statuses } }); + assert.deepEqual(result.events, [ + { cell: "C3-auto-merge", event: "gate-failure", at: "2026-09-01T00:15:00Z" }, + { cell: "C3-auto-merge", event: "gate-failure", at: "2026-09-01T00:20:00Z" }, + ]); +}); + +test("a gate failure that completes after --through is still emitted", () => { + const failing = { ...REC.checkRun, conclusion: "failure", completed_at: "2026-10-06T00:00:00Z" }; + const result = run({ pulls: [pr(903, "2026-10-04T23:00:00Z", { labels: [MECHANICAL] })], checkRuns: { 903: [failing] } }); + assert.deepEqual(result.events, [{ cell: "C2-auto-merge", event: "gate-failure", at: "2026-10-06T00:00:00Z" }]); +}); + +test("a missed-blocking label a human applied and someone later removed still counts", () => { + const result = run({ + pulls: [pr(904, "2026-09-01T00:00:00Z", { labels: [SCOPED], merger: "kyle-sexton" })], + issueEvents: { 904: [labelEvent("labeled", MISSED, "2026-09-04T00:00:00Z"), labelEvent("unlabeled", MISSED, "2026-09-05T00:00:00Z", LANE, "Bot")] }, + }); + assert.deepEqual(result.events, [{ cell: "C3-ai-review-blocking", event: "verification-divergence", at: "2026-09-04T00:00:00Z" }]); +}); + +test("the class is read as of the merge: post-merge relabeling neither adds nor drops a merge", () => { + const result = run({ + pulls: [ + pr(905, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] }), + pr(906, "2026-09-01T00:00:00Z"), + pr(907, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] }), + ], + issueEvents: { + 905: [labelEvent("labeled", MECHANICAL, "2026-09-02T00:00:00Z")], + 906: [labelEvent("labeled", MECHANICAL, "2026-08-31T00:00:00Z"), labelEvent("unlabeled", MECHANICAL, "2026-09-02T00:00:00Z", LANE, "Bot")], + // Linked issue #6007 is labeled scoped now, but gained that label after #907 merged. + 6007: [labelEvent("labeled", SCOPED, "2026-09-03T00:00:00Z")], + }, + linked: { 907: REC.linkedIssues }, + }); + assert.equal(termOf(result, "C2-auto-merge", "autonomous C2 completions").value, 2); + assert.equal(termOf(result, "C3-auto-merge", "autonomous C3 completions").value, 0); +}); + +test("20 merges inside one day fail the 14-day window, which ends at the last counted merge", () => { + const sameDay = Array.from({ length: 20 }, (_, i) => pr(1000 + i, later("2026-09-01T00:00:00Z", i), { labels: [MECHANICAL] })); + const result = run({ pulls: sameDay }); + assert.equal(termOf(result, "C2-auto-merge", "window days").pass, false); + assert.equal(result.cells["C2-auto-merge"].window.to, "2026-09-01T00:19:00.000Z"); +}); + +test("--through is clamped to now", () => { + const result = run({ pulls: [] }, { through: "2099-01-01T00:00:00Z" }); + assert.ok(Date.parse(result.through) <= Date.now()); +}); + +test("a linked-issue read with another page fails closed", () => { + const nodes = REC.linkedIssues.data.repository.pullRequest.closingIssuesReferences.nodes; + const fixture = (more) => ({ pulls: [pr(908, "2026-09-01T00:00:00Z")], linked: { 908: linkedTo(nodes, more) } }); + assert.throws(() => run(fixture(true)), (error) => error instanceof PublishError && /hasNextPage/.test(error.message)); + const labelsPaged = linkedTo([{ ...nodes[0], labels: { ...nodes[0].labels, pageInfo: { hasNextPage: true } } }]); + assert.throws(() => run({ pulls: [pr(908, "2026-09-01T00:00:00Z")], linked: { 908: labelsPaged } }), /hasNextPage/); +}); + +test("a revert seen in both the PR body and the squash commit yields one event at the earlier time", () => { + const result = run({ + pulls: [ + pr(909, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] }), + pr(910, "2026-09-02T00:00:00Z", { title: 'Revert "x"', body: `Reverts ${REPO}#909`, merger: "kyle-sexton" }), + ], + commits: [{ ...REC.commit, sha: shaOf(910), commit: { message: `Revert "x" (#910)\n\nReverts ${REPO}#909`, committer: { date: "2026-09-01T23:55:00Z" } } }], + }); + assert.deepEqual(result.events, [{ cell: "C2-auto-merge", event: "reverted-merge", at: "2026-09-01T23:55:00Z" }]); +}); + +test("a PR updated while the listing pages is not skipped", () => { + // 101 unclassed merges plus one C3 merge that sorts last by update time until it is updated. + const pulls = Array.from({ length: 101 }, (_, i) => pr(2000 + i, later("2026-09-10T00:00:00Z", i))); + pulls.push(pr(3000, "2026-09-05T00:00:00Z", { labels: [SCOPED] })); + const result = run({ pulls, bumpOnPage2: 3000 }); + assert.equal(termOf(result, "C3-auto-merge", "autonomous C3 completions").value, 1); +}); + const A = { cell: "C2-auto-merge", event: "reverted-merge", at: "2026-09-02T00:00:00Z" }; const B = { cell: "C3-auto-merge", event: "gate-failure", at: "2026-09-03T00:00:00Z" }; const hash = (text) => createHash("sha256").update(text).digest("hex"); @@ -195,6 +306,24 @@ test("an existing temp file is neither overwritten nor removed, and nothing is p assert.throws(() => readFileSync(join(out, "evidence.json")), { code: "ENOENT" }); }); +test("an events file that no longer matches its sidecar is refused and left unchanged", () => { + const out = mkdtempSync(join(base, "out-")); + publish(out, [A], THROUGH); + writeFileSync(join(out, "evidence.json"), `${JSON.stringify([A, B], null, 2)}\n`); + assert.throws(() => publish(out, [A, B], THROUGH), /refusing.*sidecar/); + assert.deepEqual(JSON.parse(readFileSync(join(out, "evidence.json"), "utf8")), [A, B]); +}); + +test("a sidecar write failure leaves the published events file unchanged", () => { + const out = mkdtempSync(join(base, "out-")); + publish(out, [A], THROUGH); + const before = readFileSync(join(out, "evidence.json"), "utf8"); + writeFileSync(join(out, `evidence.meta.json.${process.pid}.tmp`), "planted"); + assert.throws(() => publish(out, [A, B], THROUGH), { code: "EEXIST" }); + assert.equal(readFileSync(join(out, "evidence.json"), "utf8"), before); + assert.throws(() => readFileSync(join(out, `evidence.json.${process.pid}.tmp`)), { code: "ENOENT" }); +}); + test("an existing events file that is not a JSON array is refused", () => { const out = mkdtempSync(join(base, "out-")); writeFileSync(join(out, "evidence.json"), "{}"); From b1d1a529dfcb43966aeaef6d41d9e8f555ba5654 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 12 Oct 2026 23:44:08 -0400 Subject: [PATCH 04/18] fix(source-control): stop post-merge edits from hiding an autonomous merge - Rebuild the issues a PR linked at merge from its timeline: undo ConnectedEvent/DisconnectedEvent after merged_at, and treat a body edited after the merge as unknown closing-keyword links. - Keep an autonomous merge whose class resolves to none: list it in unclassedAutonomous, count it toward no positive term, and emit its contrary events against every promotable cell. - Compare logins case-insensitively with the [bot] suffix dropped, as babysit_merge_policy.py _login_key does. - Count a cancelled, action_required or stale post-merge run as a gate failure unless a later run of the same check succeeded. - Attribute autonomy as derive_auto_merge does: a merge whose latest queue event has a null or non-human actor or enqueuer, or that bypassed the queue under a non-human merged_by, is autonomous. - Note the cost of paging the full closed-PR history in the header. Refs: #4588 Co-Authored-By: Claude Opus 5.5 --- ...8-promotion-evidence-publisher-3634507d.md | 2 +- .../reference/promotion-evidence-bootstrap.md | 9 +- .../publish-promotion-evidence/recorded.json | 65 ++++-- .../scripts/publish-promotion-evidence.mjs | 187 ++++++++++++------ .../publish-promotion-evidence.test.mjs | 113 +++++++++-- 5 files changed, 280 insertions(+), 96 deletions(-) diff --git a/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md b/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md index 60aee5c89d..7dd3865615 100644 --- a/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md +++ b/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md @@ -4,4 +4,4 @@ bump: minor ### Added -- `babysit-loop` ships `scripts/publish-promotion-evidence.mjs`, an operator-side publisher for the promotion-evidence events file (#4588). It reads GitHub only, through read-only `gh api` calls, and emits the `{cell, event, at}` contrary events the security-binding checker reads (`reverted-merge`, `gate-failure`, and `verification-divergence` for a missed-blocking finding a human confirmed) plus a C3 promotion-evidence report: each promotable cell's predicate terms with counts, window and pass or fail. Report mode, the default, writes nothing. `--out ` writes `evidence.json` and an `evidence.meta.json` sidecar (`generated_at`, `through`, `sha256`, `prev_sha256`) by temp file and rename, and refuses a set that drops a previously published event or an existing file that does not match its sidecar. Work classes are read from the linked issues' and the PR's labels as of the merge, and every check run (re-runs included) and commit status on a merge commit counts toward `gate-failure`. Where it runs and under which identity is not decided yet, and the resolver does not read the sidecar; merge authority is unchanged. +- `babysit-loop` ships `scripts/publish-promotion-evidence.mjs`, an operator-side publisher for the promotion-evidence events file (#4588). It reads GitHub only, through read-only `gh api` calls, and emits the `{cell, event, at}` contrary events the security-binding checker reads (`reverted-merge`, `gate-failure`, and `verification-divergence` for a missed-blocking finding a human confirmed) plus a C3 promotion-evidence report: each promotable cell's predicate terms with counts, window and pass or fail. Report mode, the default, writes nothing. `--out ` writes `evidence.json` and an `evidence.meta.json` sidecar (`generated_at`, `through`, `sha256`, `prev_sha256`) by temp file and rename, and refuses a set that drops a previously published event or an existing file that does not match its sidecar. Work classes are read from the labels of the PR and of the issues it linked at merge, as of the merge. A merge counts as autonomous when no human enqueued it or a `--lane-login` identity merged it. An autonomous merge with no readable class is listed as unclassed and its contrary events count against every cell. Every check run (re-runs included) and commit status on a merge commit counts toward `gate-failure`, a cancelled run included unless a later run of that check succeeded. Where it runs and under which identity is not decided yet, and the resolver does not read the sidecar; merge authority is unchanged. diff --git a/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md b/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md index 40db9ac329..f762329f9f 100644 --- a/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md +++ b/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md @@ -149,10 +149,11 @@ binding. [`scripts/publish-promotion-evidence.mjs`](../scripts/publish-promotion-evidence.mjs) builds the events file from GitHub alone, through read-only `gh api` calls: merged pull requests classed by the `work-class:` labels their linked issues and the pull request carried at merge time, which of them -the `--lane-login` identities merged, reverts of those merges, every check run and commit status -on their merge commits, and the missed-blocking label a human applied. Its header comment states -what counts as each event, including the judgment mappings and how to set `--lane-login` for -merge-queue merges. +merged autonomously (no human enqueued them, or a `--lane-login` identity merged them), reverts of +those merges, every check run and commit status on their merge commits, and the missed-blocking +label a human applied. An autonomous merge whose class cannot be read is listed as unclassed, and +its contrary events count against every promotable cell. Its header comment states what counts as +each event, including the judgment mappings and how to set `--lane-login` for merge-queue merges. - **Report mode** (no `--out`) prints the events array and the C3 promotion-evidence report to stdout and writes nothing. The report gives each promotable cell's predicate terms from diff --git a/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json b/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json index 3ccad36a6e..7cc6fb4310 100644 --- a/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json +++ b/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json @@ -13,17 +13,6 @@ "type": "User" } }, - "pull": { - "labels": [], - "merge_commit_sha": "aad0b2b69bc22154c61b0857307a183d0c7821a8", - "merged_at": "2026-10-13T02:42:17Z", - "merged_by": { - "login": "kyle-sexton", - "type": "User" - }, - "number": 7352, - "title": "fix(source-control): settle unattended merge residuals before Phase 15" - }, "checkRun": { "completed_at": "2026-10-13T02:38:54Z", "conclusion": "success", @@ -61,10 +50,26 @@ "type": "Bot" } }, - "linkedIssues": { + "pullGraph": { "data": { "repository": { "pullRequest": { + "number": 7063, + "mergedAt": "2026-10-13T01:47:44Z", + "lastEditedAt": "2026-10-13T01:38:02Z", + "mergeCommit": { + "oid": "6fa4f210dd0eb904a9ca1504344d8b66294ee6f5" + }, + "mergedBy": { + "__typename": "User", + "login": "kyle-sexton" + }, + "labels": { + "pageInfo": { + "hasNextPage": false + }, + "nodes": [] + }, "closingIssuesReferences": { "pageInfo": { "hasNextPage": false @@ -93,9 +98,45 @@ } } ] + }, + "timelineItems": { + "pageInfo": { + "hasNextPage": false + }, + "nodes": [ + { + "__typename": "AddedToMergeQueueEvent", + "createdAt": "2026-10-13T01:42:44Z", + "actor": { + "__typename": "User", + "login": "kyle-sexton" + }, + "enqueuer": { + "__typename": "User", + "login": "kyle-sexton" + } + } + ] } } } } + }, + "issue": { + "labels": [ + { + "name": "priority: medium" + }, + { + "name": "status: ready" + }, + { + "name": "needs-human" + }, + { + "name": "work-class: scoped" + } + ], + "number": 6007 } } diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs index 954c3d5988..4a3c83bbeb 100644 --- a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs @@ -9,35 +9,49 @@ // [--through ] [--ai-reviewer ...] [--missed-blocking-label ] [--out ] // Exit 0 = report printed (and published with --out); 1 = publish refused; 2 = usage or GitHub read error. // -// Report mode (no --out) prints {repo, since, through, events, unresolvedReverts, cells} to +// Report mode (no --out) prints {repo, since, through, events, unresolvedReverts, unclassedAutonomous, cells} to // stdout and writes nothing. --through defaults to, and is clamped to, now. With --out it also // writes /evidence.json (the events array) and /evidence.meta.json ({generated_at, // through, sha256, prev_sha256}). Both temp files are written before either rename, the events // file renamed first. It refuses when an existing evidence.json is not a JSON array, has no // sidecar or does not match the sidecar's sha256, or holds an event the new array lacks. // -// --lane-login: GitHub records the merge-queue enqueuer as merged_by. Probed 2026-10-12 on PR -// #7352: merged_by.login was the login of the AddedToMergeQueueEvent enqueuer, not a queue bot. -// So pass the identity the lane enqueues or merges with (its App bot login, `[bot]`); a -// lane PR a human enqueued counts as a human merge. Recheck if merged_by stops matching the -// enqueuer. +// Cost: every run pages the repository's whole closed-PR history (one REST page per 100 closed +// PRs, created ascending, so no merge is skipped while paging), then reads, per merged PR in the +// window, one GraphQL query and the issue events of the PR and each linked issue, and per +// autonomous merge its check runs and statuses. +// +// Autonomy follows babysit_merge_policy.py derive_auto_merge: the latest AddedToMergeQueueEvent's +// actor and enqueuer, or merged_by when the merge bypassed the queue; a merge is autonomous when +// any of them is null or not human (a User outside --lane-login, not `github-actions`, not a +// `[bot]` login; derive_auto_merge's write-permission read is not made), or merged_by is a +// --lane-login. Logins compare case-insensitively with `[bot]` dropped. GitHub records the +// enqueuer as merged_by (probed 2026-10-12 on PR #7352: merged_by.login matched the +// AddedToMergeQueueEvent enqueuer), so --lane-login names any lane identity that runs as a User +// (a machine user); an App lane is a Bot and counts without it. Recheck if merged_by stops +// matching the enqueuer. // // What counts (judgment where the contract names no GitHub source): // - A merged PR's work class is the highest `work-class:` label (the label set -// babysit-prs/scripts/babysit_merge_policy.py WORK_CLASS_LABELS maps) on its linked issues -// (closingIssuesReferences) and on the PR, each label set rebuilt as of merged_at by undoing -// later labeled/unlabeled events, so relabeling after the merge changes nothing. A linked-issue -// or label list with another page fails closed. A PR with no class is not counted. -// - Autonomous means merged_by is a --lane-login identity. +// babysit-prs/scripts/babysit_merge_policy.py WORK_CLASS_LABELS maps) on the issues it linked +// at merge and on the PR, each label set rebuilt as of merged_at by undoing later +// labeled/unlabeled events, so relabeling after the merge changes nothing. The linked set is +// closingIssuesReferences with later ConnectedEvent/DisconnectedEvent undone; when the PR body +// was edited after the merge, its closing-keyword links at merge are unknown and the class is +// none. A list with another page fails closed. +// - A C2/C3 merge is counted. An autonomous merge with no class is listed in +// unclassedAutonomous, never counted toward a positive term, and each of its contrary events +// is emitted against every promotable cell. // - reverted-merge: an autonomous C2/C3 merge named by a merged PR body or a default-branch // commit as `Reverts #` (GitHub's revert button) or `This reverts commit // ` (git revert); one event per reverted merge, at the earliest revert seen. A merged PR // titled revert... with neither form is listed in unresolvedReverts for a human to read. // - gate-failure: every check run on an autonomous merge's merge commit (filter=all, so a run // later re-run green still counts) concluding failure, timed_out or startup_failure, and every -// commit status in state failure or error, whenever it completed. The gate-pass term also -// fails while a run is not completed with success, neutral or skipped, or a context's latest -// status is not success. +// commit status in state failure or error, whenever it completed. A run concluding cancelled, +// action_required, stale or anything else not green also counts, unless a later run of the +// same check name on that commit succeeded. The gate-pass term also fails while a run is in +// progress or a context's latest status is not success. // - verification-divergence: a human (a User actor outside --lane-login) applied // --missed-blocking-label to a merged C3 PR (cell C3-ai-review-blocking) or an autonomous C2 // PR (cell C2-auto-merge), read from the label timeline, so removing the label later does not @@ -79,7 +93,7 @@ function instant(value, name) { return ms; } -// A REST path is read with GET; "graphql" runs a query (read-only: LINKED_ISSUES is the only one). +// A REST path is read with GET; "graphql" runs a query (read-only: PULL_QUERY is the only one). function realGh(path, graphql) { const args = graphql ? ["api", "graphql", "-f", `query=${graphql.query}`, ...Object.entries(graphql.variables).flatMap(([key, value]) => ["-F", `${key}=${value}`])] @@ -89,8 +103,30 @@ function realGh(path, graphql) { return JSON.parse(run.stdout); } -const LINKED_ISSUES = - "query($owner:String!,$name:String!,$number:Int!){repository(owner:$owner,name:$name){pullRequest(number:$number){closingIssuesReferences(first:50){pageInfo{hasNextPage} nodes{number labels(first:100){pageInfo{hasNextPage} nodes{name}}}}}}}"; +const PULL_QUERY = + "query($owner:String!,$name:String!,$number:Int!){repository(owner:$owner,name:$name){pullRequest(number:$number){" + + "number mergedAt lastEditedAt mergeCommit{oid} mergedBy{__typename login} labels(first:100){pageInfo{hasNextPage} nodes{name}} " + + "closingIssuesReferences(first:50){pageInfo{hasNextPage} nodes{number labels(first:100){pageInfo{hasNextPage} nodes{name}}}} " + + "timelineItems(first:100,itemTypes:[CONNECTED_EVENT,DISCONNECTED_EVENT,ADDED_TO_MERGE_QUEUE_EVENT]){pageInfo{hasNextPage} nodes{__typename " + + "... on ConnectedEvent{createdAt subject{__typename ... on Issue{number}}} ... on DisconnectedEvent{createdAt subject{__typename ... on Issue{number}}} " + + "... on AddedToMergeQueueEvent{createdAt actor{__typename login} enqueuer{__typename login}}}}}}}"; +const CELLS = ["C2-auto-merge", "C3-ai-review-blocking", "C3-auto-merge"]; + +// A login compared across REST (`app[bot]`) and GraphQL (`app`) spellings, as +// babysit_merge_policy.py _login_key does. +function loginKey(login) { + return String(login ?? "") + .trim() + .toLowerCase() + .replace(/\[bot\]$/, ""); +} + +function onePage(connection, what) { + if (!Array.isArray(connection?.nodes) || connection.pageInfo?.hasNextPage !== false) { + throw new PublishError(`gh api graphql: ${what} unreadable or hasNextPage`); + } + return connection.nodes; +} function pages(gh, path, key) { const out = []; @@ -124,7 +160,11 @@ function eventKey(event) { export function collect({ repo, since, through, laneLogins, aiReviewers = [], missedBlockingLabel, gh = realGh }) { const sinceMs = instant(since, "--since"); const throughMs = Math.min(instant(through, "--through"), Date.now()); - const lane = new Set(laneLogins); + const lane = new Set(laneLogins.map(loginKey)); + // babysit_merge_policy.py is_human, without its permission read: a User that is not a lane + // login, `github-actions` or a `[bot]` login. A null actor is not human. + const human = (actor) => + actor?.__typename === "User" && !lane.has(loginKey(actor.login)) && loginKey(actor.login) !== "github-actions" && !/\[bot\]$/i.test(actor.login ?? ""); const inWindow = (at) => typeof at === "string" && Date.parse(at) >= sinceMs && Date.parse(at) <= throughMs; const events = new Map(); const addEvent = (cell, event, at) => { @@ -144,43 +184,54 @@ export function collect({ repo, since, through, laneLogins, aiReviewers = [], mi return issueEvents.get(number); }; const [owner, name] = repo.split("/"); - const classes = new Map(); - const classOf = (number, prLabels, mergedAt) => { - if (!classes.has(number)) { - const refs = gh("graphql", { query: LINKED_ISSUES, variables: { owner, name, number } })?.data?.repository?.pullRequest?.closingIssuesReferences; - if (!Array.isArray(refs?.nodes) || refs.pageInfo?.hasNextPage !== false) { - throw new PublishError(`gh api graphql: closingIssuesReferences for #${number} unreadable or hasNextPage`); - } - const atMs = Date.parse(mergedAt); - const names = labelsAsOf((prLabels ?? []).map((label) => label?.name), eventsOf(number), atMs); - for (const issue of refs.nodes) { - if (!Array.isArray(issue?.labels?.nodes) || issue.labels.pageInfo?.hasNextPage !== false) { - throw new PublishError(`gh api graphql: labels of issue #${issue?.number} unreadable or hasNextPage`); - } - names.push(...labelsAsOf(issue.labels.nodes.map((label) => label?.name), eventsOf(issue.number), atMs)); - } - classes.set(number, workClass(names)); + + // The class at merge: labels of the PR and of the issues linked at merge, each rebuilt as of + // mergedAt. Links: the current set with later Connected/Disconnected events undone; a body + // edited after the merge leaves the closing-keyword links unknown, so the class is too. + const classAtMerge = (g, timeline) => { + const atMs = Date.parse(g.mergedAt); + if (g.lastEditedAt && Date.parse(g.lastEditedAt) > atMs) return { cls: null, reason: "PR body edited after the merge: closing-keyword links unknown" }; + const linked = new Map(); + for (const issue of onePage(g.closingIssuesReferences, `closingIssuesReferences of #${g.number}`)) { + linked.set(issue.number, onePage(issue.labels, `labels of issue #${issue.number}`).map((label) => label?.name)); + } + const laterLinks = timeline + .filter((e) => ["ConnectedEvent", "DisconnectedEvent"].includes(e.__typename) && e.subject?.__typename === "Issue" && Date.parse(e.createdAt) > atMs) + .sort((a, b) => Date.parse(b.createdAt) - Date.parse(a.createdAt)); + for (const e of laterLinks) { + if (e.__typename === "ConnectedEvent") linked.delete(e.subject.number); + else if (!linked.has(e.subject.number)) linked.set(e.subject.number, (gh(`repos/${repo}/issues/${e.subject.number}`).labels ?? []).map((label) => label?.name)); } - return classes.get(number); + const names = labelsAsOf(onePage(g.labels, `labels of #${g.number}`).map((label) => label?.name), eventsOf(g.number), atMs); + for (const [issue, labels] of linked) names.push(...labelsAsOf(labels, eventsOf(issue), atMs)); + const cls = workClass(names); + return { cls, reason: cls ? null : "no work-class label at merge" }; + }; + + // Autonomy as babysit_merge_policy.py derive_auto_merge reads it: the latest + // AddedToMergeQueueEvent's actor and enqueuer, or merged_by when the merge bypassed the + // queue; any of them null or not human makes the merge autonomous. + const autonomousOf = (g, timeline) => { + const queued = timeline.filter((e) => e.__typename === "AddedToMergeQueueEvent").sort((a, b) => Date.parse(a.createdAt) - Date.parse(b.createdAt)); + const actors = queued.length > 0 ? [queued.at(-1).actor, queued.at(-1).enqueuer] : [g.mergedBy]; + return actors.some((actor) => !human(actor)) || lane.has(loginKey(g.mergedBy?.login)); }; const details = new Map(); const record = (number) => { if (!details.has(number)) { - const pr = gh(`repos/${repo}/pulls/${number}`); - details.set(number, { - number, - cls: classOf(number, pr.labels, pr.merged_at), - autonomous: lane.has(pr.merged_by?.login), - mergedAt: pr.merged_at, - sha: pr.merge_commit_sha, - reverted: false, - }); + const g = gh("graphql", { query: PULL_QUERY, variables: { owner, name, number } })?.data?.repository?.pullRequest; + if (!g?.mergedAt) throw new PublishError(`gh api graphql: pull request #${number} unreadable or not merged`); + const timeline = onePage(g.timelineItems, `timeline of #${number}`); + details.set(number, { number, ...classAtMerge(g, timeline), autonomous: autonomousOf(g, timeline), mergedAt: g.mergedAt, sha: g.mergeCommit?.oid, reverted: false }); } return details.get(number); }; - const counted = merged.filter((pr) => ["C2", "C3"].includes(classOf(pr.number, pr.labels, pr.merged_at))).map((pr) => record(pr.number)); - const bySha = new Map(counted.map((r) => [r.sha, r])); + // Tracked: C2/C3 merges, and autonomous merges with no class, which are never counted toward + // a positive term but whose contrary events go against every cell. + const tracked = merged.map((pr) => record(pr.number)).filter((r) => ["C2", "C3"].includes(r.cls) || (r.autonomous && r.cls === null)); + const cellsOf = (r) => (r.cls ? [`${r.cls}-auto-merge`] : CELLS); + const bySha = new Map(tracked.map((r) => [r.sha, r])); // Reverts, from merged PR bodies and default-branch commit messages; earliest per target. const own = repo.toLowerCase(); @@ -216,42 +267,53 @@ export function collect({ repo, since, through, laneLogins, aiReviewers = [], mi } for (const [number, at] of firstRevert) { const target = record(number); - if (target.autonomous && ["C2", "C3"].includes(target.cls)) { + if (target.autonomous && (["C2", "C3"].includes(target.cls) || target.cls === null)) { target.reverted = true; - addEvent(`${target.cls}-auto-merge`, "reverted-merge", at); + for (const cell of cellsOf(target)) addEvent(cell, "reverted-merge", at); } } // Post-merge gate on each autonomous merge commit: every check run and every commit status. - for (const r of counted.filter((c) => c.autonomous)) { + // A run that did not finish green or red (cancelled, action_required, stale) fails the gate + // unless a later run of the same check succeeded, so cancelling a red run cannot erase it. + for (const r of tracked.filter((c) => c.autonomous)) { const runs = pages(gh, `repos/${repo}/commits/${r.sha}/check-runs?filter=all`, "check_runs"); const statuses = pages(gh, `repos/${repo}/commits/${r.sha}/statuses`); - const failedRuns = runs.filter((run) => run.status === "completed" && GATE_FAILED.has(run.conclusion)); + const outcome = (run) => { + if (run.status !== "completed") return "unsettled"; + if (GATE_FAILED.has(run.conclusion)) return "failed"; + if (GATE_OK.has(run.conclusion)) return "ok"; + const rerunGreen = runs.some( + (o) => o !== run && o.name === run.name && o.status === "completed" && o.conclusion === "success" && Date.parse(o.completed_at) > Date.parse(run.completed_at ?? 0), + ); + return rerunGreen ? "ok" : "failed"; + }; + const failedRuns = runs.filter((run) => outcome(run) === "failed"); const failedStatuses = statuses.filter((status) => STATUS_FAILED.has(status.state)); - for (const run of failedRuns) addEvent(`${r.cls}-auto-merge`, "gate-failure", run.completed_at); - for (const status of failedStatuses) addEvent(`${r.cls}-auto-merge`, "gate-failure", status.created_at); + for (const cell of cellsOf(r)) { + for (const run of failedRuns) addEvent(cell, "gate-failure", run.completed_at ?? r.mergedAt); + for (const status of failedStatuses) addEvent(cell, "gate-failure", status.created_at); + } const latest = new Map(); for (const status of [...statuses].sort((a, b) => Date.parse(b.created_at) - Date.parse(a.created_at))) { if (!latest.has(status.context)) latest.set(status.context, status.state); } - const settled = - runs.length + latest.size > 0 && - runs.every((run) => run.status === "completed" && GATE_OK.has(run.conclusion)) && - [...latest.values()].every((state) => state === "success"); + const settled = runs.length + latest.size > 0 && runs.every((run) => outcome(run) === "ok") && [...latest.values()].every((state) => state === "success"); r.gate = failedRuns.length + failedStatuses.length > 0 ? "failed" : settled ? "passed" : "unsettled"; } // Human-confirmed missed-blocking AI-review findings, from the label timeline. - for (const r of counted.filter((c) => c.cls === "C3" || c.autonomous)) { + for (const r of tracked.filter((c) => c.cls === "C3" || c.autonomous)) { const confirmed = eventsOf(r.number).filter( - (e) => e.event === "labeled" && e.label?.name === missedBlockingLabel && e.actor?.type === "User" && !lane.has(e.actor.login) && inWindow(e.created_at), + (e) => e.event === "labeled" && e.label?.name === missedBlockingLabel && e.actor?.type === "User" && !lane.has(loginKey(e.actor.login)) && inWindow(e.created_at), ); r.missedBlocking = confirmed.length > 0 ? 1 : 0; - for (const e of confirmed) addEvent(r.cls === "C3" ? "C3-ai-review-blocking" : "C2-auto-merge", "verification-divergence", e.created_at); + const cells = r.cls === "C3" ? ["C3-ai-review-blocking"] : r.cls === "C2" ? ["C2-auto-merge"] : CELLS; + for (const e of confirmed) for (const cell of cells) addEvent(cell, "verification-divergence", e.created_at); } const reviewers = new Set(aiReviewers); if (reviewers.size > 0) { - for (const r of counted.filter((c) => c.cls === "C3")) { + for (const r of tracked.filter((c) => c.cls === "C3")) { r.aiReviewed = pages(gh, `repos/${repo}/pulls/${r.number}/reviews`).some((review) => reviewers.has(review.user?.login)); } } @@ -263,7 +325,10 @@ export function collect({ repo, since, through, laneLogins, aiReviewers = [], mi through: new Date(throughMs).toISOString(), events: sorted, unresolvedReverts, - cells: report(counted, sorted, reviewers.size > 0), + unclassedAutonomous: [...details.values()] + .filter((r) => r.autonomous && r.cls === null && (tracked.includes(r) || r.reverted)) + .map((r) => ({ number: r.number, reason: r.reason })), + cells: report(tracked, sorted, reviewers.size > 0), }; } diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs index 73a8affecc..155a5a00ca 100644 --- a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs @@ -28,9 +28,33 @@ after(() => rmSync(base, { recursive: true, force: true })); const shaOf = (number) => number.toString(16).padStart(40, "0"); const later = (at, minutes) => new Date(Date.parse(at) + minutes * 60_000).toISOString().replace(".000Z", "Z"); -function pr(number, mergedAt, { labels = [], merger = LANE, title = REC.pullListItem.title, body = REC.pullListItem.body } = {}) { +// GraphQL spells an App as `__typename: Bot` with the bare slug; REST adds `[bot]`. +const actorOf = (login) => (login === null ? null : login.endsWith("[bot]") ? { __typename: "Bot", login: login.slice(0, -5) } : { __typename: "User", login }); +const page1 = (nodes, hasNextPage = false) => ({ pageInfo: { hasNextPage }, nodes }); +// REC.pullGraph: PR #7063, which closes issue #6007 (labeled `work-class: scoped`). +const ISSUE_6007 = REC.pullGraph.data.repository.pullRequest.closingIssuesReferences.nodes; +const queued = (at, enqueuer, actor = enqueuer) => ({ __typename: "AddedToMergeQueueEvent", createdAt: at, actor: actorOf(actor), enqueuer: actorOf(enqueuer) }); +const link = (type, issue, at) => ({ __typename: type, createdAt: at, subject: { __typename: "Issue", number: issue } }); + +function pr( + number, + mergedAt, + { labels = [], merger = LANE, title = REC.pullListItem.title, body = REC.pullListItem.body, linked = [], timeline = [], lastEditedAt = null, patch = {} } = {}, +) { const names = labels.map((name) => ({ name })); return { + graph: { + ...REC.pullGraph.data.repository.pullRequest, + number, + mergedAt, + lastEditedAt, + mergeCommit: { oid: shaOf(number) }, + mergedBy: actorOf(merger), + labels: page1(names), + closingIssuesReferences: page1(linked), + timelineItems: page1(timeline), + ...patch, + }, list: { ...REC.pullListItem, number, @@ -42,11 +66,8 @@ function pr(number, mergedAt, { labels = [], merger = LANE, title = REC.pullList updated_at: later(mergedAt, 1), merge_commit_sha: shaOf(number), }, - detail: { ...REC.pull, number, title, labels: names, merged_at: mergedAt, merge_commit_sha: shaOf(number), merged_by: { login: merger, type: merger === LANE ? "Bot" : "User" } }, }; } - -const linkedTo = (nodes, hasNextPage = false) => ({ data: { repository: { pullRequest: { closingIssuesReferences: { pageInfo: { hasNextPage }, nodes } } } } }); const labelEvent = (event, name, at, login = "kyle-sexton", type = "User") => ({ ...REC.issueEvent, event, label: { name }, actor: { login, type }, created_at: at }); // A recorded GitHub: answers only the GET paths the publisher is expected to call, with the @@ -54,17 +75,18 @@ const labelEvent = (event, name, at, login = "kyle-sexton", type = "User") => ({ // newest first), and throws on any other, so an unexpected or mutating call fails the case. // A run marked `superseded` was re-run: only `filter=all` returns it. // `bumpOnPage2` updates that PR (a comment, a label) between the listing's first and second page. -function github({ pulls = [], commits = [], checkRuns = {}, statuses = {}, issueEvents = {}, reviews = {}, linked = {}, bumpOnPage2 }) { +function github({ pulls = [], commits = [], checkRuns = {}, statuses = {}, issueEvents = {}, reviews = {}, issues = {}, bumpOnPage2 }) { const page = (items, path) => { const n = Number(/[?&]page=(\d+)/.exec(path)[1]); return items.slice((n - 1) * 100, n * 100); }; - const runsFor = (number) => checkRuns[number] ?? [{ ...REC.checkRun, completed_at: later(pulls.find((p) => p.detail.number === number).detail.merged_at, 5) }]; + const runsFor = (number) => checkRuns[number] ?? [{ ...REC.checkRun, completed_at: later(pulls.find((p) => p.graph.number === number).graph.mergedAt, 5) }]; return (path, graphql) => { if (path === "graphql") { assert.deepEqual([graphql.variables.owner, graphql.variables.name], REPO.split("/")); assert.match(graphql.query, /^query\(/); - return linked[graphql.variables.number] ?? linkedTo([]); + assert.doesNotMatch(graphql.query, /mutation/); + return { data: { repository: { pullRequest: pulls.find((p) => p.graph.number === graphql.variables.number).graph } } }; } const prefix = `repos/${REPO}/`; assert.ok(path.startsWith(prefix), path); @@ -76,7 +98,7 @@ function github({ pulls = [], commits = [], checkRuns = {}, statuses = {}, issue const sign = m[2] === "asc" ? 1 : -1; return page(pulls.map((p) => p.list).sort((a, b) => sign * a[key].localeCompare(b[key])), rest); } - if ((m = /^pulls\/(\d+)$/.exec(rest))) return pulls.find((p) => p.detail.number === Number(m[1])).detail; + if ((m = /^issues\/(\d+)$/.exec(rest))) return issues[m[1]]; if ((m = /^pulls\/(\d+)\/reviews\?/.exec(rest))) return page(reviews[m[1]] ?? [], rest); if (/^commits\?since=[^&]+&until=[^&]+&/.test(rest)) return page(commits, rest); if ((m = /^commits\/([0-9a-f]{40})\/check-runs\?/.exec(rest))) { @@ -161,10 +183,8 @@ test("20 autonomous C2 merges over 14+ days pass the C2 term and 19 fail it", () }); test("a class carried only by the linked issue counts, and the higher of issue and PR class wins", () => { - // REC.linkedIssues: PR #7063 closes issue #6007, labeled `work-class: scoped`. const result = run({ - pulls: [pr(801, "2026-09-01T00:00:00Z"), pr(802, "2026-09-02T00:00:00Z", { labels: [MECHANICAL] })], - linked: { 801: REC.linkedIssues, 802: REC.linkedIssues }, + pulls: [pr(801, "2026-09-01T00:00:00Z", { linked: ISSUE_6007 }), pr(802, "2026-09-02T00:00:00Z", { labels: [MECHANICAL], linked: ISSUE_6007 })], }); assert.equal(termOf(result, "C3-auto-merge", "autonomous C3 completions").value, 2); assert.equal(termOf(result, "C2-auto-merge", "autonomous C2 completions").value, 0); @@ -215,7 +235,7 @@ test("the class is read as of the merge: post-merge relabeling neither adds nor pulls: [ pr(905, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] }), pr(906, "2026-09-01T00:00:00Z"), - pr(907, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] }), + pr(907, "2026-09-01T00:00:00Z", { labels: [MECHANICAL], linked: ISSUE_6007 }), ], issueEvents: { 905: [labelEvent("labeled", MECHANICAL, "2026-09-02T00:00:00Z")], @@ -223,7 +243,6 @@ test("the class is read as of the merge: post-merge relabeling neither adds nor // Linked issue #6007 is labeled scoped now, but gained that label after #907 merged. 6007: [labelEvent("labeled", SCOPED, "2026-09-03T00:00:00Z")], }, - linked: { 907: REC.linkedIssues }, }); assert.equal(termOf(result, "C2-auto-merge", "autonomous C2 completions").value, 2); assert.equal(termOf(result, "C3-auto-merge", "autonomous C3 completions").value, 0); @@ -242,11 +261,69 @@ test("--through is clamped to now", () => { }); test("a linked-issue read with another page fails closed", () => { - const nodes = REC.linkedIssues.data.repository.pullRequest.closingIssuesReferences.nodes; - const fixture = (more) => ({ pulls: [pr(908, "2026-09-01T00:00:00Z")], linked: { 908: linkedTo(nodes, more) } }); - assert.throws(() => run(fixture(true)), (error) => error instanceof PublishError && /hasNextPage/.test(error.message)); - const labelsPaged = linkedTo([{ ...nodes[0], labels: { ...nodes[0].labels, pageInfo: { hasNextPage: true } } }]); - assert.throws(() => run({ pulls: [pr(908, "2026-09-01T00:00:00Z")], linked: { 908: labelsPaged } }), /hasNextPage/); + const paged = (patch) => ({ pulls: [pr(908, "2026-09-01T00:00:00Z", { linked: ISSUE_6007, patch })] }); + assert.throws(() => run(paged({ closingIssuesReferences: page1(ISSUE_6007, true) })), (error) => error instanceof PublishError && /hasNextPage/.test(error.message)); + const labelsPaged = [{ ...ISSUE_6007[0], labels: { ...ISSUE_6007[0].labels, pageInfo: { hasNextPage: true } } }]; + assert.throws(() => run(paged({ closingIssuesReferences: page1(labelsPaged) })), /hasNextPage/); +}); + +test("an issue unlinked after the merge still classes the merge; one linked after it does not", () => { + const result = run({ + pulls: [ + pr(1101, "2026-09-01T00:00:00Z", { timeline: [link("ConnectedEvent", 6007, "2026-08-31T00:00:00Z"), link("DisconnectedEvent", 6007, "2026-09-02T00:00:00Z")] }), + pr(1102, "2026-09-01T00:00:00Z", { linked: ISSUE_6007, timeline: [link("ConnectedEvent", 6007, "2026-09-02T00:00:00Z")] }), + ], + issues: { 6007: REC.issue }, + }); + assert.equal(termOf(result, "C3-auto-merge", "autonomous C3 completions").value, 1); + assert.deepEqual(result.unclassedAutonomous, [{ number: 1102, reason: "no work-class label at merge" }]); +}); + +test("an autonomous merge whose links cannot be rebuilt is unclassed, and its contrary events hit every cell", () => { + const failing = { ...REC.checkRun, conclusion: "failure", completed_at: "2026-09-01T00:10:00Z" }; + const result = run({ + pulls: [pr(1103, "2026-09-01T00:00:00Z", { labels: [MECHANICAL], lastEditedAt: "2026-09-02T00:00:00Z" })], + checkRuns: { 1103: [failing] }, + }); + assert.deepEqual(result.unclassedAutonomous, [{ number: 1103, reason: "PR body edited after the merge: closing-keyword links unknown" }]); + assert.deepEqual(result.events, ["C2-auto-merge", "C3-ai-review-blocking", "C3-auto-merge"].map((cell) => ({ cell, event: "gate-failure", at: "2026-09-01T00:10:00Z" }))); + assert.equal(termOf(result, "C2-auto-merge", "autonomous C2 completions").value, 0); +}); + +test("a revert of an unclassed autonomous merge yields reverted-merge against every cell", () => { + const result = run({ + pulls: [pr(1104, "2026-09-01T00:00:00Z"), pr(1105, "2026-09-02T00:00:00Z", { body: `Reverts ${REPO}#1104`, merger: "kyle-sexton" })], + }); + assert.deepEqual(result.events, ["C2-auto-merge", "C3-ai-review-blocking", "C3-auto-merge"].map((cell) => ({ cell, event: "reverted-merge", at: "2026-09-02T00:00:00Z" }))); +}); + +test("lane logins match case-insensitively", () => { + const result = run({ pulls: [pr(1106, "2026-09-01T00:00:00Z", { labels: [MECHANICAL], merger: "Lane-User", timeline: [queued("2026-08-31T23:00:00Z", "Lane-User")] })] }, { laneLogins: ["lane-user"] }); + assert.equal(termOf(result, "C2-auto-merge", "autonomous C2 completions").value, 1); +}); + +test("a merge a bot or nobody enqueued is autonomous even under a human merged_by; a human enqueue is not", () => { + const result = run({ + pulls: [ + pr(1107, "2026-09-01T00:00:00Z", { labels: [MECHANICAL], merger: "kyle-sexton", timeline: [queued("2026-08-31T23:00:00Z", null, "some-app[bot]")] }), + pr(1108, "2026-09-01T00:00:00Z", { labels: [MECHANICAL], merger: "kyle-sexton", timeline: [queued("2026-08-31T23:00:00Z", "kyle-sexton")] }), + pr(1109, "2026-09-01T00:00:00Z", { labels: [MECHANICAL], merger: "kyle-sexton" }), + ], + }); + assert.equal(termOf(result, "C2-auto-merge", "autonomous C2 completions").value, 1); +}); + +test("a cancelled gate run counts as a failure unless a later run of the same check succeeded", () => { + const cancelled = (name, at) => ({ ...REC.checkRun, name, conclusion: "cancelled", completed_at: at }); + const result = run({ + pulls: [pr(1110, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] }), pr(1111, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] })], + checkRuns: { + 1110: [cancelled("lint-repo", "2026-09-01T00:10:00Z")], + 1111: [cancelled("lint-repo", "2026-09-01T00:10:00Z"), { ...REC.checkRun, completed_at: "2026-09-01T00:20:00Z" }], + }, + }); + assert.deepEqual(result.events, [{ cell: "C2-auto-merge", event: "gate-failure", at: "2026-09-01T00:10:00Z" }]); + assert.deepEqual(termOf(result, "C2-auto-merge", "deterministic-gate pass").detail, { passed: 1, failed: 1, unsettled: 0, total: 2 }); }); test("a revert seen in both the PR body and the squash commit yields one event at the earlier time", () => { From 4a8f38b673f912d077ce4434b6980cd2172b5721 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 12 Oct 2026 23:48:46 -0400 Subject: [PATCH 05/18] fix(source-control): read only the named post-merge gate as gate evidence The publisher read every check run and status on a merge commit, so unrelated workflows and routine cancellations became gate failures. It now requires --gate-check (repeatable) naming the check runs and status contexts that form the post-merge gate, refuses to run without one, and keeps the cancelled-without-later-success rule within those names. The report gains gateFailures: each failing check, its conclusion and time. A test pins that merges outside --since..--through reach no list, count or event. Refs: #4588 Co-Authored-By: Claude Opus 5.5 --- ...8-promotion-evidence-publisher-3634507d.md | 2 +- .../reference/promotion-evidence-bootstrap.md | 3 +- .../scripts/publish-promotion-evidence.mjs | 36 ++++++++++++------ .../publish-promotion-evidence.test.mjs | 37 ++++++++++++++++++- 4 files changed, 63 insertions(+), 15 deletions(-) diff --git a/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md b/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md index 7dd3865615..a9299823c5 100644 --- a/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md +++ b/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md @@ -4,4 +4,4 @@ bump: minor ### Added -- `babysit-loop` ships `scripts/publish-promotion-evidence.mjs`, an operator-side publisher for the promotion-evidence events file (#4588). It reads GitHub only, through read-only `gh api` calls, and emits the `{cell, event, at}` contrary events the security-binding checker reads (`reverted-merge`, `gate-failure`, and `verification-divergence` for a missed-blocking finding a human confirmed) plus a C3 promotion-evidence report: each promotable cell's predicate terms with counts, window and pass or fail. Report mode, the default, writes nothing. `--out ` writes `evidence.json` and an `evidence.meta.json` sidecar (`generated_at`, `through`, `sha256`, `prev_sha256`) by temp file and rename, and refuses a set that drops a previously published event or an existing file that does not match its sidecar. Work classes are read from the labels of the PR and of the issues it linked at merge, as of the merge. A merge counts as autonomous when no human enqueued it or a `--lane-login` identity merged it. An autonomous merge with no readable class is listed as unclassed and its contrary events count against every cell. Every check run (re-runs included) and commit status on a merge commit counts toward `gate-failure`, a cancelled run included unless a later run of that check succeeded. Where it runs and under which identity is not decided yet, and the resolver does not read the sidecar; merge authority is unchanged. +- `babysit-loop` ships `scripts/publish-promotion-evidence.mjs`, an operator-side publisher for the promotion-evidence events file (#4588). It reads GitHub only, through read-only `gh api` calls, and emits the `{cell, event, at}` contrary events the security-binding checker reads (`reverted-merge`, `gate-failure`, and `verification-divergence` for a missed-blocking finding a human confirmed) plus a C3 promotion-evidence report: each promotable cell's predicate terms with counts, window and pass or fail. Report mode, the default, writes nothing. `--out ` writes `evidence.json` and an `evidence.meta.json` sidecar (`generated_at`, `through`, `sha256`, `prev_sha256`) by temp file and rename, and refuses a set that drops a previously published event or an existing file that does not match its sidecar. Work classes are read from the labels of the PR and of the issues it linked at merge, as of the merge. A merge counts as autonomous when no human enqueued it or a `--lane-login` identity merged it. An autonomous merge with no readable class is listed as unclassed and its contrary events count against every cell. Only the post-merge gate named by the required `--gate-check` counts toward `gate-failure`: every such check run (re-runs included) and commit status on a merge commit, a cancelled run included unless a later run of that check succeeded. Where it runs and under which identity is not decided yet, and the resolver does not read the sidecar; merge authority is unchanged. diff --git a/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md b/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md index f762329f9f..1cfd48468e 100644 --- a/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md +++ b/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md @@ -150,7 +150,8 @@ binding. events file from GitHub alone, through read-only `gh api` calls: merged pull requests classed by the `work-class:` labels their linked issues and the pull request carried at merge time, which of them merged autonomously (no human enqueued them, or a `--lane-login` identity merged them), reverts of -those merges, every check run and commit status on their merge commits, and the missed-blocking +those merges, the post-merge gate checks named by the required `--gate-check` on their merge +commits, and the missed-blocking label a human applied. An autonomous merge whose class cannot be read is listed as unclassed, and its contrary events count against every promotable cell. Its header comment states what counts as each event, including the judgment mappings and how to set `--lane-login` for merge-queue merges. diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs index 4a3c83bbeb..bc3bc385d9 100644 --- a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs @@ -6,10 +6,10 @@ // GraphQL query, and never runs inside the lane: the lane must not be able to write its output. // // Usage: node publish-promotion-evidence.mjs --repo --since --lane-login [--lane-login ...] -// [--through ] [--ai-reviewer ...] [--missed-blocking-label ] [--out ] +// --gate-check [--gate-check ...] [--through ] [--ai-reviewer ...] [--missed-blocking-label ] [--out ] // Exit 0 = report printed (and published with --out); 1 = publish refused; 2 = usage or GitHub read error. // -// Report mode (no --out) prints {repo, since, through, events, unresolvedReverts, unclassedAutonomous, cells} to +// Report mode (no --out) prints {repo, since, through, events, unresolvedReverts, gateFailures, unclassedAutonomous, cells} to // stdout and writes nothing. --through defaults to, and is clamped to, now. With --out it also // writes /evidence.json (the events array) and /evidence.meta.json ({generated_at, // through, sha256, prev_sha256}). Both temp files are written before either rename, the events @@ -46,9 +46,15 @@ // commit as `Reverts #` (GitHub's revert button) or `This reverts commit // ` (git revert); one event per reverted merge, at the earliest revert seen. A merged PR // titled revert... with neither form is listed in unresolvedReverts for a human to read. -// - gate-failure: every check run on an autonomous merge's merge commit (filter=all, so a run -// later re-run green still counts) concluding failure, timed_out or startup_failure, and every -// commit status in state failure or error, whenever it completed. A run concluding cancelled, +// - gate-failure: only the post-merge gate counts: check runs and commit-status contexts on an +// autonomous merge's merge commit whose name is a --gate-check (required; none named is a +// usage error, never an empty gate). In this repository babysit-prs/scripts/babysit_postmerge.py +// reads the post-merge gate as the silent-revert canary (post-merge-detect-silent-revert.yml, +// check run `Silent-revert canary` on the merge commit) and the full pr-require-checks.yml run +// (aggregate job `ci-status`), scheduled or dispatched on a later SHA containing the merge; this +// script reads the merge commit only, where `ci-status` is the merge_group run. Among those, every check run (filter=all, so a run later +// re-run green still counts) concluding failure, timed_out or startup_failure, and every +// status in state failure or error, whenever it completed. A run concluding cancelled, // action_required, stale or anything else not green also counts, unless a later run of the // same check name on that commit succeeded. The gate-pass term also fails while a run is in // progress or a context's latest status is not success. @@ -157,7 +163,9 @@ function eventKey(event) { return JSON.stringify([event.cell, event.event, event.at]); } -export function collect({ repo, since, through, laneLogins, aiReviewers = [], missedBlockingLabel, gh = realGh }) { +export function collect({ repo, since, through, laneLogins, gateChecks, aiReviewers = [], missedBlockingLabel, gh = realGh }) { + const gate = new Set(gateChecks ?? []); + if (gate.size === 0) throw new PublishError("at least one --gate-check naming the post-merge gate is required"); const sinceMs = instant(since, "--since"); const throughMs = Math.min(instant(through, "--through"), Date.now()); const lane = new Set(laneLogins.map(loginKey)); @@ -245,6 +253,7 @@ export function collect({ repo, since, through, laneLogins, aiReviewers = [], mi }; const reverts = []; const unresolvedReverts = []; + const gateFailures = []; for (const pr of merged) { const targets = revertsOf(pr.body); for (const target of targets) reverts.push({ ...target, at: pr.merged_at }); @@ -273,12 +282,12 @@ export function collect({ repo, since, through, laneLogins, aiReviewers = [], mi } } - // Post-merge gate on each autonomous merge commit: every check run and every commit status. + // Post-merge gate on each autonomous merge commit: the --gate-check runs and status contexts. // A run that did not finish green or red (cancelled, action_required, stale) fails the gate // unless a later run of the same check succeeded, so cancelling a red run cannot erase it. for (const r of tracked.filter((c) => c.autonomous)) { - const runs = pages(gh, `repos/${repo}/commits/${r.sha}/check-runs?filter=all`, "check_runs"); - const statuses = pages(gh, `repos/${repo}/commits/${r.sha}/statuses`); + const runs = pages(gh, `repos/${repo}/commits/${r.sha}/check-runs?filter=all`, "check_runs").filter((run) => gate.has(run.name)); + const statuses = pages(gh, `repos/${repo}/commits/${r.sha}/statuses`).filter((status) => gate.has(status.context)); const outcome = (run) => { if (run.status !== "completed") return "unsettled"; if (GATE_FAILED.has(run.conclusion)) return "failed"; @@ -294,6 +303,8 @@ export function collect({ repo, since, through, laneLogins, aiReviewers = [], mi for (const run of failedRuns) addEvent(cell, "gate-failure", run.completed_at ?? r.mergedAt); for (const status of failedStatuses) addEvent(cell, "gate-failure", status.created_at); } + for (const run of failedRuns) gateFailures.push({ number: r.number, check: run.name, conclusion: run.conclusion, at: run.completed_at ?? r.mergedAt }); + for (const status of failedStatuses) gateFailures.push({ number: r.number, check: status.context, conclusion: status.state, at: status.created_at }); const latest = new Map(); for (const status of [...statuses].sort((a, b) => Date.parse(b.created_at) - Date.parse(a.created_at))) { if (!latest.has(status.context)) latest.set(status.context, status.state); @@ -325,6 +336,7 @@ export function collect({ repo, since, through, laneLogins, aiReviewers = [], mi through: new Date(throughMs).toISOString(), events: sorted, unresolvedReverts, + gateFailures, unclassedAutonomous: [...details.values()] .filter((r) => r.autonomous && r.cls === null && (tracked.includes(r) || r.reverted)) .map((r) => ({ number: r.number, reason: r.reason })), @@ -462,13 +474,14 @@ function main() { since: { type: "string" }, through: { type: "string" }, "lane-login": { type: "string", multiple: true }, + "gate-check": { type: "string", multiple: true }, "ai-reviewer": { type: "string", multiple: true }, "missed-blocking-label": { type: "string", default: "ai-review: missed-blocking" }, out: { type: "string" }, }, })); - if (!/^[\w.-]+\/[\w.-]+$/.test(values.repo ?? "") || !values.since || !values["lane-login"]?.length) { - throw new PublishError("--repo , --since and at least one --lane-login are required"); + if (!/^[\w.-]+\/[\w.-]+$/.test(values.repo ?? "") || !values.since || !values["lane-login"]?.length || !values["gate-check"]?.length) { + throw new PublishError("--repo , --since , at least one --lane-login and at least one --gate-check are required"); } } catch (error) { process.stderr.write(`usage: ${error.message}\n`); @@ -480,6 +493,7 @@ function main() { since: values.since, through: values.through ?? new Date().toISOString(), laneLogins: values["lane-login"], + gateChecks: values["gate-check"], aiReviewers: values["ai-reviewer"] ?? [], missedBlockingLabel: values["missed-blocking-label"], }); diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs index 155a5a00ca..003dda0f57 100644 --- a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs @@ -111,12 +111,44 @@ function github({ pulls = [], commits = [], checkRuns = {}, statuses = {}, issue }; } +// The recorded check run and status name the post-merge gate in every case. +const GATE = [REC.checkRun.name, REC.status.context]; + function run(fixture, extra = {}) { - return collect({ repo: REPO, since: SINCE, through: THROUGH, laneLogins: [LANE], missedBlockingLabel: MISSED, gh: github(fixture), ...extra }); + return collect({ repo: REPO, since: SINCE, through: THROUGH, laneLogins: [LANE], gateChecks: GATE, missedBlockingLabel: MISSED, gh: github(fixture), ...extra }); } const termOf = (result, cell, name) => result.cells[cell].terms.find((t) => t.term === name); +test("only the named gate checks and status contexts are gate evidence", () => { + const other = (name, conclusion) => ({ ...REC.checkRun, name, conclusion, completed_at: "2026-09-01T00:10:00Z" }); + const result = run({ + pulls: [pr(1201, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] })], + checkRuns: { 1201: [REC.checkRun, other("dispatch", "failure"), other("resolve-conflicts / run", "cancelled")] }, + statuses: { 1201: [{ ...REC.status, context: "GitGuardian", state: "error" }] }, + }); + assert.deepEqual(result.events, []); + assert.deepEqual(termOf(result, "C2-auto-merge", "deterministic-gate pass").detail, { passed: 1, failed: 0, unsettled: 0, total: 1 }); +}); + +test("no gate check named is refused rather than read as an empty gate", () => { + assert.throws(() => run({ pulls: [] }, { gateChecks: [] }), (error) => error instanceof PublishError && /--gate-check/.test(error.message)); + const cli = spawnSync(process.execPath, [join(HERE, "publish-promotion-evidence.mjs"), "--repo", REPO, "--since", SINCE, "--lane-login", LANE], { encoding: "utf8" }); + assert.equal(cli.status, 2); + assert.match(cli.stderr, /--gate-check/); +}); + +test("merges outside --since..--through appear in no list, count and event", () => { + const failing = (at) => [{ ...REC.checkRun, conclusion: "failure", completed_at: at }]; + const result = run({ + pulls: [pr(1202, "2026-07-31T00:00:00Z"), pr(1203, "2026-07-31T00:00:00Z", { labels: [MECHANICAL] }), pr(1204, "2026-10-06T00:00:00Z", { labels: [SCOPED] })], + checkRuns: { 1202: failing("2026-08-02T00:00:00Z"), 1203: failing("2026-08-02T00:00:00Z"), 1204: failing("2026-10-06T00:10:00Z") }, + }); + assert.deepEqual([result.events, result.unclassedAutonomous], [[], []]); + assert.equal(termOf(result, "C2-auto-merge", "autonomous C2 completions").value, 0); + assert.equal(termOf(result, "C3-auto-merge", "autonomous C3 completions").value, 0); +}); + test("a revert-button PR of an autonomous C2 merge yields reverted-merge at the revert's merge time", () => { const result = run({ pulls: [ @@ -209,7 +241,7 @@ test("a failure or error commit status, even one later superseded by success, yi { ...REC.status, state: "error", created_at: "2026-09-01T00:20:00Z" }, { ...REC.status, context: "other", state: "failure", created_at: "2026-09-01T00:15:00Z" }, ]; - const result = run({ pulls: [pr(902, "2026-09-01T00:00:00Z", { labels: [SCOPED] })], statuses: { 902: statuses } }); + const result = run({ pulls: [pr(902, "2026-09-01T00:00:00Z", { labels: [SCOPED] })], statuses: { 902: statuses } }, { gateChecks: [...GATE, "other"] }); assert.deepEqual(result.events, [ { cell: "C3-auto-merge", event: "gate-failure", at: "2026-09-01T00:15:00Z" }, { cell: "C3-auto-merge", event: "gate-failure", at: "2026-09-01T00:20:00Z" }, @@ -324,6 +356,7 @@ test("a cancelled gate run counts as a failure unless a later run of the same ch }); assert.deepEqual(result.events, [{ cell: "C2-auto-merge", event: "gate-failure", at: "2026-09-01T00:10:00Z" }]); assert.deepEqual(termOf(result, "C2-auto-merge", "deterministic-gate pass").detail, { passed: 1, failed: 1, unsettled: 0, total: 2 }); + assert.deepEqual(result.gateFailures, [{ number: 1110, check: "lint-repo", conclusion: "cancelled", at: "2026-09-01T00:10:00Z" }]); }); test("a revert seen in both the PR body and the squash commit yields one event at the earlier time", () => { From 05f3ce27df08478a569f70fa851b2b052f6eadd6 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Tue, 13 Oct 2026 00:05:59 -0400 Subject: [PATCH 06/18] fix(source-control): read post-merge full and canary runs on later SHAs A red post-merge full run on a later SHA that contains the merge was a missed contrary event. The publisher now reads the post-merge gate the way babysit_postmerge.py check() does: pr-require-checks.yml runs from schedule and workflow_dispatch and post-merge-detect-silent-revert.yml push runs, created at or after the merge on a SHA containing it, with attempt 1 deciding, a red full run a flake only when a lane identity's re-run passed, and runs after the merge was verified (green lane dispatch and green canary) not counted against it. Two departures, both toward a failure: a red full run still waiting on its re-run counts as red six hours after it completed, and one red run covering several merges is a gate failure of each. --gate-check still filters check runs and statuses on the merge commit. Refs: #4588 Co-Authored-By: Claude Opus 5.5 --- ...8-promotion-evidence-publisher-3634507d.md | 2 +- .../reference/promotion-evidence-bootstrap.md | 5 +- .../publish-promotion-evidence/recorded.json | 31 +++++ .../scripts/publish-promotion-evidence.mjs | 119 +++++++++++++++++- .../publish-promotion-evidence.test.mjs | 118 ++++++++++++++++- 5 files changed, 267 insertions(+), 8 deletions(-) diff --git a/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md b/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md index a9299823c5..3eab5aca58 100644 --- a/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md +++ b/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md @@ -4,4 +4,4 @@ bump: minor ### Added -- `babysit-loop` ships `scripts/publish-promotion-evidence.mjs`, an operator-side publisher for the promotion-evidence events file (#4588). It reads GitHub only, through read-only `gh api` calls, and emits the `{cell, event, at}` contrary events the security-binding checker reads (`reverted-merge`, `gate-failure`, and `verification-divergence` for a missed-blocking finding a human confirmed) plus a C3 promotion-evidence report: each promotable cell's predicate terms with counts, window and pass or fail. Report mode, the default, writes nothing. `--out ` writes `evidence.json` and an `evidence.meta.json` sidecar (`generated_at`, `through`, `sha256`, `prev_sha256`) by temp file and rename, and refuses a set that drops a previously published event or an existing file that does not match its sidecar. Work classes are read from the labels of the PR and of the issues it linked at merge, as of the merge. A merge counts as autonomous when no human enqueued it or a `--lane-login` identity merged it. An autonomous merge with no readable class is listed as unclassed and its contrary events count against every cell. Only the post-merge gate named by the required `--gate-check` counts toward `gate-failure`: every such check run (re-runs included) and commit status on a merge commit, a cancelled run included unless a later run of that check succeeded. Where it runs and under which identity is not decided yet, and the resolver does not read the sidecar; merge authority is unchanged. +- `babysit-loop` ships `scripts/publish-promotion-evidence.mjs`, an operator-side publisher for the promotion-evidence events file (#4588). It reads GitHub only, through read-only `gh api` calls, and emits the `{cell, event, at}` contrary events the security-binding checker reads (`reverted-merge`, `gate-failure`, and `verification-divergence` for a missed-blocking finding a human confirmed) plus a C3 promotion-evidence report: each promotable cell's predicate terms with counts, window and pass or fail. Report mode, the default, writes nothing. `--out ` writes `evidence.json` and an `evidence.meta.json` sidecar (`generated_at`, `through`, `sha256`, `prev_sha256`) by temp file and rename, and refuses a set that drops a previously published event or an existing file that does not match its sidecar. Work classes are read from the labels of the PR and of the issues it linked at merge, as of the merge. A merge counts as autonomous when no human enqueued it or a `--lane-login` identity merged it. An autonomous merge with no readable class is listed as unclassed and its contrary events count against every cell. Only the post-merge gate counts toward `gate-failure`. That gate is the full and canary workflow runs on any later commit containing the merge, read as `babysit_postmerge.py` reads them, plus the check runs and statuses named by the required `--gate-check` on the merge commit. A cancelled run counts unless a later run of that check succeeded, and a red full run covering several merges counts against each. Where it runs and under which identity is not decided yet, and the resolver does not read the sidecar; merge authority is unchanged. diff --git a/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md b/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md index 1cfd48468e..9d236118a5 100644 --- a/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md +++ b/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md @@ -150,8 +150,9 @@ binding. events file from GitHub alone, through read-only `gh api` calls: merged pull requests classed by the `work-class:` labels their linked issues and the pull request carried at merge time, which of them merged autonomously (no human enqueued them, or a `--lane-login` identity merged them), reverts of -those merges, the post-merge gate checks named by the required `--gate-check` on their merge -commits, and the missed-blocking +those merges, the post-merge gate (the full and canary workflow runs on any later commit that +contains the merge, read as `babysit-prs/scripts/babysit_postmerge.py` reads them, and the +checks named by the required `--gate-check` on the merge commit), and the missed-blocking label a human applied. An autonomous merge whose class cannot be read is listed as unclassed, and its contrary events count against every promotable cell. Its header comment states what counts as each event, including the judgment mappings and how to set `--lane-login` for merge-queue merges. diff --git a/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json b/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json index 7cc6fb4310..338073c6e2 100644 --- a/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json +++ b/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json @@ -138,5 +138,36 @@ } ], "number": 6007 + }, + "repo": { + "default_branch": "main" + }, + "workflowRun": { + "actor": { + "login": "github-merge-queue[bot]" + }, + "conclusion": "failure", + "created_at": "2026-10-12T16:32:14Z", + "event": "schedule", + "head_branch": "main", + "head_sha": "2dc6ef1cef19870b4f15ba796f970980bcb48cf8", + "html_url": "https://github.com/melodic-software/claude-code-plugins/actions/runs/38278079868", + "id": 38278079868, + "run_attempt": 1, + "status": "completed", + "triggering_actor": { + "login": "github-merge-queue[bot]" + }, + "updated_at": "2026-10-12T16:40:40Z" + }, + "compare": { + "ahead_by": 1, + "commits": [ + { + "sha": "aad0b2b69bc22154c61b0857307a183d0c7821a8" + } + ], + "status": "ahead", + "total_commits": 1 } } diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs index bc3bc385d9..446de42841 100644 --- a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs @@ -51,8 +51,9 @@ // usage error, never an empty gate). In this repository babysit-prs/scripts/babysit_postmerge.py // reads the post-merge gate as the silent-revert canary (post-merge-detect-silent-revert.yml, // check run `Silent-revert canary` on the merge commit) and the full pr-require-checks.yml run -// (aggregate job `ci-status`), scheduled or dispatched on a later SHA containing the merge; this -// script reads the merge commit only, where `ci-status` is the merge_group run. Among those, every check run (filter=all, so a run later +// (aggregate job `ci-status`), scheduled or dispatched on a later SHA containing the merge. This +// script reads those workflow runs the same way (postMergeRuns) and, on the merge commit itself, +// the --gate-check check runs and contexts. Among the latter, every check run (filter=all, so a run later // re-run green still counts) concluding failure, timed_out or startup_failure, and every // status in state failure or error, whenever it completed. A run concluding cancelled, // action_required, stale or anything else not green also counts, unless a later run of the @@ -163,6 +164,104 @@ function eventKey(event) { return JSON.stringify([event.cell, event.event, event.at]); } +// The post-merge gate's workflow runs, read as babysit-prs/scripts/babysit_postmerge.py check() +// and _judge() read them (same plugin, mirrored rather than imported across languages): +// FULL_RUN_WORKFLOW runs from `schedule` and `workflow_dispatch`, and CANARY_WORKFLOW runs from +// `push`, on the default branch, created at or after a merge on a SHA that contains it. Attempt 1 +// decides; a red full run is a flake only when a lane identity's attempt 2 passed. A merge is +// verified at the later of its first green lane dispatch run and first green canary run, and runs +// created after that are not its evidence. Two departures, both toward a failure: a red full run +// whose re-run has not come counts as red once DISPATCH_DEADLINE has passed since it completed +// (check() waits on it), and a red run covering several merges is a failure of each. +const FULL_RUN_WORKFLOW = "pr-require-checks.yml"; +const CANARY_WORKFLOW = "post-merge-detect-silent-revert.yml"; +const DISPATCH_DEADLINE_MS = 6 * 3_600_000; + +function postMergeRuns(gh, repo, merges, { lane, throughMs }) { + if (merges.length === 0) return () => ({ failures: [], unsettled: false, green: 0 }); + const branch = gh(`repos/${repo}`)?.default_branch; + if (!branch) throw new PublishError(`gh api repos/${repo}: no default_branch`); + const ordered = [...merges].sort((a, b) => Date.parse(a.mergedAt) - Date.parse(b.mergedAt)); + const floor = Date.parse(ordered[0].mergedAt); + const created = encodeURIComponent(`>=${new Date(floor).toISOString().replace(/\.\d{3}Z$/, "Z")}`); + const list = (workflow, event) => + pages(gh, `repos/${repo}/actions/workflows/${workflow}/runs?event=${event}&branch=${encodeURIComponent(branch)}&created=${created}`, "workflow_runs") + .filter((run) => run.head_branch === branch && Date.parse(run.created_at) >= floor && Date.parse(run.created_at) <= throughMs) + .map((run) => ({ ...run, check: `${workflow} ${event}` })); + const fullRuns = [...list(FULL_RUN_WORKFLOW, "schedule"), ...list(FULL_RUN_WORKFLOW, "workflow_dispatch")]; + const canaryRuns = list(CANARY_WORKFLOW, "push"); + + // Containment from one base, the earliest merge: on a linear (merge-queue) branch a later merge + // is in a run's history exactly when it is among the commits the run's SHA is ahead by. + const base = ordered[0].sha; + const aheadOf = new Map(); + const contains = (sha, runSha) => { + if (!aheadOf.has(runSha)) { + const shas = new Set(); + let status = null; + for (let page = 1; ; page += 1) { + const body = gh(`repos/${repo}/compare/${base}...${runSha}?per_page=${PAGE}&page=${page}`); + if (!Array.isArray(body?.commits)) throw new PublishError(`gh api compare ${base}...${runSha}: no commits array`); + status ??= body.status; + for (const commit of body.commits) shas.add(commit.sha); + if (body.commits.length < PAGE) break; + } + aheadOf.set(runSha, { status, shas }); + } + const { status, shas } = aheadOf.get(runSha); + return sha === base ? status === "ahead" || status === "identical" : shas.has(sha); + }; + + const outcomes = new Map(); + const outcome = (run, attributable) => { + const key = `${run.id}:${attributable}`; + if (!outcomes.has(key)) { + const attempt = (n) => (run.run_attempt === n ? run : gh(`repos/${repo}/actions/runs/${run.id}/attempts/${n}`)); + const first = attempt(1); + let state; + if (!first?.status) throw new PublishError(`run ${run.id}: attempt 1 is missing`); + if (first.status !== "completed") state = "pending"; + else if (first.conclusion === "success") state = "green"; + else if (!attributable) state = "red"; + else { + const second = run.run_attempt >= 2 ? attempt(2) : null; + if (!second) state = throughMs - Date.parse(first.updated_at) > DISPATCH_DEADLINE_MS ? "red" : "pending"; + else if (!lane.has(loginKey(second.triggering_actor?.login))) state = "red"; + else if (second.status !== "completed") state = "pending"; + else state = second.conclusion === "success" ? "flake" : "red"; + } + outcomes.set(key, { state, conclusion: first.conclusion }); + } + return outcomes.get(key); + }; + + return (merge) => { + const mergedMs = Date.parse(merge.mergedAt); + const relevant = (run) => Date.parse(run.created_at) >= mergedMs && contains(merge.sha, run.head_sha); + const fulls = fullRuns.filter(relevant); + const canaries = canaryRuns.filter(relevant); + const earliest = (runs) => Math.min(...runs.map((run) => Date.parse(run.created_at))); + const greenDispatch = fulls.filter( + (run) => run.event === "workflow_dispatch" && lane.has(loginKey(run.actor?.login)) && ["green", "flake"].includes(outcome(run, true).state), + ); + const greenCanary = canaries.filter((run) => outcome(run, false).state === "green"); + const verifiedAt = greenDispatch.length && greenCanary.length ? Math.max(earliest(greenDispatch), earliest(greenCanary)) : Number.POSITIVE_INFINITY; + const result = { failures: [], unsettled: false, green: 0 }; + for (const [runs, attributable] of [ + [fulls, true], + [canaries, false], + ]) { + for (const run of runs.filter((r) => Date.parse(r.created_at) <= verifiedAt)) { + const { state, conclusion } = outcome(run, attributable); + if (state === "red") result.failures.push({ check: run.check, conclusion, at: run.updated_at }); + else if (state === "pending") result.unsettled = true; + else result.green += 1; + } + } + return result; + }; +} + export function collect({ repo, since, through, laneLogins, gateChecks, aiReviewers = [], missedBlockingLabel, gh = realGh }) { const gate = new Set(gateChecks ?? []); if (gate.size === 0) throw new PublishError("at least one --gate-check naming the post-merge gate is required"); @@ -282,10 +381,18 @@ export function collect({ repo, since, through, laneLogins, gateChecks, aiReview } } + const judgeWorkflows = postMergeRuns(gh, repo, tracked.filter((c) => c.autonomous), { lane, throughMs }); + // Post-merge gate on each autonomous merge commit: the --gate-check runs and status contexts. // A run that did not finish green or red (cancelled, action_required, stale) fails the gate // unless a later run of the same check succeeded, so cancelling a red run cannot erase it. + // Then the post-merge workflow runs on later SHAs containing the merge (postMergeRuns). for (const r of tracked.filter((c) => c.autonomous)) { + const wf = judgeWorkflows(r); + for (const failure of wf.failures) { + for (const cell of cellsOf(r)) addEvent(cell, "gate-failure", failure.at); + gateFailures.push({ number: r.number, ...failure }); + } const runs = pages(gh, `repos/${repo}/commits/${r.sha}/check-runs?filter=all`, "check_runs").filter((run) => gate.has(run.name)); const statuses = pages(gh, `repos/${repo}/commits/${r.sha}/statuses`).filter((status) => gate.has(status.context)); const outcome = (run) => { @@ -309,8 +416,12 @@ export function collect({ repo, since, through, laneLogins, gateChecks, aiReview for (const status of [...statuses].sort((a, b) => Date.parse(b.created_at) - Date.parse(a.created_at))) { if (!latest.has(status.context)) latest.set(status.context, status.state); } - const settled = runs.length + latest.size > 0 && runs.every((run) => outcome(run) === "ok") && [...latest.values()].every((state) => state === "success"); - r.gate = failedRuns.length + failedStatuses.length > 0 ? "failed" : settled ? "passed" : "unsettled"; + const settled = + runs.length + latest.size + wf.green > 0 && + !wf.unsettled && + runs.every((run) => outcome(run) === "ok") && + [...latest.values()].every((state) => state === "success"); + r.gate = failedRuns.length + failedStatuses.length + wf.failures.length > 0 ? "failed" : settled ? "passed" : "unsettled"; } // Human-confirmed missed-blocking AI-review findings, from the label timeline. diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs index 003dda0f57..56f90900fa 100644 --- a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs @@ -75,7 +75,11 @@ const labelEvent = (event, name, at, login = "kyle-sexton", type = "User") => ({ // newest first), and throws on any other, so an unexpected or mutating call fails the case. // A run marked `superseded` was re-run: only `filter=all` returns it. // `bumpOnPage2` updates that PR (a comment, a label) between the listing's first and second page. -function github({ pulls = [], commits = [], checkRuns = {}, statuses = {}, issueEvents = {}, reviews = {}, issues = {}, bumpOnPage2 }) { +// `workflowRuns`: {: [run]}; `attempts`: {: {: run}}; `history`: the +// default branch's commits oldest first (linear, as a merge-queue branch is), defaulting to the +// merge commits in merge order. +function github({ pulls = [], commits = [], checkRuns = {}, statuses = {}, issueEvents = {}, reviews = {}, issues = {}, workflowRuns = {}, attempts = {}, history, bumpOnPage2 }) { + const branch = history ?? [...pulls].sort((a, b) => a.graph.mergedAt.localeCompare(b.graph.mergedAt)).map((p) => p.graph.mergeCommit.oid); const page = (items, path) => { const n = Number(/[?&]page=(\d+)/.exec(path)[1]); return items.slice((n - 1) * 100, n * 100); @@ -88,6 +92,7 @@ function github({ pulls = [], commits = [], checkRuns = {}, statuses = {}, issue assert.doesNotMatch(graphql.query, /mutation/); return { data: { repository: { pullRequest: pulls.find((p) => p.graph.number === graphql.variables.number).graph } } }; } + if (path === `repos/${REPO}`) return REC.repo; const prefix = `repos/${REPO}/`; assert.ok(path.startsWith(prefix), path); const rest = path.slice(prefix.length); @@ -107,6 +112,18 @@ function github({ pulls = [], commits = [], checkRuns = {}, statuses = {}, issue } if ((m = /^commits\/([0-9a-f]{40})\/statuses\?/.exec(rest))) return page(statuses[parseInt(m[1], 16)] ?? [], rest); if ((m = /^issues\/(\d+)\/events\?/.exec(rest))) return page(issueEvents[m[1]] ?? [], rest); + if ((m = /^actions\/workflows\/([\w.-]+)\/runs\?event=(\w+)&branch=main&created=%3E%3D[^&]+&/.exec(rest))) { + const runs = (workflowRuns[m[1]] ?? []).filter((r) => r.event === m[2]); + return { total_count: runs.length, workflow_runs: page(runs, rest) }; + } + if ((m = /^actions\/runs\/(\d+)\/attempts\/(\d+)$/.exec(rest))) return attempts[m[1]][m[2]]; + if ((m = /^compare\/([0-9a-f]{40})\.\.\.([0-9a-f]{40})\?/.exec(rest))) { + const [base, head] = [branch.indexOf(m[1]), branch.indexOf(m[2])]; + assert.ok(base >= 0 && head >= 0, `compare outside the recorded history: ${rest}`); + const status = head > base ? "ahead" : head === base ? "identical" : "behind"; + const ahead = head > base ? branch.slice(base + 1, head + 1).map((sha) => ({ sha })) : []; + return { ...REC.compare, status, ahead_by: ahead.length, total_commits: ahead.length, commits: page(ahead, rest) }; + } throw new Error(`unrecorded gh api call: ${path}`); }; } @@ -120,6 +137,105 @@ function run(fixture, extra = {}) { const termOf = (result, cell, name) => result.cells[cell].terms.find((t) => t.term === name); +const FULL = "pr-require-checks.yml"; +const CANARY = "post-merge-detect-silent-revert.yml"; +const wfRun = (id, event, sha, created, { conclusion = "success", status = "completed", attempt = 1, actor = "github-merge-queue[bot]", triggering = actor, updated = later(created, 8) } = {}) => ({ + ...REC.workflowRun, + id, + event, + head_sha: sha, + created_at: created, + updated_at: updated, + status, + conclusion: status === "completed" ? conclusion : null, + run_attempt: attempt, + actor: { login: actor }, + triggering_actor: { login: triggering }, +}); +const gateEvents = (at, ...cells) => cells.map((cell) => ({ cell, event: "gate-failure", at })); + +test("a red full run on a later SHA containing two merges yields gate-failure for each", () => { + const tip = shaOf(0xa001); + const result = run({ + pulls: [pr(1301, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] }), pr(1302, "2026-09-02T00:00:00Z", { labels: [SCOPED] })], + history: [shaOf(1301), shaOf(1302), tip], + workflowRuns: { [FULL]: [wfRun(1, "schedule", tip, "2026-09-03T00:00:00Z", { conclusion: "failure" })] }, + }); + assert.deepEqual(result.events, gateEvents("2026-09-03T00:08:00Z", "C2-auto-merge", "C3-auto-merge")); + assert.deepEqual( + result.gateFailures.map((g) => [g.number, g.check, g.conclusion]), + [ + [1301, `${FULL} schedule`, "failure"], + [1302, `${FULL} schedule`, "failure"], + ], + ); +}); + +test("a run on a SHA without the merge, or created before the merge, is not its evidence", () => { + const older = shaOf(0xa002); + // Run 3 on #1303's own SHA predates #1303's merge, so it is evidence for #1309 only. + const result = run({ + pulls: [pr(1309, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] }), pr(1303, "2026-09-02T00:00:00Z", { labels: [SCOPED] })], + history: [older, shaOf(1309), shaOf(1303)], + workflowRuns: { + [FULL]: [wfRun(2, "schedule", older, "2026-09-03T00:00:00Z", { conclusion: "failure" }), wfRun(3, "schedule", shaOf(1303), "2026-09-01T12:00:00Z", { conclusion: "failure" })], + }, + }); + assert.deepEqual(result.events, gateEvents("2026-09-01T12:08:00Z", "C2-auto-merge")); +}); + +test("a red full run is a flake only when a lane identity's re-run passed", () => { + const tip = shaOf(0xa003); + const rerun = (id, triggering) => wfRun(id, "schedule", tip, "2026-09-02T00:00:00Z", { attempt: 2, triggering, updated: "2026-09-02T01:00:00Z" }); + const firstAttempt = (id) => ({ 1: wfRun(id, "schedule", tip, "2026-09-02T00:00:00Z", { conclusion: "failure" }) }); + const result = run({ + pulls: [pr(1304, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] })], + history: [shaOf(1304), tip], + workflowRuns: { [FULL]: [rerun(4, LANE), rerun(5, "kyle-sexton")] }, + attempts: { 4: firstAttempt(4), 5: firstAttempt(5) }, + }); + assert.deepEqual(result.events, gateEvents("2026-09-02T01:00:00Z", "C2-auto-merge")); +}); + +test("a red run after a merge was verified by a green lane dispatch and canary is not that merge's", () => { + const [s1, s2] = [shaOf(0xa004), shaOf(0xa005)]; + const result = run({ + pulls: [pr(1305, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] }), pr(1306, "2026-09-01T12:00:00Z", { labels: [SCOPED] })], + history: [shaOf(1305), s1, shaOf(1306), s2], + workflowRuns: { + [FULL]: [wfRun(6, "workflow_dispatch", s1, "2026-09-01T01:00:00Z", { actor: LANE }), wfRun(7, "schedule", s2, "2026-09-02T00:00:00Z", { conclusion: "failure" })], + [CANARY]: [wfRun(8, "push", s1, "2026-09-01T01:05:00Z")], + }, + }); + assert.deepEqual(result.events, gateEvents("2026-09-02T00:08:00Z", "C3-auto-merge")); +}); + +test("a red canary run on a SHA containing the merge yields gate-failure", () => { + const result = run({ + pulls: [pr(1307, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] })], + workflowRuns: { [CANARY]: [wfRun(9, "push", shaOf(1307), "2026-09-01T00:01:00Z", { conclusion: "failure" })] }, + }); + assert.deepEqual(result.events, gateEvents("2026-09-01T00:09:00Z", "C2-auto-merge")); +}); + +test("a red full run awaiting its re-run is unsettled for 6 hours, then a failure", () => { + const tip = shaOf(0xa006); + const red = (id, updated) => wfRun(id, "schedule", tip, later(updated, -8), { conclusion: "failure", updated }); + const recent = run({ + pulls: [pr(1308, "2026-10-04T00:00:00Z", { labels: [MECHANICAL] })], + history: [shaOf(1308), tip], + workflowRuns: { [FULL]: [red(10, "2026-10-04T23:00:00Z")] }, + }); + assert.deepEqual(recent.events, []); + assert.equal(termOf(recent, "C2-auto-merge", "deterministic-gate pass").detail.unsettled, 1); + const stale = run({ + pulls: [pr(1308, "2026-10-04T00:00:00Z", { labels: [MECHANICAL] })], + history: [shaOf(1308), tip], + workflowRuns: { [FULL]: [red(10, "2026-10-04T17:00:00Z")] }, + }); + assert.deepEqual(stale.events, gateEvents("2026-10-04T17:00:00Z", "C2-auto-merge")); +}); + test("only the named gate checks and status contexts are gate evidence", () => { const other = (name, conclusion) => ({ ...REC.checkRun, name, conclusion, completed_at: "2026-09-01T00:10:00Z" }); const result = run({ From cbf2239d005ec71e1456fbc4addd601813de86ce Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Tue, 13 Oct 2026 00:40:02 -0400 Subject: [PATCH 07/18] fix(source-control): compare per merge and pass a merge only when verified - Decide containment with a compare from each merge to each run SHA, as babysit_postmerge.py _Context.contains does; a status that is neither ahead nor identical refuses the publish. The single-base compare missed a merge that is an ancestor of a same-time merge. - Read post-merge workflow runs with no upper bound, like the merge-commit path. - Count a merge as gate-passed only when check() would call it verified: a green lane dispatch run and a green canary run that contain it, with nothing red or pending; otherwise it is unsettled. - List only pull requests merged into the default branch. - Fix the link-event shape from a recorded ConnectedEvent (cli/cli #14616); an unrecognized shape after the merge leaves the class unknown. - List a revert naming an unmerged or missing pull request, or a commit with no merged pull request, in unresolvedReverts instead of failing. Refs: #4588 Co-Authored-By: Claude Opus 5.5 --- .../publish-promotion-evidence/recorded.json | 38 +++++ .../scripts/publish-promotion-evidence.mjs | 131 +++++++++++------- .../publish-promotion-evidence.test.mjs | 101 +++++++++++--- 3 files changed, 201 insertions(+), 69 deletions(-) diff --git a/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json b/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json index 338073c6e2..8fd02bdc56 100644 --- a/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json +++ b/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json @@ -1,5 +1,8 @@ { "pullListItem": { + "base": { + "ref": "main" + }, "body": "Refs: #7339", "labels": [], "created_at": "2026-10-13T00:54:35Z", @@ -142,6 +145,41 @@ "repo": { "default_branch": "main" }, + "connectedEventFromCliCli14616": { + "__typename": "ConnectedEvent", + "createdAt": "2026-10-07T08:24:54Z", + "source": { + "__typename": "PullRequest", + "number": 14616 + }, + "subject": { + "__typename": "Issue", + "number": 14614 + } + }, + "pullNotFound": { + "data": { + "repository": { + "pullRequest": null + } + }, + "errors": [ + { + "type": "NOT_FOUND", + "path": [ + "repository", + "pullRequest" + ], + "locations": [ + { + "line": 1, + "column": 66 + } + ], + "message": "Could not resolve to a PullRequest with the number of 999999." + } + ] + }, "workflowRun": { "actor": { "login": "github-merge-queue[bot]" diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs index 446de42841..17ae2c7315 100644 --- a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs @@ -45,7 +45,8 @@ // - reverted-merge: an autonomous C2/C3 merge named by a merged PR body or a default-branch // commit as `Reverts #` (GitHub's revert button) or `This reverts commit // ` (git revert); one event per reverted merge, at the earliest revert seen. A merged PR -// titled revert... with neither form is listed in unresolvedReverts for a human to read. +// titled revert... with neither form is listed in unresolvedReverts for a human to read, as is +// a named target that is not a merged pull request, so a bad reference cannot stop a publish. // - gate-failure: only the post-merge gate counts: check runs and commit-status contexts on an // autonomous merge's merge commit whose name is a --gate-check (required; none named is a // usage error, never an empty gate). In this repository babysit-prs/scripts/babysit_postmerge.py @@ -106,10 +107,20 @@ function realGh(path, graphql) { ? ["api", "graphql", "-f", `query=${graphql.query}`, ...Object.entries(graphql.variables).flatMap(([key, value]) => ["-F", `${key}=${value}`])] : ["api", "--method", "GET", path]; const run = spawnSync("gh", args, { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 }); - if (run.status !== 0) throw new PublishError(`gh api ${path} failed: ${(run.stderr || run.error?.message || "").trim()}`); + if (run.status !== 0) { + // A GraphQL NOT_FOUND (a number that is no pull request) is an answer, not a read failure. + let body = null; + try { + body = JSON.parse(run.stdout); + } catch {} + if (graphql && Array.isArray(body?.errors) && body.errors.length > 0 && body.errors.every((e) => e?.type === "NOT_FOUND")) return body; + throw new PublishError(`gh api ${path} failed: ${(run.stderr || run.error?.message || "").trim()}`); + } return JSON.parse(run.stdout); } +export class NotMergedError extends PublishError {} + const PULL_QUERY = "query($owner:String!,$name:String!,$number:Int!){repository(owner:$owner,name:$name){pullRequest(number:$number){" + "number mergedAt lastEditedAt mergeCommit{oid} mergedBy{__typename login} labels(first:100){pageInfo{hasNextPage} nodes{name}} " + @@ -167,49 +178,43 @@ function eventKey(event) { // The post-merge gate's workflow runs, read as babysit-prs/scripts/babysit_postmerge.py check() // and _judge() read them (same plugin, mirrored rather than imported across languages): // FULL_RUN_WORKFLOW runs from `schedule` and `workflow_dispatch`, and CANARY_WORKFLOW runs from -// `push`, on the default branch, created at or after a merge on a SHA that contains it. Attempt 1 -// decides; a red full run is a flake only when a lane identity's attempt 2 passed. A merge is -// verified at the later of its first green lane dispatch run and first green canary run, and runs -// created after that are not its evidence. Two departures, both toward a failure: a red full run -// whose re-run has not come counts as red once DISPATCH_DEADLINE has passed since it completed -// (check() waits on it), and a red run covering several merges is a failure of each. +// `push`, on the default branch, created at or after a merge (no upper bound), on a SHA that +// contains it: compare ..., as _Context.contains does. Attempt 1 decides; a red +// full run is a flake only when a lane identity's attempt 2 passed. A merge is verified at the +// later of its first green lane dispatch run and first green canary run, and runs created after +// that are not its evidence; only a verified merge with nothing red or pending passes the gate, +// so a merge with no lane dispatch run (check()'s 6-hour failure) stays unsettled. Departures, +// all toward a failure: a compare that is neither ahead nor identical refuses the publish +// (check() reads it as not containing); a red full run whose re-run has not come counts as red +// once DISPATCH_DEADLINE has passed since it completed (check() waits on it); a red run covering +// several merges is a failure of each. const FULL_RUN_WORKFLOW = "pr-require-checks.yml"; const CANARY_WORKFLOW = "post-merge-detect-silent-revert.yml"; const DISPATCH_DEADLINE_MS = 6 * 3_600_000; -function postMergeRuns(gh, repo, merges, { lane, throughMs }) { - if (merges.length === 0) return () => ({ failures: [], unsettled: false, green: 0 }); - const branch = gh(`repos/${repo}`)?.default_branch; - if (!branch) throw new PublishError(`gh api repos/${repo}: no default_branch`); - const ordered = [...merges].sort((a, b) => Date.parse(a.mergedAt) - Date.parse(b.mergedAt)); - const floor = Date.parse(ordered[0].mergedAt); +function postMergeRuns(gh, repo, branch, merges, { lane, throughMs }) { + if (merges.length === 0) return () => ({ failures: [], unsettled: false, verified: false }); + const floor = Math.min(...merges.map((m) => Date.parse(m.mergedAt))); const created = encodeURIComponent(`>=${new Date(floor).toISOString().replace(/\.\d{3}Z$/, "Z")}`); + const byCreated = (a, b) => Date.parse(a.created_at) - Date.parse(b.created_at); const list = (workflow, event) => pages(gh, `repos/${repo}/actions/workflows/${workflow}/runs?event=${event}&branch=${encodeURIComponent(branch)}&created=${created}`, "workflow_runs") - .filter((run) => run.head_branch === branch && Date.parse(run.created_at) >= floor && Date.parse(run.created_at) <= throughMs) + .filter((run) => run.head_branch === branch && Date.parse(run.created_at) >= floor) .map((run) => ({ ...run, check: `${workflow} ${event}` })); - const fullRuns = [...list(FULL_RUN_WORKFLOW, "schedule"), ...list(FULL_RUN_WORKFLOW, "workflow_dispatch")]; - const canaryRuns = list(CANARY_WORKFLOW, "push"); + const fullRuns = [...list(FULL_RUN_WORKFLOW, "schedule"), ...list(FULL_RUN_WORKFLOW, "workflow_dispatch")].sort(byCreated); + const canaryRuns = list(CANARY_WORKFLOW, "push").sort(byCreated); - // Containment from one base, the earliest merge: on a linear (merge-queue) branch a later merge - // is in a run's history exactly when it is among the commits the run's SHA is ahead by. - const base = ordered[0].sha; - const aheadOf = new Map(); + const compares = new Map(); const contains = (sha, runSha) => { - if (!aheadOf.has(runSha)) { - const shas = new Set(); - let status = null; - for (let page = 1; ; page += 1) { - const body = gh(`repos/${repo}/compare/${base}...${runSha}?per_page=${PAGE}&page=${page}`); - if (!Array.isArray(body?.commits)) throw new PublishError(`gh api compare ${base}...${runSha}: no commits array`); - status ??= body.status; - for (const commit of body.commits) shas.add(commit.sha); - if (body.commits.length < PAGE) break; + const key = `${sha}...${runSha}`; + if (!compares.has(key)) { + const status = gh(`repos/${repo}/compare/${key}?per_page=1&page=1`)?.status; + if (status !== "ahead" && status !== "identical") { + throw new PublishError(`refusing to publish: compare ${key} is ${JSON.stringify(status)}, so whether that post-merge run contains the merge is unknown`); } - aheadOf.set(runSha, { status, shas }); + compares.set(key, true); } - const { status, shas } = aheadOf.get(runSha); - return sha === base ? status === "ahead" || status === "identical" : shas.has(sha); + return compares.get(key); }; const outcomes = new Map(); @@ -235,27 +240,28 @@ function postMergeRuns(gh, repo, merges, { lane, throughMs }) { return outcomes.get(key); }; + // Compares run only where the answer can change the verdict: a red or pending run, a green lane + // dispatch run, and a green canary run once a lane dispatch has verified half the merge. return (merge) => { const mergedMs = Date.parse(merge.mergedAt); - const relevant = (run) => Date.parse(run.created_at) >= mergedMs && contains(merge.sha, run.head_sha); - const fulls = fullRuns.filter(relevant); - const canaries = canaryRuns.filter(relevant); - const earliest = (runs) => Math.min(...runs.map((run) => Date.parse(run.created_at))); - const greenDispatch = fulls.filter( - (run) => run.event === "workflow_dispatch" && lane.has(loginKey(run.actor?.login)) && ["green", "flake"].includes(outcome(run, true).state), + const after = (runs) => runs.filter((run) => Date.parse(run.created_at) >= mergedMs); + const relevant = (run) => contains(merge.sha, run.head_sha); + const fulls = after(fullRuns); + const canaries = after(canaryRuns); + const dispatch = fulls.find( + (run) => run.event === "workflow_dispatch" && lane.has(loginKey(run.actor?.login)) && ["green", "flake"].includes(outcome(run, true).state) && relevant(run), ); - const greenCanary = canaries.filter((run) => outcome(run, false).state === "green"); - const verifiedAt = greenDispatch.length && greenCanary.length ? Math.max(earliest(greenDispatch), earliest(greenCanary)) : Number.POSITIVE_INFINITY; - const result = { failures: [], unsettled: false, green: 0 }; + const canary = dispatch && canaries.find((run) => outcome(run, false).state === "green" && relevant(run)); + const verifiedAt = canary ? Math.max(Date.parse(dispatch.created_at), Date.parse(canary.created_at)) : Number.POSITIVE_INFINITY; + const result = { failures: [], unsettled: false, verified: Boolean(canary) }; for (const [runs, attributable] of [ [fulls, true], [canaries, false], ]) { for (const run of runs.filter((r) => Date.parse(r.created_at) <= verifiedAt)) { const { state, conclusion } = outcome(run, attributable); - if (state === "red") result.failures.push({ check: run.check, conclusion, at: run.updated_at }); - else if (state === "pending") result.unsettled = true; - else result.green += 1; + if (state === "red" && relevant(run)) result.failures.push({ check: run.check, conclusion, at: run.updated_at }); + else if (state === "pending" && relevant(run)) result.unsettled = true; } } return result; @@ -279,9 +285,12 @@ export function collect({ repo, since, through, laneLogins, gateChecks, aiReview events.set(eventKey(record), record); }; - // Created-ascending order is stable while paging: an update moves no PR between pages. + const branch = gh(`repos/${repo}`)?.default_branch; + if (!branch) throw new PublishError(`gh api repos/${repo}: no default_branch`); + // Merges into the default branch only (babysit_postmerge.py _merged_since). Created-ascending + // order is stable while paging: an update moves no PR between pages. const seen = new Set(); - const merged = pages(gh, `repos/${repo}/pulls?state=closed&sort=created&direction=asc`).filter( + const merged = pages(gh, `repos/${repo}/pulls?state=closed&base=${encodeURIComponent(branch)}&sort=created&direction=asc`).filter( (pr) => inWindow(pr.merged_at) && !seen.has(pr.number) && seen.add(pr.number), ); @@ -302,9 +311,14 @@ export function collect({ repo, since, through, laneLogins, gateChecks, aiReview for (const issue of onePage(g.closingIssuesReferences, `closingIssuesReferences of #${g.number}`)) { linked.set(issue.number, onePage(issue.labels, `labels of issue #${issue.number}`).map((label) => label?.name)); } + // A ConnectedEvent on a PR timeline names the issue as `subject` (recorded from cli/cli #14616); + // any other shape after the merge leaves the links unknown. const laterLinks = timeline - .filter((e) => ["ConnectedEvent", "DisconnectedEvent"].includes(e.__typename) && e.subject?.__typename === "Issue" && Date.parse(e.createdAt) > atMs) + .filter((e) => ["ConnectedEvent", "DisconnectedEvent"].includes(e.__typename) && Date.parse(e.createdAt) > atMs) .sort((a, b) => Date.parse(b.createdAt) - Date.parse(a.createdAt)); + if (laterLinks.some((e) => e.subject?.__typename !== "Issue" || !Number.isInteger(e.subject.number))) { + return { cls: null, reason: "a link event after the merge has an unrecognized shape: links unknown" }; + } for (const e of laterLinks) { if (e.__typename === "ConnectedEvent") linked.delete(e.subject.number); else if (!linked.has(e.subject.number)) linked.set(e.subject.number, (gh(`repos/${repo}/issues/${e.subject.number}`).labels ?? []).map((label) => label?.name)); @@ -328,7 +342,7 @@ export function collect({ repo, since, through, laneLogins, gateChecks, aiReview const record = (number) => { if (!details.has(number)) { const g = gh("graphql", { query: PULL_QUERY, variables: { owner, name, number } })?.data?.repository?.pullRequest; - if (!g?.mergedAt) throw new PublishError(`gh api graphql: pull request #${number} unreadable or not merged`); + if (!g?.mergedAt) throw new NotMergedError(`#${number} is not a merged pull request`); const timeline = onePage(g.timelineItems, `timeline of #${number}`); details.set(number, { number, ...classAtMerge(g, timeline), autonomous: autonomousOf(g, timeline), mergedAt: g.mergedAt, sha: g.mergeCommit?.oid, reverted: false }); } @@ -369,19 +383,29 @@ export function collect({ repo, since, through, laneLogins, gateChecks, aiReview if (number === undefined) { const known = bySha.get(revert.sha); number = known ? known.number : gh(`repos/${repo}/commits/${revert.sha}/pulls`).find((pr) => pr.merged_at)?.number; - if (number === undefined) continue; + if (number === undefined) { + unresolvedReverts.push({ target: revert.sha, reason: "no merged pull request for this commit" }); + continue; + } } if (!firstRevert.has(number) || Date.parse(revert.at) < Date.parse(firstRevert.get(number))) firstRevert.set(number, revert.at); } for (const [number, at] of firstRevert) { - const target = record(number); + let target; + try { + target = record(number); + } catch (error) { + if (!(error instanceof NotMergedError)) throw error; + unresolvedReverts.push({ target: number, reason: "not a merged pull request" }); + continue; + } if (target.autonomous && (["C2", "C3"].includes(target.cls) || target.cls === null)) { target.reverted = true; for (const cell of cellsOf(target)) addEvent(cell, "reverted-merge", at); } } - const judgeWorkflows = postMergeRuns(gh, repo, tracked.filter((c) => c.autonomous), { lane, throughMs }); + const judgeWorkflows = postMergeRuns(gh, repo, branch, tracked.filter((c) => c.autonomous), { lane, throughMs }); // Post-merge gate on each autonomous merge commit: the --gate-check runs and status contexts. // A run that did not finish green or red (cancelled, action_required, stale) fails the gate @@ -416,8 +440,9 @@ export function collect({ repo, since, through, laneLogins, gateChecks, aiReview for (const status of [...statuses].sort((a, b) => Date.parse(b.created_at) - Date.parse(a.created_at))) { if (!latest.has(status.context)) latest.set(status.context, status.state); } + // Passed only where check() would call the merge verified, with nothing pending. const settled = - runs.length + latest.size + wf.green > 0 && + wf.verified && !wf.unsettled && runs.every((run) => outcome(run) === "ok") && [...latest.values()].every((state) => state === "success"); diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs index 56f90900fa..ebefbeb31b 100644 --- a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs @@ -34,12 +34,14 @@ const page1 = (nodes, hasNextPage = false) => ({ pageInfo: { hasNextPage }, node // REC.pullGraph: PR #7063, which closes issue #6007 (labeled `work-class: scoped`). const ISSUE_6007 = REC.pullGraph.data.repository.pullRequest.closingIssuesReferences.nodes; const queued = (at, enqueuer, actor = enqueuer) => ({ __typename: "AddedToMergeQueueEvent", createdAt: at, actor: actorOf(actor), enqueuer: actorOf(enqueuer) }); -const link = (type, issue, at) => ({ __typename: type, createdAt: at, subject: { __typename: "Issue", number: issue } }); +// The shape of a real ConnectedEvent on a PR timeline (cli/cli #14616): `subject` is the issue. +const LINK = REC.connectedEventFromCliCli14616; +const link = (type, issue, at) => ({ ...LINK, __typename: type, createdAt: at, subject: { ...LINK.subject, number: issue } }); function pr( number, mergedAt, - { labels = [], merger = LANE, title = REC.pullListItem.title, body = REC.pullListItem.body, linked = [], timeline = [], lastEditedAt = null, patch = {} } = {}, + { labels = [], merger = LANE, title = REC.pullListItem.title, body = REC.pullListItem.body, linked = [], timeline = [], lastEditedAt = null, patch = {}, base = "main" } = {}, ) { const names = labels.map((name) => ({ name })); return { @@ -57,6 +59,7 @@ function pr( }, list: { ...REC.pullListItem, + base: { ...REC.pullListItem.base, ref: base }, number, title, body, @@ -78,7 +81,9 @@ const labelEvent = (event, name, at, login = "kyle-sexton", type = "User") => ({ // `workflowRuns`: {: [run]}; `attempts`: {: {: run}}; `history`: the // default branch's commits oldest first (linear, as a merge-queue branch is), defaulting to the // merge commits in merge order. -function github({ pulls = [], commits = [], checkRuns = {}, statuses = {}, issueEvents = {}, reviews = {}, issues = {}, workflowRuns = {}, attempts = {}, history, bumpOnPage2 }) { +// `diverged`: head SHAs a compare reports as diverged. A PR number not in `pulls` answers with the +// recorded NOT_FOUND response. +function github({ pulls = [], commits = [], checkRuns = {}, statuses = {}, issueEvents = {}, reviews = {}, issues = {}, workflowRuns = {}, attempts = {}, history, diverged = [], bumpOnPage2 }) { const branch = history ?? [...pulls].sort((a, b) => a.graph.mergedAt.localeCompare(b.graph.mergedAt)).map((p) => p.graph.mergeCommit.oid); const page = (items, path) => { const n = Number(/[?&]page=(\d+)/.exec(path)[1]); @@ -90,18 +95,20 @@ function github({ pulls = [], commits = [], checkRuns = {}, statuses = {}, issue assert.deepEqual([graphql.variables.owner, graphql.variables.name], REPO.split("/")); assert.match(graphql.query, /^query\(/); assert.doesNotMatch(graphql.query, /mutation/); - return { data: { repository: { pullRequest: pulls.find((p) => p.graph.number === graphql.variables.number).graph } } }; + const found = pulls.find((p) => p.graph.number === graphql.variables.number); + return found ? { data: { repository: { pullRequest: found.graph } } } : REC.pullNotFound; } if (path === `repos/${REPO}`) return REC.repo; const prefix = `repos/${REPO}/`; assert.ok(path.startsWith(prefix), path); const rest = path.slice(prefix.length); let m; - if ((m = /^pulls\?state=closed&sort=(created|updated)&direction=(asc|desc)&/.exec(rest))) { + if ((m = /^pulls\?state=closed(?:&base=([\w.-]+))?&sort=(created|updated)&direction=(asc|desc)&/.exec(rest))) { if (bumpOnPage2 && /[?&]page=2\b/.test(rest)) pulls.find((p) => p.list.number === bumpOnPage2).list.updated_at = "2026-10-04T00:00:00Z"; - const key = m[1] === "created" ? "created_at" : "updated_at"; - const sign = m[2] === "asc" ? 1 : -1; - return page(pulls.map((p) => p.list).sort((a, b) => sign * a[key].localeCompare(b[key])), rest); + const key = m[2] === "created" ? "created_at" : "updated_at"; + const sign = m[3] === "asc" ? 1 : -1; + const listed = pulls.map((p) => p.list).filter((p) => !m[1] || p.base.ref === m[1]); + return page(listed.sort((a, b) => sign * a[key].localeCompare(b[key])), rest); } if ((m = /^issues\/(\d+)$/.exec(rest))) return issues[m[1]]; if ((m = /^pulls\/(\d+)\/reviews\?/.exec(rest))) return page(reviews[m[1]] ?? [], rest); @@ -120,7 +127,7 @@ function github({ pulls = [], commits = [], checkRuns = {}, statuses = {}, issue if ((m = /^compare\/([0-9a-f]{40})\.\.\.([0-9a-f]{40})\?/.exec(rest))) { const [base, head] = [branch.indexOf(m[1]), branch.indexOf(m[2])]; assert.ok(base >= 0 && head >= 0, `compare outside the recorded history: ${rest}`); - const status = head > base ? "ahead" : head === base ? "identical" : "behind"; + const status = diverged.includes(m[2]) ? "diverged" : head > base ? "ahead" : head === base ? "identical" : "behind"; const ahead = head > base ? branch.slice(base + 1, head + 1).map((sha) => ({ sha })) : []; return { ...REC.compare, status, ahead_by: ahead.length, total_commits: ahead.length, commits: page(ahead, rest) }; } @@ -153,6 +160,11 @@ const wfRun = (id, event, sha, created, { conclusion = "success", status = "comp triggering_actor: { login: triggering }, }); const gateEvents = (at, ...cells) => cells.map((cell) => ({ cell, event: "gate-failure", at })); +// A green lane dispatch run and a green canary run on `tip`: check() calls each merge they contain +// verified. +const TIP = shaOf(0xf1f1); +const verifiedOn = (tip, at) => ({ [FULL]: [wfRun(9001, "workflow_dispatch", tip, at, { actor: LANE })], [CANARY]: [wfRun(9002, "push", tip, at)] }); +const mergeOrder = (pulls) => [...pulls].sort((a, b) => a.graph.mergedAt.localeCompare(b.graph.mergedAt)).map((p) => p.graph.mergeCommit.oid); test("a red full run on a later SHA containing two merges yields gate-failure for each", () => { const tip = shaOf(0xa001); @@ -171,19 +183,71 @@ test("a red full run on a later SHA containing two merges yields gate-failure fo ); }); -test("a run on a SHA without the merge, or created before the merge, is not its evidence", () => { - const older = shaOf(0xa002); +test("a run created before the merge is not its evidence", () => { // Run 3 on #1303's own SHA predates #1303's merge, so it is evidence for #1309 only. const result = run({ pulls: [pr(1309, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] }), pr(1303, "2026-09-02T00:00:00Z", { labels: [SCOPED] })], - history: [older, shaOf(1309), shaOf(1303)], - workflowRuns: { - [FULL]: [wfRun(2, "schedule", older, "2026-09-03T00:00:00Z", { conclusion: "failure" }), wfRun(3, "schedule", shaOf(1303), "2026-09-01T12:00:00Z", { conclusion: "failure" })], - }, + workflowRuns: { [FULL]: [wfRun(3, "schedule", shaOf(1303), "2026-09-01T12:00:00Z", { conclusion: "failure" })] }, }); assert.deepEqual(result.events, gateEvents("2026-09-01T12:08:00Z", "C2-auto-merge")); }); +test("a post-merge run whose SHA is behind or diverged from the merge refuses the publish", () => { + const other = shaOf(0xa002); + const behind = { pulls: [pr(1310, "2026-09-02T00:00:00Z", { labels: [MECHANICAL] })], history: [other, shaOf(1310)] }; + const red = { [FULL]: [wfRun(2, "schedule", other, "2026-09-03T00:00:00Z", { conclusion: "failure" })] }; + assert.throws(() => run({ ...behind, workflowRuns: red }), (error) => error instanceof PublishError && /behind/.test(error.message)); + const diverged = { pulls: [pr(1310, "2026-09-02T00:00:00Z", { labels: [MECHANICAL] })], history: [shaOf(1310), other], diverged: [other] }; + assert.throws(() => run({ ...diverged, workflowRuns: red }), /diverged/); +}); + +test("a merge that is an ancestor of a same-time merge listed before it is still covered by a later run", () => { + const tip = shaOf(0xa007); + const result = run({ + pulls: [pr(1401, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] }), pr(1402, "2026-09-01T00:00:00Z", { labels: [SCOPED] })], + history: [shaOf(1402), shaOf(1401), tip], + workflowRuns: { [FULL]: [wfRun(11, "schedule", tip, "2026-09-02T00:00:00Z", { conclusion: "failure" })] }, + }); + assert.deepEqual(result.events, gateEvents("2026-09-02T00:08:00Z", "C2-auto-merge", "C3-auto-merge")); +}); + +test("a post-merge run created after --through still counts", () => { + const result = run({ + pulls: [pr(1403, "2026-10-04T00:00:00Z", { labels: [MECHANICAL] })], + workflowRuns: { [CANARY]: [wfRun(12, "push", shaOf(1403), "2026-10-06T00:00:00Z", { conclusion: "failure" })] }, + }); + assert.deepEqual(result.events, gateEvents("2026-10-06T00:08:00Z", "C2-auto-merge")); +}); + +test("a merge is gate-passed only when a green lane dispatch and a green canary both contain it", () => { + const pulls = () => [pr(1404, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] })]; + const history = [shaOf(1404), TIP]; + const gateOf = (result) => termOf(result, "C2-auto-merge", "deterministic-gate pass").detail; + assert.deepEqual(gateOf(run({ pulls: pulls() })), { passed: 0, failed: 0, unsettled: 1, total: 1 }); + const dispatchOnly = { [FULL]: verifiedOn(TIP, "2026-09-02T00:00:00Z")[FULL] }; + assert.deepEqual(gateOf(run({ pulls: pulls(), history, workflowRuns: dispatchOnly })), { passed: 0, failed: 0, unsettled: 1, total: 1 }); + const canaryOnly = { [CANARY]: verifiedOn(TIP, "2026-09-02T00:00:00Z")[CANARY] }; + assert.deepEqual(gateOf(run({ pulls: pulls(), history, workflowRuns: canaryOnly })), { passed: 0, failed: 0, unsettled: 1, total: 1 }); + assert.deepEqual(gateOf(run({ pulls: pulls(), history, workflowRuns: verifiedOn(TIP, "2026-09-02T00:00:00Z") })), { passed: 1, failed: 0, unsettled: 0, total: 1 }); +}); + +test("a PR merged into a branch other than the default is not read", () => { + const result = run({ pulls: [pr(1405, "2026-09-01T00:00:00Z", { labels: [MECHANICAL], base: "release" })] }); + assert.equal(termOf(result, "C2-auto-merge", "autonomous C2 completions").value, 0); +}); + +test("a link event after the merge in an unrecognized shape leaves the class unknown", () => { + const odd = { ...link("DisconnectedEvent", 6007, "2026-09-02T00:00:00Z"), subject: { __typename: "PullRequest", number: 5 } }; + const result = run({ pulls: [pr(1406, "2026-09-01T00:00:00Z", { labels: [MECHANICAL], timeline: [odd] })] }); + assert.equal(termOf(result, "C2-auto-merge", "autonomous C2 completions").value, 0); + assert.deepEqual(result.unclassedAutonomous, [{ number: 1406, reason: "a link event after the merge has an unrecognized shape: links unknown" }]); +}); + +test("a revert naming a PR that is not merged or does not exist is listed, not fatal", () => { + const result = run({ pulls: [pr(1407, "2026-09-01T00:00:00Z", { body: `Reverts ${REPO}#999999`, merger: "kyle-sexton" })] }); + assert.deepEqual(result.unresolvedReverts, [{ target: 999999, reason: "not a merged pull request" }]); +}); + test("a red full run is a flake only when a lane identity's re-run passed", () => { const tip = shaOf(0xa003); const rerun = (id, triggering) => wfRun(id, "schedule", tip, "2026-09-02T00:00:00Z", { attempt: 2, triggering, updated: "2026-09-02T01:00:00Z" }); @@ -242,6 +306,8 @@ test("only the named gate checks and status contexts are gate evidence", () => { pulls: [pr(1201, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] })], checkRuns: { 1201: [REC.checkRun, other("dispatch", "failure"), other("resolve-conflicts / run", "cancelled")] }, statuses: { 1201: [{ ...REC.status, context: "GitGuardian", state: "error" }] }, + history: [shaOf(1201), TIP], + workflowRuns: verifiedOn(TIP, "2026-09-02T00:00:00Z"), }); assert.deepEqual(result.events, []); assert.deepEqual(termOf(result, "C2-auto-merge", "deterministic-gate pass").detail, { passed: 1, failed: 0, unsettled: 0, total: 1 }); @@ -322,7 +388,8 @@ test("a missed-blocking label a human applied to a C3 PR yields verification-div test("20 autonomous C2 merges over 14+ days pass the C2 term and 19 fail it", () => { const merges = (count) => Array.from({ length: count }, (_, i) => pr(600 + i, `2026-09-${String(i + 1).padStart(2, "0")}T00:00:00Z`, { labels: [MECHANICAL] })); - const twenty = run({ pulls: merges(20) }); + const pulls = merges(20); + const twenty = run({ pulls, history: [...mergeOrder(pulls), TIP], workflowRuns: verifiedOn(TIP, "2026-09-21T00:00:00Z") }); assert.equal(termOf(twenty, "C2-auto-merge", "autonomous C2 completions").pass, true); assert.equal(twenty.cells["C2-auto-merge"].pass, true); const nineteen = run({ pulls: merges(19) }); @@ -469,6 +536,8 @@ test("a cancelled gate run counts as a failure unless a later run of the same ch 1110: [cancelled("lint-repo", "2026-09-01T00:10:00Z")], 1111: [cancelled("lint-repo", "2026-09-01T00:10:00Z"), { ...REC.checkRun, completed_at: "2026-09-01T00:20:00Z" }], }, + history: [shaOf(1110), shaOf(1111), TIP], + workflowRuns: verifiedOn(TIP, "2026-09-02T00:00:00Z"), }); assert.deepEqual(result.events, [{ cell: "C2-auto-merge", event: "gate-failure", at: "2026-09-01T00:10:00Z" }]); assert.deepEqual(termOf(result, "C2-auto-merge", "deterministic-gate pass").detail, { passed: 1, failed: 1, unsettled: 0, total: 2 }); From a5ef5caefee14e42514d32d35a799b76ea0f1db2 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Tue, 13 Oct 2026 01:18:18 -0400 Subject: [PATCH 08/18] fix(source-control): degrade lane-written references instead of refusing Anything a lane can write into a PR or commit now degrades to a listing or to unclassed-autonomous; only GitHub being unreadable refuses. - A `This reverts commit ` naming no commit (REST 422) is listed in unresolvedReverts. - A lane PR whose labels, linked issues or timeline page is unreadable or incomplete is unclassed-autonomous. - A compare that is behind means not contained, as check() reads it; only diverged or unknown refuses. - A merge is verified only by a lane dispatch created within the 6-hour dispatch deadline of the merge. - The GraphQL NOT_FOUND answer is accepted only for the pull request itself (pullRequest null, every error path ending in pullRequest); a repository NOT_FOUND fails. - An unparsable date refuses the publish. - Document that the lane identity must not hold actions:write. Refs: #4588 Co-Authored-By: Claude Opus 5.5 --- .../reference/promotion-evidence-bootstrap.md | 3 +- .../publish-promotion-evidence/recorded.json | 24 +++ .../scripts/publish-promotion-evidence.mjs | 144 +++++++++++++----- .../publish-promotion-evidence.test.mjs | 103 +++++++++---- 4 files changed, 202 insertions(+), 72 deletions(-) diff --git a/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md b/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md index 9d236118a5..7cd3caac7e 100644 --- a/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md +++ b/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md @@ -168,7 +168,8 @@ each event, including the judgment mappings and how to set `--lane-login` for me `sha256`, or when the new events drop one the previous file held. Run it as the operator, never from the lane, and point `--out` at a directory the lane cannot -write. Which host runs it, under which identity and on what schedule is not decided yet: it waits +write. The lane identity must not hold `actions:write` on the repository: with it, the lane could +delete a red workflow run before the publisher reads it. Which host runs it, under which identity and on what schedule is not decided yet: it waits on the open decision about the single merge authority ([#4588](https://github.com/melodic-software/claude-code-plugins/issues/4588)). Until then the resolver does not read the sidecar. diff --git a/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json b/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json index 8fd02bdc56..46c5f849af 100644 --- a/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json +++ b/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json @@ -157,6 +157,30 @@ "number": 14614 } }, + "commitNotFound": { + "stdout": "{\"message\":\"No commit found for SHA: 0123456789abcdef0123456789abcdef01234567\",\"documentation_url\":\"https://docs.github.com/rest/commits/commits#list-pull-requests-associated-with-a-commit\",\"status\":\"422\"}", + "stderr": "gh: No commit found for SHA: 0123456789abcdef0123456789abcdef01234567 (HTTP 422)\n" + }, + "repoNotFound": { + "data": { + "repository": null + }, + "errors": [ + { + "type": "NOT_FOUND", + "path": [ + "repository" + ], + "locations": [ + { + "line": 1, + "column": 2 + } + ], + "message": "Could not resolve to a Repository with the name 'melodic-software/no-such-repo-xyz'." + } + ] + }, "pullNotFound": { "data": { "repository": { diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs index 17ae2c7315..7eb0abca8a 100644 --- a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs @@ -31,6 +31,13 @@ // (a machine user); an App lane is a Bot and counts without it. Recheck if merged_by stops // matching the enqueuer. // +// Deployment requirement: the lane identity must not hold actions:write on the repository. With it +// the lane could delete a red workflow run, and a deleted run is evidence this script never sees. +// +// Anything a lane can write into a PR or commit degrades to a listing or to unclassed-autonomous, +// never to a refused publish. What refuses is GitHub itself being unreadable: a failed read, an +// unparsable date, a compare that is neither ahead, identical nor behind. +// // What counts (judgment where the contract names no GitHub source): // - A merged PR's work class is the highest `work-class:` label (the label set // babysit-prs/scripts/babysit_merge_policy.py WORK_CLASS_LABELS maps) on the issues it linked @@ -38,7 +45,8 @@ // labeled/unlabeled events, so relabeling after the merge changes nothing. The linked set is // closingIssuesReferences with later ConnectedEvent/DisconnectedEvent undone; when the PR body // was edited after the merge, its closing-keyword links at merge are unknown and the class is -// none. A list with another page fails closed. +// none. A label, linked-issue or timeline list that is unreadable or has another page leaves +// the merge unclassed and counts it as autonomous. // - A C2/C3 merge is counted. An autonomous merge with no class is listed in // unclassedAutonomous, never counted toward a positive term, and each of its contrary events // is emitted against every promotable cell. @@ -46,7 +54,8 @@ // commit as `Reverts #` (GitHub's revert button) or `This reverts commit // ` (git revert); one event per reverted merge, at the earliest revert seen. A merged PR // titled revert... with neither form is listed in unresolvedReverts for a human to read, as is -// a named target that is not a merged pull request, so a bad reference cannot stop a publish. +// a named target that is not a merged pull request or a SHA no commit has, so a bad reference +// cannot stop a publish. // - gate-failure: only the post-merge gate counts: check runs and commit-status contexts on an // autonomous merge's merge commit whose name is a --gate-check (required; none named is a // usage error, never an empty gate). In this repository babysit-prs/scripts/babysit_postmerge.py @@ -107,19 +116,44 @@ function realGh(path, graphql) { ? ["api", "graphql", "-f", `query=${graphql.query}`, ...Object.entries(graphql.variables).flatMap(([key, value]) => ["-F", `${key}=${value}`])] : ["api", "--method", "GET", path]; const run = spawnSync("gh", args, { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 }); - if (run.status !== 0) { - // A GraphQL NOT_FOUND (a number that is no pull request) is an answer, not a read failure. - let body = null; - try { - body = JSON.parse(run.stdout); - } catch {} - if (graphql && Array.isArray(body?.errors) && body.errors.length > 0 && body.errors.every((e) => e?.type === "NOT_FOUND")) return body; - throw new PublishError(`gh api ${path} failed: ${(run.stderr || run.error?.message || "").trim()}`); - } + if (run.status !== 0) return ghFailure(path, graphql, run.stdout, run.stderr || run.error?.message || ""); return JSON.parse(run.stdout); } -export class NotMergedError extends PublishError {} +// A reference a lane can write (a PR number, a commit SHA) that names nothing: listed, never fatal. +export class UnresolvableError extends PublishError {} +// A page that is unreadable or has another page: the merge's class is unknown. +class IncompleteError extends PublishError {} + +// A failed `gh api` call. Only two answers are not read failures: GraphQL NOT_FOUND on the pull +// request itself (data.repository.pullRequest null, every error path ending in pullRequest), which +// is returned as the answer, and REST 422 "No commit found for SHA", which is unresolvable. Any +// other failure, a repository NOT_FOUND included, fails the publish. +export function ghFailure(path, graphql, stdout, stderr) { + let body = null; + try { + body = JSON.parse(stdout); + } catch {} + const errors = body?.errors; + if ( + graphql && + body?.data?.repository?.pullRequest === null && + Array.isArray(errors) && + errors.length > 0 && + errors.every((e) => e?.type === "NOT_FOUND" && Array.isArray(e.path) && e.path.at(-1) === "pullRequest") + ) { + return body; + } + if (!graphql && String(body?.status) === "422" && /^No commit found for SHA/.test(body?.message ?? "")) throw new UnresolvableError(body.message); + throw new PublishError(`gh api ${path} failed: ${String(stderr).trim()}`); +} + +// A date GitHub returned; one that does not parse refuses the publish. +function ms(value, what) { + const parsed = typeof value === "string" ? Date.parse(value) : Number.NaN; + if (Number.isNaN(parsed)) throw new PublishError(`refusing to publish: ${what} has an unreadable date ${JSON.stringify(value)}`); + return parsed; +} const PULL_QUERY = "query($owner:String!,$name:String!,$number:Int!){repository(owner:$owner,name:$name){pullRequest(number:$number){" + @@ -141,7 +175,7 @@ function loginKey(login) { function onePage(connection, what) { if (!Array.isArray(connection?.nodes) || connection.pageInfo?.hasNextPage !== false) { - throw new PublishError(`gh api graphql: ${what} unreadable or hasNextPage`); + throw new IncompleteError(`gh api graphql: ${what} unreadable or hasNextPage`); } return connection.nodes; } @@ -165,8 +199,8 @@ function workClass(names) { function labelsAsOf(current, events, atMs) { const set = new Set(current); const later = events - .filter((e) => (e.event === "labeled" || e.event === "unlabeled") && Date.parse(e.created_at) > atMs) - .sort((a, b) => Date.parse(b.created_at) - Date.parse(a.created_at)); + .filter((e) => (e.event === "labeled" || e.event === "unlabeled") && ms(e.created_at, "a label event") > atMs) + .sort((a, b) => ms(b.created_at, "a label event") - ms(a.created_at, "a label event")); for (const e of later) e.event === "labeled" ? set.delete(e.label?.name) : set.add(e.label?.name); return [...set]; } @@ -182,10 +216,11 @@ function eventKey(event) { // contains it: compare ..., as _Context.contains does. Attempt 1 decides; a red // full run is a flake only when a lane identity's attempt 2 passed. A merge is verified at the // later of its first green lane dispatch run and first green canary run, and runs created after -// that are not its evidence; only a verified merge with nothing red or pending passes the gate, -// so a merge with no lane dispatch run (check()'s 6-hour failure) stays unsettled. Departures, -// all toward a failure: a compare that is neither ahead nor identical refuses the publish -// (check() reads it as not containing); a red full run whose re-run has not come counts as red +// that are not its evidence; a compare that is behind means not contained. Only a verified merge +// with nothing red or pending passes the gate, and only a lane dispatch created within +// DISPATCH_DEADLINE of the merge verifies it, so a merge with no timely lane dispatch run +// (check()'s 6-hour failure) stays unsettled. Departures, all toward a failure: a compare that is +// diverged or unknown refuses the publish (check() reads it as not containing); a red full run whose re-run has not come counts as red // once DISPATCH_DEADLINE has passed since it completed (check() waits on it); a red run covering // several merges is a failure of each. const FULL_RUN_WORKFLOW = "pr-require-checks.yml"; @@ -194,13 +229,13 @@ const DISPATCH_DEADLINE_MS = 6 * 3_600_000; function postMergeRuns(gh, repo, branch, merges, { lane, throughMs }) { if (merges.length === 0) return () => ({ failures: [], unsettled: false, verified: false }); - const floor = Math.min(...merges.map((m) => Date.parse(m.mergedAt))); + const floor = Math.min(...merges.map((m) => ms(m.mergedAt, `#${m.number} mergedAt`))); const created = encodeURIComponent(`>=${new Date(floor).toISOString().replace(/\.\d{3}Z$/, "Z")}`); - const byCreated = (a, b) => Date.parse(a.created_at) - Date.parse(b.created_at); + const byCreated = (a, b) => a.createdMs - b.createdMs; const list = (workflow, event) => pages(gh, `repos/${repo}/actions/workflows/${workflow}/runs?event=${event}&branch=${encodeURIComponent(branch)}&created=${created}`, "workflow_runs") - .filter((run) => run.head_branch === branch && Date.parse(run.created_at) >= floor) - .map((run) => ({ ...run, check: `${workflow} ${event}` })); + .map((run) => ({ ...run, createdMs: ms(run.created_at, `run ${run.id}`), check: `${workflow} ${event}` })) + .filter((run) => run.head_branch === branch && run.createdMs >= floor); const fullRuns = [...list(FULL_RUN_WORKFLOW, "schedule"), ...list(FULL_RUN_WORKFLOW, "workflow_dispatch")].sort(byCreated); const canaryRuns = list(CANARY_WORKFLOW, "push").sort(byCreated); @@ -209,10 +244,11 @@ function postMergeRuns(gh, repo, branch, merges, { lane, throughMs }) { const key = `${sha}...${runSha}`; if (!compares.has(key)) { const status = gh(`repos/${repo}/compare/${key}?per_page=1&page=1`)?.status; - if (status !== "ahead" && status !== "identical") { + // behind: the run's SHA is an ancestor of the merge, so it does not contain it (as check()). + if (status !== "ahead" && status !== "identical" && status !== "behind") { throw new PublishError(`refusing to publish: compare ${key} is ${JSON.stringify(status)}, so whether that post-merge run contains the merge is unknown`); } - compares.set(key, true); + compares.set(key, status !== "behind"); } return compares.get(key); }; @@ -230,7 +266,7 @@ function postMergeRuns(gh, repo, branch, merges, { lane, throughMs }) { else if (!attributable) state = "red"; else { const second = run.run_attempt >= 2 ? attempt(2) : null; - if (!second) state = throughMs - Date.parse(first.updated_at) > DISPATCH_DEADLINE_MS ? "red" : "pending"; + if (!second) state = throughMs - ms(first.updated_at, `run ${run.id} attempt 1`) > DISPATCH_DEADLINE_MS ? "red" : "pending"; else if (!lane.has(loginKey(second.triggering_actor?.login))) state = "red"; else if (second.status !== "completed") state = "pending"; else state = second.conclusion === "success" ? "flake" : "red"; @@ -243,22 +279,27 @@ function postMergeRuns(gh, repo, branch, merges, { lane, throughMs }) { // Compares run only where the answer can change the verdict: a red or pending run, a green lane // dispatch run, and a green canary run once a lane dispatch has verified half the merge. return (merge) => { - const mergedMs = Date.parse(merge.mergedAt); - const after = (runs) => runs.filter((run) => Date.parse(run.created_at) >= mergedMs); + const mergedMs = ms(merge.mergedAt, `#${merge.number} mergedAt`); + const after = (runs) => runs.filter((run) => run.createdMs >= mergedMs); const relevant = (run) => contains(merge.sha, run.head_sha); const fulls = after(fullRuns); const canaries = after(canaryRuns); const dispatch = fulls.find( - (run) => run.event === "workflow_dispatch" && lane.has(loginKey(run.actor?.login)) && ["green", "flake"].includes(outcome(run, true).state) && relevant(run), + (run) => + run.event === "workflow_dispatch" && + run.createdMs <= mergedMs + DISPATCH_DEADLINE_MS && + lane.has(loginKey(run.actor?.login)) && + ["green", "flake"].includes(outcome(run, true).state) && + relevant(run), ); const canary = dispatch && canaries.find((run) => outcome(run, false).state === "green" && relevant(run)); - const verifiedAt = canary ? Math.max(Date.parse(dispatch.created_at), Date.parse(canary.created_at)) : Number.POSITIVE_INFINITY; + const verifiedAt = canary ? Math.max(dispatch.createdMs, canary.createdMs) : Number.POSITIVE_INFINITY; const result = { failures: [], unsettled: false, verified: Boolean(canary) }; for (const [runs, attributable] of [ [fulls, true], [canaries, false], ]) { - for (const run of runs.filter((r) => Date.parse(r.created_at) <= verifiedAt)) { + for (const run of runs.filter((r) => r.createdMs <= verifiedAt)) { const { state, conclusion } = outcome(run, attributable); if (state === "red" && relevant(run)) result.failures.push({ check: run.check, conclusion, at: run.updated_at }); else if (state === "pending" && relevant(run)) result.unsettled = true; @@ -281,6 +322,7 @@ export function collect({ repo, since, through, laneLogins, gateChecks, aiReview const inWindow = (at) => typeof at === "string" && Date.parse(at) >= sinceMs && Date.parse(at) <= throughMs; const events = new Map(); const addEvent = (cell, event, at) => { + ms(at, `a ${event} event`); const record = { cell, event, at }; events.set(eventKey(record), record); }; @@ -305,8 +347,8 @@ export function collect({ repo, since, through, laneLogins, gateChecks, aiReview // mergedAt. Links: the current set with later Connected/Disconnected events undone; a body // edited after the merge leaves the closing-keyword links unknown, so the class is too. const classAtMerge = (g, timeline) => { - const atMs = Date.parse(g.mergedAt); - if (g.lastEditedAt && Date.parse(g.lastEditedAt) > atMs) return { cls: null, reason: "PR body edited after the merge: closing-keyword links unknown" }; + const atMs = ms(g.mergedAt, `#${g.number} mergedAt`); + if (g.lastEditedAt && ms(g.lastEditedAt, `#${g.number} lastEditedAt`) > atMs) return { cls: null, reason: "PR body edited after the merge: closing-keyword links unknown" }; const linked = new Map(); for (const issue of onePage(g.closingIssuesReferences, `closingIssuesReferences of #${g.number}`)) { linked.set(issue.number, onePage(issue.labels, `labels of issue #${issue.number}`).map((label) => label?.name)); @@ -314,14 +356,18 @@ export function collect({ repo, since, through, laneLogins, gateChecks, aiReview // A ConnectedEvent on a PR timeline names the issue as `subject` (recorded from cli/cli #14616); // any other shape after the merge leaves the links unknown. const laterLinks = timeline - .filter((e) => ["ConnectedEvent", "DisconnectedEvent"].includes(e.__typename) && Date.parse(e.createdAt) > atMs) - .sort((a, b) => Date.parse(b.createdAt) - Date.parse(a.createdAt)); + .filter((e) => ["ConnectedEvent", "DisconnectedEvent"].includes(e.__typename) && ms(e.createdAt, "a link event") > atMs) + .sort((a, b) => ms(b.createdAt, "a link event") - ms(a.createdAt, "a link event")); if (laterLinks.some((e) => e.subject?.__typename !== "Issue" || !Number.isInteger(e.subject.number))) { return { cls: null, reason: "a link event after the merge has an unrecognized shape: links unknown" }; } for (const e of laterLinks) { if (e.__typename === "ConnectedEvent") linked.delete(e.subject.number); - else if (!linked.has(e.subject.number)) linked.set(e.subject.number, (gh(`repos/${repo}/issues/${e.subject.number}`).labels ?? []).map((label) => label?.name)); + else if (!linked.has(e.subject.number)) { + const labels = gh(`repos/${repo}/issues/${e.subject.number}`)?.labels; + if (!Array.isArray(labels)) throw new IncompleteError(`labels of issue #${e.subject.number} unreadable`); + linked.set(e.subject.number, labels.map((label) => label?.name)); + } } const names = labelsAsOf(onePage(g.labels, `labels of #${g.number}`).map((label) => label?.name), eventsOf(g.number), atMs); for (const [issue, labels] of linked) names.push(...labelsAsOf(labels, eventsOf(issue), atMs)); @@ -333,7 +379,7 @@ export function collect({ repo, since, through, laneLogins, gateChecks, aiReview // AddedToMergeQueueEvent's actor and enqueuer, or merged_by when the merge bypassed the // queue; any of them null or not human makes the merge autonomous. const autonomousOf = (g, timeline) => { - const queued = timeline.filter((e) => e.__typename === "AddedToMergeQueueEvent").sort((a, b) => Date.parse(a.createdAt) - Date.parse(b.createdAt)); + const queued = timeline.filter((e) => e.__typename === "AddedToMergeQueueEvent").sort((a, b) => ms(a.createdAt, "a merge-queue event") - ms(b.createdAt, "a merge-queue event")); const actors = queued.length > 0 ? [queued.at(-1).actor, queued.at(-1).enqueuer] : [g.mergedBy]; return actors.some((actor) => !human(actor)) || lane.has(loginKey(g.mergedBy?.login)); }; @@ -342,9 +388,17 @@ export function collect({ repo, since, through, laneLogins, gateChecks, aiReview const record = (number) => { if (!details.has(number)) { const g = gh("graphql", { query: PULL_QUERY, variables: { owner, name, number } })?.data?.repository?.pullRequest; - if (!g?.mergedAt) throw new NotMergedError(`#${number} is not a merged pull request`); - const timeline = onePage(g.timelineItems, `timeline of #${number}`); - details.set(number, { number, ...classAtMerge(g, timeline), autonomous: autonomousOf(g, timeline), mergedAt: g.mergedAt, sha: g.mergeCommit?.oid, reverted: false }); + if (!g?.mergedAt) throw new UnresolvableError(`#${number} is not a merged pull request`); + ms(g.mergedAt, `#${number} mergedAt`); + const base = { number, mergedAt: g.mergedAt, sha: g.mergeCommit?.oid, reverted: false }; + // A page a lane can make incomplete or unreadable leaves the merge unclassed-autonomous. + try { + const timeline = onePage(g.timelineItems, `timeline of #${number}`); + details.set(number, { ...base, ...classAtMerge(g, timeline), autonomous: autonomousOf(g, timeline) }); + } catch (error) { + if (!(error instanceof IncompleteError)) throw error; + details.set(number, { ...base, cls: null, reason: "a linked-issue, label or timeline page is unreadable or incomplete: class unknown", autonomous: true }); + } } return details.get(number); }; @@ -382,7 +436,13 @@ export function collect({ repo, since, through, laneLogins, gateChecks, aiReview let number = revert.number; if (number === undefined) { const known = bySha.get(revert.sha); - number = known ? known.number : gh(`repos/${repo}/commits/${revert.sha}/pulls`).find((pr) => pr.merged_at)?.number; + try { + number = known ? known.number : gh(`repos/${repo}/commits/${revert.sha}/pulls`).find((pr) => pr.merged_at)?.number; + } catch (error) { + if (!(error instanceof UnresolvableError)) throw error; + unresolvedReverts.push({ target: revert.sha, reason: "no commit found for this SHA" }); + continue; + } if (number === undefined) { unresolvedReverts.push({ target: revert.sha, reason: "no merged pull request for this commit" }); continue; @@ -395,7 +455,7 @@ export function collect({ repo, since, through, laneLogins, gateChecks, aiReview try { target = record(number); } catch (error) { - if (!(error instanceof NotMergedError)) throw error; + if (!(error instanceof UnresolvableError)) throw error; unresolvedReverts.push({ target: number, reason: "not a merged pull request" }); continue; } diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs index ebefbeb31b..74fa252a95 100644 --- a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs @@ -7,6 +7,7 @@ import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { after, test } from "node:test"; import { fileURLToPath } from "node:url"; +import * as publisher from "./publish-promotion-evidence.mjs"; import { collect, PublishError, publish } from "./publish-promotion-evidence.mjs"; const HERE = dirname(fileURLToPath(import.meta.url)); @@ -117,6 +118,11 @@ function github({ pulls = [], commits = [], checkRuns = {}, statuses = {}, issue const runs = runsFor(parseInt(m[1], 16)).filter((r) => /[?&]filter=all\b/.test(rest) || !r.superseded); return { total_count: runs.length, check_runs: page(runs, rest) }; } + if ((m = /^commits\/([0-9a-f]{40})\/pulls$/.exec(rest))) { + // A SHA no commit has: the recorded 422, as realGh hands it to ghFailure. + if (!branch.includes(m[1])) return publisher.ghFailure(path, undefined, REC.commitNotFound.stdout, REC.commitNotFound.stderr); + return pulls.filter((p) => p.graph.mergeCommit.oid === m[1]).map((p) => ({ number: p.graph.number, merged_at: p.graph.mergedAt })); + } if ((m = /^commits\/([0-9a-f]{40})\/statuses\?/.exec(rest))) return page(statuses[parseInt(m[1], 16)] ?? [], rest); if ((m = /^issues\/(\d+)\/events\?/.exec(rest))) return page(issueEvents[m[1]] ?? [], rest); if ((m = /^actions\/workflows\/([\w.-]+)\/runs\?event=(\w+)&branch=main&created=%3E%3D[^&]+&/.exec(rest))) { @@ -162,9 +168,12 @@ const wfRun = (id, event, sha, created, { conclusion = "success", status = "comp const gateEvents = (at, ...cells) => cells.map((cell) => ({ cell, event: "gate-failure", at })); // A green lane dispatch run and a green canary run on `tip`: check() calls each merge they contain // verified. -const TIP = shaOf(0xf1f1); -const verifiedOn = (tip, at) => ({ [FULL]: [wfRun(9001, "workflow_dispatch", tip, at, { actor: LANE })], [CANARY]: [wfRun(9002, "push", tip, at)] }); -const mergeOrder = (pulls) => [...pulls].sort((a, b) => a.graph.mergedAt.localeCompare(b.graph.mergedAt)).map((p) => p.graph.mergeCommit.oid); +// Each merge gets them on its own merge commit `minutes` after merging (within the 6-hour dispatch +// deadline by default). +const verifiedEach = (pulls, minutes = 60) => ({ + [FULL]: pulls.map((p) => wfRun(900000 + p.graph.number, "workflow_dispatch", p.graph.mergeCommit.oid, later(p.graph.mergedAt, minutes), { actor: LANE })), + [CANARY]: pulls.map((p) => wfRun(800000 + p.graph.number, "push", p.graph.mergeCommit.oid, later(p.graph.mergedAt, minutes))), +}); test("a red full run on a later SHA containing two merges yields gate-failure for each", () => { const tip = shaOf(0xa001); @@ -192,13 +201,13 @@ test("a run created before the merge is not its evidence", () => { assert.deepEqual(result.events, gateEvents("2026-09-01T12:08:00Z", "C2-auto-merge")); }); -test("a post-merge run whose SHA is behind or diverged from the merge refuses the publish", () => { +test("a post-merge run behind the merge is not its evidence; a diverged one refuses the publish", () => { const other = shaOf(0xa002); - const behind = { pulls: [pr(1310, "2026-09-02T00:00:00Z", { labels: [MECHANICAL] })], history: [other, shaOf(1310)] }; const red = { [FULL]: [wfRun(2, "schedule", other, "2026-09-03T00:00:00Z", { conclusion: "failure" })] }; - assert.throws(() => run({ ...behind, workflowRuns: red }), (error) => error instanceof PublishError && /behind/.test(error.message)); + const behind = run({ pulls: [pr(1310, "2026-09-02T00:00:00Z", { labels: [MECHANICAL] })], history: [other, shaOf(1310)], workflowRuns: red }); + assert.deepEqual(behind.events, []); const diverged = { pulls: [pr(1310, "2026-09-02T00:00:00Z", { labels: [MECHANICAL] })], history: [shaOf(1310), other], diverged: [other] }; - assert.throws(() => run({ ...diverged, workflowRuns: red }), /diverged/); + assert.throws(() => run({ ...diverged, workflowRuns: red }), (error) => error instanceof PublishError && /diverged/.test(error.message)); }); test("a merge that is an ancestor of a same-time merge listed before it is still covered by a later run", () => { @@ -219,16 +228,59 @@ test("a post-merge run created after --through still counts", () => { assert.deepEqual(result.events, gateEvents("2026-10-06T00:08:00Z", "C2-auto-merge")); }); -test("a merge is gate-passed only when a green lane dispatch and a green canary both contain it", () => { +test("a merge is gate-passed only when a green lane dispatch within 6 hours and a green canary both contain it", () => { const pulls = () => [pr(1404, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] })]; - const history = [shaOf(1404), TIP]; const gateOf = (result) => termOf(result, "C2-auto-merge", "deterministic-gate pass").detail; - assert.deepEqual(gateOf(run({ pulls: pulls() })), { passed: 0, failed: 0, unsettled: 1, total: 1 }); - const dispatchOnly = { [FULL]: verifiedOn(TIP, "2026-09-02T00:00:00Z")[FULL] }; - assert.deepEqual(gateOf(run({ pulls: pulls(), history, workflowRuns: dispatchOnly })), { passed: 0, failed: 0, unsettled: 1, total: 1 }); - const canaryOnly = { [CANARY]: verifiedOn(TIP, "2026-09-02T00:00:00Z")[CANARY] }; - assert.deepEqual(gateOf(run({ pulls: pulls(), history, workflowRuns: canaryOnly })), { passed: 0, failed: 0, unsettled: 1, total: 1 }); - assert.deepEqual(gateOf(run({ pulls: pulls(), history, workflowRuns: verifiedOn(TIP, "2026-09-02T00:00:00Z") })), { passed: 1, failed: 0, unsettled: 0, total: 1 }); + const unsettled = { passed: 0, failed: 0, unsettled: 1, total: 1 }; + assert.deepEqual(gateOf(run({ pulls: pulls() })), unsettled); + assert.deepEqual(gateOf(run({ pulls: pulls(), workflowRuns: { [FULL]: verifiedEach(pulls())[FULL] } })), unsettled); + assert.deepEqual(gateOf(run({ pulls: pulls(), workflowRuns: { [CANARY]: verifiedEach(pulls())[CANARY] } })), unsettled); + assert.deepEqual(gateOf(run({ pulls: pulls(), workflowRuns: verifiedEach(pulls()) })), { passed: 1, failed: 0, unsettled: 0, total: 1 }); + assert.deepEqual(gateOf(run({ pulls: pulls(), workflowRuns: verifiedEach(pulls(), 7 * 60) })), unsettled); +}); + +test("a bogus `This reverts commit` SHA is listed, not fatal", () => { + const bogus = "0123456789abcdef0123456789abcdef01234567"; + const result = run({ + pulls: [pr(1408, "2026-09-01T00:00:00Z", { merger: "kyle-sexton" })], + commits: [{ ...REC.commit, sha: shaOf(1408), commit: { message: `Revert "x"\n\nThis reverts commit ${bogus}.`, committer: { date: "2026-09-01T00:00:00Z" } } }], + }); + assert.deepEqual(result.unresolvedReverts, [{ target: bogus, reason: "no commit found for this SHA" }]); +}); + +test("gh failures: a missing PR is an answer, a missing commit is unresolvable, anything else fails", () => { + assert.equal(typeof publisher.ghFailure, "function"); + assert.deepEqual(publisher.ghFailure("graphql", true, JSON.stringify(REC.pullNotFound), ""), REC.pullNotFound); + assert.throws(() => publisher.ghFailure("x/pulls", false, REC.commitNotFound.stdout, REC.commitNotFound.stderr), publisher.UnresolvableError); + const fails = (body) => + assert.throws( + () => publisher.ghFailure("graphql", true, JSON.stringify(body), "gh: Could not resolve"), + (error) => error instanceof PublishError && !(error instanceof publisher.UnresolvableError), + ); + fails(REC.repoNotFound); + // Each half of the PR-NOT_FOUND test alone: a null pullRequest with a repository-level error, and + // a pullRequest-path error with the repository gone. + fails({ ...REC.pullNotFound, errors: REC.repoNotFound.errors }); + fails({ ...REC.repoNotFound, errors: REC.pullNotFound.errors }); +}); + +test("an incomplete page on a lane PR leaves it unclassed, not a refused publish", () => { + const paged = (patch) => run({ pulls: [pr(1409, "2026-09-01T00:00:00Z", { labels: [MECHANICAL], linked: ISSUE_6007, patch })] }); + const reason = "a linked-issue, label or timeline page is unreadable or incomplete: class unknown"; + for (const patch of [{ closingIssuesReferences: page1(ISSUE_6007, true) }, { timelineItems: page1([], true) }, { labels: page1([{ name: MECHANICAL }], true) }]) { + assert.deepEqual(paged(patch).unclassedAutonomous, [{ number: 1409, reason }]); + } + const issueLabelsPaged = [{ ...ISSUE_6007[0], labels: { ...ISSUE_6007[0].labels, pageInfo: { hasNextPage: true } } }]; + const noLabels = [{ number: 6007 }]; + for (const nodes of [issueLabelsPaged, noLabels]) { + assert.deepEqual(paged({ closingIssuesReferences: page1(nodes) }).unclassedAutonomous, [{ number: 1409, reason }]); + } +}); + +test("an unparsable date refuses the publish", () => { + const result = () => + run({ pulls: [pr(1410, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] })], workflowRuns: { [CANARY]: [wfRun(13, "push", shaOf(1410), "not-a-date", { updated: "not-a-date" })] } }); + assert.throws(result, (error) => error instanceof PublishError && /date/.test(error.message)); }); test("a PR merged into a branch other than the default is not read", () => { @@ -302,12 +354,12 @@ test("a red full run awaiting its re-run is unsettled for 6 hours, then a failur test("only the named gate checks and status contexts are gate evidence", () => { const other = (name, conclusion) => ({ ...REC.checkRun, name, conclusion, completed_at: "2026-09-01T00:10:00Z" }); + const pulls = [pr(1201, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] })]; const result = run({ - pulls: [pr(1201, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] })], + pulls, checkRuns: { 1201: [REC.checkRun, other("dispatch", "failure"), other("resolve-conflicts / run", "cancelled")] }, statuses: { 1201: [{ ...REC.status, context: "GitGuardian", state: "error" }] }, - history: [shaOf(1201), TIP], - workflowRuns: verifiedOn(TIP, "2026-09-02T00:00:00Z"), + workflowRuns: verifiedEach(pulls), }); assert.deepEqual(result.events, []); assert.deepEqual(termOf(result, "C2-auto-merge", "deterministic-gate pass").detail, { passed: 1, failed: 0, unsettled: 0, total: 1 }); @@ -389,7 +441,7 @@ test("a missed-blocking label a human applied to a C3 PR yields verification-div test("20 autonomous C2 merges over 14+ days pass the C2 term and 19 fail it", () => { const merges = (count) => Array.from({ length: count }, (_, i) => pr(600 + i, `2026-09-${String(i + 1).padStart(2, "0")}T00:00:00Z`, { labels: [MECHANICAL] })); const pulls = merges(20); - const twenty = run({ pulls, history: [...mergeOrder(pulls), TIP], workflowRuns: verifiedOn(TIP, "2026-09-21T00:00:00Z") }); + const twenty = run({ pulls, workflowRuns: verifiedEach(pulls) }); assert.equal(termOf(twenty, "C2-auto-merge", "autonomous C2 completions").pass, true); assert.equal(twenty.cells["C2-auto-merge"].pass, true); const nineteen = run({ pulls: merges(19) }); @@ -475,13 +527,6 @@ test("--through is clamped to now", () => { assert.ok(Date.parse(result.through) <= Date.now()); }); -test("a linked-issue read with another page fails closed", () => { - const paged = (patch) => ({ pulls: [pr(908, "2026-09-01T00:00:00Z", { linked: ISSUE_6007, patch })] }); - assert.throws(() => run(paged({ closingIssuesReferences: page1(ISSUE_6007, true) })), (error) => error instanceof PublishError && /hasNextPage/.test(error.message)); - const labelsPaged = [{ ...ISSUE_6007[0], labels: { ...ISSUE_6007[0].labels, pageInfo: { hasNextPage: true } } }]; - assert.throws(() => run(paged({ closingIssuesReferences: page1(labelsPaged) })), /hasNextPage/); -}); - test("an issue unlinked after the merge still classes the merge; one linked after it does not", () => { const result = run({ pulls: [ @@ -530,14 +575,14 @@ test("a merge a bot or nobody enqueued is autonomous even under a human merged_b test("a cancelled gate run counts as a failure unless a later run of the same check succeeded", () => { const cancelled = (name, at) => ({ ...REC.checkRun, name, conclusion: "cancelled", completed_at: at }); + const pulls = [pr(1110, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] }), pr(1111, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] })]; const result = run({ - pulls: [pr(1110, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] }), pr(1111, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] })], + pulls, checkRuns: { 1110: [cancelled("lint-repo", "2026-09-01T00:10:00Z")], 1111: [cancelled("lint-repo", "2026-09-01T00:10:00Z"), { ...REC.checkRun, completed_at: "2026-09-01T00:20:00Z" }], }, - history: [shaOf(1110), shaOf(1111), TIP], - workflowRuns: verifiedOn(TIP, "2026-09-02T00:00:00Z"), + workflowRuns: verifiedEach(pulls), }); assert.deepEqual(result.events, [{ cell: "C2-auto-merge", event: "gate-failure", at: "2026-09-01T00:10:00Z" }]); assert.deepEqual(termOf(result, "C2-auto-merge", "deterministic-gate pass").detail, { passed: 1, failed: 1, unsettled: 0, total: 2 }); From ddf4d7250f2395b8106fb8caf13a8660777d9798 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Tue, 13 Oct 2026 01:52:27 -0400 Subject: [PATCH 09/18] fix(source-control): mirror the dispatch deadline and pin read-back - Where babysit_postmerge.py check() files a Failure, emit a gate-failure: no dispatch run by a lane identity within the 6-hour deadline (at the deadline), and for a lane-enqueued merge the pin read-back: no unedited lane pin once the deadline passed (at the deadline) or a pinned SHA that is not headRefOid (at the merge). The header maps each Failure to its event. - A lane-enqueued merge passes the gate only when its pin matches. - Bound a revert's PR number to a positive GraphQL Int before any lookup, without parsing long digit strings into a Number; others go to unresolvedReverts. - Rewrap the publisher section of promotion-evidence-bootstrap.md. Refs: #4588 Co-Authored-By: Claude Opus 5.5 --- ...8-promotion-evidence-publisher-3634507d.md | 2 +- .../reference/promotion-evidence-bootstrap.md | 27 +++--- .../publish-promotion-evidence/recorded.json | 25 +++++ .../scripts/publish-promotion-evidence.mjs | 94 +++++++++++++++++-- .../publish-promotion-evidence.test.mjs | 85 +++++++++++++++-- 5 files changed, 199 insertions(+), 34 deletions(-) diff --git a/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md b/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md index 3eab5aca58..b05f84be2c 100644 --- a/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md +++ b/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md @@ -4,4 +4,4 @@ bump: minor ### Added -- `babysit-loop` ships `scripts/publish-promotion-evidence.mjs`, an operator-side publisher for the promotion-evidence events file (#4588). It reads GitHub only, through read-only `gh api` calls, and emits the `{cell, event, at}` contrary events the security-binding checker reads (`reverted-merge`, `gate-failure`, and `verification-divergence` for a missed-blocking finding a human confirmed) plus a C3 promotion-evidence report: each promotable cell's predicate terms with counts, window and pass or fail. Report mode, the default, writes nothing. `--out ` writes `evidence.json` and an `evidence.meta.json` sidecar (`generated_at`, `through`, `sha256`, `prev_sha256`) by temp file and rename, and refuses a set that drops a previously published event or an existing file that does not match its sidecar. Work classes are read from the labels of the PR and of the issues it linked at merge, as of the merge. A merge counts as autonomous when no human enqueued it or a `--lane-login` identity merged it. An autonomous merge with no readable class is listed as unclassed and its contrary events count against every cell. Only the post-merge gate counts toward `gate-failure`. That gate is the full and canary workflow runs on any later commit containing the merge, read as `babysit_postmerge.py` reads them, plus the check runs and statuses named by the required `--gate-check` on the merge commit. A cancelled run counts unless a later run of that check succeeded, and a red full run covering several merges counts against each. Where it runs and under which identity is not decided yet, and the resolver does not read the sidecar; merge authority is unchanged. +- `babysit-loop` ships `scripts/publish-promotion-evidence.mjs`, an operator-side publisher for the promotion-evidence events file (#4588). It reads GitHub only, through read-only `gh api` calls, and emits the `{cell, event, at}` contrary events the security-binding checker reads (`reverted-merge`, `gate-failure`, and `verification-divergence` for a missed-blocking finding a human confirmed) plus a C3 promotion-evidence report: each promotable cell's predicate terms with counts, window and pass or fail. Report mode, the default, writes nothing. `--out ` writes `evidence.json` and an `evidence.meta.json` sidecar (`generated_at`, `through`, `sha256`, `prev_sha256`) by temp file and rename, and refuses a set that drops a previously published event or an existing file that does not match its sidecar. Work classes are read from the labels of the PR and of the issues it linked at merge, as of the merge. A merge counts as autonomous when no human enqueued it or a `--lane-login` identity merged it. An autonomous merge with no readable class is listed as unclassed and its contrary events count against every cell. Only the post-merge gate counts toward `gate-failure`. That gate is the full and canary workflow runs on any later commit containing the merge, read as `babysit_postmerge.py` reads them, plus the check runs and statuses named by the required `--gate-check` on the merge commit. A cancelled run counts unless a later run of that check succeeded, and a red full run covering several merges counts against each. As in `babysit_postmerge.py`, a merge with no lane dispatch run within 6 hours, or a lane-enqueued merge whose pin comment is missing, edited or names another head, is a gate failure. Where it runs and under which identity is not decided yet, and the resolver does not read the sidecar; merge authority is unchanged. diff --git a/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md b/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md index 7cd3caac7e..bdfa0dee42 100644 --- a/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md +++ b/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md @@ -147,15 +147,16 @@ binding. #### The publisher [`scripts/publish-promotion-evidence.mjs`](../scripts/publish-promotion-evidence.mjs) builds the -events file from GitHub alone, through read-only `gh api` calls: merged pull requests classed by the -`work-class:` labels their linked issues and the pull request carried at merge time, which of them -merged autonomously (no human enqueued them, or a `--lane-login` identity merged them), reverts of -those merges, the post-merge gate (the full and canary workflow runs on any later commit that -contains the merge, read as `babysit-prs/scripts/babysit_postmerge.py` reads them, and the -checks named by the required `--gate-check` on the merge commit), and the missed-blocking -label a human applied. An autonomous merge whose class cannot be read is listed as unclassed, and -its contrary events count against every promotable cell. Its header comment states what counts as -each event, including the judgment mappings and how to set `--lane-login` for merge-queue merges. +events file from GitHub alone, through read-only `gh api` calls: merged pull requests classed by +the `work-class:` labels their linked issues and the pull request carried at merge time, which of +them merged autonomously (no human enqueued them, or a `--lane-login` identity merged them), +reverts of those merges, the post-merge gate (the full and canary workflow runs on any later +commit that contains the merge, the dispatch deadline and the pin read-back, read as +`babysit-prs/scripts/babysit_postmerge.py` reads them, and the checks named by the required +`--gate-check` on the merge commit), and the missed-blocking label a human applied. An autonomous +merge whose class cannot be read is listed as unclassed, and its contrary events count against +every promotable cell. Its header comment states what counts as each event, including the +judgment mappings and how to set `--lane-login` for merge-queue merges. - **Report mode** (no `--out`) prints the events array and the C3 promotion-evidence report to stdout and writes nothing. The report gives each promotable cell's predicate terms from @@ -169,10 +170,10 @@ each event, including the judgment mappings and how to set `--lane-login` for me Run it as the operator, never from the lane, and point `--out` at a directory the lane cannot write. The lane identity must not hold `actions:write` on the repository: with it, the lane could -delete a red workflow run before the publisher reads it. Which host runs it, under which identity and on what schedule is not decided yet: it waits -on the open decision about the single merge authority -([#4588](https://github.com/melodic-software/claude-code-plugins/issues/4588)). Until then the -resolver does not read the sidecar. +delete a red workflow run before the publisher reads it. Which host runs it, under which identity +and on what schedule is not decided yet: it waits on the open decision about the single merge +authority ([#4588](https://github.com/melodic-software/claude-code-plugins/issues/4588)). Until +then the resolver does not read the sidecar. ### `promotion_evidence_checker` diff --git a/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json b/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json index 46c5f849af..bf6cef188d 100644 --- a/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json +++ b/plugins/source-control/skills/babysit-loop/scripts/fixtures/publish-promotion-evidence/recorded.json @@ -58,6 +58,7 @@ "repository": { "pullRequest": { "number": 7063, + "headRefOid": "b6888664c45a19abc91c9cd819770406a6e7616d", "mergedAt": "2026-10-13T01:47:44Z", "lastEditedAt": "2026-10-13T01:38:02Z", "mergeCommit": { @@ -145,6 +146,30 @@ "repo": { "default_branch": "main" }, + "commentsPage": { + "data": { + "repository": { + "pullRequest": { + "comments": { + "pageInfo": { + "hasNextPage": true, + "endCursor": "Y3Vyc29yOnYyOpHPAAAAAWv4HUQ=" + }, + "nodes": [ + { + "author": { + "__typename": "User", + "login": "kyle-sexton" + }, + "body": "`ci-status` is red because this PR is a draft, not because of the change.", + "lastEditedAt": null + } + ] + } + } + } + } + }, "connectedEventFromCliCli14616": { "__typename": "ConnectedEvent", "createdAt": "2026-10-07T08:24:54Z", diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs index 7eb0abca8a..d2b1daff99 100644 --- a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs @@ -157,7 +157,7 @@ function ms(value, what) { const PULL_QUERY = "query($owner:String!,$name:String!,$number:Int!){repository(owner:$owner,name:$name){pullRequest(number:$number){" + - "number mergedAt lastEditedAt mergeCommit{oid} mergedBy{__typename login} labels(first:100){pageInfo{hasNextPage} nodes{name}} " + + "number headRefOid mergedAt lastEditedAt mergeCommit{oid} mergedBy{__typename login} labels(first:100){pageInfo{hasNextPage} nodes{name}} " + "closingIssuesReferences(first:50){pageInfo{hasNextPage} nodes{number labels(first:100){pageInfo{hasNextPage} nodes{name}}}} " + "timelineItems(first:100,itemTypes:[CONNECTED_EVENT,DISCONNECTED_EVENT,ADDED_TO_MERGE_QUEUE_EVENT]){pageInfo{hasNextPage} nodes{__typename " + "... on ConnectedEvent{createdAt subject{__typename ... on Issue{number}}} ... on DisconnectedEvent{createdAt subject{__typename ... on Issue{number}}} " + @@ -219,14 +219,57 @@ function eventKey(event) { // that are not its evidence; a compare that is behind means not contained. Only a verified merge // with nothing red or pending passes the gate, and only a lane dispatch created within // DISPATCH_DEADLINE of the merge verifies it, so a merge with no timely lane dispatch run -// (check()'s 6-hour failure) stays unsettled. Departures, all toward a failure: a compare that is -// diverged or unknown refuses the publish (check() reads it as not containing); a red full run whose re-run has not come counts as red -// once DISPATCH_DEADLINE has passed since it completed (check() waits on it); a red run covering -// several merges is a failure of each. +// (check()'s 6-hour failure) is not passed. Departures, all toward a failure: a compare that is +// diverged or unknown refuses the publish (check() reads it as not containing); a red full run +// whose re-run has not come counts as red once DISPATCH_DEADLINE has passed since it completed +// (check() waits on it); a red run covering several merges is a failure of each. +// +// Where check() files a Failure, this emits a gate-failure, so demotion counts match: a red full +// or canary run (at the run's update time); no dispatch run by a lane identity created within +// DISPATCH_DEADLINE of the merge, once that deadline has passed (at the deadline); and, for a +// merge a lane identity enqueued, the pin read-back (read_pin): no unedited lane pin once the +// deadline has passed (at the deadline), or a pinned SHA that is not headRefOid (at the merge). const FULL_RUN_WORKFLOW = "pr-require-checks.yml"; const CANARY_WORKFLOW = "post-merge-detect-silent-revert.yml"; const DISPATCH_DEADLINE_MS = 6 * 3_600_000; +function isoSeconds(msValue) { + return new Date(msValue).toISOString().replace(/\.000Z$/, "Z"); +} + +// babysit_postmerge.py read_pin: the newest comment by the identity whose first line is the pin +// marker, read through at most PIN_PAGE_LIMIT pages; none, an edited one, or a longer thread is +// no pin. +const PIN_RE = /^$/; +const PIN_QUERY = + "query($owner:String!,$name:String!,$number:Int!,$c:String){repository(owner:$owner,name:$name){pullRequest(number:$number){" + + "comments(first:100,after:$c){pageInfo{hasNextPage endCursor} nodes{author{__typename login} body lastEditedAt}}}}}"; +const PIN_PAGE_LIMIT = 30; + +function readPin(gh, owner, name, number, laneExact) { + // GraphQL spells an App `app` with __typename Bot; REST and --lane-login spell it `app[bot]`. + const author = (a) => { + const login = String(a?.login ?? "").trim().toLowerCase(); + return a?.__typename === "Bot" && login && !login.endsWith("[bot]") ? `${login}[bot]` : login; + }; + const firstLine = (text) => String(text ?? "").trimStart().split("\n", 1)[0].trim(); + const pins = []; + let cursor = null; + for (let page = 0; page < PIN_PAGE_LIMIT; page += 1) { + const variables = { owner, name, number, ...(cursor ? { c: cursor } : {}) }; + const comments = gh("graphql", { query: PIN_QUERY, variables })?.data?.repository?.pullRequest?.comments; + if (!Array.isArray(comments?.nodes) || typeof comments.pageInfo?.hasNextPage !== "boolean") throw new PublishError(`#${number} pin: no comment page`); + pins.push(...comments.nodes.filter((c) => laneExact.has(author(c?.author)) && PIN_RE.test(firstLine(c?.body)))); + if (!comments.pageInfo.hasNextPage) { + const last = pins.at(-1); + return last && !last.lastEditedAt ? PIN_RE.exec(firstLine(last.body))[1] : null; + } + cursor = comments.pageInfo.endCursor; + if (typeof cursor !== "string" || !cursor) throw new PublishError(`#${number} pin: a next page with no cursor`); + } + return null; +} + function postMergeRuns(gh, repo, branch, merges, { lane, throughMs }) { if (merges.length === 0) return () => ({ failures: [], unsettled: false, verified: false }); const floor = Math.min(...merges.map((m) => ms(m.mergedAt, `#${m.number} mergedAt`))); @@ -294,7 +337,12 @@ function postMergeRuns(gh, repo, branch, merges, { lane, throughMs }) { ); const canary = dispatch && canaries.find((run) => outcome(run, false).state === "green" && relevant(run)); const verifiedAt = canary ? Math.max(dispatch.createdMs, canary.createdMs) : Number.POSITIVE_INFINITY; + const deadline = mergedMs + DISPATCH_DEADLINE_MS; + const timely = fulls.some((run) => run.event === "workflow_dispatch" && run.createdMs <= deadline && lane.has(loginKey(run.actor?.login)) && relevant(run)); const result = { failures: [], unsettled: false, verified: Boolean(canary) }; + if (!timely && throughMs > deadline) { + result.failures.push({ check: "dispatch deadline", conclusion: "no dispatch run by a lane identity within 6 h", at: isoSeconds(deadline) }); + } for (const [runs, attributable] of [ [fulls, true], [canaries, false], @@ -381,7 +429,8 @@ export function collect({ repo, since, through, laneLogins, gateChecks, aiReview const autonomousOf = (g, timeline) => { const queued = timeline.filter((e) => e.__typename === "AddedToMergeQueueEvent").sort((a, b) => ms(a.createdAt, "a merge-queue event") - ms(b.createdAt, "a merge-queue event")); const actors = queued.length > 0 ? [queued.at(-1).actor, queued.at(-1).enqueuer] : [g.mergedBy]; - return actors.some((actor) => !human(actor)) || lane.has(loginKey(g.mergedBy?.login)); + const byIdentity = actors.some((actor) => actor && lane.has(loginKey(actor.login))); + return { autonomous: actors.some((actor) => !human(actor)) || lane.has(loginKey(g.mergedBy?.login)), byIdentity }; }; const details = new Map(); @@ -390,14 +439,15 @@ export function collect({ repo, since, through, laneLogins, gateChecks, aiReview const g = gh("graphql", { query: PULL_QUERY, variables: { owner, name, number } })?.data?.repository?.pullRequest; if (!g?.mergedAt) throw new UnresolvableError(`#${number} is not a merged pull request`); ms(g.mergedAt, `#${number} mergedAt`); - const base = { number, mergedAt: g.mergedAt, sha: g.mergeCommit?.oid, reverted: false }; + const base = { number, mergedAt: g.mergedAt, sha: g.mergeCommit?.oid, head: g.headRefOid, reverted: false }; // A page a lane can make incomplete or unreadable leaves the merge unclassed-autonomous. try { const timeline = onePage(g.timelineItems, `timeline of #${number}`); - details.set(number, { ...base, ...classAtMerge(g, timeline), autonomous: autonomousOf(g, timeline) }); + details.set(number, { ...base, ...classAtMerge(g, timeline), ...autonomousOf(g, timeline) }); } catch (error) { if (!(error instanceof IncompleteError)) throw error; - details.set(number, { ...base, cls: null, reason: "a linked-issue, label or timeline page is unreadable or incomplete: class unknown", autonomous: true }); + // Who enqueued it is unknown too, so it also owes a pin. + details.set(number, { ...base, cls: null, reason: "a linked-issue, label or timeline page is unreadable or incomplete: class unknown", autonomous: true, byIdentity: true }); } } return details.get(number); @@ -413,7 +463,11 @@ export function collect({ repo, since, through, laneLogins, gateChecks, aiReview const revertsOf = (text) => { const targets = []; for (const m of (text ?? "").matchAll(/\bReverts\s+(?:([\w.-]+\/[\w.-]+))?#(\d+)/g)) { - if (!m[1] || m[1].toLowerCase() === own) targets.push({ number: Number(m[2]) }); + // A GraphQL Int is at most 2^31 - 1; a longer digit string is never parsed into a Number. + if (!m[1] || m[1].toLowerCase() === own) { + const valid = /^[1-9]\d{0,9}$/.test(m[2]) && BigInt(m[2]) <= 2147483647n; + targets.push(valid ? { number: Number(m[2]) } : { invalid: m[2] }); + } } for (const m of (text ?? "").matchAll(/This reverts commit ([0-9a-f]{40})/g)) targets.push({ sha: m[1] }); return targets; @@ -433,6 +487,10 @@ export function collect({ repo, since, through, laneLogins, gateChecks, aiReview const firstRevert = new Map(); for (const revert of reverts) { if (!inWindow(revert.at)) continue; + if (revert.invalid !== undefined) { + unresolvedReverts.push({ target: revert.invalid, reason: "not a pull request number (a positive GraphQL Int)" }); + continue; + } let number = revert.number; if (number === undefined) { const known = bySha.get(revert.sha); @@ -466,6 +524,19 @@ export function collect({ repo, since, through, laneLogins, gateChecks, aiReview } const judgeWorkflows = postMergeRuns(gh, repo, branch, tracked.filter((c) => c.autonomous), { lane, throughMs }); + const laneExact = new Set(laneLogins.map((login) => String(login).trim().toLowerCase())); + + // check()'s read-back for a merge a lane identity enqueued. + const readBack = (r) => { + const deadline = ms(r.mergedAt, `#${r.number} mergedAt`) + DISPATCH_DEADLINE_MS; + const pinned = readPin(gh, owner, name, r.number, laneExact); + if (pinned === null) { + return throughMs > deadline + ? { failure: { check: "pin read-back", conclusion: "no unedited pin comment by a lane identity", at: isoSeconds(deadline) } } + : { pending: true }; + } + return pinned === r.head ? {} : { failure: { check: "pin read-back", conclusion: `headRefOid ${r.head} is not the pinned ${pinned}`, at: r.mergedAt } }; + }; // Post-merge gate on each autonomous merge commit: the --gate-check runs and status contexts. // A run that did not finish green or red (cancelled, action_required, stale) fails the gate @@ -473,6 +544,9 @@ export function collect({ repo, since, through, laneLogins, gateChecks, aiReview // Then the post-merge workflow runs on later SHAs containing the merge (postMergeRuns). for (const r of tracked.filter((c) => c.autonomous)) { const wf = judgeWorkflows(r); + const pin = r.byIdentity ? readBack(r) : {}; + if (pin.failure) wf.failures.push(pin.failure); + if (pin.pending) wf.unsettled = true; for (const failure of wf.failures) { for (const cell of cellsOf(r)) addEvent(cell, "gate-failure", failure.at); gateFailures.push({ number: r.number, ...failure }); diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs index 74fa252a95..88c52483ec 100644 --- a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs @@ -42,13 +42,14 @@ const link = (type, issue, at) => ({ ...LINK, __typename: type, createdAt: at, s function pr( number, mergedAt, - { labels = [], merger = LANE, title = REC.pullListItem.title, body = REC.pullListItem.body, linked = [], timeline = [], lastEditedAt = null, patch = {}, base = "main" } = {}, + { labels = [], merger = LANE, title = REC.pullListItem.title, body = REC.pullListItem.body, linked = [], timeline = [], lastEditedAt = null, patch = {}, base = "main", head = shaOf(number + 0x100000) } = {}, ) { const names = labels.map((name) => ({ name })); return { graph: { ...REC.pullGraph.data.repository.pullRequest, number, + headRefOid: head, mergedAt, lastEditedAt, mergeCommit: { oid: shaOf(number) }, @@ -84,7 +85,15 @@ const labelEvent = (event, name, at, login = "kyle-sexton", type = "User") => ({ // merge commits in merge order. // `diverged`: head SHAs a compare reports as diverged. A PR number not in `pulls` answers with the // recorded NOT_FOUND response. -function github({ pulls = [], commits = [], checkRuns = {}, statuses = {}, issueEvents = {}, reviews = {}, issues = {}, workflowRuns = {}, attempts = {}, history, diverged = [], bumpOnPage2 }) { +// The lane posts a pin comment on every PR it enqueues and dispatches a full run after every merge +// (babysit_merge.py --unattended, babysit_postmerge.py), so each PR gets a correct pin and a green +// lane dispatch run on its merge commit an hour after merging, unless `comments` sets that PR's +// comments or `noDefaultDispatch` is set; an explicit run with the same id replaces the default. +const pinComment = (sha) => ({ author: actorOf(LANE), body: `\nThe unattended merge path enqueued this pull request at head \`${sha}\`.`, lastEditedAt: null }); +function github({ pulls = [], commits = [], checkRuns = {}, statuses = {}, issueEvents = {}, reviews = {}, issues = {}, workflowRuns: explicitRuns = {}, attempts = {}, history, diverged = [], comments = {}, noDefaultDispatch = false, bumpOnPage2 }) { + const defaultDispatch = noDefaultDispatch ? [] : pulls.map((p) => wfRun(900000 + p.graph.number, "workflow_dispatch", p.graph.mergeCommit.oid, later(p.graph.mergedAt, 60), { actor: LANE })); + const explicitIds = new Set(Object.values(explicitRuns).flat().map((r) => r.id)); + const workflowRuns = { ...explicitRuns, [FULL]: [...defaultDispatch.filter((r) => !explicitIds.has(r.id)), ...(explicitRuns[FULL] ?? [])] }; const branch = history ?? [...pulls].sort((a, b) => a.graph.mergedAt.localeCompare(b.graph.mergedAt)).map((p) => p.graph.mergeCommit.oid); const page = (items, path) => { const n = Number(/[?&]page=(\d+)/.exec(path)[1]); @@ -96,6 +105,14 @@ function github({ pulls = [], commits = [], checkRuns = {}, statuses = {}, issue assert.deepEqual([graphql.variables.owner, graphql.variables.name], REPO.split("/")); assert.match(graphql.query, /^query\(/); assert.doesNotMatch(graphql.query, /mutation/); + if (/comments\(/.test(graphql.query)) { + const pull = pulls.find((p) => p.graph.number === graphql.variables.number); + const all = comments[graphql.variables.number] ?? [pinComment(pull.graph.headRefOid)]; + const n = Number(graphql.variables.c ?? 0); + const nodes = all.slice(n * 100, (n + 1) * 100); + const more = all.length > (n + 1) * 100; + return { data: { repository: { pullRequest: { comments: { ...REC.commentsPage.data.repository.pullRequest.comments, pageInfo: { hasNextPage: more, endCursor: more ? String(n + 1) : null }, nodes } } } } }; + } const found = pulls.find((p) => p.graph.number === graphql.variables.number); return found ? { data: { repository: { pullRequest: found.graph } } } : REC.pullNotFound; } @@ -228,15 +245,63 @@ test("a post-merge run created after --through still counts", () => { assert.deepEqual(result.events, gateEvents("2026-10-06T00:08:00Z", "C2-auto-merge")); }); -test("a merge is gate-passed only when a green lane dispatch within 6 hours and a green canary both contain it", () => { +const gateOf = (result, cell = "C2-auto-merge") => termOf(result, cell, "deterministic-gate pass").detail; +const PASSED = { passed: 1, failed: 0, unsettled: 0, total: 1 }; +const FAILED = { passed: 0, failed: 1, unsettled: 0, total: 1 }; +const UNSETTLED = { passed: 0, failed: 0, unsettled: 1, total: 1 }; + +test("a merge passes the gate only with a lane dispatch within 6 hours and a green canary; no timely dispatch is a gate failure", () => { const pulls = () => [pr(1404, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] })]; - const gateOf = (result) => termOf(result, "C2-auto-merge", "deterministic-gate pass").detail; - const unsettled = { passed: 0, failed: 0, unsettled: 1, total: 1 }; - assert.deepEqual(gateOf(run({ pulls: pulls() })), unsettled); - assert.deepEqual(gateOf(run({ pulls: pulls(), workflowRuns: { [FULL]: verifiedEach(pulls())[FULL] } })), unsettled); - assert.deepEqual(gateOf(run({ pulls: pulls(), workflowRuns: { [CANARY]: verifiedEach(pulls())[CANARY] } })), unsettled); - assert.deepEqual(gateOf(run({ pulls: pulls(), workflowRuns: verifiedEach(pulls()) })), { passed: 1, failed: 0, unsettled: 0, total: 1 }); - assert.deepEqual(gateOf(run({ pulls: pulls(), workflowRuns: verifiedEach(pulls(), 7 * 60) })), unsettled); + const noDispatch = gateEvents("2026-09-01T06:00:00Z", "C2-auto-merge"); + const bare = run({ pulls: pulls(), noDefaultDispatch: true }); + assert.deepEqual([gateOf(bare), bare.events], [FAILED, noDispatch]); + assert.deepEqual(bare.gateFailures, [{ number: 1404, check: "dispatch deadline", conclusion: "no dispatch run by a lane identity within 6 h", at: "2026-09-01T06:00:00Z" }]); + const dispatchOnly = run({ pulls: pulls(), workflowRuns: { [FULL]: verifiedEach(pulls())[FULL] } }); + assert.deepEqual([gateOf(dispatchOnly), dispatchOnly.events], [UNSETTLED, []]); + const canaryOnly = run({ pulls: pulls(), noDefaultDispatch: true, workflowRuns: { [CANARY]: verifiedEach(pulls())[CANARY] } }); + assert.deepEqual([gateOf(canaryOnly), canaryOnly.events], [FAILED, noDispatch]); + assert.deepEqual(gateOf(run({ pulls: pulls(), workflowRuns: verifiedEach(pulls()) })), PASSED); + const late = run({ pulls: pulls(), workflowRuns: verifiedEach(pulls(), 7 * 60) }); + assert.deepEqual([gateOf(late), late.events], [FAILED, noDispatch]); +}); + +test("a lane-enqueued merge passes only when its unedited lane pin names headRefOid; otherwise a gate failure", () => { + const pulls = () => [pr(1501, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] })]; + const head = shaOf(1501 + 0x100000); + const withComments = (list) => run({ pulls: pulls(), workflowRuns: verifiedEach(pulls()), comments: { 1501: list } }); + assert.deepEqual(gateOf(withComments([pinComment(head)])), PASSED); + const filler = Array.from({ length: 120 }, () => REC.commentsPage.data.repository.pullRequest.comments.nodes[0]); + assert.deepEqual(gateOf(withComments([...filler, pinComment(head)])), PASSED); + const noPin = gateEvents("2026-09-01T06:00:00Z", "C2-auto-merge"); + for (const list of [[], [{ ...pinComment(head), lastEditedAt: "2026-09-01T00:30:00Z" }], [{ ...pinComment(head), author: actorOf("kyle-sexton") }]]) { + const result = withComments(list); + assert.deepEqual([gateOf(result), result.events], [FAILED, noPin]); + } + const mismatch = withComments([pinComment(shaOf(0xbad))]); + assert.deepEqual([gateOf(mismatch), mismatch.events], [FAILED, gateEvents("2026-09-01T00:00:00Z", "C2-auto-merge")]); + // Before the deadline a missing pin is pending, as in check(). + const fresh = [pr(1502, "2026-10-04T22:00:00Z", { labels: [MECHANICAL] })]; + const pending = run({ pulls: fresh, workflowRuns: verifiedEach(fresh, 30), comments: { 1502: [] } }); + assert.deepEqual([gateOf(pending), pending.events], [UNSETTLED, []]); + // A merge another bot enqueued is autonomous but not the lane's: it has no pin to read back. + const bot = [pr(1503, "2026-09-01T00:00:00Z", { labels: [MECHANICAL], merger: "kyle-sexton", timeline: [queued("2026-08-31T23:00:00Z", null, "some-app[bot]")] })]; + assert.deepEqual(gateOf(run({ pulls: bot, workflowRuns: verifiedEach(bot), comments: { 1503: [] } })), PASSED); +}); + +test("a revert number that is not a GraphQL Int is listed without a lookup", () => { + const result = run({ + pulls: [ + pr(1601, "2026-09-01T00:00:00Z", { body: `Reverts ${REPO}#99999999999999999999`, merger: "kyle-sexton" }), + pr(1602, "2026-09-01T00:00:00Z", { body: `Reverts ${REPO}#2147483648`, merger: "kyle-sexton" }), + pr(1603, "2026-09-01T00:00:00Z", { body: `Reverts ${REPO}#0`, merger: "kyle-sexton" }), + ], + }); + const reason = "not a pull request number (a positive GraphQL Int)"; + assert.deepEqual(result.unresolvedReverts, [ + { target: "99999999999999999999", reason }, + { target: "2147483648", reason }, + { target: "0", reason }, + ]); }); test("a bogus `This reverts commit` SHA is listed, not fatal", () => { From 48df0180083421c3a16ee0a3308617999c18ca0c Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Tue, 13 Oct 2026 02:27:22 -0400 Subject: [PATCH 10/18] docs(source-control): record the actions:write decision as open and test the pin page cap - Test that a pin past the 30th comment page is no pin, so a lane-enqueued merge past the deadline is a gate failure. - State in the header that the pin read-back needs --lane-login spelled `name[bot]` for an App lane, while autonomy and the dispatch check use the stripped key. - Name what the test fixtures hold, including that the comment page body is trimmed to its first line. - Replace the "lane must not hold actions:write" requirement with an open deployment decision: GitHub has no dispatch-without-delete permission and the post-merge dispatch runs under the lane identity, so either that dispatch moves to an identity the lane cannot use or a deleted red run stays a named residual. Refs: #4588 Co-Authored-By: Claude Opus 5.5 --- .../reference/promotion-evidence-bootstrap.md | 15 ++++++++++----- .../scripts/publish-promotion-evidence.mjs | 11 ++++++++--- .../publish-promotion-evidence.test.mjs | 18 +++++++++++++++--- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md b/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md index bdfa0dee42..322ad9a3b3 100644 --- a/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md +++ b/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md @@ -169,11 +169,16 @@ judgment mappings and how to set `--lane-login` for merge-queue merges. `sha256`, or when the new events drop one the previous file held. Run it as the operator, never from the lane, and point `--out` at a directory the lane cannot -write. The lane identity must not hold `actions:write` on the repository: with it, the lane could -delete a red workflow run before the publisher reads it. Which host runs it, under which identity -and on what schedule is not decided yet: it waits on the open decision about the single merge -authority ([#4588](https://github.com/melodic-software/claude-code-plugins/issues/4588)). Until -then the resolver does not read the sidecar. +write. Which host runs it, under which identity and on what schedule is not decided yet: it waits +on the open decision about the single merge authority +([#4588](https://github.com/melodic-software/claude-code-plugins/issues/4588)). Until then the +resolver does not read the sidecar. + +One more deployment decision is open. An identity with `actions:write` can delete a red workflow +run before the publisher reads it, and GitHub offers no permission that allows dispatching a +workflow without also allowing run deletion. The post-merge check dispatches its full run under +the lane identity, so either that dispatch moves to an identity the lane cannot use, or a deleted +red run stays a named residual risk. ### `promotion_evidence_checker` diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs index d2b1daff99..8e398d9aa3 100644 --- a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs @@ -29,10 +29,15 @@ // enqueuer as merged_by (probed 2026-10-12 on PR #7352: merged_by.login matched the // AddedToMergeQueueEvent enqueuer), so --lane-login names any lane identity that runs as a User // (a machine user); an App lane is a Bot and counts without it. Recheck if merged_by stops -// matching the enqueuer. +// matching the enqueuer. Autonomy and the dispatch check compare that stripped key; the pin +// read-back compares exactly, so for an App lane pass --lane-login as GraphQL spells the App +// plus `[bot]` (`name[bot]`), or its pins are not found. // -// Deployment requirement: the lane identity must not hold actions:write on the repository. With it -// the lane could delete a red workflow run, and a deleted run is evidence this script never sees. +// Open deployment decision: an identity with actions:write can delete a red workflow run, which +// this script then never sees, and GitHub has no permission that allows dispatching a workflow +// without also allowing run deletion. babysit_postmerge.py dispatches the post-merge full run under +// the lane identity, so either that dispatch moves to an identity the lane cannot use, or a +// deleted red run remains a named residual. Not decided here. // // Anything a lane can write into a PR or commit degrades to a listing or to unclassed-autonomous, // never to a refused publish. What refuses is GitHub itself being unreadable: a failed read, an diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs index 88c52483ec..bc982f80f4 100644 --- a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs @@ -11,9 +11,11 @@ import * as publisher from "./publish-promotion-evidence.mjs"; import { collect, PublishError, publish } from "./publish-promotion-evidence.mjs"; const HERE = dirname(fileURLToPath(import.meta.url)); -// Real `gh api` responses from this repository (PR #7352, its merge commit, a check run, an -// issue event and a review), recorded 2026-10-12 and trimmed to the fields the publisher reads. -// Each case copies a record and varies number, labels, merger and timestamps. +// Real `gh api` responses, recorded 2026-10-12 and 2026-10-13 and trimmed to the fields the +// publisher reads: from this repository (PRs #7352 and #7063, a merge commit, a check run, a +// status, an issue event, a review, a comment page, workflow runs, a compare, NOT_FOUND and 422 +// answers) and one ConnectedEvent from cli/cli #14616. The comment page's body is trimmed to its +// first line. Each case copies a record and varies number, labels, merger and timestamps. const REC = JSON.parse(readFileSync(join(HERE, "fixtures", "publish-promotion-evidence", "recorded.json"), "utf8")); const REPO = "melodic-software/claude-code-plugins"; const LANE = "lane-bot[bot]"; @@ -288,6 +290,16 @@ test("a lane-enqueued merge passes only when its unedited lane pin names headRef assert.deepEqual(gateOf(run({ pulls: bot, workflowRuns: verifiedEach(bot), comments: { 1503: [] } })), PASSED); }); +test("a pin past the 30th comment page is no pin, as in read_pin", () => { + const pulls = () => [pr(1504, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] })]; + const filler = (count) => Array.from({ length: count }, () => REC.commentsPage.data.repository.pullRequest.comments.nodes[0]); + const withComments = (list) => run({ pulls: pulls(), workflowRuns: verifiedEach(pulls()), comments: { 1504: list } }); + const head = shaOf(1504 + 0x100000); + assert.deepEqual(gateOf(withComments([...filler(2999), pinComment(head)])), PASSED); + const beyond = withComments([...filler(3000), pinComment(head)]); + assert.deepEqual([gateOf(beyond), beyond.events], [FAILED, gateEvents("2026-09-01T06:00:00Z", "C2-auto-merge")]); +}); + test("a revert number that is not a GraphQL Int is listed without a lookup", () => { const result = run({ pulls: [ From fd50e312827d57fc24227a9604b4f5e7a1194dc2 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Tue, 13 Oct 2026 02:34:19 -0400 Subject: [PATCH 11/18] fix(source-control): clear lint-repo failures on the evidence publisher Mark publish-promotion-evidence.mjs and its test executable in the index, since both carry a shebang, and drop the pull and issue numbers from three comments that the comment-hygiene scan flags as tracker references. Co-Authored-By: Claude Opus 5.5 --- .../babysit-loop/scripts/publish-promotion-evidence.mjs | 2 +- .../babysit-loop/scripts/publish-promotion-evidence.test.mjs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) mode change 100644 => 100755 plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs mode change 100644 => 100755 plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs old mode 100644 new mode 100755 index 8e398d9aa3..e746a8794d --- a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs @@ -26,7 +26,7 @@ // any of them is null or not human (a User outside --lane-login, not `github-actions`, not a // `[bot]` login; derive_auto_merge's write-permission read is not made), or merged_by is a // --lane-login. Logins compare case-insensitively with `[bot]` dropped. GitHub records the -// enqueuer as merged_by (probed 2026-10-12 on PR #7352: merged_by.login matched the +// enqueuer as merged_by (probed 2026-10-12 on a queue-merged pull: merged_by.login matched the // AddedToMergeQueueEvent enqueuer), so --lane-login names any lane identity that runs as a User // (a machine user); an App lane is a Bot and counts without it. Recheck if merged_by stops // matching the enqueuer. Autonomy and the dispatch check compare that stripped key; the pin diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs old mode 100644 new mode 100755 index bc982f80f4..715addf8c8 --- a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs @@ -34,7 +34,7 @@ const later = (at, minutes) => new Date(Date.parse(at) + minutes * 60_000).toISO // GraphQL spells an App as `__typename: Bot` with the bare slug; REST adds `[bot]`. const actorOf = (login) => (login === null ? null : login.endsWith("[bot]") ? { __typename: "Bot", login: login.slice(0, -5) } : { __typename: "User", login }); const page1 = (nodes, hasNextPage = false) => ({ pageInfo: { hasNextPage }, nodes }); -// REC.pullGraph: PR #7063, which closes issue #6007 (labeled `work-class: scoped`). +// REC.pullGraph: a recorded pull whose one closing reference, number 6007, is labeled `work-class: scoped`. const ISSUE_6007 = REC.pullGraph.data.repository.pullRequest.closingIssuesReferences.nodes; const queued = (at, enqueuer, actor = enqueuer) => ({ __typename: "AddedToMergeQueueEvent", createdAt: at, actor: actorOf(actor), enqueuer: actorOf(enqueuer) }); // The shape of a real ConnectedEvent on a PR timeline (cli/cli #14616): `subject` is the issue. @@ -584,7 +584,7 @@ test("the class is read as of the merge: post-merge relabeling neither adds nor issueEvents: { 905: [labelEvent("labeled", MECHANICAL, "2026-09-02T00:00:00Z")], 906: [labelEvent("labeled", MECHANICAL, "2026-08-31T00:00:00Z"), labelEvent("unlabeled", MECHANICAL, "2026-09-02T00:00:00Z", LANE, "Bot")], - // Linked issue #6007 is labeled scoped now, but gained that label after #907 merged. + // The linked 6007 is labeled scoped now, but gained that label after 907 merged. 6007: [labelEvent("labeled", SCOPED, "2026-09-03T00:00:00Z")], }, }); From 315b31724807bec6bd36ce6ae6eb1895bd439606 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Tue, 13 Oct 2026 02:42:13 -0400 Subject: [PATCH 12/18] docs(source-control): point at GitHub's Actions permissions instead of restating them The publisher section of the promotion-evidence bootstrap restated GitHub's Actions permission model and tied the publisher's deployment to an open tracker item with an "until then". State the deployment choice and the dispatch-versus-deletion question in our own words, and point at the Actions repository permissions table with an as-of date and recheck trigger. Co-Authored-By: Claude Opus 5.5 --- .../reference/promotion-evidence-bootstrap.md | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md b/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md index 322ad9a3b3..826e6a77bc 100644 --- a/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md +++ b/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md @@ -169,16 +169,15 @@ judgment mappings and how to set `--lane-login` for merge-queue merges. `sha256`, or when the new events drop one the previous file held. Run it as the operator, never from the lane, and point `--out` at a directory the lane cannot -write. Which host runs it, under which identity and on what schedule is not decided yet: it waits -on the open decision about the single merge authority -([#4588](https://github.com/melodic-software/claude-code-plugins/issues/4588)). Until then the -resolver does not read the sidecar. - -One more deployment decision is open. An identity with `actions:write` can delete a red workflow -run before the publisher reads it, and GitHub offers no permission that allows dispatching a -workflow without also allowing run deletion. The post-merge check dispatches its full run under -the lane identity, so either that dispatch moves to an identity the lane cannot use, or a deleted -red run stays a named residual risk. +write. Which host runs it, under which identity and on what schedule is the operator's +deployment choice. The resolver does not read the sidecar. + +The post-merge check dispatches its full run under the lane identity. Where the permission that +dispatch needs also lets that identity delete a red run before the publisher reads it, either the +dispatch moves to an identity the lane cannot use, or a deleted red run stays a named residual +risk. Which permission covers dispatch and which covers run deletion: GitHub's +[Actions repository permissions table](https://docs.github.com/en/rest/authentication/permissions-required-for-github-apps#repository-permissions-for-actions) +(as of 2026-10-13; recheck when GitHub adds or splits an Actions permission). ### `promotion_evidence_checker` From 4cb4dde86150a061e594e389a695d3ac4f6f03e8 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Tue, 13 Oct 2026 02:42:13 -0400 Subject: [PATCH 13/18] fix(source-control): match lane identities exactly and restore events on a sidecar failure The publisher stripped `[bot]` from every login, so with an App lane `foo[bot]` a human User `foo` counted as the lane and their merge as autonomous. Compare in REST spelling instead, adding `[bot]` to a GraphQL Bot actor as babysit_merge_policy.py actor_key does. When the sidecar rename failed after the events file was replaced, the pair no longer matched and every later publish refused. Keep a copy of the previous events file and put it back (or remove a first file) when that rename fails. Co-Authored-By: Claude Opus 5.5 --- ...8-promotion-evidence-publisher-3634507d.md | 2 +- .../scripts/publish-promotion-evidence.mjs | 70 +++++++++++-------- .../publish-promotion-evidence.test.mjs | 23 +++++- 3 files changed, 62 insertions(+), 33 deletions(-) diff --git a/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md b/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md index b05f84be2c..80bfa73545 100644 --- a/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md +++ b/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md @@ -4,4 +4,4 @@ bump: minor ### Added -- `babysit-loop` ships `scripts/publish-promotion-evidence.mjs`, an operator-side publisher for the promotion-evidence events file (#4588). It reads GitHub only, through read-only `gh api` calls, and emits the `{cell, event, at}` contrary events the security-binding checker reads (`reverted-merge`, `gate-failure`, and `verification-divergence` for a missed-blocking finding a human confirmed) plus a C3 promotion-evidence report: each promotable cell's predicate terms with counts, window and pass or fail. Report mode, the default, writes nothing. `--out ` writes `evidence.json` and an `evidence.meta.json` sidecar (`generated_at`, `through`, `sha256`, `prev_sha256`) by temp file and rename, and refuses a set that drops a previously published event or an existing file that does not match its sidecar. Work classes are read from the labels of the PR and of the issues it linked at merge, as of the merge. A merge counts as autonomous when no human enqueued it or a `--lane-login` identity merged it. An autonomous merge with no readable class is listed as unclassed and its contrary events count against every cell. Only the post-merge gate counts toward `gate-failure`. That gate is the full and canary workflow runs on any later commit containing the merge, read as `babysit_postmerge.py` reads them, plus the check runs and statuses named by the required `--gate-check` on the merge commit. A cancelled run counts unless a later run of that check succeeded, and a red full run covering several merges counts against each. As in `babysit_postmerge.py`, a merge with no lane dispatch run within 6 hours, or a lane-enqueued merge whose pin comment is missing, edited or names another head, is a gate failure. Where it runs and under which identity is not decided yet, and the resolver does not read the sidecar; merge authority is unchanged. +- `babysit-loop` ships `scripts/publish-promotion-evidence.mjs`, an operator-side publisher for the promotion-evidence events file (#4588). It reads GitHub only, through read-only `gh api` calls, and emits the `{cell, event, at}` contrary events the security-binding checker reads (`reverted-merge`, `gate-failure`, and `verification-divergence` for a missed-blocking finding a human confirmed) plus a C3 promotion-evidence report: each promotable cell's predicate terms with counts, window and pass or fail. Report mode, the default, writes nothing. `--out ` writes `evidence.json` and an `evidence.meta.json` sidecar (`generated_at`, `through`, `sha256`, `prev_sha256`) by temp file and rename, puts the previous events file back when the sidecar rename fails, and refuses a set that drops a previously published event or an existing file that does not match its sidecar. Work classes are read from the labels of the PR and of the issues it linked at merge, as of the merge. A merge counts as autonomous when no human enqueued it or a `--lane-login` identity merged it. An autonomous merge with no readable class is listed as unclassed and its contrary events count against every cell. Only the post-merge gate counts toward `gate-failure`. That gate is the full and canary workflow runs on any later commit containing the merge, read as `babysit_postmerge.py` reads them, plus the check runs and statuses named by the required `--gate-check` on the merge commit. A cancelled run counts unless a later run of that check succeeded, and a red full run covering several merges counts against each. As in `babysit_postmerge.py`, a merge with no lane dispatch run within 6 hours, or a lane-enqueued merge whose pin comment is missing, edited or names another head, is a gate failure. Where it runs and under which identity is not decided yet, and the resolver does not read the sidecar; merge authority is unchanged. diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs index e746a8794d..9927a1e3ec 100755 --- a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs @@ -25,19 +25,18 @@ // actor and enqueuer, or merged_by when the merge bypassed the queue; a merge is autonomous when // any of them is null or not human (a User outside --lane-login, not `github-actions`, not a // `[bot]` login; derive_auto_merge's write-permission read is not made), or merged_by is a -// --lane-login. Logins compare case-insensitively with `[bot]` dropped. GitHub records the +// --lane-login. Logins compare case-insensitively and exactly in REST spelling: a GraphQL `Bot` +// actor gains `[bot]`, so a User owning an App's bare slug is never that App. GitHub records the // enqueuer as merged_by (probed 2026-10-12 on a queue-merged pull: merged_by.login matched the // AddedToMergeQueueEvent enqueuer), so --lane-login names any lane identity that runs as a User // (a machine user); an App lane is a Bot and counts without it. Recheck if merged_by stops -// matching the enqueuer. Autonomy and the dispatch check compare that stripped key; the pin -// read-back compares exactly, so for an App lane pass --lane-login as GraphQL spells the App -// plus `[bot]` (`name[bot]`), or its pins are not found. +// matching the enqueuer. For an App lane pass --lane-login as `name[bot]`, or its dispatch runs +// and pins are not found. // -// Open deployment decision: an identity with actions:write can delete a red workflow run, which -// this script then never sees, and GitHub has no permission that allows dispatching a workflow -// without also allowing run deletion. babysit_postmerge.py dispatches the post-merge full run under -// the lane identity, so either that dispatch moves to an identity the lane cannot use, or a -// deleted red run remains a named residual. Not decided here. +// A red workflow run deleted before this script reads it is never seen. babysit_postmerge.py +// dispatches the post-merge full run under the lane identity; whether that identity can also +// delete runs is the deployment question promotion-evidence-bootstrap.md records with its pointer. +// Not decided here. // // Anything a lane can write into a PR or commit degrades to a listing or to unclassed-autonomous, // never to a refused publish. What refuses is GitHub itself being unreadable: a failed read, an @@ -169,13 +168,18 @@ const PULL_QUERY = "... on AddedToMergeQueueEvent{createdAt actor{__typename login} enqueuer{__typename login}}}}}}}"; const CELLS = ["C2-auto-merge", "C3-ai-review-blocking", "C3-auto-merge"]; -// A login compared across REST (`app[bot]`) and GraphQL (`app`) spellings, as -// babysit_merge_policy.py _login_key does. +// A REST login (`app[bot]` for an App), compared exactly and case-insensitively. function loginKey(login) { return String(login ?? "") .trim() - .toLowerCase() - .replace(/\[bot\]$/, ""); + .toLowerCase(); +} + +// A GraphQL actor in REST spelling, as babysit_merge_policy.py actor_key: a `Bot` gains the +// `[bot]` GraphQL omits, so a User who owns the bare slug `app` is never the App `app[bot]`. +function actorKey(actor) { + const login = loginKey(actor?.login); + return actor?.__typename === "Bot" && login && !login.endsWith("[bot]") ? `${login}[bot]` : login; } function onePage(connection, what) { @@ -251,12 +255,7 @@ const PIN_QUERY = "comments(first:100,after:$c){pageInfo{hasNextPage endCursor} nodes{author{__typename login} body lastEditedAt}}}}}"; const PIN_PAGE_LIMIT = 30; -function readPin(gh, owner, name, number, laneExact) { - // GraphQL spells an App `app` with __typename Bot; REST and --lane-login spell it `app[bot]`. - const author = (a) => { - const login = String(a?.login ?? "").trim().toLowerCase(); - return a?.__typename === "Bot" && login && !login.endsWith("[bot]") ? `${login}[bot]` : login; - }; +function readPin(gh, owner, name, number, lane) { const firstLine = (text) => String(text ?? "").trimStart().split("\n", 1)[0].trim(); const pins = []; let cursor = null; @@ -264,7 +263,7 @@ function readPin(gh, owner, name, number, laneExact) { const variables = { owner, name, number, ...(cursor ? { c: cursor } : {}) }; const comments = gh("graphql", { query: PIN_QUERY, variables })?.data?.repository?.pullRequest?.comments; if (!Array.isArray(comments?.nodes) || typeof comments.pageInfo?.hasNextPage !== "boolean") throw new PublishError(`#${number} pin: no comment page`); - pins.push(...comments.nodes.filter((c) => laneExact.has(author(c?.author)) && PIN_RE.test(firstLine(c?.body)))); + pins.push(...comments.nodes.filter((c) => lane.has(actorKey(c?.author)) && PIN_RE.test(firstLine(c?.body)))); if (!comments.pageInfo.hasNextPage) { const last = pins.at(-1); return last && !last.lastEditedAt ? PIN_RE.exec(firstLine(last.body))[1] : null; @@ -371,7 +370,7 @@ export function collect({ repo, since, through, laneLogins, gateChecks, aiReview // babysit_merge_policy.py is_human, without its permission read: a User that is not a lane // login, `github-actions` or a `[bot]` login. A null actor is not human. const human = (actor) => - actor?.__typename === "User" && !lane.has(loginKey(actor.login)) && loginKey(actor.login) !== "github-actions" && !/\[bot\]$/i.test(actor.login ?? ""); + actor?.__typename === "User" && !lane.has(actorKey(actor)) && loginKey(actor.login) !== "github-actions" && !/\[bot\]$/i.test(actor.login ?? ""); const inWindow = (at) => typeof at === "string" && Date.parse(at) >= sinceMs && Date.parse(at) <= throughMs; const events = new Map(); const addEvent = (cell, event, at) => { @@ -434,8 +433,8 @@ export function collect({ repo, since, through, laneLogins, gateChecks, aiReview const autonomousOf = (g, timeline) => { const queued = timeline.filter((e) => e.__typename === "AddedToMergeQueueEvent").sort((a, b) => ms(a.createdAt, "a merge-queue event") - ms(b.createdAt, "a merge-queue event")); const actors = queued.length > 0 ? [queued.at(-1).actor, queued.at(-1).enqueuer] : [g.mergedBy]; - const byIdentity = actors.some((actor) => actor && lane.has(loginKey(actor.login))); - return { autonomous: actors.some((actor) => !human(actor)) || lane.has(loginKey(g.mergedBy?.login)), byIdentity }; + const byIdentity = actors.some((actor) => actor && lane.has(actorKey(actor))); + return { autonomous: actors.some((actor) => !human(actor)) || lane.has(actorKey(g.mergedBy)), byIdentity }; }; const details = new Map(); @@ -529,12 +528,11 @@ export function collect({ repo, since, through, laneLogins, gateChecks, aiReview } const judgeWorkflows = postMergeRuns(gh, repo, branch, tracked.filter((c) => c.autonomous), { lane, throughMs }); - const laneExact = new Set(laneLogins.map((login) => String(login).trim().toLowerCase())); // check()'s read-back for a merge a lane identity enqueued. const readBack = (r) => { const deadline = ms(r.mergedAt, `#${r.number} mergedAt`) + DISPATCH_DEADLINE_MS; - const pinned = readPin(gh, owner, name, r.number, laneExact); + const pinned = readPin(gh, owner, name, r.number, lane); if (pinned === null) { return throughMs > deadline ? { failure: { check: "pin read-back", conclusion: "no unedited pin comment by a lane identity", at: isoSeconds(deadline) } } @@ -696,7 +694,7 @@ function readIfPresent(path) { } } -export function publish(outDir, events, through, now = new Date()) { +export function publish(outDir, events, through, now = new Date(), { rename = renameSync } = {}) { const file = join(outDir, "evidence.json"); const metaFile = join(outDir, "evidence.meta.json"); const previous = readIfPresent(file); @@ -727,16 +725,26 @@ export function publish(outDir, events, through, now = new Date()) { const meta = { generated_at: now.toISOString(), through, sha256: sha256(text), prev_sha256: previous === null ? null : sha256(previous) }; const eventsTemp = writeTemp(file, text); let metaTemp; + let backup; try { metaTemp = writeTemp(metaFile, `${JSON.stringify(meta, null, 2)}\n`); - renameSync(eventsTemp, file); + if (previous !== null) backup = writeTemp(`${file}.prev`, previous); + rename(eventsTemp, file); + } catch (error) { + for (const temp of [eventsTemp, metaTemp, backup]) if (temp) rmSync(temp, { force: true }); + throw error; + } + try { + rename(metaTemp, metaFile); } catch (error) { - rmSync(eventsTemp, { force: true }); - if (metaTemp) rmSync(metaTemp, { force: true }); + // Put the pair back as it was. An interruption before this point still leaves a pair the + // next run refuses on the sha256 check: fail closed. + if (backup) rename(backup, file); + else rmSync(file, { force: true }); + rmSync(metaTemp, { force: true }); throw error; } - // A failure here leaves a pair the next run refuses on the sha256 check: fail closed. - renameSync(metaTemp, metaFile); + if (backup) rmSync(backup, { force: true }); return meta; } diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs index 715addf8c8..79f6c66d6e 100755 --- a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs @@ -2,7 +2,7 @@ import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { mkdtempSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { after, test } from "node:test"; @@ -634,6 +634,11 @@ test("a revert of an unclassed autonomous merge yields reverted-merge against ev assert.deepEqual(result.events, ["C2-auto-merge", "C3-ai-review-blocking", "C3-auto-merge"].map((cell) => ({ cell, event: "reverted-merge", at: "2026-09-02T00:00:00Z" }))); }); +test("a User owning the bare slug of an App lane is human, not the lane", () => { + const result = run({ pulls: [pr(1110, "2026-09-01T00:00:00Z", { labels: [MECHANICAL], merger: "lane-bot", timeline: [queued("2026-08-31T23:00:00Z", "lane-bot")] })] }); + assert.equal(termOf(result, "C2-auto-merge", "autonomous C2 completions").value, 0); +}); + test("lane logins match case-insensitively", () => { const result = run({ pulls: [pr(1106, "2026-09-01T00:00:00Z", { labels: [MECHANICAL], merger: "Lane-User", timeline: [queued("2026-08-31T23:00:00Z", "Lane-User")] })] }, { laneLogins: ["lane-user"] }); assert.equal(termOf(result, "C2-auto-merge", "autonomous C2 completions").value, 1); @@ -741,6 +746,22 @@ test("a sidecar write failure leaves the published events file unchanged", () => assert.throws(() => readFileSync(join(out, `evidence.json.${process.pid}.tmp`)), { code: "ENOENT" }); }); +test("a failed sidecar rename puts the previous events file back, or removes a first one", () => { + const failMeta = (from, to) => { + if (to.endsWith("evidence.meta.json")) throw Object.assign(new Error("rename failed"), { code: "EIO" }); + renameSync(from, to); + }; + const out = mkdtempSync(join(base, "out-")); + assert.throws(() => publish(out, [A], THROUGH, new Date(), { rename: failMeta }), { code: "EIO" }); + assert.deepEqual(readdirSync(out), []); + publish(out, [A], THROUGH); + const before = readFileSync(join(out, "evidence.json"), "utf8"); + assert.throws(() => publish(out, [A, B], THROUGH, new Date(), { rename: failMeta }), { code: "EIO" }); + assert.equal(readFileSync(join(out, "evidence.json"), "utf8"), before); + assert.deepEqual(readdirSync(out).sort(), ["evidence.json", "evidence.meta.json"]); + publish(out, [A, B], THROUGH); +}); + test("an existing events file that is not a JSON array is refused", () => { const out = mkdtempSync(join(base, "out-")); writeFileSync(join(out, "evidence.json"), "{}"); From 3585c7164b9bf12112129150e7953d62680ebaa9 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Tue, 13 Oct 2026 02:49:24 -0400 Subject: [PATCH 14/18] docs(source-control): align the publisher fragment with the deployment text Co-Authored-By: Claude Opus 5.5 --- .../feat-4588-promotion-evidence-publisher-3634507d.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md b/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md index 80bfa73545..8a6fdc8a15 100644 --- a/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md +++ b/.changes/source-control/feat-4588-promotion-evidence-publisher-3634507d.md @@ -4,4 +4,4 @@ bump: minor ### Added -- `babysit-loop` ships `scripts/publish-promotion-evidence.mjs`, an operator-side publisher for the promotion-evidence events file (#4588). It reads GitHub only, through read-only `gh api` calls, and emits the `{cell, event, at}` contrary events the security-binding checker reads (`reverted-merge`, `gate-failure`, and `verification-divergence` for a missed-blocking finding a human confirmed) plus a C3 promotion-evidence report: each promotable cell's predicate terms with counts, window and pass or fail. Report mode, the default, writes nothing. `--out ` writes `evidence.json` and an `evidence.meta.json` sidecar (`generated_at`, `through`, `sha256`, `prev_sha256`) by temp file and rename, puts the previous events file back when the sidecar rename fails, and refuses a set that drops a previously published event or an existing file that does not match its sidecar. Work classes are read from the labels of the PR and of the issues it linked at merge, as of the merge. A merge counts as autonomous when no human enqueued it or a `--lane-login` identity merged it. An autonomous merge with no readable class is listed as unclassed and its contrary events count against every cell. Only the post-merge gate counts toward `gate-failure`. That gate is the full and canary workflow runs on any later commit containing the merge, read as `babysit_postmerge.py` reads them, plus the check runs and statuses named by the required `--gate-check` on the merge commit. A cancelled run counts unless a later run of that check succeeded, and a red full run covering several merges counts against each. As in `babysit_postmerge.py`, a merge with no lane dispatch run within 6 hours, or a lane-enqueued merge whose pin comment is missing, edited or names another head, is a gate failure. Where it runs and under which identity is not decided yet, and the resolver does not read the sidecar; merge authority is unchanged. +- `babysit-loop` ships `scripts/publish-promotion-evidence.mjs`, an operator-side publisher for the promotion-evidence events file (#4588). It reads GitHub only, through read-only `gh api` calls, and emits the `{cell, event, at}` contrary events the security-binding checker reads (`reverted-merge`, `gate-failure`, and `verification-divergence` for a missed-blocking finding a human confirmed) plus a C3 promotion-evidence report: each promotable cell's predicate terms with counts, window and pass or fail. Report mode, the default, writes nothing. `--out ` writes `evidence.json` and an `evidence.meta.json` sidecar (`generated_at`, `through`, `sha256`, `prev_sha256`) by temp file and rename, puts the previous events file back when the sidecar rename fails, and refuses a set that drops a previously published event or an existing file that does not match its sidecar. Work classes are read from the labels of the PR and of the issues it linked at merge, as of the merge. A merge counts as autonomous when no human enqueued it or a `--lane-login` identity merged it. An autonomous merge with no readable class is listed as unclassed and its contrary events count against every cell. Only the post-merge gate counts toward `gate-failure`. That gate is the full and canary workflow runs on any later commit containing the merge, read as `babysit_postmerge.py` reads them, plus the check runs and statuses named by the required `--gate-check` on the merge commit. A cancelled run counts unless a later run of that check succeeded, and a red full run covering several merges counts against each. As in `babysit_postmerge.py`, a merge with no lane dispatch run within 6 hours, or a lane-enqueued merge whose pin comment is missing, edited or names another head, is a gate failure. Where it runs and under which identity is the operator's deployment choice (an App lane is passed as `--lane-login name[bot]`), and the resolver does not read the sidecar; merge authority is unchanged. From 27a5eeb8b9b7d65ff1fb26f73105cdf79589827b Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Tue, 13 Oct 2026 09:05:02 -0400 Subject: [PATCH 15/18] fix(source-control): count a revert only when a human merged it The owner decided the revert rule for the promotion-evidence publisher. - A revert PR counts as reverted-merge only when a human merged or enqueued it, judged by the same actor rule the publisher uses for autonomy. A revert PR a bot or lane identity merged or enqueued, and the squash commit it left, is listed in unresolvedReverts and not counted. - A revert commit pushed straight to the default branch, with no PR, still counts: its git author can be forged, so it is not trusted to be harmless. - The bootstrap doc names the confirmed deployment defaults: the missed-blocking label `ai-review: missed-blocking` and the review bots `claude[bot]` and `chatgpt-codex-connector[bot]` for --ai-reviewer. Refs: #4588 Co-Authored-By: Claude Opus 5.5 --- .../reference/promotion-evidence-bootstrap.md | 5 +++ .../scripts/publish-promotion-evidence.mjs | 42 +++++++++++++++++-- .../publish-promotion-evidence.test.mjs | 33 ++++++++++++++- 3 files changed, 74 insertions(+), 6 deletions(-) diff --git a/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md b/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md index 826e6a77bc..da5949a48c 100644 --- a/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md +++ b/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md @@ -163,6 +163,11 @@ judgment mappings and how to set `--lane-login` for merge-queue merges. [work-classes.md](https://raw.githubusercontent.com/melodic-software/claude-code-plugins/main/plugins/autonomy/reference/guardrails/work-classes.md) "Suggested default predicates" with counts, window and pass or fail, and lists merged reverts it could not match to a merge. +- **Deployment defaults:** `--missed-blocking-label ai-review: missed-blocking` (the script's + default), and `--ai-reviewer claude[bot] --ai-reviewer chatgpt-codex-connector[bot]`, the REST + spellings of the review bots on this repository's pull requests (read from #7361's reviews, + 2026-10-13). A revert counts as `reverted-merge` only when a human merged or enqueued the revert + PR; a revert commit pushed with no PR still counts. - **`--out `** writes `evidence.json` and the sidecar `evidence.meta.json` (`generated_at`, `through`, `sha256` of the events file, `prev_sha256`), each by a temp file and a rename. It refuses to publish when the previous events file does not match its sidecar's diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs index 9927a1e3ec..95477f1831 100755 --- a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs @@ -59,7 +59,12 @@ // ` (git revert); one event per reverted merge, at the earliest revert seen. A merged PR // titled revert... with neither form is listed in unresolvedReverts for a human to read, as is // a named target that is not a merged pull request or a SHA no commit has, so a bad reference -// cannot stop a publish. +// cannot stop a publish. A revert PR counts only when a human merged or enqueued it (the +// autonomy rule above: no non-human actor, merged_by not a --lane-login); one a bot or lane +// identity merged or enqueued is listed in unresolvedReverts and not counted, and so is the +// squash commit such a PR left. A revert commit pushed straight to the default branch, with no +// PR, counts: its git author can be forged, so it is not trusted to be harmless (conservative +// toward demotion). // - gate-failure: only the post-merge gate counts: check runs and commit-status contexts on an // autonomous merge's merge commit whose name is a --gate-check (required; none named is a // usage error, never an empty gate). In this repository babysit-prs/scripts/babysit_postmerge.py @@ -479,15 +484,30 @@ export function collect({ repo, since, through, laneLogins, gateChecks, aiReview const reverts = []; const unresolvedReverts = []; const gateFailures = []; + // A revert PR counts only when a human merged or enqueued it (the autonomy rule above, negated). + const humanRevert = (number) => !record(number).autonomous; for (const pr of merged) { const targets = revertsOf(pr.body); - for (const target of targets) reverts.push({ ...target, at: pr.merged_at }); + const human = targets.length > 0 && humanRevert(pr.number); + for (const target of targets) reverts.push({ ...target, at: pr.merged_at, human }); if (targets.length === 0 && /^revert\b/i.test(pr.title ?? "")) unresolvedReverts.push({ number: pr.number, title: pr.title }); } const commits = pages(gh, `repos/${repo}/commits?since=${new Date(sinceMs).toISOString()}&until=${new Date(throughMs).toISOString()}`); + // A revert commit that is a PR's merge commit follows that PR's rule. One pushed straight to the + // default branch counts: its git author can be forged, so it is not trusted to be harmless. for (const commit of commits) { - for (const target of revertsOf(commit.commit?.message)) reverts.push({ ...target, at: commit.commit?.committer?.date }); + const targets = revertsOf(commit.commit?.message); + if (targets.length === 0) continue; + let viaPr; + try { + viaPr = gh(`repos/${repo}/commits/${commit.sha}/pulls`).find((p) => p.merged_at && p.merge_commit_sha === commit.sha); + } catch (error) { + if (!(error instanceof UnresolvableError)) throw error; + } + const human = viaPr ? humanRevert(viaPr.number) : true; + for (const target of targets) reverts.push({ ...target, at: commit.commit?.committer?.date, human }); } + const notHuman = new Map(); const firstRevert = new Map(); for (const revert of reverts) { if (!inWindow(revert.at)) continue; @@ -510,7 +530,21 @@ export function collect({ repo, since, through, laneLogins, gateChecks, aiReview continue; } } - if (!firstRevert.has(number) || Date.parse(revert.at) < Date.parse(firstRevert.get(number))) firstRevert.set(number, revert.at); + if (!revert.human) notHuman.set(number, true); + else if (!firstRevert.has(number) || Date.parse(revert.at) < Date.parse(firstRevert.get(number))) firstRevert.set(number, revert.at); + } + // A revert no human merged is listed when it names a merge it would otherwise have counted. + for (const number of notHuman.keys()) { + let target; + try { + target = record(number); + } catch (error) { + if (!(error instanceof UnresolvableError)) throw error; + continue; + } + if (target.autonomous && (["C2", "C3"].includes(target.cls) || target.cls === null)) { + unresolvedReverts.push({ target: number, reason: "the revert was merged or enqueued by a bot or lane identity, not a human" }); + } } for (const [number, at] of firstRevert) { let target; diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs index 79f6c66d6e..1768122685 100755 --- a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs @@ -140,7 +140,7 @@ function github({ pulls = [], commits = [], checkRuns = {}, statuses = {}, issue if ((m = /^commits\/([0-9a-f]{40})\/pulls$/.exec(rest))) { // A SHA no commit has: the recorded 422, as realGh hands it to ghFailure. if (!branch.includes(m[1])) return publisher.ghFailure(path, undefined, REC.commitNotFound.stdout, REC.commitNotFound.stderr); - return pulls.filter((p) => p.graph.mergeCommit.oid === m[1]).map((p) => ({ number: p.graph.number, merged_at: p.graph.mergedAt })); + return pulls.filter((p) => p.graph.mergeCommit.oid === m[1]).map((p) => ({ number: p.graph.number, merged_at: p.graph.mergedAt, merge_commit_sha: m[1] })); } if ((m = /^commits\/([0-9a-f]{40})\/statuses\?/.exec(rest))) return page(statuses[parseInt(m[1], 16)] ?? [], rest); if ((m = /^issues\/(\d+)\/events\?/.exec(rest))) return page(issueEvents[m[1]] ?? [], rest); @@ -471,14 +471,43 @@ test("a revert-button PR of an autonomous C2 merge yields reverted-merge at the assert.deepEqual(termOf(result, "C2-auto-merge", "human-reverted merges"), { term: "human-reverted merges", value: 1, threshold: "0", pass: false }); }); -test("a git-revert commit of an autonomous C3 merge yields reverted-merge for C3 auto-merge", () => { +test("a git-revert commit pushed straight to the default branch yields reverted-merge for C3 auto-merge", () => { + // No PR merged the revert commit, and its git author can be forged, so it counts. const result = run({ pulls: [pr(201, "2026-09-01T00:00:00Z", { labels: [SCOPED] })], + history: [shaOf(201), shaOf(999)], commits: [{ ...REC.commit, sha: shaOf(999), commit: { message: `Revert "x"\n\nThis reverts commit ${shaOf(201)}.`, committer: { date: "2026-09-03T00:00:00Z" } } }], }); assert.deepEqual(result.events, [{ cell: "C3-auto-merge", event: "reverted-merge", at: "2026-09-03T00:00:00Z" }]); }); +const NOT_HUMAN = "the revert was merged or enqueued by a bot or lane identity, not a human"; + +test("a revert PR a bot or lane identity merged or enqueued is listed, not counted", () => { + const result = run({ + pulls: [ + pr(1701, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] }), + pr(1702, "2026-09-02T00:00:00Z", { title: 'Revert "x"', body: `Reverts ${REPO}#1701` }), + pr(1703, "2026-09-01T00:00:00Z", { labels: [SCOPED] }), + pr(1704, "2026-09-02T00:00:00Z", { body: `Reverts ${REPO}#1703`, merger: "kyle-sexton", timeline: [queued("2026-09-01T23:00:00Z", null, "some-app[bot]")] }), + ], + }); + assert.deepEqual(result.events.filter((e) => e.event === "reverted-merge"), []); + assert.deepEqual(result.unresolvedReverts, [ + { target: 1701, reason: NOT_HUMAN }, + { target: 1703, reason: NOT_HUMAN }, + ]); +}); + +test("the squash commit of a lane-merged revert PR does not count as a direct revert", () => { + const result = run({ + pulls: [pr(1705, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] }), pr(1706, "2026-09-02T00:00:00Z", { body: `Reverts ${REPO}#1705` })], + commits: [{ ...REC.commit, sha: shaOf(1706), commit: { message: `Revert "x" (#1706)\n\nReverts ${REPO}#1705`, committer: { date: "2026-09-02T00:00:00Z" } } }], + }); + assert.deepEqual(result.events.filter((e) => e.event === "reverted-merge"), []); + assert.deepEqual(result.unresolvedReverts, [{ target: 1705, reason: NOT_HUMAN }]); +}); + test("a revert of a human merge, and a revert written only in prose, emit no event; prose is listed unresolved", () => { const result = run({ pulls: [ From 07af269c41a2c7f54d4ba8ed772893245acbbb9c Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Tue, 13 Oct 2026 09:14:36 -0400 Subject: [PATCH 16/18] fix(source-control): treat an unresolvable revert PR as a direct push When the REST commits/{sha}/pulls lookup names a merged PR for a revert commit but the GraphQL lookup of that PR cannot resolve it, the publisher threw and refused the whole publish. It now treats the commit as one pushed with no PR, which counts, matching the script's rule that a bad reference degrades and never stops a publish. Refs: #4588 Co-Authored-By: Claude Opus 5.5 --- .../scripts/publish-promotion-evidence.mjs | 7 ++++--- .../scripts/publish-promotion-evidence.test.mjs | 13 +++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs index 95477f1831..77cbd1eb28 100755 --- a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs @@ -498,13 +498,14 @@ export function collect({ repo, since, through, laneLogins, gateChecks, aiReview for (const commit of commits) { const targets = revertsOf(commit.commit?.message); if (targets.length === 0) continue; - let viaPr; + // A PR the REST lookup names but GraphQL cannot resolve is treated as no PR. + let human = true; try { - viaPr = gh(`repos/${repo}/commits/${commit.sha}/pulls`).find((p) => p.merged_at && p.merge_commit_sha === commit.sha); + const viaPr = gh(`repos/${repo}/commits/${commit.sha}/pulls`).find((p) => p.merged_at && p.merge_commit_sha === commit.sha); + if (viaPr) human = humanRevert(viaPr.number); } catch (error) { if (!(error instanceof UnresolvableError)) throw error; } - const human = viaPr ? humanRevert(viaPr.number) : true; for (const target of targets) reverts.push({ ...target, at: commit.commit?.committer?.date, human }); } const notHuman = new Map(); diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs index 1768122685..8b10030bdf 100755 --- a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs @@ -481,6 +481,19 @@ test("a git-revert commit pushed straight to the default branch yields reverted- assert.deepEqual(result.events, [{ cell: "C3-auto-merge", event: "reverted-merge", at: "2026-09-03T00:00:00Z" }]); }); +test("a revert commit whose REST-named PR GraphQL cannot resolve counts as a direct push, not a crash", () => { + const fixture = { + pulls: [pr(201, "2026-09-01T00:00:00Z", { labels: [SCOPED] })], + history: [shaOf(201), shaOf(999)], + commits: [{ ...REC.commit, sha: shaOf(999), commit: { message: `Revert "x"\n\nThis reverts commit ${shaOf(201)}.`, committer: { date: "2026-09-03T00:00:00Z" } } }], + }; + const base = github(fixture); + const gh = (path, graphql) => + path === `repos/${REPO}/commits/${shaOf(999)}/pulls` ? [{ number: 998, merged_at: "2026-09-03T00:00:00Z", merge_commit_sha: shaOf(999) }] : base(path, graphql); + const result = run(fixture, { gh }); + assert.deepEqual(result.events, [{ cell: "C3-auto-merge", event: "reverted-merge", at: "2026-09-03T00:00:00Z" }]); +}); + const NOT_HUMAN = "the revert was merged or enqueued by a bot or lane identity, not a human"; test("a revert PR a bot or lane identity merged or enqueued is listed, not counted", () => { From 3855bcfe6934bb673fe6e084d0976a23dd254332 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:10:12 -0400 Subject: [PATCH 17/18] fix(source-control): count every revert, whoever merged it The owner decided the revert rule again (#4588): every revert PR counts as reverted-merge, whoever merged or enqueued it (a human, a bot or the lane), the same as a revert commit pushed straight to the default branch. The human-only filter and its "merged or enqueued by a bot or lane identity" listing are gone; unresolvedReverts keeps only references that cannot be resolved. The trade-off, stated in the header and the bootstrap doc: a bot able to merge reverts can hold a cell down, which fails safe because a revert can only demote. It closes the hole where the lane could keep a revert out of the count by auto-merging the revert PR itself. Refs: #4588 Co-Authored-By: Claude Opus 5.5 --- .../reference/promotion-evidence-bootstrap.md | 7 ++- .../scripts/publish-promotion-evidence.mjs | 47 ++++--------------- .../publish-promotion-evidence.test.mjs | 18 ++++--- 3 files changed, 21 insertions(+), 51 deletions(-) diff --git a/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md b/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md index aaf13ce8d6..71406c6658 100644 --- a/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md +++ b/plugins/source-control/skills/babysit-loop/reference/promotion-evidence-bootstrap.md @@ -169,8 +169,11 @@ judgment mappings and how to set `--lane-login` for merge-queue merges. - **Deployment defaults:** `--missed-blocking-label ai-review: missed-blocking` (the script's default), and `--ai-reviewer claude[bot] --ai-reviewer chatgpt-codex-connector[bot]`, the REST spellings of the review bots on this repository's pull requests (read from #7361's reviews, - 2026-10-13). A revert counts as `reverted-merge` only when a human merged or enqueued the revert - PR; a revert commit pushed with no PR still counts. + 2026-10-13). +- **Every revert counts** as `reverted-merge`, whoever merged or enqueued the revert pull request + (a human, a bot or the lane), and so does a revert commit pushed with no pull request. The + trade-off: a bot able to merge reverts can hold a cell down. That fails safe, because a revert + can only demote. - **`--out `** writes `evidence.json` and the sidecar `evidence.meta.json` (`generated_at`, `through`, `sha256` of the events file, `prev_sha256`), each by a temp file and a rename. It refuses to publish when the previous events file does not match its sidecar's diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs index 77cbd1eb28..921cf2ed64 100755 --- a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs @@ -59,12 +59,10 @@ // ` (git revert); one event per reverted merge, at the earliest revert seen. A merged PR // titled revert... with neither form is listed in unresolvedReverts for a human to read, as is // a named target that is not a merged pull request or a SHA no commit has, so a bad reference -// cannot stop a publish. A revert PR counts only when a human merged or enqueued it (the -// autonomy rule above: no non-human actor, merged_by not a --lane-login); one a bot or lane -// identity merged or enqueued is listed in unresolvedReverts and not counted, and so is the -// squash commit such a PR left. A revert commit pushed straight to the default branch, with no -// PR, counts: its git author can be forged, so it is not trusted to be harmless (conservative -// toward demotion). +// cannot stop a publish. Every revert counts, whoever merged or enqueued the revert PR (a +// human, a bot or the lane) and whoever pushed a revert commit straight to the default branch +// (its git author can be forged anyway). The trade-off: a bot able to merge reverts can hold a +// cell down, which fails safe (it can only demote). // - gate-failure: only the post-merge gate counts: check runs and commit-status contexts on an // autonomous merge's merge commit whose name is a --gate-check (required; none named is a // usage error, never an empty gate). In this repository babysit-prs/scripts/babysit_postmerge.py @@ -484,31 +482,16 @@ export function collect({ repo, since, through, laneLogins, gateChecks, aiReview const reverts = []; const unresolvedReverts = []; const gateFailures = []; - // A revert PR counts only when a human merged or enqueued it (the autonomy rule above, negated). - const humanRevert = (number) => !record(number).autonomous; + // Every revert counts, whoever merged or enqueued it (owner decision, #4588). for (const pr of merged) { const targets = revertsOf(pr.body); - const human = targets.length > 0 && humanRevert(pr.number); - for (const target of targets) reverts.push({ ...target, at: pr.merged_at, human }); + for (const target of targets) reverts.push({ ...target, at: pr.merged_at }); if (targets.length === 0 && /^revert\b/i.test(pr.title ?? "")) unresolvedReverts.push({ number: pr.number, title: pr.title }); } const commits = pages(gh, `repos/${repo}/commits?since=${new Date(sinceMs).toISOString()}&until=${new Date(throughMs).toISOString()}`); - // A revert commit that is a PR's merge commit follows that PR's rule. One pushed straight to the - // default branch counts: its git author can be forged, so it is not trusted to be harmless. for (const commit of commits) { - const targets = revertsOf(commit.commit?.message); - if (targets.length === 0) continue; - // A PR the REST lookup names but GraphQL cannot resolve is treated as no PR. - let human = true; - try { - const viaPr = gh(`repos/${repo}/commits/${commit.sha}/pulls`).find((p) => p.merged_at && p.merge_commit_sha === commit.sha); - if (viaPr) human = humanRevert(viaPr.number); - } catch (error) { - if (!(error instanceof UnresolvableError)) throw error; - } - for (const target of targets) reverts.push({ ...target, at: commit.commit?.committer?.date, human }); + for (const target of revertsOf(commit.commit?.message)) reverts.push({ ...target, at: commit.commit?.committer?.date }); } - const notHuman = new Map(); const firstRevert = new Map(); for (const revert of reverts) { if (!inWindow(revert.at)) continue; @@ -531,21 +514,7 @@ export function collect({ repo, since, through, laneLogins, gateChecks, aiReview continue; } } - if (!revert.human) notHuman.set(number, true); - else if (!firstRevert.has(number) || Date.parse(revert.at) < Date.parse(firstRevert.get(number))) firstRevert.set(number, revert.at); - } - // A revert no human merged is listed when it names a merge it would otherwise have counted. - for (const number of notHuman.keys()) { - let target; - try { - target = record(number); - } catch (error) { - if (!(error instanceof UnresolvableError)) throw error; - continue; - } - if (target.autonomous && (["C2", "C3"].includes(target.cls) || target.cls === null)) { - unresolvedReverts.push({ target: number, reason: "the revert was merged or enqueued by a bot or lane identity, not a human" }); - } + if (!firstRevert.has(number) || Date.parse(revert.at) < Date.parse(firstRevert.get(number))) firstRevert.set(number, revert.at); } for (const [number, at] of firstRevert) { let target; diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs index 8b10030bdf..fecee72b7c 100755 --- a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs @@ -494,9 +494,7 @@ test("a revert commit whose REST-named PR GraphQL cannot resolve counts as a dir assert.deepEqual(result.events, [{ cell: "C3-auto-merge", event: "reverted-merge", at: "2026-09-03T00:00:00Z" }]); }); -const NOT_HUMAN = "the revert was merged or enqueued by a bot or lane identity, not a human"; - -test("a revert PR a bot or lane identity merged or enqueued is listed, not counted", () => { +test("a revert PR counts whoever merged or enqueued it, a lane or another bot included", () => { const result = run({ pulls: [ pr(1701, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] }), @@ -505,20 +503,20 @@ test("a revert PR a bot or lane identity merged or enqueued is listed, not count pr(1704, "2026-09-02T00:00:00Z", { body: `Reverts ${REPO}#1703`, merger: "kyle-sexton", timeline: [queued("2026-09-01T23:00:00Z", null, "some-app[bot]")] }), ], }); - assert.deepEqual(result.events.filter((e) => e.event === "reverted-merge"), []); - assert.deepEqual(result.unresolvedReverts, [ - { target: 1701, reason: NOT_HUMAN }, - { target: 1703, reason: NOT_HUMAN }, + assert.deepEqual(result.events.filter((e) => e.event === "reverted-merge"), [ + { cell: "C2-auto-merge", event: "reverted-merge", at: "2026-09-02T00:00:00Z" }, + { cell: "C3-auto-merge", event: "reverted-merge", at: "2026-09-02T00:00:00Z" }, ]); + assert.deepEqual(result.unresolvedReverts, []); }); -test("the squash commit of a lane-merged revert PR does not count as a direct revert", () => { +test("a lane-merged revert PR and its squash commit count once", () => { const result = run({ pulls: [pr(1705, "2026-09-01T00:00:00Z", { labels: [MECHANICAL] }), pr(1706, "2026-09-02T00:00:00Z", { body: `Reverts ${REPO}#1705` })], commits: [{ ...REC.commit, sha: shaOf(1706), commit: { message: `Revert "x" (#1706)\n\nReverts ${REPO}#1705`, committer: { date: "2026-09-02T00:00:00Z" } } }], }); - assert.deepEqual(result.events.filter((e) => e.event === "reverted-merge"), []); - assert.deepEqual(result.unresolvedReverts, [{ target: 1705, reason: NOT_HUMAN }]); + assert.deepEqual(result.events.filter((e) => e.event === "reverted-merge"), [{ cell: "C2-auto-merge", event: "reverted-merge", at: "2026-09-02T00:00:00Z" }]); + assert.deepEqual(result.unresolvedReverts, []); }); test("a revert of a human merge, and a revert written only in prose, emit no event; prose is listed unresolved", () => { From 0057c55c058df59e47253bb6701b51b3964948ff Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:16:36 -0400 Subject: [PATCH 18/18] fix(source-control): name the revert term for every revert Co-Authored-By: Claude Opus 5.5 --- .../babysit-loop/scripts/publish-promotion-evidence.mjs | 4 ++-- .../babysit-loop/scripts/publish-promotion-evidence.test.mjs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs index 921cf2ed64..28b5ac6b08 100755 --- a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.mjs @@ -658,7 +658,7 @@ function report(counted, events, reviewsMeasured) { term("autonomous C2 completions", c2.length, ">= 20", c2.length >= 20), term("window days", c2Window.days, ">= 14", c2Window.days >= 14), gateTerm("deterministic-gate pass", c2), - term("human-reverted merges", reverted(c2), "0", reverted(c2) === 0), + term("reverted merges", reverted(c2), "0", reverted(c2) === 0), ]), "C3-auto-merge": cell(windowOf([...c2, ...c3]), [ term("autonomous C2 merges", c2.length, ">= 20", c2.length >= 20), @@ -666,7 +666,7 @@ function report(counted, events, reviewsMeasured) { term("C2 demotion events", c2Demotions, "0", c2Demotions === 0), term("autonomous C3 completions", c3.length, ">= 10", c3.length >= 10), gateTerm("C3 deterministic-gate pass", c3), - term("C3 human-reverted merges", reverted(c3), "0", reverted(c3) === 0), + term("C3 reverted merges", reverted(c3), "0", reverted(c3) === 0), term("C3 human-confirmed missed-blocking findings", missed(c3), "0", missed(c3) === 0), ]), "C3-ai-review-blocking": cell(windowOf(c3All), [ diff --git a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs index fecee72b7c..763aa3a5c4 100755 --- a/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs +++ b/plugins/source-control/skills/babysit-loop/scripts/publish-promotion-evidence.test.mjs @@ -468,7 +468,7 @@ test("a revert-button PR of an autonomous C2 merge yields reverted-merge at the ], }); assert.deepEqual(result.events, [{ cell: "C2-auto-merge", event: "reverted-merge", at: "2026-09-02T00:00:00Z" }]); - assert.deepEqual(termOf(result, "C2-auto-merge", "human-reverted merges"), { term: "human-reverted merges", value: 1, threshold: "0", pass: false }); + assert.deepEqual(termOf(result, "C2-auto-merge", "reverted merges"), { term: "reverted merges", value: 1, threshold: "0", pass: false }); }); test("a git-revert commit pushed straight to the default branch yields reverted-merge for C3 auto-merge", () => {