From 1e60c323e75b69503089e3952dd900c9b8c21c32 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 25 Oct 2026 23:59:15 -0400 Subject: [PATCH 01/33] fix(pr-pipeline): route pr-unblock stops to a failing status, not needs-human, unless a person must decide One table, .github/scripts/pr-unblock/escalation-routes.json, gives each pr-unblock stop reason its activity and route. report-escalation.sh, gate-source.mjs and clear-escalations.mjs read it. `hold` keeps the needs-human label and resume-key comment for SAFETY reasons and for questions only a person settles. `fail` adds no label: report-stop-status posts a failing pr-unblock/ (or pr-unblock/conflict) status on the gated head, which a new head does not carry, and one lane-stopped comment per activity is edited in place. UNCERTAIN and VALID-defer replies now go through the existing adjudicator. A pr-verify note left open ends feedback-verifier-open, and a stood finding left open again ends feedback-stood-open. Both route to fail. clear-escalations lifts labels that older runs set for reasons now routed to fail. Co-Authored-By: Claude Opus 5.5 --- .../fix-pr-unblock-route-not-park-5c1e7a20.md | 9 + .../actions/resolve-config/callers.test.mjs | 70 +++-- .../scripts/pr-merge/clear-escalations.mjs | 35 ++- .../pr-merge/clear-escalations.test.mjs | 43 +++ .../scripts/pr-unblock/address-feedback.mjs | 5 +- .../pr-unblock/adjudicate-feedback.mjs | 68 +++-- .../pr-unblock/adjudicate-feedback.prompt.md | 6 +- .../pr-unblock/adjudicate-feedback.test.mjs | 66 +++-- .../scripts/pr-unblock/escalation-routes.json | 41 +++ .github/scripts/pr-unblock/gate-source.mjs | 31 +-- .../scripts/pr-unblock/gate-source.test.mjs | 34 ++- .../scripts/pr-unblock/publish-feedback.mjs | 33 ++- .../pr-unblock/publish-feedback.test.mjs | 40 +-- .../scripts/pr-unblock/report-escalation.sh | 244 +++++++++--------- .../pr-unblock/report-escalation.test.sh | 202 ++++++++++----- .../pr-unblock/workflow-gates.test.mjs | 2 +- .github/workflows/pr-run-activity-read.yml | 7 +- .github/workflows/pr-run-activity-write.yml | 10 +- .github/workflows/pr-unblock.yml | 82 ++++-- docs/conventions/pr-pipeline.yaml | 5 +- docs/conventions/pr-pipeline/README.md | 34 ++- .../pr-pipeline/pr-run-activity.md | 45 ++-- 22 files changed, 755 insertions(+), 357 deletions(-) create mode 100644 .changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md create mode 100644 .github/scripts/pr-unblock/escalation-routes.json diff --git a/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md b/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md new file mode 100644 index 0000000000..0aebea606e --- /dev/null +++ b/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md @@ -0,0 +1,9 @@ +--- +bump: minor +--- + +### Changed + +- A `pr-unblock` run that stops now ends on the route its reason has in [`escalation-routes.json`](https://github.com/melodic-software/claude-code-plugins/blob/main/.github/scripts/pr-unblock/escalation-routes.json), the one routing table that `report-escalation.sh`, `gate-source.mjs` and `clear-escalations.mjs` read. Only a reason a person must settle keeps the `needs-human` label: a path the lane token may not push or a supply-chain refusal, an intent, design or security question, a person's own thread or hold, or a finding no adjudicator may judge (security-review or CRITICAL). Every other reason (for example `ci-fix-escalated`, `feedback-not-landed`, the round and repeat caps) ends with no label, a failing commit status on the head the activity gated (`pr-unblock/conflict` for `resolve-conflicts`, `pr-unblock/` otherwise, posted by the `report-stop-status` job that replaces `report-conflict-status`), and one lanes App comment per activity that the next stop edits in place. A new head carries no status. +- An `UNCERTAIN` or `VALID-defer` review reply now goes to the adjudicator as an `INCORRECT` one does, instead of to a person. A pr-verify note left open ends `feedback-verifier-open`, and a finding that stood at adjudication and was left open again ends `feedback-stood-open`; both route to a failing status. A round's reason held for a person now outranks any reason routed to a failing status. +- `clear-escalations` takes off a `needs-human` an older run set for a reason now routed to a failing status: a `ci-fix-*` reason at once, any other once the head moved past the one the lane stopped on. diff --git a/.github/actions/resolve-config/callers.test.mjs b/.github/actions/resolve-config/callers.test.mjs index a18a1ab829..4ee9df80bb 100644 --- a/.github/actions/resolve-config/callers.test.mjs +++ b/.github/actions/resolve-config/callers.test.mjs @@ -419,30 +419,52 @@ test("intake-triage.yml: the model may not apply good first issue or help wanted } }); -test("pr-unblock.yml: report-conflict-status runs only for an unresolved-conflict escalation, from the select, push or scan-refusal job, with a gated head", () => { +test("pr-unblock.yml: report-stop-status runs when report-escalation does, on its reason, with a gated head and never on a cancelled run", () => { const head = "c".repeat(40); const { jobs } = workflow("pr-unblock.yml"); - const job = jobs["report-conflict-status"]; - const context = (select, apply, headSha, refused = "") => ({ - "needs.resolve-conflicts-select.outputs.escalation-reason": select, - "needs.apply-conflict-patch.outputs.escalation-reason": apply, - "needs.conflict-scan-refusal.outputs.escalation-reason": refused, - "needs.resolve-conflicts-select.outputs.head-sha": headSha, + const job = jobs["report-stop-status"]; + const escalation = jobs["report-escalation"]; + assert.deepEqual(job.needs, escalation.needs); + const SOURCES = [ + "resolve-conflicts-select", + "apply-conflict-patch", + "conflict-scan-refusal", + "address-feedback-select", + "publish-feedback", + "review-dependency-select", + "apply-dependency-verdict", + "fix-ci-context", + "apply-patch", + ]; + const HEADS = ["resolve-conflicts-select", "address-feedback-select", "review-dependency-select", "fix-ci-context"]; + const context = ({ from = "", reason = "", headFrom = "resolve-conflicts-select", sha = head, rungUp = "", rungUpResult = "skipped", rungUpReason = "", rungUpSkip = "" } = {}) => ({ + ...Object.fromEntries(SOURCES.map((id) => [`needs.${id}.outputs.escalation-reason`, id === from ? reason : ""])), + ...Object.fromEntries(HEADS.map((id) => [`needs.${id}.outputs.head-sha`, id === headFrom ? sha : ""])), + "needs.apply-patch.outputs.rung-up": rungUp, + "needs.apply-patch-rung-up.result": rungUpResult, + "needs.apply-patch-rung-up.outputs.escalation-reason": rungUpReason, + "needs.apply-patch-rung-up.outputs.patch-skip-reason": rungUpSkip, }); - assert.deepEqual(job.needs, ["resolve-conflicts-select", "apply-conflict-patch", "conflict-scan-refusal"]); - assert.equal(evaluate(job.if, context("unresolved-conflict", "", head)), true, "before any model turn"); - assert.equal(evaluate(job.if, context("", "unresolved-conflict", head)), true, "a conflict the patch left"); - assert.equal(evaluate(job.if, context("", "", head, "unresolved-conflict")), true, "a scan that refused the model's output"); - for (const reason of ["repeat-cap", "protected-path-conflict", "workflow-staged", "conflict-patch-invalid", ""]) { - assert.equal(evaluate(job.if, context(reason, "", head)), false, reason); - assert.equal(evaluate(job.if, context("", reason, head)), false, reason); - } - assert.equal(evaluate(job.if, context("", "", "", "unresolved-conflict")), false, "no gated head to post on"); - assert.equal(evaluate(job.if, context("unresolved-conflict", "", "")), false, "no gated head to post on"); - // The head is the select job's, never a model call's. - const post = job.steps.find((step) => step.name === "Post the conflict status"); - assert.equal(post.env.HEAD_SHA, "${{ needs.resolve-conflicts-select.outputs.head-sha }}"); - assert.equal(post.env.ESCALATION_REASON, "unresolved-conflict"); + for (const from of SOURCES) { + assert.equal(evaluate(job.if, context({ from, reason: "x" })), true, from); + } + for (const headFrom of HEADS) { + assert.equal(evaluate(job.if, context({ from: "publish-feedback", reason: "x", headFrom })), true, headFrom); + } + assert.equal(evaluate(job.if, context()), false, "no reason"); + assert.equal(evaluate(job.if, context({ from: "apply-patch", reason: "x", sha: "" })), false, "no gated head to post on"); + assert.match(job.if, /^\$\{\{ !cancelled\(\) && /, "a cancelled run posts nothing"); + assert.doesNotMatch(job.if, /always\(\)/); + // The rung-up retry: a stop unless its patch applied cleanly. + assert.equal(evaluate(job.if, context({ headFrom: "fix-ci-context", rungUp: "true", rungUpResult: "failure" })), true); + assert.equal(evaluate(job.if, context({ headFrom: "fix-ci-context", rungUp: "true", rungUpResult: "success", rungUpReason: "ci-fix-invalid" })), true); + assert.equal(evaluate(job.if, context({ headFrom: "fix-ci-context", rungUp: "true", rungUpResult: "success" })), false, "the retry's patch applied"); + // The head is a select job's, never a model call's; the reason is the one + // report-escalation reads. + const post = job.steps.find((step) => step.name === "Post the stop status"); + assert.equal(post.env.HEAD_SHA.trim(), HEADS.map((id) => `needs.${id}.outputs.head-sha`).join(" || ").replace(/^/, "${{ ").replace(/$/, " }}")); + assert.equal(post.env.ESCALATION_REASON, escalation.with["escalation-reason"]); + assert.equal(post.run, "bash .base/.github/scripts/pr-unblock/report-escalation.sh status"); assert.deepEqual(job.permissions, { contents: "read", statuses: "write" }); assert.equal(job.uses, undefined, "a script job, never the write runner"); }); @@ -2458,12 +2480,14 @@ test("leak scan: a dependency verdict is scanned and hashed, and only a known ve // address-feedback: whether the scanned replies hold an INCORRECT one, the // only reply adjudicate-feedback has work for. A line that is not JSON reads // as disputed, so the adjudicator's own checks decide it. -test("leak scan: replies-disputed is true only for scanned replies holding an INCORRECT reply or an unreadable line", async (t) => { +test("leak scan: replies-disputed is true only for scanned replies holding a reply that leaves its thread open or an unreadable line", async (t) => { const plantReplies = (text) => ({ temp }) => writeFileSync(join(temp, "skill-reply", "replies.jsonl"), text); const row = (disposition) => JSON.stringify({ thread: "PRRT_a", disposition, evidence: [], fix_sha: "", note: "n" }); for (const [text, want] of [ [`${row("VALID")}\n${row("INCORRECT")}\n`, "true"], - [`${row("VALID")}\n${row("UNCERTAIN")}\n`, ""], + [`${row("VALID")}\n${row("UNCERTAIN")}\n`, "true"], + [`${row("VALID-defer")}\n`, "true"], + [`${row("VALID")}\n`, ""], [`${row("VALID")}\n{not json\n`, "true"], [`${JSON.stringify({ disposition: "incorrect" })}\n`, ""], ]) { diff --git a/.github/scripts/pr-merge/clear-escalations.mjs b/.github/scripts/pr-merge/clear-escalations.mjs index 641460695a..b69714d068 100755 --- a/.github/scripts/pr-merge/clear-escalations.mjs +++ b/.github/scripts/pr-merge/clear-escalations.mjs @@ -32,6 +32,10 @@ // unresolved-conflict, GitHub reports the PR mergeable: no conflict // protected-path-conflict is left to resolve // feedback-protected-path the PR changes nothing under .github/workflows/ +// a reason escalation-routes.json routes `fail` (a lane run before that +// route set the label): a fix-ci reason at once, the head's own CI +// checks staying red while it fails; any other once the head is not one +// its escalations name as their resume key // A listing that may be cut short, or a mergeable state not yet computed, // counts as the reason still holding. // Any other reason is decided by a person and is never cleared here. @@ -72,14 +76,18 @@ import { CLEARED, ESCALATION, escalationsSince, + escalationActivity, + escalationRoute, foreignMarker, isApp, labelHistory, + LANE_STOPPED, laneHold, lines, openReviewHold, PAIRED_MS, RESUME_KEY, + ROUTES, same, spoken, } from "../pr-unblock/gate-source.mjs"; @@ -135,6 +143,23 @@ const CHECKS = Object.freeze({ ? null : "the pull request changes nothing under .github/workflows/"; }, + // A reason escalation-routes.json now routes to a failing status rather + // than a person, on a label an older lane run set: fix-ci's at once, since + // the head's own CI checks stay red while it fails; any other once the head + // is not one the lane stopped on, since the lane checks a new head itself. + ...Object.fromEntries( + Object.keys(ROUTES) + .filter((reason) => escalationRoute(reason) === "fail" && reason !== "unresolved-conflict") + .map((reason) => [ + reason, + async ({ pull, keys }) => + escalationActivity(reason) === "fix-ci" + ? "the lane now ends it in a failing status and the head's own red checks, not a person" + : keys.length > 0 && !keys.includes(pull.head.sha) + ? "the lane now ends it in a failing status, not a person, and checks the new head itself" + : null, + ]), + ), }); function mergeable(pull) { @@ -170,7 +195,7 @@ function pairedWithPrMerge(appComments, since) { let newest = -Infinity; let ours = false; for (const item of appComments) { - const markers = lines(item.body).filter((line) => line.includes("\nStopped.", "2026-10-18T09:00:00Z", "2026-10-18T10:30:00Z"), + ], + }); + await run(w); + assert.equal(w.writes().at(-1)?.method, "DELETE"); +}); + test("a conflict reason clears only once GitHub reports the PR mergeable", async () => { const timeline = [labeled(BOT, "2026-10-18T10:00:00Z"), escalated("unresolved-conflict")]; const unknown = world({ timeline, mergeable: null }); diff --git a/.github/scripts/pr-unblock/address-feedback.mjs b/.github/scripts/pr-unblock/address-feedback.mjs index 1db8e21b96..493a4b3c9e 100644 --- a/.github/scripts/pr-unblock/address-feedback.mjs +++ b/.github/scripts/pr-unblock/address-feedback.mjs @@ -138,9 +138,10 @@ export function reviewedHeads(issueComments) { } // The first line of the reply publish-feedback.mjs posts on a thread it -// leaves open for a person: every disposition but VALID. +// leaves open: every disposition but VALID; older threads still carry the +// `for a person` forms. const ESCALATION_REPLY = - /^\*\*pr-unblock \/ address-feedback: (INCORRECT, left open|UNCERTAIN, left open for a person|VALID \(defer\), left open for a person)\*\*$/; + /^\*\*pr-unblock \/ address-feedback: (INCORRECT, left open|UNCERTAIN, left open(?: for a person)?|VALID \(defer\), left open(?: for a person)?)\*\*$/; // The first line of the reply publish-feedback.mjs posts when an adjudicator // finds a disputed finding stands (adjudicate-feedback.mjs): the next round // works that thread again and accepts no second INCORRECT on it. diff --git a/.github/scripts/pr-unblock/adjudicate-feedback.mjs b/.github/scripts/pr-unblock/adjudicate-feedback.mjs index 87778fae8f..3be49bda0b 100755 --- a/.github/scripts/pr-unblock/adjudicate-feedback.mjs +++ b/.github/scripts/pr-unblock/adjudicate-feedback.mjs @@ -1,6 +1,7 @@ #!/usr/bin/env node // adjudicate-feedback.mjs: the pr-unblock lane's adjudication of a disputed -// review finding, one an address-feedback round answered INCORRECT. A +// review finding, one an address-feedback round answered INCORRECT, UNCERTAIN +// or VALID-defer (DISPUTED): any reply that leaves the thread open. A // fresh-context model sees the whole review comment, the locations both the // reviewer and the lane cited and those files at the head, never the lane's // reasoning, and returns a verdict as structured output; this file stages its @@ -10,7 +11,7 @@ // with the job's read-only token, before any model turn: reads the // select job's threads.json and the scan job's replies.jsonl, each // checked against the SHA-256 its writer recorded, and for each -// INCORRECT reply on an adjudicable thread no adjudication has +// DISPUTED reply on an adjudicable thread no adjudication has // stood before and no trusted person directed (up to MAX_THREADS) // writes // //: finding.json (path, line, head, and the @@ -24,7 +25,7 @@ // entry per indexed thread; an absent or malformed verdict is // null. Outputs `sha256`. // -// settleDisputes is publish-feedback.mjs's INCORRECT branch: after the lane's +// settleDisputes is publish-feedback.mjs's DISPUTED branch: after the lane's // replies are posted it applies each verdict (adjudicationReplies, fixed text // only) and returns the round's escalation reason. @@ -39,6 +40,9 @@ import { createGitHubApi, FINDING_STANDS, need, SHA } from "./address-feedback.m // model step per slot. A thread past it is not adjudicated, and the round // escalates feedback-unadjudicated. export const MAX_THREADS = 3; +// The dispositions that leave a thread open, each one an adjudicator settles +// on an adjudicable thread instead of a person. +export const DISPUTED = Object.freeze(["INCORRECT", "UNCERTAIN", "VALID-defer"]); export const VERDICTS = ["lane", "reviewer", "human"]; export const QUESTIONS = ["intent", "design", "security"]; // The first line of each adjudication reply the lanes App posts: this, or @@ -122,7 +126,7 @@ function readChecked(path, expected, what) { return text; } -// The slots to adjudicate: each INCORRECT reply whose thread the select job +// The slots to adjudicate: each DISPUTED reply whose thread the select job // kept, is adjudicable, no adjudication has stood and no trusted person // directed, in reply order, one slot per thread. Returns [{thread, finding, // evidence, cited, paths}] and the threads past MAX_THREADS. @@ -134,7 +138,7 @@ export function disputes(threadFile, repliesText, repository = "") { for (const line of repliesText.split("\n").filter((row) => row.trim() !== "")) { const reply = JSON.parse(line); const thread = kept.get(reply?.thread); - if (reply?.disposition !== "INCORRECT" || !thread || !THREAD_ID.test(thread.id) || seen.has(thread.id) || !adjudicable(thread.comments) || stood(thread.comments) || directed(thread.comments)) { + if (!DISPUTED.includes(reply?.disposition) || !thread || !THREAD_ID.test(thread.id) || seen.has(thread.id) || !adjudicable(thread.comments) || stood(thread.comments) || directed(thread.comments)) { continue; } seen.add(thread.id); @@ -266,45 +270,55 @@ export function adjudicationReply(verdict, evidence, headSha) { FINDING_STANDS, "**pr-unblock / adjudicate-feedback: the finding stands**", "", - `${seen} and found that the finding holds. The next address-feedback round fixes it; another INCORRECT reply on this thread is not accepted and hands the pull request to a person.`, + `${seen} and found that the finding holds. The next address-feedback round fixes it; a round that leaves this thread open again ends the lane in a failing status.`, "", evidenceLine(evidence), ].join("\n"); } // The round's escalation reasons, most urgent first; one round records one. +// A reason escalation-routes.json holds for a person outranks every reason it +// routes to a failing status, so a question for a person is never lost. const ORDER = [ - "feedback-partial", - "feedback-unadjudicated", "feedback-question-security", "feedback-question-intent", "feedback-question-design", "feedback-disputed", "feedback-uncertain", + "feedback-partial", + "feedback-unadjudicated", + "feedback-stood-open", + "feedback-verifier-open", "", ]; +// Whether a thread is a pr-verify verifier note (threadAuthors tags it). +const verifierNote = (comments) => comments?.[0]?.author === "verifier"; + // The reply job's skip reason for a round that left a finding standing with // the PR head where the select job read it: no review run follows to start // the round that fixes it, so feedback-dispatch.mjs's sweep dispatches that // head again. export const FINDING_STANDS_SKIP = "finding-stands"; -// publish-feedback.mjs's INCORRECT branch, after the lane's replies posted: +// publish-feedback.mjs's DISPUTED branch, after the lane's replies posted: // - a thread an adjudication stood before (stood) that this round left open -// is the cap: feedback-disputed, and no second adjudication; -// - an INCORRECT thread a trusted person directed (directed) is theirs: +// is the cap: feedback-stood-open, and no second adjudication; +// - a DISPUTED thread a trusted person directed (directed) is theirs: // feedback-disputed, and no adjudication; -// - an INCORRECT thread that is not adjudicable (a CRITICAL or -// security-review finding, a verifier note, or one of unknown lane) goes to -// a person: feedback-disputed, and any verdict for it is ignored; -// - an INCORRECT thread with no verdict escalates feedback-unadjudicated; +// - a DISPUTED pr-verify verifier note: feedback-verifier-open (pr-verify's +// own FAIL holds the merge, and it judges the next head afresh); +// - any other DISPUTED thread that is not adjudicable (a CRITICAL or +// security-review finding, or one of unknown lane) goes to a person: +// feedback-disputed when the reply is INCORRECT, else feedback-uncertain, +// and any verdict for it is ignored; +// - a DISPUTED thread with no verdict escalates feedback-unadjudicated; // - human: no reply, feedback-question-; a question // only the owner answers stays the owner's; // - lane: the fixed reply, then the thread is resolved; -// - reviewer: the fixed FINDING_STANDS reply; when the head did not move -// this round (`pushed` false) and nothing escalates, the round skips -// FINDING_STANDS_SKIP. +// - reviewer: the fixed FINDING_STANDS reply, which the next round reads as +// a finding to fix; when the head did not move this round (`pushed` +// false) and nothing escalates, the round skips FINDING_STANDS_SKIP. // A thread that changed, a moved head or a failed write stops here as // feedback-partial. Returns {escalationReason, skipReason}. export async function settleDisputes({ api, pr, adjudicatedSha, posted, replies, shown, now, verdicts, pushed, headMoved, resolveAfterReply, escalationReason, log }) { @@ -317,24 +331,30 @@ export async function settleDisputes({ api, pr, adjudicatedSha, posted, replies, }; const byThread = new Map(replies.map((reply) => [reply.thread, reply])); for (const { thread, disposition } of posted) { - if (stood(now.get(thread)?.comments)) { + const comments = now.get(thread)?.comments; + if (stood(comments)) { if (disposition !== "VALID") { log(`${thread}: the finding stood at adjudication and this round left it open (${disposition})`); - raise("feedback-disputed"); + raise("feedback-stood-open"); } continue; } - if (disposition !== "INCORRECT") { + if (!DISPUTED.includes(disposition)) { continue; } - if (directed(now.get(thread)?.comments)) { + if (directed(comments)) { log(`${thread}: a trusted person directed this thread; it goes back to them, not to an adjudicator`); raise("feedback-disputed"); continue; } - if (!adjudicable(now.get(thread)?.comments)) { + if (verifierNote(comments)) { + log(`${thread}: a pr-verify note left open (${disposition}); pr-verify judges the next head`); + raise("feedback-verifier-open"); + continue; + } + if (!adjudicable(comments)) { log(`${thread}: not a non-CRITICAL code-review finding; it goes to a person, not to an adjudicator`); - raise("feedback-disputed"); + raise(disposition === "INCORRECT" ? "feedback-disputed" : "feedback-uncertain"); continue; } const verdict = verdicts.get(thread); diff --git a/.github/scripts/pr-unblock/adjudicate-feedback.prompt.md b/.github/scripts/pr-unblock/adjudicate-feedback.prompt.md index c7e8166e81..e70e726f6f 100644 --- a/.github/scripts/pr-unblock/adjudicate-feedback.prompt.md +++ b/.github/scripts/pr-unblock/adjudicate-feedback.prompt.md @@ -1,8 +1,8 @@ # Adjudicate one disputed review finding -A review lane posted a finding on a pull request. The lane that fixes findings answered that the -finding is incorrect and cited locations in the code as its evidence. You decide which side the -code supports. You see the whole review comment, the locations each side cited and the files at +A review lane posted a finding on a pull request. The lane that fixes findings left it open: it +answered that the finding is incorrect, that it could not tell, or that the fix belongs elsewhere, +and may have cited locations in the code as its evidence. You decide which side the code supports. You see the whole review comment, the locations each side cited and the files at the pull request head. You do not see the fixing lane's reasoning. No one can answer a question, and you change nothing: your structured output is the only result, and a script acts on it. diff --git a/.github/scripts/pr-unblock/adjudicate-feedback.test.mjs b/.github/scripts/pr-unblock/adjudicate-feedback.test.mjs index bebfa753d2..ebd7045aac 100644 --- a/.github/scripts/pr-unblock/adjudicate-feedback.test.mjs +++ b/.github/scripts/pr-unblock/adjudicate-feedback.test.mjs @@ -5,7 +5,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { test } from "node:test"; -import { adjudicationReply, collect, disputes, MAX_THREADS, main, parseVerdicts, prepare, reviewerEvidence, settleDisputes, verdictOf } from "./adjudicate-feedback.mjs"; +import { adjudicationReply, collect, DISPUTED, disputes, MAX_THREADS, main, parseVerdicts, prepare, reviewerEvidence, settleDisputes, verdictOf } from "./adjudicate-feedback.mjs"; import { noteComments } from "./verify-notes.mjs"; const HEAD = "a".repeat(40); @@ -116,26 +116,52 @@ test("disputes never adjudicates a CRITICAL code-review finding or a verifier no assert.deepEqual(disputes(fileOf(important), incorrect("PRRT_i")).slots.map((slot) => slot.thread), ["PRRT_i"]); }); -test("settleDisputes sends an INCORRECT reply on a CRITICAL code-review finding or a verifier note to a person, whatever verdict arrived", async () => { - const api = new Proxy({}, { get: (_, name) => () => assert.fail(`no API call expected, got ${String(name)}`) }); - for (const now of [new Map([["PRRT_a", criticalThread("PRRT_a")]]), new Map([["PRRT_a", verifierThread("PRRT_a")]])]) { - const result = await settleDisputes({ - api, - pr: 42, - adjudicatedSha: HEAD, - posted: [{ thread: "PRRT_a", disposition: "INCORRECT" }], - replies: [], - shown: new Map([["PRRT_a", { comment_ids: [1, 2] }]]), - now, - verdicts: new Map([["PRRT_a", { verdict: "lane", question: "none" }]]), - pushed: false, - headMoved: async () => false, - resolveAfterReply: async () => assert.fail("not resolved"), - escalationReason: "", - log: () => {}, - }); - assert.deepEqual(result, { escalationReason: "feedback-disputed", skipReason: "" }); +// settleDisputes with no API call allowed, on one posted thread. +const settleOffline = (now, disposition, verdicts = new Map([["PRRT_a", { verdict: "lane", question: "none" }]]), escalationReason = "") => + settleDisputes({ + api: new Proxy({}, { get: (_, name) => () => assert.fail(`no API call expected, got ${String(name)}`) }), + pr: 42, + adjudicatedSha: HEAD, + posted: [{ thread: "PRRT_a", disposition }], + replies: [], + shown: new Map([["PRRT_a", { comment_ids: [1, 2] }]]), + now, + verdicts, + pushed: false, + headMoved: async () => false, + resolveAfterReply: async () => assert.fail("not resolved"), + escalationReason, + log: () => {}, + }); + +test("settleDisputes holds a CRITICAL code-review finding left open for a person, whatever verdict arrived: disputed when INCORRECT, else uncertain", async () => { + const now = new Map([["PRRT_a", criticalThread("PRRT_a")]]); + assert.deepEqual(await settleOffline(now, "INCORRECT"), { escalationReason: "feedback-disputed", skipReason: "" }); + for (const disposition of ["UNCERTAIN", "VALID-defer"]) { + assert.deepEqual(await settleOffline(now, disposition), { escalationReason: "feedback-uncertain", skipReason: "" }, disposition); + } +}); + +test("settleDisputes ends a verifier note left open in feedback-verifier-open, a failing status and no person, whatever verdict arrived", async () => { + const now = new Map([["PRRT_a", verifierThread("PRRT_a")]]); + for (const disposition of DISPUTED) { + assert.deepEqual(await settleOffline(now, disposition), { escalationReason: "feedback-verifier-open", skipReason: "" }, disposition); + } +}); + +test("a reason held for a person outranks every fail-routed reason in one round", async () => { + const now = new Map([["PRRT_a", criticalThread("PRRT_a")]]); + assert.deepEqual(await settleOffline(now, "UNCERTAIN", undefined, "feedback-partial"), { escalationReason: "feedback-uncertain", skipReason: "" }); + const verifier = new Map([["PRRT_a", verifierThread("PRRT_a")]]); + assert.deepEqual(await settleOffline(verifier, "UNCERTAIN", undefined, "feedback-partial"), { escalationReason: "feedback-partial", skipReason: "" }); +}); + +test("disputes stages an UNCERTAIN or VALID-defer reply on an adjudicable thread as it does an INCORRECT one", () => { + for (const disposition of ["UNCERTAIN", "VALID-defer"]) { + const { slots } = disputes(fileOf(thread("PRRT_a"), criticalThread("PRRT_c")), [incorrect("PRRT_a", { disposition }), incorrect("PRRT_c", { disposition })].join("\n")); + assert.deepEqual(slots.map((slot) => slot.thread), ["PRRT_a"], disposition); } + assert.deepEqual(disputes(fileOf(thread("PRRT_a")), incorrect("PRRT_a", { disposition: "VALID" })).slots, []); }); test("the adjudication reply says what the adjudicator read: the review comment, not the lane's reasoning", () => { diff --git a/.github/scripts/pr-unblock/escalation-routes.json b/.github/scripts/pr-unblock/escalation-routes.json new file mode 100644 index 0000000000..1d7f60ec5f --- /dev/null +++ b/.github/scripts/pr-unblock/escalation-routes.json @@ -0,0 +1,41 @@ +{ + "protected-path-conflict": { "activity": "resolve-conflicts", "route": "hold", "why": "a conflicted path is a workflow, submodule config or a settings file, which the lane does not resolve" }, + "workflow-staged": { "activity": "resolve-conflicts", "route": "hold", "why": "the merge stages a change under .github/workflows/, which the lane does not push" }, + "unexpected-path-staged": { "activity": "resolve-conflicts", "route": "fail", "why": "the conflict resolution changes a path outside the conflicted ones, which the lane does not push" }, + "repeat-cap": { "activity": "resolve-conflicts", "route": "fail", "why": "the lane has resolved conflicts on this branch as many times in a row as its cap allows" }, + "unresolved-conflict": { "activity": "resolve-conflicts", "route": "fail", "why": "the lane could not resolve every conflict" }, + "conflict-patch-invalid": { "activity": "resolve-conflicts", "route": "fail", "why": "the conflict resolution the lane wrote did not pass validation, so the merge was not pushed" }, + "conflict-patch-not-applied": { "activity": "resolve-conflicts", "route": "fail", "why": "the conflict resolution did not apply to the merge it was written for, so the merge was not pushed" }, + "conflict-patch-protected-path": { "activity": "resolve-conflicts", "route": "hold", "why": "the conflict resolution changes a path the lane does not push: .github/workflows/, CODEOWNERS, .gitmodules or a settings file" }, + "feedback-protected-path": { "activity": "address-feedback", "route": "hold", "why": "the pull request changes .github/workflows/, which the lane token cannot push, so the lane does not address its review threads" }, + "feedback-round-cap": { "activity": "address-feedback", "route": "hold", "why": "the lane has addressed review threads on this pull request as many rounds as its cap allows, and an open thread is a security or intent finding or a person holds the pull request, so a person decides" }, + "feedback-not-converging": { "activity": "address-feedback", "route": "fail", "why": "review rounds are not converging: after the round cap and one fresh fixer round the code review still has open findings, so the fixer and the reviewer disagree on what the change should be" }, + "feedback-uncertain": { "activity": "address-feedback", "route": "hold", "why": "the lane could not settle a security-review or CRITICAL finding, or one no known review lane wrote, which no adjudicator judges, so a person decides it" }, + "feedback-verifier-open": { "activity": "address-feedback", "route": "fail", "why": "the lane left a pr-verify note open (uncertain, deferred or answered incorrect); pr-verify judges the next head afresh" }, + "feedback-not-landed": { "activity": "address-feedback", "route": "fail", "why": "a fix the lane recorded did not reach the branch, so none of its review replies was posted" }, + "feedback-invalid": { "activity": "address-feedback", "route": "fail", "why": "the review replies the lane recorded did not pass validation, so none was posted" }, + "feedback-partial": { "activity": "address-feedback", "route": "fail", "why": "posting the review replies stopped partway, or a fixed thread could not be resolved" }, + "feedback-incomplete": { "activity": "address-feedback", "route": "fail", "why": "the lane did not record a disposition for every review thread it kept, so none was posted" }, + "feedback-disputed": { "activity": "address-feedback", "route": "hold", "why": "a disputed review finding is one only a person decides (a security-review or CRITICAL finding, one no known review lane wrote, or a thread a person directed)" }, + "feedback-stood-open": { "activity": "address-feedback", "route": "fail", "why": "an adjudicator found a review finding stands, and the next round left it open again" }, + "feedback-unadjudicated": { "activity": "address-feedback", "route": "fail", "why": "the lane disputed a review finding and no adjudicator verdict on that thread was recorded" }, + "feedback-question-intent": { "activity": "address-feedback", "route": "hold", "why": "an adjudicator found that a disputed review finding turns on what the change is meant to do, which a person decides" }, + "feedback-question-design": { "activity": "address-feedback", "route": "hold", "why": "an adjudicator found that a disputed review finding turns on a design choice, which a person decides" }, + "feedback-question-security": { "activity": "address-feedback", "route": "hold", "why": "an adjudicator found that a disputed review finding turns on a security judgment, which a person decides" }, + "dependency-round-cap": { "activity": "review-dependency", "route": "fail", "why": "the lane has reviewed this dependency update as many rounds as its cap allows" }, + "dependency-invalid": { "activity": "review-dependency", "route": "fail", "why": "the dependency review verdict did not pass validation, so the lane acted on none of it" }, + "dependency-fix-not-landed": { "activity": "review-dependency", "route": "fail", "why": "the dependency review chose a fix, but no fix commit reached the branch (the lane lands a fix only when it changes nothing but the versions in npm or pip manifest and lock files outside .github/)" }, + "dependency-security-close": { "activity": "review-dependency", "route": "hold", "why": "the dependency review would close a security update, which the lane never closes" }, + "dependency-escalate": { "activity": "review-dependency", "route": "hold", "why": "the dependency review found a risk the lane does not decide alone" }, + "dependency-config-unreadable": { "activity": "review-dependency", "route": "fail", "why": "the lane could not read the base .github/dependabot.yml, so it does not mark this dependency update for merge" }, + "dependency-files-truncated": { "activity": "review-dependency", "route": "fail", "why": "the dependency review chose merge, but the files listing it read was cut short, so the lane does not mark this dependency update for merge" }, + "dependency-fix-refused": { "activity": "review-dependency", "route": "hold", "why": "a commit on the dependency update other than Dependabot's changes more than a version of an npm or pip manifest or lock file, as a regular file outside .github/, so the lane does not act on the review" }, + "dependency-control-uncited": { "activity": "review-dependency", "route": "fail", "why": "the dependency review chose merge for a review-only dependency without quoting its control, so the lane does not mark it for merge" }, + "dependency-workflow-unpinned": { "activity": "review-dependency", "route": "hold", "why": "the dependency review chose merge, but the update changes a workflow or action file by more than swapping one full-SHA action pin for another, so the lane does not mark it for merge" }, + "ci-fix-round-cap": { "activity": "fix-ci", "route": "fail", "why": "the lane has pushed as many CI fixes to this pull request as its cap allows, and CI still fails" }, + "ci-fix-empty": { "activity": "fix-ci", "route": "fail", "why": "the lane changed nothing that fixes the failing checks" }, + "ci-fix-invalid": { "activity": "fix-ci", "route": "fail", "why": "the CI fix the lane wrote did not pass validation, so it was not pushed" }, + "ci-fix-protected-path": { "activity": "fix-ci", "route": "hold", "why": "the CI fix changes a path the lane does not push: .github/workflows/, CODEOWNERS, .gitmodules or a settings file" }, + "ci-fix-not-applied": { "activity": "fix-ci", "route": "fail", "why": "the CI fix did not apply to the head it was written for, so it was not pushed" }, + "ci-fix-escalated": { "activity": "fix-ci", "route": "fail", "why": "the lane could not fix every failing check, after one retry a model rung up; its run log names each one and why" } +} diff --git a/.github/scripts/pr-unblock/gate-source.mjs b/.github/scripts/pr-unblock/gate-source.mjs index fe380fb4ad..872b35c40f 100644 --- a/.github/scripts/pr-unblock/gate-source.mjs +++ b/.github/scripts/pr-unblock/gate-source.mjs @@ -17,6 +17,8 @@ // `label`, `created_at`, `updated_at`. Text written by anyone but those // authors is never read. +import { readFileSync } from "node:fs"; + export const ESCALATION = ""; export const RESUME_KEY = /^$/; // Any escalation marker, of any lane: one that is not a whole ESCALATION @@ -27,6 +29,9 @@ const ANY_ESCALATION = ""; const MODEL_RUN = /^$/; export const CLEARED = /^$/; +// report-escalation.sh's comment for a fail-routed stop, one per activity, +// edited in place; it adds no label. +export const LANE_STOPPED = /^$/; // A review lane's hold comment opens a line with this marker. const REVIEW_HOLD = "$/; @@ -129,7 +134,7 @@ export function gateSource({ timeline, label, head, isLanesApp }) { // A whole escalation's two marker lines and address-feedback's own round // record are known; any other marker, another lane's escalation or an // escalation line out of place included, is not. - const known = (line) => line === ESCALATION || RESUME_KEY.test(line) || line === ROUND_MARKER || MODEL_RUN.test(line); + const known = (line) => line === ESCALATION || RESUME_KEY.test(line) || line === ROUND_MARKER || MODEL_RUN.test(line) || LANE_STOPPED.test(line); const malformed = appComments .filter((item) => writtenSince(item, since)) .map((item) => lines(item.body)) @@ -144,21 +149,17 @@ export function gateSource({ timeline, label, head, isLanesApp }) { // The Reason line report-escalation.sh writes into its escalation comment. const REASON_LINE = /^- Reason: `([a-z][a-z-]*)`$/; -// resolve-conflicts's reasons, which report-escalation.sh maps by default. -const CONFLICT_REASONS = new Set(["protected-path-conflict", "workflow-staged", "unexpected-path-staged", "repeat-cap", "unresolved-conflict"]); +// escalation-routes.json, the one routing table: each reason's activity and +// route (`hold` hands the PR to a person with the label; `fail` ends in a +// failing status and no label), as report-escalation.sh reads it. +export const ROUTES = Object.freeze(JSON.parse(readFileSync(new URL("./escalation-routes.json", import.meta.url), "utf8"))); + +// The activity a reason resumes, or null for a reason the table does not +// list, which holds every activity. +export const escalationActivity = (reason) => (Object.hasOwn(ROUTES, reason) ? ROUTES[reason].activity : null); -// The activity a reason resumes, as report-escalation.sh maps it, or null for -// a reason it does not list, which holds every activity. -export const escalationActivity = (reason) => - reason.startsWith("feedback-") - ? "address-feedback" - : reason.startsWith("dependency-") - ? "review-dependency" - : reason.startsWith("ci-fix-") - ? "fix-ci" - : CONFLICT_REASONS.has(reason) || reason.startsWith("conflict-patch-") - ? "resolve-conflicts" - : null; +// A reason's route, or null for a reason the table does not list. +export const escalationRoute = (reason) => (Object.hasOwn(ROUTES, reason) ? ROUTES[reason].route : null); // When a timeline item was written, and by whom, for a comment or a review // (clear-escalations.mjs reads people speaking the same way). diff --git a/.github/scripts/pr-unblock/gate-source.test.mjs b/.github/scripts/pr-unblock/gate-source.test.mjs index 209b224303..cdcc013c11 100644 --- a/.github/scripts/pr-unblock/gate-source.test.mjs +++ b/.github/scripts/pr-unblock/gate-source.test.mjs @@ -2,9 +2,10 @@ // review lanes' hold marker (docs/conventions/pr-pipeline/README.md, "Review-lane // holds"), and from GitHub's GraphQL LabeledEvent and UnlabeledEvent shapes. import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; import { test } from "node:test"; -import { gateSource, laneEscalationAllows, listLabelEvents, prHold } from "./gate-source.mjs"; +import { escalationActivity, escalationRoute, gateSource, laneEscalationAllows, listLabelEvents, prHold, ROUTES } from "./gate-source.mjs"; const APP = { login: "melodic-automation-lanes[bot]", id: 337595936, type: "Bot" }; const ACTIONS = { login: "github-actions[bot]", id: 41898282, type: "Bot" }; @@ -51,6 +52,37 @@ test("another lane's marker on a lanes App comment since the label is unknown", assert.equal(source(timeline), "unknown"); }); +test("report-escalation.sh's lane-stopped comment, posted or edited since the label, leaves a lane escalation workable", () => { + const timeline = [ + label("labeled", APP, "2026-10-18T10:00:00Z"), + escalation(), + { ...commented(APP, "\nStopped.", "2026-10-18T09:00:00Z"), updated_at: "2026-10-18T10:30:00Z" }, + ]; + assert.equal(source(timeline), "lane-escalation"); + // Only that marker's exact shape is known. + const odd = [label("labeled", APP, "2026-10-18T10:00:00Z"), escalation(), commented(APP, "", "2026-10-18T10:30:00Z")]; + assert.equal(source(odd), "unknown"); +}); + +// The write runner's escalation enum (pr-run-activity-write.yml, its first +// step) is the list of reasons a run may hand report-escalation.sh; the +// routing table must route each one, and route nothing else. +test("escalation-routes.json routes every reason the write runner accepts, each to one activity and one route", () => { + const workflow = readFileSync(new URL("../../workflows/pr-run-activity-write.yml", import.meta.url), "utf8"); + const block = workflow.slice(workflow.indexOf("case $ESCALATION_REASON in"), workflow.indexOf("patch-empty | patch-invalid")); + const accepted = [...block.matchAll(/(?<=^|[\s|])([a-z]+(?:-[a-z]+)+)(?=\s*(?:\||\)))/gm)].map((match) => match[1]); + assert.ok(accepted.length > 30, `read ${accepted.length} reasons from the enum`); + assert.deepEqual([...new Set(accepted)].sort(), Object.keys(ROUTES).sort()); + for (const [reason, entry] of Object.entries(ROUTES)) { + assert.ok(["resolve-conflicts", "address-feedback", "review-dependency", "fix-ci"].includes(entry.activity), reason); + assert.ok(["hold", "fail"].includes(entry.route), reason); + assert.equal(escalationActivity(reason), entry.activity); + assert.equal(escalationRoute(reason), entry.route); + } + assert.equal(escalationActivity("bogus"), null); + assert.equal(escalationRoute("constructor"), null); +}); + test("address-feedback's own round records since the escalation leave it workable", () => { const timeline = [ label("labeled", APP, "2026-10-18T10:00:00Z"), diff --git a/.github/scripts/pr-unblock/publish-feedback.mjs b/.github/scripts/pr-unblock/publish-feedback.mjs index 5805601824..0a0f41f2ff 100755 --- a/.github/scripts/pr-unblock/publish-feedback.mjs +++ b/.github/scripts/pr-unblock/publish-feedback.mjs @@ -38,11 +38,12 @@ // refusal or a named upload feedback-invalid. Otherwise escalates // feedback-invalid, feedback-incomplete (a kept thread has no // disposition), feedback-not-landed, feedback-partial (posting stopped -// after a reply), feedback-uncertain (an UNCERTAIN or VALID-defer reply), -// or a reason adjudicate-feedback.mjs settleDisputes gives for an -// INCORRECT reply: feedback-unadjudicated, feedback-question-intent, -// feedback-question-design, feedback-question-security, or -// feedback-disputed (the finding stood at an earlier adjudication and +// after a reply), or a reason adjudicate-feedback.mjs settleDisputes +// gives for an INCORRECT, UNCERTAIN or VALID-defer reply: +// feedback-unadjudicated, feedback-question-intent, +// feedback-question-design, feedback-question-security, +// feedback-disputed, feedback-uncertain, feedback-verifier-open, or +// feedback-stood-open (the finding stood at an earlier adjudication and // this round left it open). A finding that stands with the head unmoved // skips finding-stands instead, which feedback-dispatch.mjs's sweep // dispatches again. @@ -91,7 +92,7 @@ import { SHA, trustedHumansFrom, } from "./address-feedback.mjs"; -import { parseVerdicts, settleDisputes } from "./adjudicate-feedback.mjs"; +import { DISPUTED, parseVerdicts, settleDisputes } from "./adjudicate-feedback.mjs"; import { inThreadScope } from "./thread-scope.mjs"; // bytes: replies.jsonl's cap, the read runner's carry, seal and scan cap too; @@ -118,8 +119,8 @@ const KEYS = ["disposition", "evidence", "fix_sha", "note", "thread"]; const LABELS = { VALID: "VALID, fixed", INCORRECT: "INCORRECT, left open", - UNCERTAIN: "UNCERTAIN, left open for a person", - "VALID-defer": "VALID (defer), left open for a person", + UNCERTAIN: "UNCERTAIN, left open", + "VALID-defer": "VALID (defer), left open", }; class Invalid extends Error {} @@ -562,9 +563,6 @@ async function postReplies({ api, pr, headSha, headRef, appId, patchState = "", return { escalationReason, posted: [] }; } replies = replies.map((reply) => (reply.disposition === "VALID" ? { ...reply, fixSha: landed.get(reply.thread) } : reply)); - if (replies.some((reply) => reply.disposition === "UNCERTAIN" || reply.disposition === "VALID-defer")) { - escalationReason = "feedback-uncertain"; - } const posted = []; for (const reply of replies) { const ids = shown.get(reply.thread).comment_ids; @@ -608,11 +606,12 @@ async function postReplies({ api, pr, headSha, headRef, appId, patchState = "", } log(`replied on ${reply.thread} (${reply.disposition})${reply.disposition === "VALID" ? "; resolved" : ""}`); } - // An INCORRECT thread goes to the adjudicator's verdict, not straight to - // a person (owner decision): it is resolved, handed back to the next round, - // or escalated with the question the verdict names; a thread whose finding - // stood at an earlier adjudication and is still open escalates - // feedback-disputed (adjudicate-feedback.mjs settleDisputes). The + // A thread left open (INCORRECT, UNCERTAIN or VALID-defer) goes to the + // adjudicator's verdict, not straight to a person (owner decision): it is + // resolved, handed back to the next round, or escalated with the question + // the verdict names; a thread whose finding stood at an earlier + // adjudication and is still open ends feedback-stood-open + // (adjudicate-feedback.mjs settleDisputes). The // adjudicator read the select job's head, which this round's fix push // may have moved past headSha. // A head that moved this round gets a review run, which starts the next @@ -625,7 +624,7 @@ async function postReplies({ api, pr, headSha, headRef, appId, patchState = "", replies, shown, now, - verdicts: posted.some((entry) => entry.disposition === "INCORRECT") ? parseVerdicts(readAdjudication(), run.head_sha) : new Map(), + verdicts: posted.some((entry) => DISPUTED.includes(entry.disposition)) ? parseVerdicts(readAdjudication(), run.head_sha) : new Map(), pushed: run.head_sha !== headSha, headMoved, resolveAfterReply, diff --git a/.github/scripts/pr-unblock/publish-feedback.test.mjs b/.github/scripts/pr-unblock/publish-feedback.test.mjs index 601078377b..e441e39745 100644 --- a/.github/scripts/pr-unblock/publish-feedback.test.mjs +++ b/.github/scripts/pr-unblock/publish-feedback.test.mjs @@ -244,7 +244,7 @@ test("every reply the reply job leaves open reads as an escalation; a VALID one assert.equal(isEscalationReply({ user: { id: APP }, body: body("VALID") }, APP), false); }); -test("INCORRECT is posted and left open; UNCERTAIN and VALID-defer are posted, left open, and escalate", async () => { +test("INCORRECT, UNCERTAIN and VALID-defer are posted, left open, and go to the adjudicator, not a person", async () => { const api = fakeApi(); const result = await run( api, @@ -253,20 +253,27 @@ test("INCORRECT is posted and left open; UNCERTAIN and VALID-defer are posted, l reply({ thread: "PRRT_b", disposition: "UNCERTAIN", evidence: [], note: "Needs the vendor docs." }), ), undefined, - // The finding stands, so INCORRECT leaves the thread open; UNCERTAIN - // still escalates. + // The finding stands, so INCORRECT leaves the thread open; the UNCERTAIN + // thread has no verdict, so the round ends unadjudicated: a failing + // status, not a person. verdicts(judged("PRRT_a", "reviewer")), ); - assert.equal(result.escalationReason, "feedback-uncertain"); + assert.equal(result.escalationReason, "feedback-unadjudicated"); assert.deepEqual(api.resolved(), []); assert.match(api.posted()[0][3], /^\*\*pr-unblock \/ address-feedback: INCORRECT, left open\*\*\n\nEvidence: `src\/a.js:3`, `docs\/b.md:1`\n/); - assert.match(api.posted()[1][3], /UNCERTAIN, left open for a person\*\*\n\nEvidence: none in the checkout\n/); + assert.match(api.posted()[1][3], /UNCERTAIN, left open\*\*\n\nEvidence: none in the checkout\n/); assert.deepEqual(api.calls.find((call) => call[0] === "compare"), ["compare", GATED, NOW], "a moved branch is checked even with no VALID reply"); + // An adjudicator that finds the code answers a deferred finding resolves + // it; one that finds an UNCERTAIN finding stands hands it to the next round. const defer = fakeApi(); - assert.equal((await run(defer, lines(reply({ disposition: "VALID-defer" })))).escalationReason, "feedback-uncertain"); - assert.match(defer.posted()[0][3], /VALID \(defer\), left open for a person/); - assert.deepEqual(defer.resolved(), []); + assert.equal((await run(defer, lines(reply({ disposition: "VALID-defer" })), undefined, verdicts(judged("PRRT_a", "lane")))).escalationReason, ""); + assert.match(defer.posted()[0][3], /VALID \(defer\), left open\*\*/); + assert.deepEqual(defer.resolved(), ["PRRT_a"]); + const unsure = fakeApi(); + assert.equal((await run(unsure, lines(reply({ disposition: "UNCERTAIN" })), undefined, verdicts(judged("PRRT_a", "reviewer")))).escalationReason, ""); + assert.match(unsure.posted()[1][3], /finding-stands/); + assert.deepEqual(unsure.resolved(), []); }); test("the round marker is posted once, as the first line, numbered after the App's earlier rounds", async () => { @@ -533,7 +540,7 @@ test("a reviewer verdict posts the finding-stands reply and leaves the thread op const stands = api.posted()[2][3].split("\n"); assert.equal(stands[0], ""); assert.equal(stands[1], "**pr-unblock / adjudicate-feedback: the finding stands**"); - assert.match(stands[3], /another INCORRECT reply on this thread is not accepted/); + assert.match(stands[3], /a round that leaves this thread open again ends the lane in a failing status/); assert.equal(stands[5], "Evidence: `docs/b.md:1`"); }); @@ -552,7 +559,8 @@ test("a finding that stands with the head unmoved skips finding-stands and escal assert.equal(api.posted().length, 2, "the lane's reply and the finding-stands reply"); assert.deepEqual(api.resolved(), []); - // An UNCERTAIN reply in the same round still goes to a person. + // An UNCERTAIN reply in the same round with no verdict of its own ends the + // round unadjudicated. const uncertain = fakeApi({ compare: unmoved }); const both = await run( uncertain, @@ -560,7 +568,7 @@ test("a finding that stands with the head unmoved skips finding-stands and escal atNow(), verdictsAt(NOW, judged("PRRT_a", "reviewer")), ); - assert.deepEqual([both.escalationReason, both.skipReason], ["feedback-uncertain", ""]); + assert.deepEqual([both.escalationReason, both.skipReason], ["feedback-unadjudicated", ""]); }); // #9560's fallback lands a VALID reply on an earlier lane commit: nothing is @@ -629,20 +637,20 @@ const stoodApi = (overrides = {}) => }); const stoodRun = JSON.stringify({ ...RUN, threads: [{ id: "PRRT_a", path: "src/a.js", comment_ids: [100, 101, 102] }] }); -test("no second dispute: an INCORRECT reply on a thread whose finding stood escalates feedback-disputed and is not adjudicated", async () => { +test("no second dispute: an INCORRECT reply on a thread whose finding stood ends feedback-stood-open and is not adjudicated", async () => { const api = stoodApi({ compare: { status: "identical", commits: [], files: [] } }); const result = await run(api, lines(reply()), stoodRun, verdicts(judged("PRRT_a", "lane"))); - assert.equal(result.escalationReason, "feedback-disputed"); + assert.equal(result.escalationReason, "feedback-stood-open"); assert.equal(api.posted().length, 1, "the lane's reply, and no adjudication reply"); assert.match(api.posted()[0][3], /INCORRECT, left open/); assert.deepEqual(api.resolved(), []); }); -test("the cap: a stood thread the next round leaves open escalates feedback-disputed; one it fixes resolves", async () => { +test("the cap: a stood thread the next round leaves open ends feedback-stood-open; one it fixes resolves", async () => { for (const disposition of ["UNCERTAIN", "VALID-defer"]) { const api = stoodApi({ compare: { status: "identical", commits: [], files: [] } }); const result = await run(api, lines(reply({ disposition, evidence: [] })), stoodRun); - assert.equal(result.escalationReason, "feedback-disputed", disposition); + assert.equal(result.escalationReason, "feedback-stood-open", disposition); assert.deepEqual(api.resolved(), []); } const fixed = stoodApi(); @@ -1131,7 +1139,7 @@ test("main writes the escalation and skip reason files and refuses a symlinked a }; const api = fakeApi(); assert.equal(await main(env, { api, log: () => {} }), 0); - assert.equal(readFileSync(env.ESCALATION_REASON_FILE, "utf8"), "feedback-uncertain\n"); + assert.equal(readFileSync(env.ESCALATION_REASON_FILE, "utf8"), "feedback-unadjudicated\n"); // replies.jsonl must be the bytes the scan job uploaded, whose SHA-256 it // passed out. diff --git a/.github/scripts/pr-unblock/report-escalation.sh b/.github/scripts/pr-unblock/report-escalation.sh index c7f509f3d1..d24f041587 100755 --- a/.github/scripts/pr-unblock/report-escalation.sh +++ b/.github/scripts/pr-unblock/report-escalation.sh @@ -1,38 +1,27 @@ #!/usr/bin/env bash -# report-escalation.sh: the pr-unblock lane's report-escalation leaf. Hands the -# PR to a person: one machine-marked comment with the reason, the resume key -# and how to resume, plus the human-gated role label. Run from the base copy; -# it reads only the trusted base checkout and the GitHub API, never a PR head +# report-escalation.sh: the pr-unblock lane's report-escalation leaf. Ends a +# lane run that stopped, on the route escalation-routes.json (beside this +# script, the one routing table) gives its reason. Run from the base copy; it +# reads only the trusted base checkout and the GitHub API, never a PR head # file, and never writes to the branch. # +# - route `hold` (SAFETY, or a question for a person: intent, design, HIGH or +# CRITICAL security, a person's own hold): hands the PR to a person with +# one machine-marked comment holding the reason, the resume key and how to +# resume, plus the human-gated role label. +# - route `fail` (a defect the lane's own bounded retries did not mend): no +# label. One lanes App comment per activity, marked LANE_STOPPED, is posted +# once and edited in place after, naming the reason and the head; status +# mode (below) puts a failing commit status on that head. A new head carries +# no status, so the lane's next run there starts clean. +# # Environment: # GH_TOKEN token with pull-requests: write and issues: write # (statuses: write in status mode) # GITHUB_REPOSITORY owner/repo # PR_NUMBER the pull request # HEAD_SHA the gated head; it is the resume key -# ESCALATION_REASON protected-path-conflict | workflow-staged -# | unexpected-path-staged | repeat-cap -# | unresolved-conflict | feedback-protected-path -# | feedback-round-cap | feedback-uncertain -# | feedback-not-landed | feedback-invalid -# | feedback-partial | feedback-incomplete -# | feedback-disputed | feedback-unadjudicated -# | feedback-question-intent | feedback-question-design -# | feedback-question-security | feedback-not-converging -# | dependency-round-cap -# | dependency-invalid | dependency-fix-not-landed -# | dependency-security-close | dependency-escalate -# | dependency-config-unreadable -# | dependency-files-truncated -# | dependency-control-uncited -# | dependency-workflow-unpinned -# | dependency-fix-refused | ci-fix-round-cap -# | ci-fix-empty | ci-fix-invalid -# | ci-fix-protected-path | ci-fix-not-applied -# | ci-fix-escalated | conflict-patch-invalid -# | conflict-patch-not-applied -# | conflict-patch-protected-path +# ESCALATION_REASON a key of escalation-routes.json # TRACKER_CONFIG base .work-item-tracker.json # (default .base/.work-item-tracker.json) # LANES_BOT_LOGIN the lanes App bot login, the only author whose marker @@ -40,20 +29,22 @@ # # The role label is config.role_labels["human-gated"]. A missing file or entry # means needs-human; a present entry that is not a non-empty string fails. -# Idempotent: with the label on the PR and a LANES_BOT_LOGIN marker for this -# resume key and reason, it writes nothing; another reason on the same head -# gets its own comment, so gate-source.mjs reads every reason the label holds -# for. Without the label (a person removed it to resume), it +# Hold is idempotent: with the label on the PR and a LANES_BOT_LOGIN marker +# for this resume key and reason, it writes nothing; another reason on the +# same head gets its own comment, so gate-source.mjs reads every reason the +# label holds for. Without the label (a person removed it to resume), it # escalates again. The label is written before the comment, so a retry after a # failure between the two finds the label and posts at most one comment. The # comment write is tried three times, so a transient failure does not leave the -# label without its explanation. +# label without its explanation. Fail writes nothing when its comment already +# reads as it would. # -# With the argument `status`, it instead posts one failing `pr-unblock/conflict` -# commit status on HEAD_SHA when the reason is unresolved-conflict, and nothing -# for any other reason. GitHub starts no pull_request workflow on a conflicted -# PR, so this status is what shows the head why its CI is missing. That mode -# needs GH_TOKEN with statuses: write and reads TARGET_URL (optional, the run). +# With the argument `status`, it instead posts one failing commit status on +# HEAD_SHA for a fail-routed reason, and nothing for a held one: context +# `pr-unblock/conflict` for resolve-conflicts (GitHub starts no pull_request +# workflow on a conflicted PR, so this status is also what shows the head why +# its CI is missing), else `pr-unblock/`. That mode needs GH_TOKEN +# with statuses: write and reads TARGET_URL (optional, the run). # # Exit: 0 when escalated or already escalated; 1 on bad input or API failure. set -euo pipefail @@ -63,52 +54,13 @@ die() { exit 1 } -case ${ESCALATION_REASON:-} in - protected-path-conflict) why='a conflicted path is a workflow, submodule config or a settings file, which the lane does not resolve' ;; - workflow-staged) why='the merge stages a change under .github/workflows/, which the lane does not push' ;; - unexpected-path-staged) why='the conflict resolution changes a path outside the conflicted ones, which the lane does not push' ;; - repeat-cap) why='the lane has resolved conflicts on this branch as many times in a row as its cap allows' ;; - unresolved-conflict) why='the lane could not resolve every conflict' ;; - feedback-protected-path) why='the pull request changes .github/workflows/, which the lane token cannot push, so the lane does not address its review threads' ;; - feedback-round-cap) why='the lane has addressed review threads on this pull request as many rounds as its cap allows, and an open thread is a security or intent finding or a person holds the pull request, so a person decides' ;; - feedback-not-converging) why='review rounds are not converging: after the round cap and one fresh fixer round the code review still has open findings, so the fixer and the reviewer disagree on what the change should be' ;; - feedback-uncertain) why='a review finding is uncertain or deferred, so a person decides it' ;; - feedback-not-landed) why='a fix the lane recorded did not reach the branch, so none of its review replies was posted' ;; - feedback-invalid) why='the review replies the lane recorded did not pass validation, so none was posted' ;; - feedback-partial) why='posting the review replies stopped partway, or a fixed thread could not be resolved, so a person finishes them' ;; - feedback-incomplete) why='the lane did not record a disposition for every review thread it kept, so none was posted' ;; - feedback-disputed) why='a disputed review finding is one only a person decides (a security-review finding, a thread a person directed, or one an adjudicator found stands and the next round left open again)' ;; - feedback-unadjudicated) why='the lane replied with evidence that a review finding is incorrect, and no adjudicator verdict on that thread was recorded, so a person decides it' ;; - feedback-question-intent) why='an adjudicator found that a disputed review finding turns on what the change is meant to do, which a person decides' ;; - feedback-question-design) why='an adjudicator found that a disputed review finding turns on a design choice, which a person decides' ;; - feedback-question-security) why='an adjudicator found that a disputed review finding turns on a security judgment, which a person decides' ;; - dependency-round-cap) why='the lane has reviewed this dependency update as many rounds as its cap allows' ;; - dependency-invalid) why='the dependency review verdict did not pass validation, so the lane acted on none of it' ;; - dependency-fix-not-landed) why='the dependency review chose a fix, but no fix commit reached the branch (the lane lands a fix only when it changes nothing but the versions in npm or pip manifest and lock files outside .github/)' ;; - dependency-security-close) why='the dependency review would close a security update, which the lane never closes' ;; - dependency-escalate) why='the dependency review found a risk the lane does not decide alone' ;; - dependency-config-unreadable) why='the lane could not read the base .github/dependabot.yml, so it does not mark this dependency update for merge' ;; - dependency-files-truncated) why='the dependency review chose merge, but the files listing it read was cut short, so the lane does not mark this dependency update for merge' ;; - dependency-fix-refused) why='a commit on the dependency update other than Dependabot'"'"'s changes more than a version of an npm or pip manifest or lock file, as a regular file outside .github/, so the lane does not act on the review' ;; - dependency-control-uncited) why='the dependency review chose merge for a review-only dependency without quoting its control, so the lane does not mark it for merge' ;; - dependency-workflow-unpinned) why='the dependency review chose merge, but the update changes a workflow or action file by more than swapping one full-SHA action pin for another, so the lane does not mark it for merge' ;; - ci-fix-round-cap) why='the lane has pushed as many CI fixes to this pull request as its cap allows, and CI still fails' ;; - ci-fix-empty) why='the lane changed nothing that fixes the failing checks' ;; - ci-fix-invalid) why='the CI fix the lane wrote did not pass validation, so it was not pushed' ;; - ci-fix-protected-path) why='the CI fix changes a path the lane does not push: .github/workflows/, CODEOWNERS, .gitmodules or a settings file' ;; - ci-fix-not-applied) why='the CI fix did not apply to the head it was written for, so it was not pushed' ;; - ci-fix-escalated) why='the lane could not fix every failing check; its run log names each one and why' ;; - conflict-patch-invalid) why='the conflict resolution the lane wrote did not pass validation, so the merge was not pushed' ;; - conflict-patch-not-applied) why='the conflict resolution did not apply to the merge it was written for, so the merge was not pushed' ;; - conflict-patch-protected-path) why='the conflict resolution changes a path the lane does not push: .github/workflows/, CODEOWNERS, .gitmodules or a settings file' ;; - *) die "unknown escalation reason [${ESCALATION_REASON:-}]" ;; -esac -case $ESCALATION_REASON in - feedback-*) activity=address-feedback ;; - dependency-*) activity=review-dependency ;; - ci-fix-*) activity=fix-ci ;; - *) activity=resolve-conflicts ;; -esac +routes=$(dirname "${BASH_SOURCE[0]}")/escalation-routes.json +[[ ${ESCALATION_REASON:-} =~ ^[a-z]+(-[a-z]+)*$ ]] || die "unknown escalation reason [${ESCALATION_REASON:-}]" +entry=$(jq -ce --arg r "$ESCALATION_REASON" 'if has($r) then .[$r] else null end' "$routes") || + die "unknown escalation reason [$ESCALATION_REASON]" +why=$(jq -er '.why | strings' <<<"$entry") || die "no why for $ESCALATION_REASON in $routes" +activity=$(jq -er '.activity | strings' <<<"$entry") || die "no activity for $ESCALATION_REASON in $routes" +route=$(jq -er '.route | select(. == "hold" or . == "fail")' <<<"$entry") || die "no route for $ESCALATION_REASON in $routes" [[ ${PR_NUMBER:-} =~ ^[1-9][0-9]*$ ]] || die "PR_NUMBER is not a pull request number" [[ ${HEAD_SHA:-} =~ ^([0-9a-f]{40}|[0-9a-f]{64})$ ]] || die "HEAD_SHA is not a commit id" [[ ${GITHUB_REPOSITORY:-} =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || die "GITHUB_REPOSITORY is not owner/repo" @@ -116,43 +68,64 @@ esac case ${1:-} in '') ;; status) - if [[ $ESCALATION_REASON != unresolved-conflict ]]; then - echo "no conflict status for $ESCALATION_REASON" + if [[ $route != fail ]]; then + echo "no status for $ESCALATION_REASON: it holds for a person" exit 0 fi - args=(-f state=failure -f context=pr-unblock/conflict - -f "description=Conflicts with the base branch; CI does not run until they are resolved") + if [[ $activity == resolve-conflicts ]]; then + context=pr-unblock/conflict + else + context=pr-unblock/$activity + fi + if [[ $ESCALATION_REASON == unresolved-conflict ]]; then + description='Conflicts with the base branch; CI does not run until they are resolved' + else + description="Lane stopped: $ESCALATION_REASON; checked again on a new head" + fi + args=(-f state=failure -f "context=$context" -f "description=$description") [[ -n ${TARGET_URL:-} ]] && args+=(-f "target_url=$TARGET_URL") gh api -X POST "repos/$GITHUB_REPOSITORY/statuses/$HEAD_SHA" "${args[@]}" >/dev/null - echo "posted pr-unblock/conflict failure on $HEAD_SHA" + echo "posted $context failure on $HEAD_SHA" exit 0 ;; *) die "unknown mode [$1]" ;; esac -tracker=${TRACKER_CONFIG:-.base/.work-item-tracker.json} -label=needs-human -if [[ -f $tracker ]]; then - label=$(jq -er '.config.role_labels // {} - | if has("human-gated") then .["human-gated"] | strings | select(length > 0) else "needs-human" end' "$tracker") || - die "cannot read the human-gated role label from $tracker" -fi - bot=${LANES_BOT_LOGIN:-melodic-automation-lanes[bot]} issue="repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER" -marker='' -key_line="" - -labeled=$(gh api "$issue" | jq --arg l "$label" '[.labels[].name] | index($l) != null') -if [[ $labeled == true ]]; then - marked=$(gh api --paginate "$issue/comments?per_page=100" | jq -s --arg bot "$bot" --arg m "$marker" --arg k "$key_line" \ - --arg r "- Reason: \`$ESCALATION_REASON\`" \ - '[add[] | select(.user.login == $bot) | (.body // "") | split("\n") | map(rtrimstr("\r")) - | select(.[0] == $m and (.[1] // "") == $k and any(.[]; . == $r))] | length > 0') - if [[ $marked == true ]]; then - echo "already escalated $ESCALATION_REASON for resume key $HEAD_SHA" - exit 0 +dispatch="gh workflow run pr-unblock.yml -R $GITHUB_REPOSITORY -f pr-number=$PR_NUMBER -f activity=$activity" + +if [[ $route == hold ]]; then + tracker=${TRACKER_CONFIG:-.base/.work-item-tracker.json} + label=needs-human + if [[ -f $tracker ]]; then + label=$(jq -er '.config.role_labels // {} + | if has("human-gated") then .["human-gated"] | strings | select(length > 0) else "needs-human" end' "$tracker") || + die "cannot read the human-gated role label from $tracker" fi + marker='' + key_line="" + labeled=$(gh api "$issue" | jq --arg l "$label" '[.labels[].name] | index($l) != null') + if [[ $labeled == true ]]; then + marked=$(gh api --paginate "$issue/comments?per_page=100" | jq -s --arg bot "$bot" --arg m "$marker" --arg k "$key_line" \ + --arg r "- Reason: \`$ESCALATION_REASON\`" \ + '[add[] | select(.user.login == $bot) | (.body // "") | split("\n") | map(rtrimstr("\r")) + | select(.[0] == $m and (.[1] // "") == $k and any(.[]; . == $r))] | length > 0') + if [[ $marked == true ]]; then + echo "already escalated $ESCALATION_REASON for resume key $HEAD_SHA" + exit 0 + fi + fi +else + # gate-source.mjs LANE_STOPPED: one per activity, edited in place. + marker="" + if [[ $activity == resolve-conflicts ]]; then + context=pr-unblock/conflict + else + context=pr-unblock/$activity + fi + existing=$(gh api --paginate "$issue/comments?per_page=100" | jq -sc --arg bot "$bot" --arg m "$marker" \ + '[add[] | select(.user.login == $bot and ((.body // "") | split("\n")[0] | rtrimstr("\r")) == $m)] | last // {} | {id, body}') fi # For a review that is not converging: each open review thread. The count @@ -161,7 +134,6 @@ fi # path is data: backticks, line breaks and angle brackets are dropped so it # stays one code span and cannot open an HTML comment (a marker line). details='' -resume="To resume, deal with the reason above, remove the \`$label\` label (and \`do-not-merge\`, if set), then dispatch the \`pr-unblock\` workflow with \`pr-number=$PR_NUMBER\` and \`activity=$activity\`:" if [[ $ESCALATION_REASON == feedback-not-converging ]]; then # Page by page, as address-feedback.mjs listThreads reads them: an open # thread past the first 100 can be the one that stopped the lane. Anyone @@ -198,16 +170,54 @@ if [[ $ESCALATION_REASON == feedback-not-converging ]]; then threads='- the open review threads could not be read; see the pull request'"'"'s Files changed tab' fi details=$'\n'"The review rounds are not converging after the round cap and one fresh fixer round. Open review threads:"$'\n\n'"$threads"$'\n' - # The rounds already run keep counting, so removing the label alone would - # stop again on the same head; only the threads move it. - resume="Removing the \`$label\` label alone does not resume: the rounds already run still count, so the lane stops again on this head. Do one of these instead: - -1. Resolve each open review thread yourself (fix it, or decide it stays as it is), then remove the \`$label\` label (and \`do-not-merge\`, if set). -2. From an account with write access, reply on a thread with a line that starts \`PR-fix lane: please fix this finding\`, then dispatch the \`pr-unblock\` workflow with \`pr-number=$PR_NUMBER\` and \`activity=$activity\`. The lane runs one round on the threads you asked it to fix:" fi body=$(mktemp) trap 'rm -f "$body"' EXIT + +if [[ $route == fail ]]; then + # The rounds already run keep counting for a capped reason, so a new head + # meets the cap again; a write-access fix request, or a person resolving + # the threads, moves a review that is not converging. + more='' + if [[ $ESCALATION_REASON == feedback-not-converging ]]; then + more=" A reply on a thread, from an account with write access, starting \`PR-fix lane: please fix this finding\` grants one more round on that thread." + fi + cat >"$body" </dev/null + echo "stopped: $ESCALATION_REASON on $HEAD_SHA, comment $id edited" + exit 0 + fi + for attempt in 1 2 3; do + gh api -X POST "$issue/comments" -F "body=@$body" >/dev/null && break + ((attempt < 3)) || die "cannot post the lane-stopped comment after $attempt attempts" + sleep $((attempt * 5)) + done + echo "stopped: $ESCALATION_REASON on $HEAD_SHA, no label" + exit 0 +fi + cat >"$body" <&2; exit 1; } echo '{}' @@ -107,10 +110,48 @@ run() { RC=$? set -e } -writes() { grep -c '^POST' "$STUB/requests" || true; } -comment() { printf '[{"body":%s,"user":{"login":"%s","type":"%s"}}]' "$(jq -Rs . <<<"$1")" "$2" "$3"; } +writes() { grep -cE '^(POST|PATCH)' "$STUB/requests" || true; } +# comment [id] +comment() { printf '[{"id":%s,"body":%s,"user":{"login":"%s","type":"%s"}}]' "${4:-1}" "$(jq -Rs . <<<"$1")" "$2" "$3"; } # bot_marker [reason]: the lanes App's escalation comment for that head and reason. -bot_marker() { comment "$MARKER"$'\n'""$'\nbody\n\n- Reason: `'"${2:-unresolved-conflict}"'`' 'melodic-automation-lanes[bot]' Bot; } +bot_marker() { comment "$MARKER"$'\n'""$'\nbody\n\n- Reason: `'"${2:-protected-path-conflict}"'`' 'melodic-automation-lanes[bot]' Bot; } +activity_of() { + case $1 in + feedback-*) echo address-feedback ;; + dependency-*) echo review-dependency ;; + ci-fix-*) echo fix-ci ;; + *) echo resolve-conflicts ;; + esac +} +context_of() { + local activity + activity=$(activity_of "$1") + if [[ $activity == resolve-conflicts ]]; then echo pr-unblock/conflict; else echo "pr-unblock/$activity"; fi +} + +# The routes: a person only for SAFETY (a path the lane token may not push, +# supply-chain trust) or a question only a person answers (intent, design, +# security, a person's own thread or hold); every other stop is a failing +# status and no label. +HOLD_REASONS=(protected-path-conflict workflow-staged conflict-patch-protected-path + feedback-protected-path feedback-round-cap feedback-uncertain feedback-disputed + feedback-question-intent feedback-question-design feedback-question-security + dependency-security-close dependency-escalate dependency-fix-refused dependency-workflow-unpinned + ci-fix-protected-path) +FAIL_REASONS=(unexpected-path-staged repeat-cap unresolved-conflict conflict-patch-invalid conflict-patch-not-applied + feedback-not-converging feedback-verifier-open feedback-not-landed feedback-invalid feedback-partial + feedback-incomplete feedback-stood-open feedback-unadjudicated + dependency-round-cap dependency-invalid dependency-fix-not-landed dependency-config-unreadable + dependency-files-truncated dependency-control-uncited + ci-fix-round-cap ci-fix-empty ci-fix-invalid ci-fix-not-applied ci-fix-escalated) +ROUTES="$SCRIPT_DIR/escalation-routes.json" +check "the routing table holds exactly these reasons" \ + "$(printf '%s\n' "${HOLD_REASONS[@]}" "${FAIL_REASONS[@]}" | sort)" "$(jq -r 'keys[]' "$ROUTES" | sort)" +check "the routing table holds these for a person" \ + "$(printf '%s\n' "${HOLD_REASONS[@]}" | sort)" "$(jq -r 'to_entries[] | select(.value.route == "hold") | .key' "$ROUTES" | sort)" +for reason in "${HOLD_REASONS[@]}" "${FAIL_REASONS[@]}"; do + check "$reason: the table's activity" "$(activity_of "$reason")" "$(jq -r --arg r "$reason" '.[$r].activity' "$ROUTES")" +done # Rejected reasons: workflow-files and dependency-workflow-held are retired names, the rest are not in the enum. for reason in workflow-files dependency-workflow-held bogus '' 'repeat-cap ' REPEAT-CAP; do @@ -120,15 +161,8 @@ for reason in workflow-files dependency-workflow-held bogus '' 'repeat-cap ' REP check "rejects reason [$reason]: no gh call" '' "$(cat "$STUB/calls")" done -# Accepted reasons: one comment, one label, both on the PR. -for reason in protected-path-conflict workflow-staged unexpected-path-staged repeat-cap unresolved-conflict \ - feedback-protected-path feedback-round-cap feedback-uncertain feedback-not-landed feedback-invalid \ - feedback-partial feedback-incomplete feedback-disputed feedback-not-converging \ - feedback-unadjudicated feedback-question-intent feedback-question-design feedback-question-security \ - dependency-round-cap dependency-invalid dependency-fix-not-landed dependency-security-close dependency-escalate \ - dependency-config-unreadable dependency-files-truncated dependency-control-uncited dependency-workflow-unpinned dependency-fix-refused \ - ci-fix-round-cap ci-fix-empty ci-fix-invalid ci-fix-protected-path ci-fix-not-applied ci-fix-escalated \ - conflict-patch-invalid conflict-patch-not-applied conflict-patch-protected-path; do +# Held reasons: one comment, one label, both on the PR. +for reason in "${HOLD_REASONS[@]}"; do setup '[]' '[]' run "$reason" check "$reason: exit 0" 0 "$RC" @@ -141,27 +175,64 @@ for reason in protected-path-conflict workflow-staged unexpected-path-staged rep has "$reason: comment says to remove the label" 'remove the `needs-human` label' "$STUB/bodies" has "$reason: comment says to dispatch pr-unblock" 'pr-unblock' "$STUB/bodies" has "$reason: comment names the PR to dispatch" 'pr-number=42' "$STUB/bodies" - case $reason in - feedback-*) want=address-feedback ;; - dependency-*) want=review-dependency ;; - ci-fix-*) want=fix-ci ;; - *) want=resolve-conflicts ;; - esac + want=$(activity_of "$reason") has "$reason: comment names the activity to dispatch" "-f pr-number=42 -f activity=$want" "$STUB/bodies" has "$reason: label defaults to needs-human" 'labels[]=needs-human' "$STUB/calls" check "$reason: the comment run posts no commit status" 0 "$(grep -c statuses "$STUB/requests" || true)" done -ALL_REASONS=(protected-path-conflict workflow-staged unexpected-path-staged repeat-cap unresolved-conflict - feedback-protected-path feedback-round-cap feedback-uncertain feedback-not-landed feedback-invalid - feedback-partial feedback-incomplete feedback-disputed - feedback-unadjudicated feedback-question-intent feedback-question-design feedback-question-security - dependency-round-cap dependency-invalid dependency-fix-not-landed dependency-security-close dependency-escalate - dependency-config-unreadable dependency-files-truncated dependency-control-uncited dependency-workflow-unpinned dependency-fix-refused - ci-fix-round-cap ci-fix-empty ci-fix-invalid ci-fix-protected-path ci-fix-not-applied ci-fix-escalated - conflict-patch-invalid conflict-patch-not-applied conflict-patch-protected-path) +# Fail-routed reasons: one lanes-stopped comment, no label, no status from +# the comment run. +for reason in "${FAIL_REASONS[@]}"; do + setup '[]' '[]' + run "$reason" + want=$(activity_of "$reason") + check "$reason, fail: exit 0" 0 "$RC" + check "$reason, fail: one comment, no label" 'POST repos/o/r/issues/42/comments' "$(grep -E '^(POST|PATCH)' "$STUB/requests")" + check "$reason, fail: marker is the first line" "" "$(head -n 1 "$STUB/bodies")" + check "$reason, fail: no escalation marker" 0 "$(grep -c 'work-items:escalation' "$STUB/bodies" || true)" + has "$reason, fail: names the reason" "- Reason: \`$reason\`" "$STUB/bodies" + has "$reason, fail: names the head" "- Head: \`$HEAD_A\`" "$STUB/bodies" + has "$reason, fail: names the failing status" "failing \`$(context_of "$reason")\` status" "$STUB/bodies" + has "$reason, fail: says a new head is checked again" 'the lane checks again on the next commit' "$STUB/bodies" + has "$reason, fail: gives the dispatch" "-f pr-number=42 -f activity=$want" "$STUB/bodies" + check "$reason, fail: names no label" 0 "$(grep -c 'needs-human' "$STUB/bodies" "$STUB/calls" | awk -F: '{s+=$2} END {print s}')" + check "$reason, fail: reads no issue labels" 0 "$(grep -c '^GET repos/o/r/issues/42$' "$STUB/requests" || true)" + check "$reason, fail: no commit status" 0 "$(grep -c statuses "$STUB/requests" || true)" +done -# Status mode: a failing pr-unblock/conflict status on the head, only for unresolved-conflict. +# The fail comment is one per activity, edited in place: unchanged, nothing +# is written; on a new head it is edited, never posted again. +STOP_MARKER='' +setup '[]' '[]' +run ci-fix-escalated +fail_body=$(sed '/^=====$/d' "$STUB/bodies") +setup '[]' "$(comment "$fail_body" 'melodic-automation-lanes[bot]' Bot 77)" +run ci-fix-escalated +check "fail, same head, comment unchanged: exit 0" 0 "$RC" +check "fail, same head, comment unchanged: no write" 0 "$(writes)" + +setup '[]' "$(comment "$STOP_MARKER"$'\nolder text' 'melodic-automation-lanes[bot]' Bot 77)" +run ci-fix-escalated "$HEAD_B" +check "fail, new head: edits the comment in place" 'PATCH repos/o/r/issues/comments/77' "$(grep -E '^(POST|PATCH)' "$STUB/requests")" +has "fail, new head: the edit names the new head" "- Head: \`$HEAD_B\`" "$STUB/bodies" + +setup '[]' "$(comment "$STOP_MARKER"$'\nolder text' someone User 77)" +run ci-fix-escalated +check "fail: a marker a user wrote is not the lane's comment" 'POST repos/o/r/issues/42/comments' "$(grep -E '^(POST|PATCH)' "$STUB/requests")" + +setup '[]' "$(comment ''$'\nolder' 'melodic-automation-lanes[bot]' Bot 77)" +run ci-fix-escalated +check "fail: another activity's comment is left alone" 'POST repos/o/r/issues/42/comments' "$(grep -E '^(POST|PATCH)' "$STUB/requests")" + +# A held reason on a PR that already shows a fail comment still holds. +setup '[]' "$(comment "$STOP_MARKER"$'\nolder text' 'melodic-automation-lanes[bot]' Bot 77)" +run ci-fix-protected-path +check "hold beside a fail comment: label then a new comment" \ + $'POST repos/o/r/issues/42/labels\nPOST repos/o/r/issues/42/comments' "$(grep -E '^(POST|PATCH)' "$STUB/requests")" + +# Status mode: a failing commit status on the head for a fail-routed reason, +# pr-unblock/conflict for resolve-conflicts, pr-unblock/ otherwise. setup '[]' '[]' MODE=status TARGET_URL=https://example.test/run/1 run unresolved-conflict check "status, unresolved-conflict: exit 0" 0 "$RC" @@ -173,17 +244,29 @@ has "status, unresolved-conflict: description says CI does not run" \ 'description=Conflicts with the base branch; CI does not run until they are resolved' "$STUB/calls" has "status, unresolved-conflict: links the run" 'target_url=https://example.test/run/1' "$STUB/calls" +# Re-armed on a new head: the status is the head's, so a later head starts +# with none and gets its own only if the lane stops there too. setup '[]' '[]' MODE=status run unresolved-conflict "$HEAD_B" check "status on another head: posted on that head" "POST repos/o/r/statuses/$HEAD_B" "$(cat "$STUB/requests")" check "status without TARGET_URL: no target_url" 0 "$(grep -c target_url "$STUB/calls" || true)" -for reason in "${ALL_REASONS[@]}"; do +for reason in "${FAIL_REASONS[@]}"; do [[ $reason == unresolved-conflict ]] && continue setup '[]' '[]' MODE=status run "$reason" check "status, $reason: exit 0" 0 "$RC" - check "status, $reason: no gh call" '' "$(cat "$STUB/calls")" + check "status, $reason: one status POST on the head" "POST repos/o/r/statuses/$HEAD_A" "$(cat "$STUB/requests")" + has "status, $reason: state failure" 'state=failure' "$STUB/calls" + has "status, $reason: context" "context=$(context_of "$reason")" "$STUB/calls" + has "status, $reason: description names the reason" "description=Lane stopped: $reason; checked again on a new head" "$STUB/calls" +done + +for reason in "${HOLD_REASONS[@]}"; do + setup '[]' '[]' + MODE=status run "$reason" + check "status, held $reason: exit 0" 0 "$RC" + check "status, held $reason: no gh call" '' "$(cat "$STUB/calls")" done setup '[]' '[]' @@ -204,36 +287,36 @@ check "unknown mode: no gh call" '' "$(cat "$STUB/calls")" # The role label resolves from the base tracker config, not from the workspace root. setup '[]' '[]' '{"config":{"role_labels":{"human-gated":"status: blocked-human"}}}' printf '{"config":{"role_labels":{"human-gated":"from-head"}}}' >"$STUB/ws/.work-item-tracker.json" -run repeat-cap +run workflow-staged check "role label: exit 0" 0 "$RC" has "role label: label from config" 'labels[]=status: blocked-human' "$STUB/calls" has "role label: comment names it" 'remove the `status: blocked-human` label' "$STUB/bodies" check "role label: workspace-root config ignored" 0 "$(grep -c from-head "$STUB/calls" || true)" setup '[]' '[]' '{"config":{"role_labels":{"autonomous-eligible":"go"}}}' -run repeat-cap +run workflow-staged has "role label: absent role defaults to needs-human" 'labels[]=needs-human' "$STUB/calls" setup '[]' '[]' '{not json' -run repeat-cap +run workflow-staged check "role label: unparsable config fails" 1 "$([[ $RC -ne 0 ]] && echo 1 || echo 0)" check "role label: unparsable config writes nothing" 0 "$(writes)" for entry in false null 7 '""' '["needs-human"]'; do setup '[]' '[]' "{\"config\":{\"role_labels\":{\"human-gated\":$entry}}}" - run repeat-cap + run workflow-staged check "role label: present entry $entry fails" 1 "$([[ $RC -ne 0 ]] && echo 1 || echo 0)" check "role label: present entry $entry makes no gh call" '' "$(cat "$STUB/calls")" done # Idempotency. setup '[{"name":"needs-human"}]' "$(bot_marker "$HEAD_A")" -run unresolved-conflict +run protected-path-conflict check "re-run on the same head: exit 0" 0 "$RC" check "re-run on the same head: no write" 0 "$(writes)" setup '[{"name":"needs-human"}]' "$(bot_marker "$HEAD_B")" -run unresolved-conflict +run protected-path-conflict check "labeled, marker for another head: comment only" 'POST repos/o/r/issues/42/comments' "$(grep '^POST' "$STUB/requests")" # Another activity's escalation on the same head is recorded too, so @@ -244,7 +327,7 @@ check "labeled, same head, another reason: comment only" 'POST repos/o/r/issues/ has "labeled, same head, another reason: comment names the new reason" 'ci-fix-round-cap' "$STUB/bodies" setup '[]' "$(bot_marker "$HEAD_A")" -run unresolved-conflict +run protected-path-conflict check "label removed by a human, same head: escalates again" \ $'POST repos/o/r/issues/42/labels\nPOST repos/o/r/issues/42/comments' "$(grep '^POST' "$STUB/requests")" @@ -253,13 +336,13 @@ check "label removed by a human, same head: escalates again" \ LABEL_THEN_3_COMMENTS=$'POST repos/o/r/issues/42/labels\nPOST repos/o/r/issues/42/comments\nPOST repos/o/r/issues/42/comments\nPOST repos/o/r/issues/42/comments' setup '[]' '[]' touch "$STUB/fail_labels" -run unresolved-conflict +run protected-path-conflict check "label write fails: exit 1" 1 "$([[ $RC -ne 0 ]] && echo 1 || echo 0)" check "label write fails: no comment posted" 0 "$(grep -c '^POST repos/o/r/issues/42/comments$' "$STUB/requests" || true)" setup '[]' '[]' echo 1 >"$STUB/fail_comments" -run unresolved-conflict +run protected-path-conflict check "comment write fails once: exit 0" 0 "$RC" check "comment write fails once: label, then the comment twice" \ $'POST repos/o/r/issues/42/labels\nPOST repos/o/r/issues/42/comments\nPOST repos/o/r/issues/42/comments' \ @@ -272,7 +355,7 @@ check "comment write fails once: backs off once" 'sleep 5' "$(cat "$STUB/sleeps" exhausted() { setup '[]' '[]' echo 99 >"$STUB/fail_comments" - run unresolved-conflict + run protected-path-conflict check "$1: comment write fails every attempt: exit 1" 1 "$([[ $RC -ne 0 ]] && echo 1 || echo 0)" check "$1: comment write fails every attempt: label on, three comment attempts" \ "$LABEL_THEN_3_COMMENTS" "$(grep '^POST' "$STUB/requests")" @@ -283,33 +366,33 @@ exhausted() { } exhausted "retry after a failed comment" -run unresolved-conflict +run protected-path-conflict check "retry after a failed comment: exit 0" 0 "$RC" check "retry after a failed comment: one comment, no label" 'POST repos/o/r/issues/42/comments' "$(grep '^POST' "$STUB/requests")" # A label left by a failed comment write is explained by the next run, on a later head too. exhausted "label left, later head" -run unresolved-conflict "$HEAD_B" +run protected-path-conflict "$HEAD_B" check "label left, later head: exit 0" 0 "$RC" check "label left, later head: explains the label" 'POST repos/o/r/issues/42/comments' "$(grep '^POST' "$STUB/requests")" has "label left, later head: comment carries the later resume key" "resume-key=$HEAD_B" "$STUB/bodies" setup '[{"name":"needs-human"}]' "$(comment "$MARKER"$'\n'"" someone User)" -run unresolved-conflict +run protected-path-conflict check "marker written by a user does not count" 'POST repos/o/r/issues/42/comments' "$(grep '^POST' "$STUB/requests")" setup '[{"name":"needs-human"}]' "$(comment "$MARKER"$'\n'"" 'other-app[bot]' Bot)" -run unresolved-conflict +run protected-path-conflict check "marker written by another bot does not count" 'POST repos/o/r/issues/42/comments' "$(grep '^POST' "$STUB/requests")" -setup '[{"name":"needs-human"}]' "$(comment "$MARKER"$'\n'""$'\n- Reason: `unresolved-conflict`' 'other-app[bot]' Bot)" -LBL='other-app[bot]' run unresolved-conflict +setup '[{"name":"needs-human"}]' "$(comment "$MARKER"$'\n'""$'\n- Reason: `protected-path-conflict`' 'other-app[bot]' Bot)" +LBL='other-app[bot]' run protected-path-conflict check "LANES_BOT_LOGIN names the lanes identity: no write" 0 "$(writes)" -# feedback-not-converging: the comment names each open review thread, gives -# no count of its own (select's count, which also reads model check runs, -# is on the neutral checks), and resumes through the threads, never through -# removing the label alone, which would stop again on the same head. +# feedback-not-converging, a cap: no label; the comment names each open +# review thread, gives no count of its own (select's count, which also reads +# model check runs, is on the neutral checks), and names the write-access fix +# request that grants one more round. ROUND='' rounds_json=$(jq -n --arg r "$ROUND" '[range(4) | {body: ($r + "\npr-unblock / address-feedback finished round \(. + 1) of max 4 on x."), user: {login: "melodic-automation-lanes[bot]", type: "Bot"}}]') setup '[]' "$rounds_json" @@ -330,10 +413,9 @@ has "not-converging: a path is shown without backticks or line breaks" '- `docs/ check "not-converging: a path cannot open an HTML comment" 0 "$(grep -c ''$'\nold' 'melodic-automation-lanes[bot]' Bot 9)" + run "$reason" + check "$reason: never runs git" 0 "$(grep -c '^git' "$STUB/calls" || true)" + check "$reason: never calls the contents API" 0 "$(grep -c 'contents/' "$STUB/calls" || true)" + check "$reason: writes only the issue, its comments and labels" 0 \ + "$(grep -vE '^(GET|POST) repos/o/r/issues/42(/comments(\?per_page=100)?|/labels)?$|^PATCH repos/o/r/issues/comments/9$' "$STUB/requests" | grep -c . || true)" +done exit "$FAILED" diff --git a/.github/scripts/pr-unblock/workflow-gates.test.mjs b/.github/scripts/pr-unblock/workflow-gates.test.mjs index 7a40c4bbcc..ca2a893c0e 100644 --- a/.github/scripts/pr-unblock/workflow-gates.test.mjs +++ b/.github/scripts/pr-unblock/workflow-gates.test.mjs @@ -416,7 +416,7 @@ const CONFLICT_JOBS = [ "resolve-conflicts-handoff", "resolve-conflicts-subscription", "apply-conflict-patch", - "report-conflict-status", + "report-stop-status", ]; // As for review-dependency: a job that needs a skipped subscription call and diff --git a/.github/workflows/pr-run-activity-read.yml b/.github/workflows/pr-run-activity-read.yml index 6a5bf90bfb..3d1c343c62 100644 --- a/.github/workflows/pr-run-activity-read.yml +++ b/.github/workflows/pr-run-activity-read.yml @@ -2424,11 +2424,12 @@ jobs: if (patch) core.setOutput('patch-sha256', crypto.createHash('sha256').update(patch).digest('hex')); if (replies) { core.setOutput('replies-sha256', crypto.createHash('sha256').update(replies).digest('hex')); - // adjudicate-feedback's gate: an INCORRECT reply, or a line - // that is not JSON, which the adjudicator's own checks refuse. + // adjudicate-feedback's gate: a reply that leaves its thread + // open (adjudicate-feedback.mjs DISPUTED), or a line that is + // not JSON, which the adjudicator's own checks refuse. const disputed = replies.toString('utf8').split('\n').filter((line) => line.trim() !== '').some((line) => { try { - return JSON.parse(line)?.disposition === 'INCORRECT'; + return ['INCORRECT', 'UNCERTAIN', 'VALID-defer'].includes(JSON.parse(line)?.disposition); } catch { return true; } diff --git a/.github/workflows/pr-run-activity-write.yml b/.github/workflows/pr-run-activity-write.yml index 33123d41a2..629f74ac85 100644 --- a/.github/workflows/pr-run-activity-write.yml +++ b/.github/workflows/pr-run-activity-write.yml @@ -400,10 +400,12 @@ on: outputs: escalation-reason: description: >- - Why the activity handed the PR to a person: for the + Why the activity stopped (report-escalation routes it to a person + or to a failing status, escalation-routes.json): for the review-replies script feedback-uncertain, feedback-not-landed or feedback-invalid, feedback-partial, feedback-incomplete, - feedback-disputed, feedback-unadjudicated, feedback-question-intent, + feedback-disputed, feedback-stood-open, feedback-verifier-open, + feedback-unadjudicated, feedback-question-intent, feedback-question-design or feedback-question-security; for the dependency-verdict script dependency-invalid, dependency-fix-not-landed, dependency-config-unreadable, @@ -583,7 +585,7 @@ jobs: feedback-protected-path | feedback-round-cap | feedback-not-converging | feedback-uncertain | feedback-not-landed | feedback-invalid) ;; feedback-partial | feedback-incomplete | feedback-disputed) ;; feedback-unadjudicated | feedback-question-intent | feedback-question-design) ;; - feedback-question-security) ;; + feedback-question-security | feedback-stood-open | feedback-verifier-open) ;; dependency-round-cap | dependency-invalid | dependency-fix-not-landed) ;; dependency-security-close | dependency-escalate | dependency-config-unreadable) ;; dependency-files-truncated | dependency-control-uncited | dependency-workflow-unpinned) ;; @@ -1940,7 +1942,7 @@ jobs: ?*:feedback-uncertain | ?*:feedback-not-landed | ?*:feedback-invalid) ;; ?*:feedback-partial | ?*:feedback-incomplete | ?*:feedback-disputed) ;; ?*:feedback-unadjudicated | ?*:feedback-question-intent | ?*:feedback-question-design) ;; - ?*:feedback-question-security) ;; + ?*:feedback-question-security | ?*:feedback-stood-open | ?*:feedback-verifier-open) ;; :ci-fix-invalid | :ci-fix-protected-path | :ci-fix-not-applied | :patch-invalid | :patch-protected-path | :patch-not-applied | :patch-empty) [ "$PREPARE" = apply-patch ] || { echo "::error::the escalation reason file holds no reason this script may give"; exit 1; } ;; diff --git a/.github/workflows/pr-unblock.yml b/.github/workflows/pr-unblock.yml index 2d44df1c8b..8a67ec98c6 100644 --- a/.github/workflows/pr-unblock.yml +++ b/.github/workflows/pr-unblock.yml @@ -807,7 +807,7 @@ jobs: # without force. It runs only on a written patch: no change (every thread # INCORRECT or UNCERTAIN) needs no push. report-escalation does not need it: # a patch that does not land leaves no fix commit, and publish-feedback - # hands the PR to a person as feedback-not-landed. + # ends the round feedback-not-landed. apply-feedback-patch: # After adjudicate-feedback too, so the push lands right before the reply # job: a review run on the pushed head that finishes while this run still @@ -855,9 +855,10 @@ jobs: reply-attempt: ${{ needs.address-feedback-subscription.result != 'skipped' && needs.address-feedback-subscription.outputs.scan-reply-attempt || needs.address-feedback-subscription.result == 'skipped' && needs.address-feedback.outputs.scan-reply-attempt || '' }} # The adjudicator, after the model job and before the reply job, only when - # the answering leg's scan job read an INCORRECT reply (feedback-disputed); - # skipping it leaves any such reply with no verdict, which escalates - # feedback-unadjudicated. For each INCORRECT reply on a non-CRITICAL + # the answering leg's scan job read a reply that leaves its thread open + # (INCORRECT, UNCERTAIN or VALID-defer: feedback-disputed); skipping it + # leaves any such reply with no verdict, which ends + # feedback-unadjudicated. For each such reply on a non-CRITICAL # code-review thread no adjudication stood before, up to three # (adjudicate-feedback.mjs MAX_THREADS), a fresh-context model reads the # whole review comment, the locations the reviewer and the lane cited and @@ -2097,20 +2098,45 @@ jobs: head-sha-fallback: ${{ needs.fix-ci-context.outputs.head-sha }} github-token: ${{ github.token }} - # GitHub starts no pull_request workflow on a conflicted PR, so a head the - # lane could not resolve gets no CI and no sign why. This script job, with - # no model, posts a failing pr-unblock/conflict commit status on that head, - # whether the select job escalated before any model turn, the push job - # found a conflict left, or the scan refused the model's output. It is the - # lane's only statuses: write, on its own GITHUB_TOKEN: the lanes App token - # the escalation comment uses carries no statuses permission. - report-conflict-status: - needs: [resolve-conflicts-select, apply-conflict-patch, conflict-scan-refusal] + # A run that stopped on a reason escalation-routes.json routes `fail` ends + # in a failing commit status on the head its activity gated, never a + # person: pr-unblock/conflict for resolve-conflicts (GitHub starts no + # pull_request workflow on a conflicted PR, so it also shows that head why + # its CI is missing), pr-unblock/ otherwise; a new head carries + # none. It runs when report-escalation does, on the same reason, except + # on a cancelled run, and report-escalation.sh status posts nothing for a + # reason held for a person. This script job, with no model, is the lane's only + # statuses: write, on its own GITHUB_TOKEN: the lanes App token the + # escalation comment uses carries no statuses permission. + report-stop-status: + needs: + - resolve-conflicts-select + - apply-conflict-patch + - conflict-scan-refusal + - address-feedback-select + - publish-feedback + - review-dependency-select + - apply-dependency-verdict + - fix-ci-context + - apply-patch + - apply-patch-rung-up if: >- - ${{ !cancelled() && needs.resolve-conflicts-select.outputs.head-sha != '' - && (needs.resolve-conflicts-select.outputs.escalation-reason == 'unresolved-conflict' - || needs.apply-conflict-patch.outputs.escalation-reason == 'unresolved-conflict' - || needs.conflict-scan-refusal.outputs.escalation-reason == 'unresolved-conflict') }} + ${{ !cancelled() + && (needs.resolve-conflicts-select.outputs.head-sha || needs.address-feedback-select.outputs.head-sha + || needs.review-dependency-select.outputs.head-sha || needs.fix-ci-context.outputs.head-sha || '') != '' + && (needs.resolve-conflicts-select.outputs.escalation-reason != '' + || needs.apply-conflict-patch.outputs.escalation-reason != '' + || needs.conflict-scan-refusal.outputs.escalation-reason != '' + || needs.address-feedback-select.outputs.escalation-reason != '' + || needs.publish-feedback.outputs.escalation-reason != '' + || needs.review-dependency-select.outputs.escalation-reason != '' + || needs.apply-dependency-verdict.outputs.escalation-reason != '' + || needs.fix-ci-context.outputs.escalation-reason != '' + || needs.apply-patch.outputs.escalation-reason != '' + || needs.apply-patch.outputs.rung-up == 'true' + && !(needs.apply-patch-rung-up.result == 'success' + && needs.apply-patch-rung-up.outputs.escalation-reason == '' + && needs.apply-patch-rung-up.outputs.patch-skip-reason == '')) }} runs-on: ubuntu-24.04 timeout-minutes: 10 permissions: @@ -2124,12 +2150,28 @@ jobs: path: .base persist-credentials: false - - name: Post the conflict status + # One activity runs per dispatch, so one select job's head is set; a + # reason is a fixed enum value from a base script or this run's jobs, + # which report-escalation.sh rejects unless the routing table lists it. + - name: Post the stop status env: GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ inputs.pr-number }} - HEAD_SHA: ${{ needs.resolve-conflicts-select.outputs.head-sha }} - ESCALATION_REASON: unresolved-conflict + HEAD_SHA: >- + ${{ needs.resolve-conflicts-select.outputs.head-sha || needs.address-feedback-select.outputs.head-sha + || needs.review-dependency-select.outputs.head-sha || needs.fix-ci-context.outputs.head-sha }} + ESCALATION_REASON: >- + ${{ needs.resolve-conflicts-select.outputs.escalation-reason + || needs.apply-conflict-patch.outputs.escalation-reason + || needs.conflict-scan-refusal.outputs.escalation-reason + || needs.address-feedback-select.outputs.escalation-reason + || needs.publish-feedback.outputs.escalation-reason + || needs.review-dependency-select.outputs.escalation-reason + || needs.apply-dependency-verdict.outputs.escalation-reason + || needs.fix-ci-context.outputs.escalation-reason + || needs.apply-patch.outputs.escalation-reason + || needs.apply-patch-rung-up.outputs.escalation-reason + || needs.apply-patch.outputs.rung-up == 'true' && 'ci-fix-escalated' || '' }} TARGET_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: bash .base/.github/scripts/pr-unblock/report-escalation.sh status diff --git a/docs/conventions/pr-pipeline.yaml b/docs/conventions/pr-pipeline.yaml index 48339b9ac6..8a04b575a6 100644 --- a/docs/conventions/pr-pipeline.yaml +++ b/docs/conventions/pr-pipeline.yaml @@ -383,7 +383,10 @@ activities: reads-untrusted: false inputs: [issue] report-escalation: - # Labels the PR needs-human and posts the reason with a resume key. + # Ends a stopped run on its reason's route (escalation-routes.json): + # hold labels the PR needs-human and posts the reason with a resume key; + # fail posts or edits one lane-stopped comment, and report-stop-status a + # failing status on the head. script: .github/scripts/pr-unblock/report-escalation.sh effect: publish-artifact gating: advisory diff --git a/docs/conventions/pr-pipeline/README.md b/docs/conventions/pr-pipeline/README.md index db5816514f..f6398de3ab 100644 --- a/docs/conventions/pr-pipeline/README.md +++ b/docs/conventions/pr-pipeline/README.md @@ -152,14 +152,13 @@ merge with the head and the pinned base as parents ([`pr-run-activity.md`](pr-run-activity.md#prepare-merge-base)). GitHub starts no `pull_request` workflow on a PR with a merge conflict, so a conflicted head gets -no CI. When `resolve-conflicts` escalates with `unresolved-conflict`, `pr-unblock.yml`'s -`report-conflict-status` job (a script job with no model, the lane's only `statuses: write`, on its -own `GITHUB_TOKEN`) posts a `failure` commit status with context `pr-unblock/conflict` on that -head, saying CI does not run until the conflicts are resolved. A push that resolves them is a new -head, which the status does not reach. A conflict that clears with the head unchanged (main moved -again) leaves the failing status on it until a push or a person replaces it. The other -`resolve-conflicts` escalations post no status; they hand the PR to a person through the comment -and label alone. +no CI. When `resolve-conflicts` stops on a reason routed to a failing status (below), +`pr-unblock.yml`'s `report-stop-status` job (a script job with no model, the lane's only +`statuses: write`, on its own `GITHUB_TOKEN`) posts a `failure` commit status with context +`pr-unblock/conflict` on that head; for `unresolved-conflict` it says CI does not run until the +conflicts are resolved. A push that resolves them is a new head, which the status does not reach. +A conflict that clears with the head unchanged (main moved again) leaves the failing status on it +until a push or a person replaces it. `pr-unblock`'s `address-feedback` dispatches come from `pr-feedback-dispatch.yml`, also scripted, on `workflow_run` `completed` of `pr-review-hosted` and `pr-review-security-hosted`. It takes only @@ -301,8 +300,9 @@ head's CI decides, and a red new head gets its own run. The one exception: a run `model-rung` 0 that changed nothing and escalated only `unfixed` or `not-reproducible` checks skips `apply-patch` with `rung-up`, and the same `pr-unblock` run runs `fix-ci` and `apply-patch` once more at `model-rung` 1 (`fix-ci-rung-up`, `fix-ci-rung-up-subscription`, `apply-patch-rung-up`; [Models](#models)). `report-escalation` -hands the PR to a person unless that retry's patch applied cleanly: the retry's own reason, else -`ci-fix-escalated` (a neutral or failed retry). A cancelled run applies no label: the write +ends the run unless that retry's patch applied cleanly: the retry's own reason, else +`ci-fix-escalated` (a neutral or failed retry), each routed to a failing `pr-unblock/fix-ci` +status but `ci-fix-protected-path`, which goes to a person. A cancelled run applies no label: the write workflow's `run` job has no status function in its `if:`, so it skips on a cancel, nothing pushes and `ci-status` stays red. In a run that changed nothing, `protected-path`, `weaken-required` and `flaky` go to a person without a retry; beside a pushed patch, the new @@ -514,6 +514,19 @@ An activity is idempotent: a rerun on the same commit gives the same verdict or mutating activity with nothing to change makes no commit, and each commit it makes names the activity in a trailer. A verdict is bound to the commit it judged and is not reused for any other commit. +A `pr-unblock` run that stops ends on the route +[`escalation-routes.json`](../../../.github/scripts/pr-unblock/escalation-routes.json), the one +routing table, gives its reason; `report-escalation.sh` reads it. Route `hold` is for a reason only +a person settles: a path the lane token may not push or a supply-chain refusal (SAFETY), an intent, +design or security question, a person's own thread or hold, or a finding no adjudicator may judge (a +security-review or CRITICAL one). Every other reason is a defect the lane's bounded retries (the +`fix-ci` rung-up, the adjudicator, the round, repeat and per-head caps) did not mend, routed `fail`: +no label, a failing commit status on the head the activity gated (`pr-unblock/conflict` for +`resolve-conflicts`, `pr-unblock/` otherwise; `report-stop-status` posts it), and one +lanes App comment per activity (marked ``) +that the next stop edits in place. A new head carries no status, so the lane's next run there +starts clean. + When an activity needs a human, it escalates with a resume key and exits instead of blocking. An activity whose `labels-absent` lists `needs-human` or `do-not-merge` skips with `awaiting-human` while that label is on the PR; a human removes the label, then dispatches the lane, to resume it @@ -535,6 +548,7 @@ head: | `workflow-staged` | main changes nothing under `.github/workflows/` since its merge base with the head | | `unresolved-conflict`, `protected-path-conflict` | GitHub reports the PR mergeable | | `feedback-protected-path` | the PR changes nothing under `.github/workflows/` | +| a reason `escalation-routes.json` routes `fail`, on a label set before it did | a `ci-fix-*` reason at once (the head's own CI checks stay red while it fails); any other once the head is not one its escalations name as the resume key | | `head-moved` (pr-merge's `babysit-merge-verify-escalation`, `babysit-merge-risk` and `babysit-merge-shared-copy-escalation`, with no pr-unblock escalation since the label, and the marked App comment written last before the label is pr-merge's and within a minute of it) | the newest App comment carrying each marker names, as ``at head `` ``, a 40-hex head other than the current one; pr-merge then judges the new head and escalates again if its refusal still holds | It removes only a label whose newest `labeled` event has the lanes App as its actor, and keeps it diff --git a/docs/conventions/pr-pipeline/pr-run-activity.md b/docs/conventions/pr-pipeline/pr-run-activity.md index 30d5a8ada5..c38e98d38e 100644 --- a/docs/conventions/pr-pipeline/pr-run-activity.md +++ b/docs/conventions/pr-pipeline/pr-run-activity.md @@ -820,9 +820,9 @@ the repeat cap counts by), `loop.no-progress` as `LOOP_NO_PROGRESS` and a job output, and escalates `unresolved-conflict` when one says the scan refused or the download failed (`conflict-scan-refusal.mjs`, through `scanRefusal`); no marker escalates nothing. -6. `report-conflict-status` posts a failing `pr-unblock/conflict` status on the select job's head - when the select, push or `conflict-scan-refusal` job escalates `unresolved-conflict`; `report-escalation` - takes any of their reasons. +6. `report-stop-status` posts a failing `pr-unblock/conflict` status on the select job's head + when the select, push or `conflict-scan-refusal` job stops on a reason routed `fail`; + `report-escalation` takes any of their reasons. The three merges of one round must agree on the conflict tree; one that does not fails red and nothing is resolved or pushed. The kill switch (step 3) stops new runs only; a run already past it @@ -1028,9 +1028,9 @@ the base's `.github/scripts/pr-unblock/address-feedback.mjs`, `apply-patch.mjs` second line names the workflow run and `review-replies-attempt`, and a re-run of the reply job alone finds a lanes App marker with that line and posts none, so one round counts once, while a re-run of the model job counts another. A round with no replies attempt names none, - and its marker is always posted. It escalates - `feedback-uncertain` for an `UNCERTAIN` or `VALID-defer` reply; - - after the replies, settles each posted `INCORRECT` reply by the adjudicator's verdict + and its marker is always posted; + - after the replies, settles each posted `INCORRECT`, `UNCERTAIN` or `VALID-defer` reply by + the adjudicator's verdict ([below](#the-adjudicator)): `lane` posts the fixed finding-answered reply and resolves the thread; `reviewer` posts the fixed finding-stands reply and leaves the thread open for the next round; when the PR head did not move this round, no review run follows to start that @@ -1040,8 +1040,14 @@ the base's `.github/scripts/pr-unblock/address-feedback.mjs`, `apply-patch.mjs` `feedback-question-intent`, `feedback-question-design` or `feedback-question-security`; no verdict escalates `feedback-unadjudicated`. A thread whose finding stood at an earlier adjudication is not adjudicated again: any reply but `VALID` on it escalates - `feedback-disputed`. A thread or head that changed after the lane's reply, or a failed write, - escalates `feedback-partial`. One round records one reason, the most urgent in that order. + `feedback-stood-open`. A thread no adjudicator may judge goes elsewhere: a pr-verify note + left open escalates `feedback-verifier-open`, a thread a trusted person directed, or an + `INCORRECT` reply on a security-review, CRITICAL or unknown-lane finding, escalates + `feedback-disputed`, and an `UNCERTAIN` or `VALID-defer` reply on one escalates + `feedback-uncertain`. A thread or head that changed after the lane's reply, or a failed + write, escalates `feedback-partial`. One round records one reason: one held for a person + (`feedback-question-*`, `feedback-disputed`, `feedback-uncertain`) before any routed to a + failing status. The script writes its reason to `ESCALATION_REASON_FILE`; the runner accepts only those reasons, and only with `review-replies`, as the run's `escalation-reason` output. @@ -1063,12 +1069,13 @@ model turn. #### The adjudicator pr-unblock.yml's `adjudicate-feedback` job runs after the model job, before the push and reply jobs, when -the leg the reply job reads uploaded replies holding an `INCORRECT` reply or a line that is not -JSON (the read runner's scan job sets `feedback-disputed` from the bytes it uploaded); otherwise it -is skipped, and an `INCORRECT` reply with no verdict still escalates `feedback-unadjudicated`. It +the leg the reply job reads uploaded replies holding a reply that leaves its thread open +(`INCORRECT`, `UNCERTAIN` or `VALID-defer`) or a line that is not JSON (the read runner's scan job +sets `feedback-disputed` from the bytes it uploaded); otherwise it is skipped, and such a reply +with no verdict still escalates `feedback-unadjudicated`. It runs only base code with a read-only token and no OIDC token. `adjudicate-feedback.mjs prepare` checks the select job's thread file and the scan job's replies against the SHA-256 each writer -recorded, then stages each `INCORRECT` reply on a thread no adjudication stood before and no +recorded, then stages each such reply on a thread no adjudication stood before and no trusted person directed, up to three: the finding's path and line, the whole review comment, the locations the comment cites (`path:line` tokens and line-numbered permalinks into this repository, at most ten), the locations the lane cited, at most ten (never its note), and those files at the @@ -1076,15 +1083,17 @@ select job's head, read through the contents API with each response's rate-limit the job's end (`.github/actions/log-api-budget`), each up to 200 KiB and staged with a `.txt` suffix so a cited `CLAUDE.md` or `.claude/**` file is never loaded as instructions. A thread a trusted person directed (a fix request, or a reply after an escalation) stays theirs: an -`INCORRECT` reply on it escalates `feedback-disputed` with no adjudication. Only a +reply that leaves it open escalates `feedback-disputed` with no adjudication. Only a github-actions[bot] review comment whose footer names the code-review lane (`claude-lane: claude-review`; the select job tags the thread file's comment with that `lane`, and a footer in a claude[bot] comment, whose body the model writes, names none) and that carries no -`severity: CRITICAL` line is adjudicated. A CRITICAL finding, a security-review finding, one whose -lane its root comment does not name, and a pr-verify verifier note always go to a person: an -`INCORRECT` reply on one escalates `feedback-disputed`, takes no slot, and any verdict for it is -ignored. A verifier note is never adjudicated because resolving its thread clears nothing: the -FAIL still holds the merge, and pr-verify judges each head once. Each thread gets one fresh +`severity: CRITICAL` line is adjudicated. A CRITICAL finding, a security-review finding and one +whose lane its root comment does not name go to a person: an `INCORRECT` reply on one escalates +`feedback-disputed`, an `UNCERTAIN` or `VALID-defer` one `feedback-uncertain`; it takes no slot, +and any verdict for it is ignored. A pr-verify verifier note is never adjudicated because +resolving its thread clears nothing: the FAIL still holds the merge, and pr-verify judges each +head once; a reply that leaves one open ends `feedback-verifier-open`, a failing status and no +person. Each thread gets one fresh `run-claude-cli` call, after `apply-claude-managed-settings` and runner isolation, on the pr-unblock API key (`PR_UNBLOCK_CLAUDE_API_KEY`), under `dontAsk` with Read alone, `Read(./adjudicate/**)` the one allow rule, Read denied on the base checkout and the runner's temp directory (which holds the lane's replies and From e2ce4cc3abf4effe924e457e3c1be6e137ebe139 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 26 Oct 2026 01:04:26 -0400 Subject: [PATCH 02/33] fix(pr-pipeline): retry a pr-unblock stop on the same head under the existing caps, and report the ones left red A stop routed to a failing status now skips lane-stopped, which is a new skip reason in the schema. On a select job's own check it ends the head, so a deterministic stop is no longer re-dispatched blindly. On the push, reply or verdict job's check, the activity's dispatcher releases the head for a same-head retry, through limit-stop.mjs stopRelease. Each activity keeps the per-head cap it already has: - resolve-conflicts and review-dependency: limit-stops-per-head - fix-ci: fix-ci-restarts and maxRunsPerHead - address-feedback: SWEEP_HEAD_CAP The fix-ci and address-feedback retries run one model rung up. The rung is read in the run from the head's own stop record by the GitHub Actions app, never from a dispatch input. A retry that does not stop posts success over the failing status, through report-escalation.sh clear. Each stop comment now names its run, so every stop edits it. fix-ci model escalations of protected-path or weaken-required stay with a person, as ci-fix-protected-path and the new ci-fix-weaken-required. fix-ci's round-cap stop now posts its own check instead of leaving the head undispatched-but-unrecorded. The monitoring lane gains signal 10, stalled lane stop. It lists open pull requests, of any author, whose stop comment names the current head and has had no attempt of that activity on the head for over 2 hours. Co-Authored-By: Claude Opus 5.5 --- .../fix-pr-unblock-route-not-park-5c1e7a20.md | 7 +++ .../actions/resolve-config/callers.test.mjs | 55 +++++++++++++++---- .github/scripts/apply-patch/apply-patch.mjs | 29 +++++++++- .../scripts/apply-patch/apply-patch.test.mjs | 32 ++++++----- .../maintenance-monitor-signals/collect.mjs | 40 +++++++++++++- .../collect.test.mjs | 45 +++++++++++++++ .../maintenance-monitor-signals/issues.mjs | 2 + .../issues.test.mjs | 1 + .../maintenance-monitor-signals/signals.mjs | 24 ++++++++ .../signals.test.mjs | 21 +++++++ .../scripts/pr-unblock/address-feedback.mjs | 8 ++- .../pr-unblock/address-feedback.test.mjs | 29 ++++++++-- .../pr-unblock/apply-dependency-verdict.mjs | 3 +- .../apply-dependency-verdict.test.mjs | 2 +- .../pr-unblock/dependency-dispatch.mjs | 14 ++++- .../pr-unblock/dependency-dispatch.test.mjs | 35 ++++++++++++ .github/scripts/pr-unblock/dispatch.mjs | 21 ++++++- .github/scripts/pr-unblock/dispatch.test.mjs | 55 ++++++++++++++++++- .../scripts/pr-unblock/escalation-routes.json | 1 + .../scripts/pr-unblock/escalation-routes.mjs | 45 +++++++++++++++ .../scripts/pr-unblock/feedback-dispatch.mjs | 8 ++- .../pr-unblock/feedback-dispatch.test.mjs | 23 ++++++++ .github/scripts/pr-unblock/fix-ci-context.mjs | 13 +++-- .../pr-unblock/fix-ci-context.test.mjs | 11 ++++ .../scripts/pr-unblock/fix-ci-dispatch.mjs | 27 ++++++++- .../pr-unblock/fix-ci-dispatch.test.mjs | 33 ++++++++++- .github/scripts/pr-unblock/gate-source.mjs | 14 +---- .github/scripts/pr-unblock/limit-stop.mjs | 21 +++++++ .../scripts/pr-unblock/publish-feedback.mjs | 3 +- .../pr-unblock/publish-feedback.test.mjs | 2 +- .../scripts/pr-unblock/report-escalation.sh | 35 +++++++++++- .../pr-unblock/report-escalation.test.sh | 42 +++++++++++++- .../scripts/pr-unblock/review-dependency.mjs | 3 +- .github/workflows/pr-run-activity-write.yml | 9 ++- .github/workflows/pr-unblock.yml | 54 ++++++++++++++---- docs/conventions/pr-pipeline/README.md | 25 +++++++-- .../pr-pipeline/pr-pipeline.schema.json | 3 +- 37 files changed, 702 insertions(+), 93 deletions(-) create mode 100644 .github/scripts/pr-unblock/escalation-routes.mjs diff --git a/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md b/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md index 0aebea606e..1a314347ab 100644 --- a/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md +++ b/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md @@ -7,3 +7,10 @@ bump: minor - A `pr-unblock` run that stops now ends on the route its reason has in [`escalation-routes.json`](https://github.com/melodic-software/claude-code-plugins/blob/main/.github/scripts/pr-unblock/escalation-routes.json), the one routing table that `report-escalation.sh`, `gate-source.mjs` and `clear-escalations.mjs` read. Only a reason a person must settle keeps the `needs-human` label: a path the lane token may not push or a supply-chain refusal, an intent, design or security question, a person's own thread or hold, or a finding no adjudicator may judge (security-review or CRITICAL). Every other reason (for example `ci-fix-escalated`, `feedback-not-landed`, the round and repeat caps) ends with no label, a failing commit status on the head the activity gated (`pr-unblock/conflict` for `resolve-conflicts`, `pr-unblock/` otherwise, posted by the `report-stop-status` job that replaces `report-conflict-status`), and one lanes App comment per activity that the next stop edits in place. A new head carries no status. - An `UNCERTAIN` or `VALID-defer` review reply now goes to the adjudicator as an `INCORRECT` one does, instead of to a person. A pr-verify note left open ends `feedback-verifier-open`, and a finding that stood at adjudication and was left open again ends `feedback-stood-open`; both route to a failing status. A round's reason held for a person now outranks any reason routed to a failing status. - `clear-escalations` takes off a `needs-human` an older run set for a reason now routed to a failing status: a `ci-fix-*` reason at once, any other once the head moved past the one the lane stopped on. +- A pr-unblock stop routed to a failing status now skips `lane-stopped` (a new `skip-reason`) instead of `awaiting-human`. On a select job's own check (a cap, or a conflict no model can resolve) it ends the head, so the dispatchers stop re-running a deterministic stop. On the push, reply or verdict job's check, the dispatcher retries the same head under the cap it already keeps: + - `limit-stops-per-head` for resolve-conflicts and review-dependency + - `fix-ci-restarts` and the per-head cap for fix-ci + - `SWEEP_HEAD_CAP` for address-feedback + + The fix-ci and address-feedback retries run one model rung up, read in the run from the head's own stop record. A retry that does not stop posts success over the failing status. fix-ci's model escalations `protected-path` and `weaken-required` stay with a person, as `ci-fix-protected-path` and the new `ci-fix-weaken-required`. +- The monitoring lane gains signal 10, `stalled lane stop`. Each open pull request whose pr-unblock stop has had no new attempt on its head for over 2 hours is listed in one rolling issue. diff --git a/.github/actions/resolve-config/callers.test.mjs b/.github/actions/resolve-config/callers.test.mjs index 4ee9df80bb..0ae980b7d3 100644 --- a/.github/actions/resolve-config/callers.test.mjs +++ b/.github/actions/resolve-config/callers.test.mjs @@ -23,6 +23,7 @@ import { parse } from "yaml"; import { loadRules, rowFor, select as selectPatch } from "../../scripts/apply-patch/apply-patch.mjs"; import { DEFAULTS as CI_FIX_DEFAULTS, skipReason } from "../../scripts/pr-unblock/fix-ci-dispatch.mjs"; import { VERDICTS } from "../../scripts/pr-unblock/review-dependency.mjs"; +import { stopSkip } from "../../scripts/pr-unblock/escalation-routes.mjs"; import { main as checkTrustedTrigger } from "../check-trusted-trigger/check-trusted-trigger.mjs"; import { generate as generateCarrierKey } from "../open-lane-output/keygen.mjs"; import { decide } from "../report-check-run/decide.mjs"; @@ -419,7 +420,7 @@ test("intake-triage.yml: the model may not apply good first issue or help wanted } }); -test("pr-unblock.yml: report-stop-status runs when report-escalation does, on its reason, with a gated head and never on a cancelled run", () => { +test("pr-unblock.yml: report-stop-status runs when report-escalation does, or after an attempt that did not stop, with a gated head and never on a cancelled run", () => { const head = "c".repeat(40); const { jobs } = workflow("pr-unblock.yml"); const job = jobs["report-stop-status"]; @@ -437,8 +438,10 @@ test("pr-unblock.yml: report-stop-status runs when report-escalation does, on it "apply-patch", ]; const HEADS = ["resolve-conflicts-select", "address-feedback-select", "review-dependency-select", "fix-ci-context"]; - const context = ({ from = "", reason = "", headFrom = "resolve-conflicts-select", sha = head, rungUp = "", rungUpResult = "skipped", rungUpReason = "", rungUpSkip = "" } = {}) => ({ + const EMITTERS = ["apply-conflict-patch", "publish-feedback", "apply-dependency-verdict", "apply-patch"]; + const context = ({ from = "", reason = "", headFrom = "resolve-conflicts-select", sha = head, rungUp = "", rungUpResult = "skipped", rungUpReason = "", rungUpSkip = "", landed = "" } = {}) => ({ ...Object.fromEntries(SOURCES.map((id) => [`needs.${id}.outputs.escalation-reason`, id === from ? reason : ""])), + ...Object.fromEntries(EMITTERS.map((id) => [`needs.${id}.result`, id === landed ? "success" : "skipped"])), ...Object.fromEntries(HEADS.map((id) => [`needs.${id}.outputs.head-sha`, id === headFrom ? sha : ""])), "needs.apply-patch.outputs.rung-up": rungUp, "needs.apply-patch-rung-up.result": rungUpResult, @@ -458,13 +461,23 @@ test("pr-unblock.yml: report-stop-status runs when report-escalation does, on it // The rung-up retry: a stop unless its patch applied cleanly. assert.equal(evaluate(job.if, context({ headFrom: "fix-ci-context", rungUp: "true", rungUpResult: "failure" })), true); assert.equal(evaluate(job.if, context({ headFrom: "fix-ci-context", rungUp: "true", rungUpResult: "success", rungUpReason: "ci-fix-invalid" })), true); - assert.equal(evaluate(job.if, context({ headFrom: "fix-ci-context", rungUp: "true", rungUpResult: "success" })), false, "the retry's patch applied"); + // An attempt that did not stop runs it too, for report-escalation.sh clear. + assert.equal(evaluate(job.if, context({ headFrom: "fix-ci-context", rungUp: "true", rungUpResult: "success" })), true, "the retry's patch applied: clear"); + for (const landed of EMITTERS) { + assert.equal(evaluate(job.if, context({ landed })), true, `${landed} ended with no stop: clear`); + } // The head is a select job's, never a model call's; the reason is the one // report-escalation reads. const post = job.steps.find((step) => step.name === "Post the stop status"); assert.equal(post.env.HEAD_SHA.trim(), HEADS.map((id) => `needs.${id}.outputs.head-sha`).join(" || ").replace(/^/, "${{ ").replace(/$/, " }}")); - assert.equal(post.env.ESCALATION_REASON, escalation.with["escalation-reason"]); - assert.equal(post.run, "bash .base/.github/scripts/pr-unblock/report-escalation.sh status"); + // report-escalation's reason, except that a rung-up retry whose patch + // applied is no stop. + const reason = (extra) => evaluate(post.env.ESCALATION_REASON, context(extra)); + assert.equal(reason({ from: "publish-feedback", reason: "feedback-not-landed" }), "feedback-not-landed"); + assert.equal(reason({ rungUp: "true", rungUpResult: "failure" }), "ci-fix-escalated"); + assert.equal(reason({ rungUp: "true", rungUpResult: "success" }), ""); + assert.equal(post.env.ACTIVITY, "${{ inputs.activity }}"); + assert.match(post.run, /report-escalation\.sh "\$\(\[ -n "\$ESCALATION_REASON" \] && echo status \|\| echo clear\)"$/); assert.deepEqual(job.permissions, { contents: "read", statuses: "write" }); assert.equal(job.uses, undefined, "a script job, never the write runner"); }); @@ -561,9 +574,20 @@ test("pr-unblock.yml: a rung-up stop runs fix-ci and apply-patch again at model- assert.equal(up.with["model-rung"], "1"); assert.equal(upSubscription.with["model-rung"], "1"); assert.equal(apply.with["patch-model-rung"], "1"); - // The first pass runs at rung 0: no rung input, so a dispatch cannot skip it. - assert.equal(jobs["fix-ci"].with["model-rung"], undefined); - assert.equal(jobs["apply-patch"].with["patch-model-rung"], undefined); + // The first pass runs at the rung fix-ci-context read from the head's own + // trusted stop record (escalation-routes.mjs retryRung): no dispatch input + // sets it, so a dispatch cannot skip rung 0 or climb. + assert.equal(jobs["fix-ci"].with["model-rung"], "${{ needs.fix-ci-context.outputs.model-rung }}"); + assert.equal(jobs["fix-ci-subscription"].with["model-rung"], "${{ needs.fix-ci-context.outputs.model-rung }}"); + assert.equal(jobs["apply-patch"].with["patch-model-rung"], "${{ needs.fix-ci-context.outputs.model-rung }}"); + for (const id of ["address-feedback", "address-feedback-subscription"]) { + assert.equal(jobs[id].with["model-rung"], "${{ needs.address-feedback-select.outputs.model-rung }}", id); + } + // review-dependency's profile names no rung above it, and resolve-conflicts + // retries as a fresh session at its own rung. + for (const id of ["review-dependency", "review-dependency-subscription", "resolve-conflicts", "resolve-conflicts-subscription"]) { + assert.equal(jobs[id].with["model-rung"], undefined, id); + } assert.equal(workflow("pr-unblock.yml").on.workflow_dispatch.inputs["model-rung"], undefined); }); @@ -622,7 +646,7 @@ test("pr-unblock.yml: fix-ci is reported neutral when fix-ci-context skips, and assert.equal(evaluate(fixCi.if, context("true", "", head)), true); assert.equal(evaluate(notRun.if, context("true", "", head)), false); assert.equal(evaluate(notRun.if, context("false", "", head)), true); - assert.equal(evaluate(notRun.if, context("false", "ci-fix-round-cap", head)), false, "report-escalation owns it"); + assert.equal(evaluate(notRun.if, context("false", "ci-fix-round-cap", head)), true, "a stop posts its own skip, so the head's fix-ci runs record it"); assert.equal(evaluate(notRun.if, context("", "", "")), false, "a kill-switch stop names no head"); assert.equal(evaluate(notRun.if, context("", "", "", "resolve-conflicts")), false); @@ -635,6 +659,9 @@ test("pr-unblock.yml: fix-ci is reported neutral when fix-ci-context skips, and // A human-gated hold fix-ci-context read posts awaiting-human instead. const held = await postNotRunCheck(notRun, context("false", "", head, "fix-ci", "awaiting-human")); assert.equal(held[0].output.title, "Skipped: awaiting-human"); + // The round cap, routed to a failing status: lane-stopped ends the head. + const capped = await postNotRunCheck(notRun, context("false", "ci-fix-round-cap", head, "fix-ci", "lane-stopped")); + assert.equal(capped[0].output.title, "Skipped: lane-stopped"); // fix-ci-dispatch.mjs dispatches a head with no fix-ci check run; the // posted one, as the listing returns it, means the head was handled. @@ -5002,7 +5029,9 @@ test("pr-unblock.yml: resolve-conflicts is reported neutral when the select job const head = "d".repeat(40); const { jobs } = workflow("pr-unblock.yml"); const notRun = jobs["resolve-conflicts-not-run"]; - const context = (proceed, headSha, { skip = "", state = "", activity = "resolve-conflicts" } = {}) => ({ + // The "Read the stop's skip reason" step runs escalation-routes.mjs skip. + const context = (proceed, headSha, { skip = "", state = "", activity = "resolve-conflicts", escalation = "" } = {}) => ({ + "steps.stop.outputs.skip": stopSkip(escalation), "inputs.activity": activity, "needs.resolve-conflicts-select.outputs.proceed": proceed, "needs.resolve-conflicts-select.outputs.head-sha": headSha, @@ -5019,7 +5048,9 @@ test("pr-unblock.yml: resolve-conflicts is reported neutral when the select job [{ skip: "not-applicable" }, "Skip reason: not-applicable"], [{ skip: "awaiting-human" }, "Skip reason: awaiting-human"], [{ state: "clean" }, "Skip reason: clean-merge"], - [{ state: "escalate" }, "Skip reason: awaiting-human"], + [{ state: "escalate", escalation: "protected-path-conflict" }, "Skip reason: awaiting-human"], + [{ state: "escalate", escalation: "repeat-cap" }, "Skip reason: lane-stopped"], + [{ state: "escalate", escalation: "unresolved-conflict" }, "Skip reason: lane-stopped"], ]) { const calls = await postNotRunChecks(notRun, context("false", head, given)); assert.deepEqual( @@ -5717,7 +5748,7 @@ test("pr-unblock.yml: a refused fix-ci collection runs apply-patch, whose select // select, with the empty patch-state a refused collection passes, escalates. const row = rowFor(loadRules(fileURLToPath(new URL("../../../", import.meta.url))), { prepare: "ci-context" }); const decision = selectPatch({ row, patchState: "", patchSha256: "", patchHeadSha: TRUSTED, headSha: TRUSTED, patchDir: "/nonexistent", reportDir: "/nonexistent" }); - assert.deepEqual(decision, { proceed: false, skipReason: "awaiting-human", escalationReason: "ci-fix-invalid" }); + assert.deepEqual(decision, { proceed: false, skipReason: "lane-stopped", escalationReason: "ci-fix-invalid" }); // report-escalation runs on that reason, and on nothing else here. const escalation = jobs["report-escalation"]; const quiet = Object.fromEntries([...escalation.if.matchAll(/needs\.[\w-]+\.(?:outputs\.[\w-]+|result)/g)].map(([ref]) => [ref, ""])); diff --git a/.github/scripts/apply-patch/apply-patch.mjs b/.github/scripts/apply-patch/apply-patch.mjs index 2f6c3b769d..b4c0fa9585 100755 --- a/.github/scripts/apply-patch/apply-patch.mjs +++ b/.github/scripts/apply-patch/apply-patch.mjs @@ -67,6 +67,7 @@ import { join } from "node:path"; import { pathToFileURL } from "node:url"; import { fixRefusal, parseConfig } from "../pr-unblock/dependency-classify.mjs"; +import { stopSkip } from "../pr-unblock/escalation-routes.mjs"; import { PREDICATES } from "./predicates.mjs"; export const MAX_PATCH_BYTES = 1024 * 1024; @@ -168,6 +169,20 @@ export function rungUpEscalation(text) { return reasons.length > 0 && reasons.length === escalatedChecks(text).length && reasons.every((reason) => RUNG_UP_REASONS.has(reason)); } +// An escalation only a person settles, and the reason it stops with: a fix +// that needs a path the lane token may not push, or one that would weaken a +// required check (escalation-routes.json holds both). +const HELD_ESCALATIONS = [ + ["protected-path", "ci-fix-protected-path"], + ["weaken-required", "ci-fix-weaken-required"], +]; + +// The held reason a report's escalations name, or "". +export function heldEscalation(text) { + const reasons = reportLines(text).flatMap((line) => ESCALATED_REASON.exec(line)?.[1] ?? []); + return HELD_ESCALATIONS.find(([reason]) => reasons.includes(reason))?.[1] ?? ""; +} + const reportText = (reportDir) => { const file = reportDir ? join(reportDir, "reply.txt") : ""; return Boolean(file) && existsSync(file) && lstatSync(file).isFile() ? readFileSync(file, "utf8") : ""; @@ -179,6 +194,7 @@ const REPORTS = { "ci-escalations": { escalated: (reportDir) => escalatedChecks(reportText(reportDir)).length > 0, rungUp: (reportDir) => rungUpEscalation(reportText(reportDir)), + held: (reportDir) => heldEscalation(reportText(reportDir)), }, }; @@ -396,7 +412,12 @@ export function patchProblem(file, expected) { // the caller runs the producer once more at rung 1, instead of escalating. export function select({ row, patchState, patchSha256, patchHeadSha, headSha, patchDir, reportDir, rung = 0, mergeState = "" }) { checkRow(row); - const decide = (proceed, skipReason, escalationReason) => ({ proceed, skipReason, escalationReason }); + // A stop routed to a failing status skips lane-stopped (escalation-routes.mjs). + const decide = (proceed, skipReason, escalationReason) => ({ + proceed, + skipReason: skipReason === "awaiting-human" && escalationReason ? stopSkip(escalationReason) : skipReason, + escalationReason, + }); // A merge row: a clean merge commits with no patch; a conflicted one needs // the model's patch, and an empty one leaves the conflicts as git left them. if (isMerge(row)) { @@ -425,7 +446,8 @@ export function select({ row, patchState, patchSha256, patchHeadSha, headSha, pa if (escalated && rung === 0 && report.rungUp(reportDir)) { return decide(false, "rung-up", ""); } - return decide(false, "awaiting-human", reasonFor(row, escalated ? "escalated" : "empty")); + const held = escalated ? report.held(reportDir) : ""; + return decide(false, "awaiting-human", held || reasonFor(row, escalated ? "escalated" : "empty")); } let names = []; try { @@ -1111,7 +1133,8 @@ export async function main(argv, env, { api, cwd = process.cwd(), gitBin, log = appendFileSync(env.GITHUB_OUTPUT, `commit=${result.commit}\nbranch=${result.branch}\n`); } if (result.outcome !== "pushed") { - writeFileSync(env.SKIP_REASON_FILE, result.outcome === "superseded" ? "superseded-sha\n" : "awaiting-human\n"); + const skip = result.outcome === "superseded" ? "superseded-sha" : result.outcome === "escalate" ? stopSkip(result.escalationReason) : "awaiting-human"; + writeFileSync(env.SKIP_REASON_FILE, `${skip}\n`); } log(`apply-patch (${row.name}): ${result.outcome}${result.escalationReason ? ` (${result.escalationReason})` : ""}`); return 0; diff --git a/.github/scripts/apply-patch/apply-patch.test.mjs b/.github/scripts/apply-patch/apply-patch.test.mjs index 469108331e..89ab29a65d 100644 --- a/.github/scripts/apply-patch/apply-patch.test.mjs +++ b/.github/scripts/apply-patch/apply-patch.test.mjs @@ -928,7 +928,7 @@ const patchDirOf = (t, text) => { }; const selectCi = (dir, sum, extra = {}) => select({ row: row("ci-context"), patchState: "written", patchSha256: sum, patchHeadSha: "a".repeat(40), headSha: "a".repeat(40), patchDir: dir, ...extra }); -const INVALID = { proceed: false, skipReason: "awaiting-human", escalationReason: "ci-fix-invalid" }; +const INVALID = { proceed: false, skipReason: "lane-stopped", escalationReason: "ci-fix-invalid" }; test("a draft set during the run drops the fix uncommitted", async (t) => { const fix = repoWithPatch(t, (repo) => writeFileSync(join(repo, "README.md"), "# s\n")); @@ -1068,21 +1068,25 @@ test("an empty patch-state whose report escalated a check names ci-fix-escalated writeFileSync(join(reportDir, "reply.txt"), "lint-shell: escalated not-reproducible needs a secret\n"); const head = "a".repeat(40); const base = { row: row("ci-context"), patchState: "empty", patchSha256: "", patchHeadSha: head, headSha: head, patchDir: scratch(t) }; - assert.deepEqual(select({ ...base, reportDir, rung: 1 }), { proceed: false, skipReason: "awaiting-human", escalationReason: "ci-fix-escalated" }); + assert.deepEqual(select({ ...base, reportDir, rung: 1 }), { proceed: false, skipReason: "lane-stopped", escalationReason: "ci-fix-escalated" }); assert.equal(select({ ...base, reportDir: scratch(t) }).escalationReason, "ci-fix-empty"); }); // A fix-ci run that changed nothing and escalated only // unfixed or not-reproducible checks retries once, one model rung up, instead -// of handing the PR to a person; protected-path, weaken-required and flaky -// stay with a person, and a rung-1 run never retries again. +// of stopping; flaky ends in a failing status (the dispatcher retries the +// head), protected-path and weaken-required stay with a person, and a rung-1 +// run never retries again in the same run. for (const [why, reply, rung, want] of [ ["unfixed at rung 0", "test-node: escalated unfixed three fixes failed\n", 0, { proceed: false, skipReason: "rung-up", escalationReason: "" }], ["unfixed and not-reproducible at rung 0", "a: escalated unfixed x\nb: escalated not-reproducible y\nc: fixed\n", 0, { proceed: false, skipReason: "rung-up", escalationReason: "" }], - ["unfixed at rung 1", "test-node: escalated unfixed three fixes failed\n", 1, { proceed: false, skipReason: "awaiting-human", escalationReason: "ci-fix-escalated" }], - ["protected-path at rung 0", "lint: escalated protected-path edits .github\n", 0, { proceed: false, skipReason: "awaiting-human", escalationReason: "ci-fix-escalated" }], - ["weaken-required beside unfixed at rung 0", "a: escalated unfixed x\nb: escalated weaken-required y\n", 0, { proceed: false, skipReason: "awaiting-human", escalationReason: "ci-fix-escalated" }], - ["flaky at rung 0", "a: escalated flaky x\n", 0, { proceed: false, skipReason: "awaiting-human", escalationReason: "ci-fix-escalated" }], + ["unfixed at rung 1", "test-node: escalated unfixed three fixes failed\n", 1, { proceed: false, skipReason: "lane-stopped", escalationReason: "ci-fix-escalated" }], + // A fix that needs a path the lane token may not push, or one that weakens + // a required check, stays with a person. + ["protected-path at rung 0", "lint: escalated protected-path edits .github\n", 0, { proceed: false, skipReason: "awaiting-human", escalationReason: "ci-fix-protected-path" }], + ["weaken-required beside unfixed at rung 0", "a: escalated unfixed x\nb: escalated weaken-required y\n", 0, { proceed: false, skipReason: "awaiting-human", escalationReason: "ci-fix-weaken-required" }], + ["protected-path beside weaken-required at rung 1", "a: escalated weaken-required x\nb: escalated protected-path y\n", 1, { proceed: false, skipReason: "awaiting-human", escalationReason: "ci-fix-protected-path" }], + ["flaky at rung 0", "a: escalated flaky x\n", 0, { proceed: false, skipReason: "lane-stopped", escalationReason: "ci-fix-escalated" }], ]) { test(`an empty fix-ci patch escalating ${why} selects ${want.skipReason}`, (t) => { const reportDir = scratch(t); @@ -1127,7 +1131,7 @@ test("a written fix escalates nothing, and a run that writes no patch escalates writeFileSync(join(nextReport, "reply.txt"), reply); const next = "b".repeat(40); const red = { row: row("ci-context"), patchState: "empty", patchSha256: "", patchHeadSha: next, headSha: next, patchDir: scratch(t), reportDir: nextReport }; - assert.deepEqual(select(red), { proceed: false, skipReason: "awaiting-human", escalationReason: "ci-fix-escalated" }); + assert.deepEqual(select(red), { proceed: false, skipReason: "lane-stopped", escalationReason: "ci-fix-escalated" }); }); test("select reads the rung from MODEL_RUNG and refuses any other value", async (t) => { @@ -1144,7 +1148,7 @@ test("select reads the rung from MODEL_RUNG and refuses any other value", async await main(["select"], env, { log: () => {} }); return readFileSync(out, "utf8"); }; - assert.match(await outputFor("1"), /^skip-reason=awaiting-human$/m); + assert.match(await outputFor("1"), /^skip-reason=lane-stopped$/m); assert.match(await outputFor("0"), /^skip-reason=rung-up$/m); assert.match(await outputFor(""), /^skip-reason=rung-up$/m); await assert.rejects(() => outputFor("2"), /MODEL_RUNG/); @@ -1721,12 +1725,12 @@ test("select under the merge-base row: a clean merge proceeds with no patch, an const at = (mergeState, patchState, extra = {}) => select({ row: row("merge-base"), mergeState, patchState, patchSha256: sum, patchHeadSha: head, headSha: head, patchDir: dir, ...extra }); assert.deepEqual(at("clean", ""), { proceed: true, skipReason: "", escalationReason: "" }); - assert.deepEqual(at("clean", "written"), { proceed: false, skipReason: "awaiting-human", escalationReason: "conflict-patch-invalid" }); + assert.deepEqual(at("clean", "written"), { proceed: false, skipReason: "lane-stopped", escalationReason: "conflict-patch-invalid" }); assert.deepEqual(at("conflicted", "written"), { proceed: true, skipReason: "", escalationReason: "" }); - assert.deepEqual(at("conflicted", "empty"), { proceed: false, skipReason: "awaiting-human", escalationReason: "unresolved-conflict" }); - assert.deepEqual(at("conflicted", ""), { proceed: false, skipReason: "awaiting-human", escalationReason: "conflict-patch-invalid" }); + assert.deepEqual(at("conflicted", "empty"), { proceed: false, skipReason: "lane-stopped", escalationReason: "unresolved-conflict" }); + assert.deepEqual(at("conflicted", ""), { proceed: false, skipReason: "lane-stopped", escalationReason: "conflict-patch-invalid" }); for (const state of ["", "escalate", "Clean"]) { - assert.deepEqual(at(state, ""), { proceed: false, skipReason: "awaiting-human", escalationReason: "conflict-patch-invalid" }, state); + assert.deepEqual(at(state, ""), { proceed: false, skipReason: "lane-stopped", escalationReason: "conflict-patch-invalid" }, state); } assert.deepEqual(at("clean", "", { patchHeadSha: "b".repeat(40) }), { proceed: false, skipReason: "superseded-sha", escalationReason: "" }); assert.deepEqual(at("conflicted", "written", { headSha: "b".repeat(40) }), { proceed: false, skipReason: "superseded-sha", escalationReason: "" }); diff --git a/.github/scripts/maintenance-monitor-signals/collect.mjs b/.github/scripts/maintenance-monitor-signals/collect.mjs index 279ca46553..bc70e02047 100644 --- a/.github/scripts/maintenance-monitor-signals/collect.mjs +++ b/.github/scripts/maintenance-monitor-signals/collect.mjs @@ -3,7 +3,7 @@ // that fails to read is recorded in `gaps` and the others still run. Every // list is bounded, and `get` spends the run's call budget. import { parseOutcomeMarker } from "../work-resolve-issue/report-outcome.mjs"; -import { parseBudget } from "./signals.mjs"; +import { parseBudget, STALLED_HOURS } from "./signals.mjs"; export const LIMITS = { prMergeRuns: 12, @@ -14,7 +14,14 @@ export const LIMITS = { mergeGroupRuns: 10, sweepArtifacts: 5, commentPages: 5, + // Open pull requests the stalled-stop read covers, oldest update first; + // one comment page each, and one check-run read per candidate. + stalledPulls: 40, }; +// report-escalation.sh's comment for a stop routed to a failing status, one +// per activity, edited at each stop (gate-source.mjs LANE_STOPPED). +const LANE_STOPPED = /^$/; +const STOP_FIELD = (name) => new RegExp(`^- ${name}: \`([^\`]+)\``, "m"); export const AUDIT_WORKFLOWS = ["maintenance-audit-test-selection", "maintenance-check-upstream-drift"]; // Pull-request and merge-group runs hold no lane token, and a merge-group // failure is counted as an ejection instead. @@ -237,6 +244,37 @@ export async function collect({ get, getStatus, readArtifact, repo, now, windowS if (unread) gaps.push(`lane stops: over ${LIMITS.commentPages * 100} comments in the window; the oldest were not read`); }); + // Open pull requests whose head carries a stop the pr-unblock lane routed to + // a failing status (its lane-stopped comment, by the lanes App, names the + // PR's current head) with no attempt of that activity started on the head + // since the comment's last edit: its per-head retries are spent. Any PR, + // not only the lanes App's: review-dependency works Dependabot's, fix-ci + // adopted owners'. The status itself needs statuses: read, which this job's + // token lacks, so the comment and the activity's check runs stand for it. + await source("stalled stops", async () => { + data.stalledStops = []; + const cutoff = now.getTime() - STALLED_HOURS * 3600_000; + const pulls = await get(`/repos/${repo}/pulls?state=open&base=main&sort=updated&direction=asc&per_page=100`); + if (pulls.length > LIMITS.stalledPulls) { + gaps.push(`stalled stops: ${pulls.length} open pull requests, only the ${LIMITS.stalledPulls} least recently updated read`); + } + for (const pull of pulls.slice(0, LIMITS.stalledPulls)) { + const head = pull.head?.sha; + const comments = await get(`/repos/${repo}/issues/${pull.number}/comments?per_page=100`); + for (const c of comments) { + const lines = String(c.body ?? "").split("\n").map((line) => line.replace(/\r$/, "")); + const activity = LANE_STOPPED.exec(lines[0])?.[1]; + const body = lines.join("\n"); + if (c.user?.login !== agentIdentity || !activity || STOP_FIELD("Head").exec(body)?.[1] !== head) continue; + const stoppedAt = c.updated_at ?? c.created_at; + if (!(Date.parse(stoppedAt) < cutoff)) continue; + const runs = (await get(`/repos/${repo}/commits/${head}/check-runs?check_name=${encodeURIComponent(`pr-unblock / ${activity}`)}&filter=all&per_page=100`)).check_runs ?? []; + if (runs.some((run) => Date.parse(run.started_at) > Date.parse(stoppedAt))) continue; + data.stalledStops.push({ pr: pull.number, head, activity, reason: STOP_FIELD("Reason").exec(body)?.[1] ?? "unknown", stoppedAt, url: c.html_url }); + } + } + }); + await source("audits", async () => { data.audits = []; for (const workflow of AUDIT_WORKFLOWS) { diff --git a/.github/scripts/maintenance-monitor-signals/collect.test.mjs b/.github/scripts/maintenance-monitor-signals/collect.test.mjs index ce24fe3a7d..01a715a99d 100644 --- a/.github/scripts/maintenance-monitor-signals/collect.test.mjs +++ b/.github/scripts/maintenance-monitor-signals/collect.test.mjs @@ -73,7 +73,35 @@ const laneComment = (id, body, extra = {}) => ({ ...extra, }); +// report-escalation.sh's lane-stopped comment for `activity` at `head`, last +// edited at `at`. +const sha = (c) => c.repeat(40); +const stopComment = (pr, head, at, { activity = "fix-ci", reason = "ci-fix-invalid", login = "melodic-automation-lanes[bot]" } = {}) => ({ + id: pr * 10, + body: `\nThe pr-unblock lane stopped.\n\n- Reason: \`${reason}\`\n- Head: \`${head}\`\n`, + user: { login }, + created_at: "2026-10-18T09:00:00Z", + updated_at: at, + html_url: `https://github.com/o/r/pull/${pr}#issuecomment-${pr * 10}`, +}); +// Open PRs: #8301 stalled (stopped at 12:00, no attempt since); #8302 stopped +// 20 minutes ago; #8303 stopped long ago but retried since; #8304 +// Dependabot's, stalled on review-dependency; #8305's comment names an +// older head; #8306's marker is a person's. +const stallRoutes = () => [ + [/\/pulls\?state=open/, [8301, 8302, 8303, 8304, 8305, 8306].map((number) => ({ number, head: { sha: sha(String(number % 10)) } }))], + [/\/issues\/8301\/comments/, [stopComment(8301, sha("1"), "2026-10-19T12:00:00Z")]], + [/\/issues\/8302\/comments/, [stopComment(8302, sha("2"), "2026-10-19T16:20:00Z")]], + [/\/issues\/8303\/comments/, [stopComment(8303, sha("3"), "2026-10-19T12:00:00Z")]], + [/\/issues\/8304\/comments/, [stopComment(8304, sha("4"), "2026-10-19T11:30:00Z", { activity: "review-dependency", reason: "dependency-invalid" })]], + [/\/issues\/8305\/comments/, [stopComment(8305, sha("9"), "2026-10-19T10:00:00Z")]], + [/\/issues\/8306\/comments/, [stopComment(8306, sha("6"), "2026-10-19T10:00:00Z", { login: "someone" })]], + [/\/commits\/3{40}\/check-runs/, { check_runs: [{ started_at: "2026-10-19T12:30:00Z" }] }], + [/\/commits\/[0-9a-f]{40}\/check-runs/, { check_runs: [{ started_at: "2026-10-19T11:00:00Z" }] }], +]; + const routes = () => [ + ...stallRoutes(), [/workflows\/pr-merge\.yml\/runs/, { workflow_runs: [run(10)] }], [/runs\/10\/jobs/, { jobs: [{ id: 100, name: "merge", conclusion: "success" }, { id: 101, name: "post-merge-dispatch" }] }], [ @@ -176,6 +204,14 @@ test("collect reads every source into the signals' data shape", async () => { data.status.incidents.map((i) => [i.id, i.components]), [["8tzbpcnl75jp", ["Actions", "Webhooks"]]], ); + // A stop with no attempt since, past STALLED_HOURS, on any author's PR. + assert.deepEqual( + data.stalledStops.map((s) => [s.pr, s.activity, s.reason]), + [ + [8301, "fix-ci", "ci-fix-invalid"], + [8304, "review-dependency", "dependency-invalid"], + ], + ); // Only the lanes App's markers created in the window, at the comment's start. assert.deepEqual( data.laneStops.map((s) => [s.id, s.outcome, s.issue]), @@ -382,3 +418,12 @@ test("listFiled lists only github-actions[bot]'s issues", async () => { await listFiled({ get: api.get, repo }); assert.deepEqual(api.calls.map((path) => new URL(path, "https://x").searchParams.get("creator")), ["github-actions[bot]"]); }); + +test("stalled stops: more open PRs than the read covers is a gap, and the least recently updated are read", async () => { + const many = Array.from({ length: 45 }, (_, i) => ({ number: 9000 + i, head: { sha: sha("e") } })); + const api = fakeApi([[/\/pulls\?state=open/, many], [/\/issues\/9\d{3}\/comments/, []], ...routes()]); + const data = await collect(input(api)); + assert.ok(data.gaps.some((g) => g.startsWith("stalled stops: 45 open pull requests"))); + assert.equal(api.calls.filter((p) => /\/issues\/9\d{3}\/comments/.test(p)).length, 40); + assert.ok(api.calls.some((p) => p.includes("sort=updated&direction=asc"))); +}); diff --git a/.github/scripts/maintenance-monitor-signals/issues.mjs b/.github/scripts/maintenance-monitor-signals/issues.mjs index 04ff58b75a..975a6b51a7 100644 --- a/.github/scripts/maintenance-monitor-signals/issues.mjs +++ b/.github/scripts/maintenance-monitor-signals/issues.mjs @@ -15,6 +15,7 @@ export const SIGNALS = [ "6 report-only audit", "8 throughput", "9 lane stop", + "10 stalled lane stop", ]; export const LOG_ONLY = new Set(["7 platform incident"]); const MUTED = new Set(["not_planned", "duplicate"]); @@ -118,6 +119,7 @@ const SOURCES = { "pr-merge sweep": ["5 held PRs", "8 throughput"], "lane merges": ["8 throughput"], "lane stops": ["9 lane stop"], + "stalled stops": ["10 stalled lane stop"], audits: ["6 report-only audit"], "githubstatus.com": [], }; diff --git a/.github/scripts/maintenance-monitor-signals/issues.test.mjs b/.github/scripts/maintenance-monitor-signals/issues.test.mjs index 58774d7056..85e2652dd6 100644 --- a/.github/scripts/maintenance-monitor-signals/issues.test.mjs +++ b/.github/scripts/maintenance-monitor-signals/issues.test.mjs @@ -200,6 +200,7 @@ test("every detector's signal is a rolling family or logged only, and an unknown audits: [{ workflow: "w", run: { id: 1, url: "u", conclusion: "failure", createdAt: "t" } }], status: { components: [{ name: "Actions", status: "major_outage" }], incidents: [] }, laneStops: [{ id: 1, outcome: "funding", issue: 3, createdAt: "t", url: "u" }], + stalledStops: [{ pr: 5, head: "a", activity: "fix-ci", reason: "ci-fix-invalid", stoppedAt: "t", url: "u" }], }); assert.deepEqual(new Set(all.map((f) => f.signal)), new Set([...SIGNALS, ...LOG_ONLY])); const plan = planIssues({ findings: [{ ...findingFor([1]), signal: "10 renamed" }], existing: [] }); diff --git a/.github/scripts/maintenance-monitor-signals/signals.mjs b/.github/scripts/maintenance-monitor-signals/signals.mjs index 9e6ae03c9d..78b430dde7 100644 --- a/.github/scripts/maintenance-monitor-signals/signals.mjs +++ b/.github/scripts/maintenance-monitor-signals/signals.mjs @@ -13,6 +13,11 @@ export const ACTIONS_JOB_LIMIT = 60; export const EJECTIONS_PER_DAY = 2; export const HELD_HOURS = 24; export const STALL_HOURS = 3; +// pr-unblock-dispatch.yml runs every 30 minutes (cron 9,39), and the feedback +// sweep runs off each of those ticks, so a stop the lane will retry gets a new +// attempt within one tick; four ticks with none mean the per-head retries are +// spent (or a dispatcher refused). +export const STALLED_HOURS = 2; const SWEEP_MAX_AGE_MINUTES = 70; const MAX_FACTS = 25; @@ -315,6 +320,24 @@ export function laneStopFindings({ laneStops = [] }) { ); } +// Pull requests a pr-unblock stop left red with no retry left on their head +// (collect.mjs, "stalled stops"): no person was asked, so this tells one. +export function stalledFindings({ stalledStops = [] }) { + if (stalledStops.length === 0) return []; + const stops = [...stalledStops].sort((a, b) => a.pr - b.pr || a.activity.localeCompare(b.activity)); + return [ + finding({ + key: "stalled-lane-stop", + signal: "10 stalled lane stop", + title: `Stalled: ${stops.length} pull request stops the PR fix lane left red for over ${STALLED_HOURS} hours`, + area: "PR fix lane (pr-unblock)", + summary: `${stops.length} pull requests carry a failing pr-unblock status the lane can no longer retry on their head; each needs a new commit or a look at its stop comment.`, + facts: stops.map((s) => `#${s.pr} ${s.activity} ${s.reason} since ${s.stoppedAt}: ${s.url}`), + evidence: ["stalled-lane-stop", stops.map((s) => [s.pr, s.head, s.activity, s.reason])], + }), + ]; +} + const DETECTORS = [ mainRedFindings, ceilingFindings, @@ -325,6 +348,7 @@ const DETECTORS = [ platformFindings, throughputFindings, laneStopFindings, + stalledFindings, ]; export function detect(data) { diff --git a/.github/scripts/maintenance-monitor-signals/signals.test.mjs b/.github/scripts/maintenance-monitor-signals/signals.test.mjs index e750d78848..52f02b9cce 100644 --- a/.github/scripts/maintenance-monitor-signals/signals.test.mjs +++ b/.github/scripts/maintenance-monitor-signals/signals.test.mjs @@ -14,6 +14,7 @@ import { mainRedFindings, parseBudget, platformFindings, + stalledFindings, throttleFindings, throughputFindings, } from "./signals.mjs"; @@ -429,3 +430,23 @@ test("every finding has a marker-safe key, a hex fingerprint, an area and inert for (const line of f.facts) assert.doesNotMatch(line, /`|\n/); } }); + +// collect.mjs hands on only stops past STALLED_HOURS with no attempt since; +// one finding names each, and its fingerprint moves only with the set. +test("stalled stops are one finding, a fact per PR, whose fingerprint moves with the stopped heads only", () => { + const stop = (pr, extra = {}) => ({ pr, head: "a".repeat(40), activity: "fix-ci", reason: "ci-fix-invalid", stoppedAt: "2026-10-19T12:00:00Z", url: `https://github.com/o/r/pull/${pr}#c`, ...extra }); + assert.deepEqual(stalledFindings({ stalledStops: [] }), []); + const [f] = stalledFindings({ stalledStops: [stop(9), stop(4, { activity: "review-dependency", reason: "dependency-invalid" })] }); + assert.equal(f.key, "stalled-lane-stop"); + assert.equal(f.signal, "10 stalled lane stop"); + assert.match(f.title, /^Stalled: 2 pull request stops/); + assert.deepEqual(f.facts, [ + "#4 review-dependency dependency-invalid since 2026-10-19T12:00:00Z: https://github.com/o/r/pull/4#c", + "#9 fix-ci ci-fix-invalid since 2026-10-19T12:00:00Z: https://github.com/o/r/pull/9#c", + ]); + const [again] = stalledFindings({ stalledStops: [stop(4, { activity: "review-dependency", reason: "dependency-invalid", stoppedAt: "later" }), stop(9)] }); + assert.equal(again.fingerprint, f.fingerprint, "the stop's time is not evidence"); + const [moved] = stalledFindings({ stalledStops: [stop(9, { head: "b".repeat(40) })] }); + assert.notEqual(moved.fingerprint, f.fingerprint); + assert.ok(detect({ ...base, stalledStops: [stop(9)] }).some((x) => x.key === "stalled-lane-stop")); +}); diff --git a/.github/scripts/pr-unblock/address-feedback.mjs b/.github/scripts/pr-unblock/address-feedback.mjs index 493a4b3c9e..92863f6f8c 100644 --- a/.github/scripts/pr-unblock/address-feedback.mjs +++ b/.github/scripts/pr-unblock/address-feedback.mjs @@ -37,6 +37,7 @@ import { pathToFileURL } from "node:url"; import { commentItems, gateSource, listLabelEvents, ROUND_MARKER } from "./gate-source.mjs"; import { isProtectedPath, namedInScope, prPathsOf } from "./thread-scope.mjs"; +import { retryRung, stopSkip } from "./escalation-routes.mjs"; // The round cap is the config's `loop.review-rounds`, passed in as // REVIEW_ROUNDS; the resolver's default is the fallback. @@ -104,6 +105,8 @@ export function countRounds(issueComments, appId) { // or broker refusal after the push moved the head); a forged one only stops // the lane sooner. export const MODEL_CHECK = "pr-unblock / address-feedback"; +// The reply job's check: a lane stop there runs the head's retry a rung up. +export const REPLY_CHECK = "pr-unblock / publish-feedback"; const isModelRun = (run, conclusions) => run.name === MODEL_CHECK && run.app?.slug === "github-actions" && conclusions.includes(run.conclusion); export function countCheckRounds(checkRunsByCommit) { @@ -546,6 +549,8 @@ export async function select({ } return { decision: "proceed", + // A retry after the reply job stopped on this head runs one rung up. + modelRung: retryRung(await api.listCheckRuns(headSha, REPLY_CHECK)), rounds, maxRounds: bound, // True drops the human-gated label from the write runner's holds: every @@ -909,7 +914,8 @@ export async function main(argv, env, { api, log = (line) => console.log(line) } // when this is true: every kept thread carries a write-access fix request, // or a lane escalation put the label on. output("fix-requested", String(result.fixRequested === true)); - output("skip-reason", result.decision === "skip" ? result.skipReason : result.decision === "escalate" ? "awaiting-human" : ""); + output("model-rung", result.modelRung ?? ""); + output("skip-reason", result.decision === "skip" ? result.skipReason : result.decision === "escalate" ? stopSkip(result.escalationReason) : ""); output("escalation-reason", result.escalationReason ?? ""); // The neutral checks' summary line when the PR stopped at the round cap. output("not-run-detail", result.decision === "escalate" && result.maxRounds ? roundLine(result.rounds, result.maxRounds) : ""); diff --git a/.github/scripts/pr-unblock/address-feedback.test.mjs b/.github/scripts/pr-unblock/address-feedback.test.mjs index 942344209a..b8403ba2c0 100644 --- a/.github/scripts/pr-unblock/address-feedback.test.mjs +++ b/.github/scripts/pr-unblock/address-feedback.test.mjs @@ -858,7 +858,7 @@ test("main select writes outputs and the thread file; escalation writes no threa TRUSTED_ACTORS: actors, }); await main(["select"], env("ok"), { api: fakeApi(), log: () => {} }); - assert.equal(readFileSync(join(dir, "ok.out"), "utf8"), "proceed=true\nfix-requested=false\nskip-reason=\nescalation-reason=\nnot-run-detail=\n"); + assert.equal(readFileSync(join(dir, "ok.out"), "utf8"), "proceed=true\nfix-requested=false\nmodel-rung=\nskip-reason=\nescalation-reason=\nnot-run-detail=\n"); const okFile = JSON.parse(readFileSync(join(dir, "ok.json"), "utf8")); assert.equal(okFile.threads[0].id, "PRRT_1"); // No lane branch: the model job commits nothing. @@ -873,7 +873,7 @@ test("main select writes outputs and the thread file; escalation writes no threa // The people on the list come from TRUSTED_ACTORS. await main(["select"], env("decided"), { api: fakeApi(escalated()), log: () => {} }); - assert.equal(readFileSync(join(dir, "decided.out"), "utf8"), "proceed=true\nfix-requested=false\nskip-reason=\nescalation-reason=\nnot-run-detail=\n"); + assert.equal(readFileSync(join(dir, "decided.out"), "utf8"), "proceed=true\nfix-requested=false\nmodel-rung=\nskip-reason=\nescalation-reason=\nnot-run-detail=\n"); assert.equal(JSON.parse(readFileSync(join(dir, "decided.json"), "utf8")).threads[0].comments[2].direction, true); // The human-gated label comes from TRACKER_CONFIG; a fix request sets fix-requested. @@ -881,15 +881,15 @@ test("main select writes outputs and the thread file; escalation writes no threa writeFileSync(tracker, JSON.stringify({ config: { role_labels: { "human-gated": "status: blocked-human" } } })); const gatedEnv = { ...env("gated"), TRACKER_CONFIG: tracker }; await main(["select"], gatedEnv, { api: fakeApi({ ...escalated({ replies: [[102, HUMAN, REQUEST]] }), labels: ["status: blocked-human"] }), log: () => {} }); - assert.equal(readFileSync(join(dir, "gated.out"), "utf8"), "proceed=true\nfix-requested=true\nskip-reason=\nescalation-reason=\nnot-run-detail=\n"); + assert.equal(readFileSync(join(dir, "gated.out"), "utf8"), "proceed=true\nfix-requested=true\nmodel-rung=\nskip-reason=\nescalation-reason=\nnot-run-detail=\n"); const heldEnv = { ...env("held"), TRACKER_CONFIG: tracker }; await main(["select"], heldEnv, { api: fakeApi({ ...escalated(), labels: ["status: blocked-human"] }), log: () => {} }); - assert.equal(readFileSync(join(dir, "held.out"), "utf8"), "proceed=false\nfix-requested=false\nskip-reason=awaiting-human\nescalation-reason=\nnot-run-detail=\n"); + assert.equal(readFileSync(join(dir, "held.out"), "utf8"), "proceed=false\nfix-requested=false\nmodel-rung=\nskip-reason=awaiting-human\nescalation-reason=\nnot-run-detail=\n"); await main(["select"], env("esc"), { api: fakeApi({ files: [{ filename: ".github/workflows/x.yml" }] }), log: () => {} }); assert.equal( readFileSync(join(dir, "esc.out"), "utf8"), - "proceed=false\nfix-requested=false\nskip-reason=awaiting-human\nescalation-reason=feedback-protected-path\nnot-run-detail=\n", + "proceed=false\nfix-requested=false\nmodel-rung=\nskip-reason=awaiting-human\nescalation-reason=feedback-protected-path\nnot-run-detail=\n", ); assert.throws(() => readFileSync(join(dir, "esc.json"))); assert.throws(() => readFileSync(join(dir, "esc.run.json")), "an escalation writes no run.json"); @@ -907,7 +907,7 @@ test("main select writes outputs and the thread file; escalation writes no threa await main(["select"], { ...env("bound"), REVIEW_ROUNDS: "1" }, { api: bound, log: () => {} }); assert.equal( readFileSync(join(dir, "bound.out"), "utf8"), - "proceed=false\nfix-requested=false\nskip-reason=awaiting-human\nescalation-reason=feedback-not-converging\nnot-run-detail=round 2 of max 2\n", + "proceed=false\nfix-requested=false\nmodel-rung=\nskip-reason=lane-stopped\nescalation-reason=feedback-not-converging\nnot-run-detail=round 2 of max 2\n", ); }); @@ -1111,3 +1111,20 @@ test("a leading or trailing _ that belongs to the name is kept; only a matched _ const result = await run(named([[101, HUMAN, body]], { trees }).api); assert.deepEqual([...result.file.threads[0].named_paths].sort(), [`${T}/__init__.py`, `${T}/_utils.js`, `${T}/run-transcript.test.js`]); }); + +// A retry after the reply job stopped on this head (a lane-stopped +// publish-feedback check from the GitHub Actions app) runs one model rung up; +// a first round, or a stop record from another app, runs at the profile's own. +test("select asks for model rung 1 only after a lane stop on the head's reply check", async () => { + const stop = { status: "completed", conclusion: "neutral", output: { title: "Skipped: lane-stopped" }, app: { id: 15368 } }; + const withReply = (replyRuns) => { + const api = fakeApi(); + const base = api.listCheckRuns; + return { ...api, listCheckRuns: async (sha, name) => (name === "pr-unblock / publish-feedback" ? replyRuns : base(sha, name)) }; + }; + const rungOf = async (replyRuns) => (await select({ api: withReply(replyRuns), pr: 42, headSha: HEAD, repository: "o/r", appId: APP, reviewerId: REVIEWER, maxRounds: 3 })).modelRung; + assert.equal(await rungOf([stop]), "1"); + assert.equal(await rungOf([]), ""); + assert.equal(await rungOf([{ ...stop, app: { id: 1 } }]), ""); + assert.equal(await rungOf([{ ...stop, output: { title: "Skipped: no-model-turn" } }]), ""); +}); diff --git a/.github/scripts/pr-unblock/apply-dependency-verdict.mjs b/.github/scripts/pr-unblock/apply-dependency-verdict.mjs index 12fe51c637..656319f972 100755 --- a/.github/scripts/pr-unblock/apply-dependency-verdict.mjs +++ b/.github/scripts/pr-unblock/apply-dependency-verdict.mjs @@ -92,6 +92,7 @@ import { WEB_FLOW_ID, } from "./dependency-classify.mjs"; import { FILES_CAP, roundLine, rounds, VERDICTS } from "./review-dependency.mjs"; +import { stopSkip } from "./escalation-routes.mjs"; export const LIMITS = { bytes: 65536, reasons: 10, reason: 500, evidence: 20, evidenceItem: 300, config: 200 }; // The compare listing stops at 250 commits and 300 files @@ -560,7 +561,7 @@ export async function main(env, { api, log = (line) => console.log(line) } = {}) }); if (escalationReason) { writeFileSync(env.ESCALATION_REASON_FILE, `${escalationReason}\n`); - writeFileSync(env.SKIP_REASON_FILE, "awaiting-human\n"); + writeFileSync(env.SKIP_REASON_FILE, `${stopSkip(escalationReason)}\n`); } else if (skipReason) { writeFileSync(env.SKIP_REASON_FILE, `${skipReason}\n`); } diff --git a/.github/scripts/pr-unblock/apply-dependency-verdict.test.mjs b/.github/scripts/pr-unblock/apply-dependency-verdict.test.mjs index 80c63603ab..c598d46262 100644 --- a/.github/scripts/pr-unblock/apply-dependency-verdict.test.mjs +++ b/.github/scripts/pr-unblock/apply-dependency-verdict.test.mjs @@ -600,7 +600,7 @@ test("main counts a round with no verdict and skips no-model-turn only on the sc ["a marker download that failed", { SCAN_REFUSED_DIR: "/nonexistent", SCAN_REFUSED_READ: "failure" }, fakeApi(), "dependency-invalid"], ]) { const result = await outcome(mainFiles({ verdict: null, env }), api); - assert.deepEqual([result.escalation, result.skip], [want, "awaiting-human"], why); + assert.deepEqual([result.escalation, result.skip], [want, "lane-stopped"], why); assert.equal(result.comments.length, 1, `${why}: the round counts`); } // An empty marker directory is no refusal. diff --git a/.github/scripts/pr-unblock/dependency-dispatch.mjs b/.github/scripts/pr-unblock/dependency-dispatch.mjs index b7fb662b89..bfe494bc78 100644 --- a/.github/scripts/pr-unblock/dependency-dispatch.mjs +++ b/.github/scripts/pr-unblock/dependency-dispatch.mjs @@ -22,7 +22,8 @@ import { DEPENDABOT_ID, DEPENDABOT_LOGIN, RED } from "./dependency-classify.mjs"; import { emptyRecord, isHandled, markHandled } from "./dispatch-record.mjs"; -import { anyHandles, limitResetsAt, permanentlyHandles } from "./limit-stop.mjs"; +import { anyHandles, LANE_STOP, limitResetsAt, permanentlyHandles, releasedBy, stopRelease } from "./limit-stop.mjs"; +import { LIMIT_DEFAULTS } from "../work-resolve-issue/limits.mjs"; import { ciStatusOf, classifyPull, commitFilesReader } from "./review-dependency.mjs"; export const DEFAULTS = Object.freeze({ @@ -43,7 +44,7 @@ const RUN_ID = /^[1-9][0-9]{0,14}$/; // The neutral titles review-dependency-not-run posts from the select job's // skip reasons (review-dependency.mjs select): no verdict job follows them, // so no no-model-turn verdict can release the head. -const TERMINAL_SKIPS = new Set(["Skipped: awaiting-human", "Skipped: not-applicable", "Skipped: superseded-sha"]); +const TERMINAL_SKIPS = new Set(["Skipped: awaiting-human", "Skipped: not-applicable", "Skipped: superseded-sha", LANE_STOP]); // NaN for a run with no readable start: it compares false either way. const startOf = (run) => Date.parse(run.started_at ?? ""); @@ -140,6 +141,15 @@ export async function dispatchDependencyReviews({ const cut = startOf(newest); existing = existing.filter((run) => limitResetsAt(run, limits) !== null || !(startOf(run) < cut)); } + // A verdict job that stopped on a reason routed to a failing status + // releases the head too, a fresh review on the same head, until + // limit-stops-per-head such stops stand on it. + const cap = (limits ?? LIMIT_DEFAULTS)["limit-stops-per-head"]; + const { cut, stops } = stopRelease(verdicts, cap); + existing = releasedBy(existing, cut, limits); + if (cut === null && stops >= cap) { + log(`::warning::#${node.number}: head ${node.headRefOid}'s ${verdictCheckName} stopped ${stops} time(s), the most one head gets; no review-dependency retry until a new push`); + } } // Until a verdict check started after the newest review-dependency // check, a no-model-turn verdict may still release the head, so it is diff --git a/.github/scripts/pr-unblock/dependency-dispatch.test.mjs b/.github/scripts/pr-unblock/dependency-dispatch.test.mjs index 3024e66d43..f43a399a30 100644 --- a/.github/scripts/pr-unblock/dependency-dispatch.test.mjs +++ b/.github/scripts/pr-unblock/dependency-dispatch.test.mjs @@ -409,3 +409,38 @@ test("a review run still going, or an untrusted-trigger skip, is not recorded", markHandled(record, CHECK, a.headRefOid); assert.deepEqual((await run(fakeApi({ nodes: [a, b], checks }), { record })).dispatched.map((d) => d.number), [13]); }); + +// A verdict job that stopped on a reason routed to a failing status +// (dependency-invalid, -fix-not-landed, -config-unreadable, -files-truncated, +// -control-uncited) releases the head as no-model-turn does: a fresh review on +// the same head, until limit-stops-per-head such stops stand. A select job's +// own stop (the round cap) on the review-dependency check is final. +const laneStopped = (started) => roundRun("apply-dependency-verdict", "neutral", "Skipped: lane-stopped", started); +const reviewed = (started) => roundRun("review-dependency", "success", "Passed", started); + +test("a verdict job's lane stop retries the head until limit-stops-per-head, and a select job's stop is final", async () => { + const node = pr(61); + const once = await run(fakeApi({ nodes: [node], checks: headOf(node, [reviewed("2026-10-17T10:00:00Z"), laneStopped("2026-10-17T10:05:00Z")]) })); + assert.deepEqual(once.dispatched, [{ number: 61, workflowRunId: "7061" }]); + const record = emptyRecord(); + const lines = []; + const capped = await run( + fakeApi({ + nodes: [node], + checks: headOf(node, [ + reviewed("2026-10-17T10:00:00Z"), + laneStopped("2026-10-17T10:05:00Z"), + reviewed("2026-10-17T10:30:00Z"), + laneStopped("2026-10-17T10:35:00Z"), + reviewed("2026-10-17T11:00:00Z"), + laneStopped("2026-10-17T11:05:00Z"), + ]), + }), + { record, log: (line) => lines.push(line) }, + ); + assert.deepEqual(capped.dispatched, []); + assert.ok(lines.some((line) => line.includes("stopped 3 time(s)")), lines.join("\n")); + assert.ok(record.handled.get(CHECK)?.has(node.headRefOid)); + const terminal = await run(fakeApi({ nodes: [node], checks: headOf(node, [roundRun("review-dependency", "neutral", "Skipped: lane-stopped", "2026-10-17T10:00:00Z")]) })); + assert.deepEqual(terminal.dispatched, []); +}); diff --git a/.github/scripts/pr-unblock/dispatch.mjs b/.github/scripts/pr-unblock/dispatch.mjs index 26e4137bc3..e5125eb0f7 100644 --- a/.github/scripts/pr-unblock/dispatch.mjs +++ b/.github/scripts/pr-unblock/dispatch.mjs @@ -37,6 +37,7 @@ import { pathToFileURL } from "node:url"; import { loadTrustedList } from "../../actions/check-trusted-trigger/check-trusted-trigger.mjs"; import { testedRun } from "../maintenance-reconcile-lost-runs/infra-failure.mjs"; +import { LIMIT_DEFAULTS } from "../work-resolve-issue/limits.mjs"; import { DEPENDABOT_ID, POLICY_PATH } from "./dependency-classify.mjs"; import { dispatchDependencyReviews } from "./dependency-dispatch.mjs"; import { emptyRecord, isHandled, markHandled, pruneRecord, readRecord, writeRecord } from "./dispatch-record.mjs"; @@ -44,7 +45,7 @@ import { prHold } from "./gate-source.mjs"; import { runTitle, shareWorkflowRuns } from "./feedback-dispatch.mjs"; import { DEFAULTS as CI_FIX_DEFAULTS, dispatchCiFixes, humanGatedLabel, LIST_QUERY as CI_FIX_QUERY } from "./fix-ci-dispatch.mjs"; import { dispatchDocsFixes, HISTORY_QUERY as DOCS_FIX_HISTORY_QUERY, LIST_QUERY as DOCS_FIX_QUERY } from "./fix-docs-dispatch.mjs"; -import { anyHandles, cappedAt, permanentlyHandles } from "./limit-stop.mjs"; +import { anyHandles, cappedAt, permanentlyHandles, releasedBy, stopRelease } from "./limit-stop.mjs"; import { budgetedFetch } from "./request-budget.mjs"; import { readConfig, readPolicy } from "./review-dependency.mjs"; @@ -53,6 +54,9 @@ export const DEFAULTS = Object.freeze({ workflow: "pr-unblock.yml", activity: "resolve-conflicts", checkName: "pr-unblock / resolve-conflicts", + // The push job's check: its lane-stopped skip releases the head for a + // same-head retry (limit-stop.mjs stopRelease). + stopCheckName: "pr-unblock / apply-conflict-patch", // The GitHub Actions app: only a check run it created counts, so another // app cannot suppress a dispatch by posting a run under the same name. appId: 15368, @@ -119,6 +123,7 @@ export async function dispatchConflicting({ workflow = DEFAULTS.workflow, activity = DEFAULTS.activity, checkName = DEFAULTS.checkName, + stopCheckName = DEFAULTS.stopCheckName, appId = DEFAULTS.appId, maxDispatch = DEFAULTS.maxDispatch, unknownRetries = DEFAULTS.unknownRetries, @@ -204,8 +209,20 @@ export async function dispatchConflicting({ log(`skip #${node.number}: ${why}`); continue; } - const existing = await api.listCheckRuns({ sha: node.headRefOid, checkName, appId }); + let existing = await api.listCheckRuns({ sha: node.headRefOid, checkName, appId }); const options = { limits, failuresRetry: true, heldSkipsRelease: true }; + // A push job that stopped on a reason routed to a failing status + // releases the head's earlier checks: the next run retries the same + // head, a fresh model session, until limit-stops-per-head such stops + // stand on it. + if (anyHandles(existing, { now, ...options })) { + const cap = (limits ?? LIMIT_DEFAULTS)["limit-stops-per-head"]; + const { cut, stops } = stopRelease(await api.listCheckRuns({ sha: node.headRefOid, checkName: stopCheckName, appId }), cap); + existing = releasedBy(existing, cut, limits); + if (cut === null && stops >= cap) { + log(`::warning::#${node.number}: head ${node.headRefOid}'s ${stopCheckName} stopped ${stops} time(s), the most one head gets; no ${activity} retry until a new push`); + } + } handled = anyHandles(existing, { now, ...options }); if (permanentlyHandles(existing, options)) { // Recording it means this warning shows once per head. diff --git a/.github/scripts/pr-unblock/dispatch.test.mjs b/.github/scripts/pr-unblock/dispatch.test.mjs index 4bf05b2226..de3fb84b66 100644 --- a/.github/scripts/pr-unblock/dispatch.test.mjs +++ b/.github/scripts/pr-unblock/dispatch.test.mjs @@ -64,8 +64,10 @@ function pr(number, overrides = {}) { // or a Map from head SHA to the check runs GitHub returns for it. `runIds` // maps a PR number to the workflow_run_id its dispatch returns, undefined // included; any other PR gets 9000 plus its number. `workflowRuns` is the -// pr-unblock.yml run listing. +// pr-unblock.yml run listing. `stops` maps a head SHA to its +// pr-unblock / apply-conflict-patch check runs, read into calls.stops. function fakeApi({ + stops = new Map(), pages = [[]], requeries = [], checked = new Set(), @@ -78,7 +80,7 @@ function fakeApi({ comments = new Map(), labelEvents = new Map(), } = {}) { - const calls = { list: [], get: [], checks: [], dispatch: [], runs: [], timeline: [] }; + const calls = { list: [], get: [], checks: [], stops: [], dispatch: [], runs: [], timeline: [] }; let round = 0; return { calls, @@ -115,6 +117,10 @@ function fakeApi({ return answer.filter((node) => numbers.includes(node.number)); }, async listCheckRuns({ sha, checkName, appId }) { + if (checkName === "pr-unblock / apply-conflict-patch") { + calls.stops.push({ sha, appId }); + return stops.get(sha) ?? []; + } calls.checks.push({ sha, checkName, appId }); if (failChecks.has(sha)) { throw new Error("check-runs read failed"); @@ -1169,3 +1175,48 @@ test("main records a fix-ci head at the per-head cap in DISPATCH_RECORD, warns o rmSync(dir, { recursive: true, force: true }); } }); + +// A same-head retry after a stop routed to a failing status: the push job's +// lane-stopped skip (escalation-routes.mjs stopSkip) releases the head's +// earlier resolve-conflicts checks, up to limit-stops-per-head stops. +const at = (time) => `2026-10-26T${time}:00Z`; +const modelRun = (time) => ({ ...SUCCESS_RUN, started_at: at(time), completed_at: at(time) }); +const laneStopRun = (time) => ({ status: "completed", conclusion: "neutral", output: { title: "Skipped: lane-stopped" }, started_at: at(time) }); + +test("a push job's lane stop releases the head for a same-head retry, up to limit-stops-per-head stops", async () => { + const head = pr(1).headRefOid; + const once = harness({ pages: [[pr(1)]], checked: new Map([[head, [modelRun("10:00")]]]), stops: new Map([[head, [laneStopRun("10:05")]]]) }); + await once.run(); + assert.deepEqual(dispatchedNumbers(once.api), ["1"], "one stop: retried"); + + const record = emptyRecord(); + const three = harness( + { + pages: [[pr(1)]], + checked: new Map([[head, [modelRun("10:00"), modelRun("10:30"), modelRun("11:00")]]]), + stops: new Map([[head, [laneStopRun("10:05"), laneStopRun("10:35"), laneStopRun("11:05")]]]), + }, + { record }, + ); + await three.run(); + assert.deepEqual(dispatchedNumbers(three.api), [], "three stops: the cap"); + assert.ok(three.lines.some((line) => line.startsWith("::warning::#1:") && line.includes("stopped 3 time(s)"))); + assert.ok(record.handled.get(CHECK_NAME).has(head), "the capped head is recorded"); +}); + +test("a retry that did not stop, or a select job's own lane stop, handles the head", async () => { + const head = pr(1).headRefOid; + // The newest push job check is not a stop: the retry landed. + const landed = harness({ + pages: [[pr(1)]], + checked: new Map([[head, [modelRun("10:00"), modelRun("10:30")]]]), + stops: new Map([[head, [laneStopRun("10:05"), { ...SUCCESS_RUN, started_at: at("10:35") }]]]), + }); + await landed.run(); + assert.deepEqual(dispatchedNumbers(landed.api), []); + // A select job's stop (the repeat cap) is deterministic: no retry. + const terminal = harness({ pages: [[pr(1)]], checked: new Map([[head, [laneStopRun("10:00")]]]) }); + await terminal.run(); + assert.deepEqual(dispatchedNumbers(terminal.api), []); + assert.deepEqual(terminal.api.calls.stops.map((call) => call.sha), [head], "the push job's checks are read, and hold no stop"); +}); diff --git a/.github/scripts/pr-unblock/escalation-routes.json b/.github/scripts/pr-unblock/escalation-routes.json index 1d7f60ec5f..a8ba8fe220 100644 --- a/.github/scripts/pr-unblock/escalation-routes.json +++ b/.github/scripts/pr-unblock/escalation-routes.json @@ -36,6 +36,7 @@ "ci-fix-empty": { "activity": "fix-ci", "route": "fail", "why": "the lane changed nothing that fixes the failing checks" }, "ci-fix-invalid": { "activity": "fix-ci", "route": "fail", "why": "the CI fix the lane wrote did not pass validation, so it was not pushed" }, "ci-fix-protected-path": { "activity": "fix-ci", "route": "hold", "why": "the CI fix changes a path the lane does not push: .github/workflows/, CODEOWNERS, .gitmodules or a settings file" }, + "ci-fix-weaken-required": { "activity": "fix-ci", "route": "hold", "why": "the lane could fix a failing check only by weakening a required check, which a person decides" }, "ci-fix-not-applied": { "activity": "fix-ci", "route": "fail", "why": "the CI fix did not apply to the head it was written for, so it was not pushed" }, "ci-fix-escalated": { "activity": "fix-ci", "route": "fail", "why": "the lane could not fix every failing check, after one retry a model rung up; its run log names each one and why" } } diff --git a/.github/scripts/pr-unblock/escalation-routes.mjs b/.github/scripts/pr-unblock/escalation-routes.mjs new file mode 100644 index 0000000000..49cb784fa8 --- /dev/null +++ b/.github/scripts/pr-unblock/escalation-routes.mjs @@ -0,0 +1,45 @@ +// escalation-routes.json, the one routing table: each stop reason's activity +// and route (`hold` hands the PR to a person with the label; `fail` ends in a +// failing status and no label), as report-escalation.sh reads it. No +// network; imported by gate-source.mjs, the select and emitter scripts, and +// apply-patch.mjs (which copies this file and the table beside its own copy). +import { readFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; + +export const ROUTES = Object.freeze(JSON.parse(readFileSync(new URL("./escalation-routes.json", import.meta.url), "utf8"))); + +// The activity a reason resumes, or null for a reason the table does not +// list, which holds every activity. +export const escalationActivity = (reason) => (Object.hasOwn(ROUTES, reason) ? ROUTES[reason].activity : null); + +// A reason's route, or null for a reason the table does not list. +export const escalationRoute = (reason) => (Object.hasOwn(ROUTES, reason) ? ROUTES[reason].route : null); + +// The skip reason a job that stops on `reason` posts on its check: a stop +// routed to a failing status is `lane-stopped`, which limit-stop.mjs reads; +// one held for a person, or one the table does not list, `awaiting-human`. +export const stopSkip = (reason) => (escalationRoute(reason) === "fail" ? "lane-stopped" : "awaiting-human"); + +// A stop routed to a failing status, posted after a model turn: the emitter +// job's neutral check carrying stopSkip's lane-stopped (limit-stop.mjs +// releases the head on it). The same skip on the activity's own check is a +// select job's stop, which handles the head for good like any other check. +export const LANE_STOP = "Skipped: lane-stopped"; +export const laneStop = (run) => run?.status === "completed" && run.conclusion === "neutral" && run.output?.title === LANE_STOP; + +// The GitHub Actions app: only its check runs are a trusted stop record. +export const ACTIONS_APP_ID = 15368; + +// The model rung a same-head retry runs at: "1" (the profile's escalate-to, +// resolve.mjs) once the head carries a lane stop from the emitter job +// (`stopRuns`, REST shape, by the GitHub Actions app), else "" (its own). +export const retryRung = (stopRuns) => (stopRuns.some((run) => laneStop(run) && run.app?.id === ACTIONS_APP_ID) ? "1" : ""); + +// `node escalation-routes.mjs skip` prints stopSkip of the REASON variable, +// for a workflow job that posts a select job's stop. +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + if (process.argv[2] !== "skip") { + throw new Error("usage: escalation-routes.mjs skip"); + } + process.stdout.write(stopSkip(process.env.REASON ?? "")); +} diff --git a/.github/scripts/pr-unblock/feedback-dispatch.mjs b/.github/scripts/pr-unblock/feedback-dispatch.mjs index 735e8a93e5..627e104b6f 100644 --- a/.github/scripts/pr-unblock/feedback-dispatch.mjs +++ b/.github/scripts/pr-unblock/feedback-dispatch.mjs @@ -77,7 +77,7 @@ import { commitPulls } from "../../actions/check-trusted-trigger/check-trusted-t import { fixRequesters, humanGatedLabelFrom, threadAuthors, trustedHumansFrom } from "./address-feedback.mjs"; import { commentItems, gateSource, listLabelEvents } from "./gate-source.mjs"; import { emptyRecord, isHandled, markHandled, readRecord, writeRecord } from "./dispatch-record.mjs"; -import { anyHandles, limitResetsAt, permanentlyHandles } from "./limit-stop.mjs"; +import { anyHandles, LANE_STOP, limitResetsAt, permanentlyHandles } from "./limit-stop.mjs"; import { budgetedFetch } from "./request-budget.mjs"; export const DEFAULTS = Object.freeze({ @@ -121,8 +121,10 @@ export const SWEEP_WORKFLOW = ".github/workflows/pr-unblock-dispatch.yml"; export const SWEEP_HEAD_CAP = 3; const startOf = (run) => Date.parse(run.started_at ?? ""); -// The reply check skips that leave the head for another round. -const RELEASES = new Set(["Skipped: no-model-turn", "Skipped: finding-stands"]); +// The reply check skips that leave the head for another round: a lane stop +// (a stop escalation-routes.json routes to a failing status) is retried on +// the same head, a fresh model session, under SWEEP_HEAD_CAP with the rest. +const RELEASES = new Set(["Skipped: no-model-turn", "Skipped: finding-stands", LANE_STOP]); const releases = (run) => run?.status === "completed" && run.conclusion === "neutral" && RELEASES.has(run.output?.title); // The SHA the newest marker of `lane` written by the marker author names, or "". diff --git a/.github/scripts/pr-unblock/feedback-dispatch.test.mjs b/.github/scripts/pr-unblock/feedback-dispatch.test.mjs index dbdafe08af..009d86a63c 100644 --- a/.github/scripts/pr-unblock/feedback-dispatch.test.mjs +++ b/.github/scripts/pr-unblock/feedback-dispatch.test.mjs @@ -1271,3 +1271,26 @@ test("a PR the sweep cannot read is logged and the next PR is still swept", asyn assert.equal(output, "run-id=38779200001\nupstream-run-id=14\n"); assert.ok(lines.some((line) => line.startsWith("::warning::#42:") && line.includes("502")), lines.join("\n")); }); + +// A round whose reply job stopped on a reason routed to a failing status +// (feedback-not-landed, -invalid, -partial, -incomplete, -unadjudicated and +// the rest): its lane-stopped skip releases the head like no-model-turn, so +// the tick retries it on the same head, a fresh model session, under +// SWEEP_HEAD_CAP with the other releasing skips. +const laneStopped = (started) => roundRun("publish-feedback", "neutral", "Skipped: lane-stopped", started); + +test("a scheduled tick retries a head whose reply job stopped, until SWEEP_HEAD_CAP stops stand", async (t) => { + const api = sweepApi({ checksAt: { [HEAD]: [answeredRound, laneStopped("2026-10-17T10:05:00Z")] } }); + await sweepMain(t, api); + assert.deepEqual(api.calls.dispatch.map((call) => call.inputs["pr-number"]), ["42"]); + const capped = sweepApi({ + checksAt: { [HEAD]: [answeredRound, laneStopped("2026-10-17T10:05:00Z"), laneStopped("2026-10-17T11:05:00Z"), laneStopped("2026-10-17T12:05:00Z")] }, + }); + assert.equal((await sweepMain(t, capped)).output, ""); + assert.deepEqual(capped.calls.dispatch, []); + // A select job's own stop (the round cap) posts on the address-feedback + // check, after the reply check: it holds the head. + const terminal = sweepApi({ checksAt: { [HEAD]: [answeredRound, laneStopped("2026-10-17T10:05:00Z"), roundRun("address-feedback", "neutral", "Skipped: lane-stopped", "2026-10-17T11:00:00Z")] } }); + await sweepMain(t, terminal); + assert.deepEqual(terminal.calls.dispatch, []); +}); diff --git a/.github/scripts/pr-unblock/fix-ci-context.mjs b/.github/scripts/pr-unblock/fix-ci-context.mjs index 4c88cc33fd..e65d4ccf73 100644 --- a/.github/scripts/pr-unblock/fix-ci-context.mjs +++ b/.github/scripts/pr-unblock/fix-ci-context.mjs @@ -22,6 +22,7 @@ import { pathToFileURL } from "node:url"; import { createGitHubApi, humanGatedLabelFrom, keepThreads, need, positive, reviewedHeads, reviewerIdFrom, SHA } from "./address-feedback.mjs"; import { prHold } from "./gate-source.mjs"; +import { retryRung, stopSkip } from "./escalation-routes.mjs"; // The trailer apply-patch.mjs writes on every fix commit, from the ci-context // row of patch-rules.json; a round is one. @@ -176,7 +177,9 @@ export async function select({ api, pr, appId, reviewerId, maxRounds, humanGated } catch (error) { log(`::warning::the review findings could not be read (${error.message}); the context holds none`); } - return { decision: "proceed", rounds, headSha, laneEscalated, file: { pr, head_sha: headSha, checks, review_findings } }; + // A retry after the push job stopped on this head runs one rung up. + const modelRung = retryRung(await api.listCheckRuns(headSha, "pr-unblock / apply-patch")); + return { decision: "proceed", rounds, headSha, laneEscalated, modelRung, file: { pr, head_sha: headSha, checks, review_findings } }; } // The base config's resolved loop caps, through the base checkout's @@ -217,10 +220,12 @@ export async function main(env, { api, log = (line) => console.log(line) } = {}) output("head-sha", result.headSha); output("escalation-reason", result.escalationReason ?? ""); output("context-sha256", sha256); - // fix-ci-not-run's check, and whether apply-patch may push past the - // human-gated label a lane escalation put on. - output("skip-reason", result.decision === "skip" ? result.skipReason : ""); + // fix-ci-not-run's check (a stop's own skip, so the head's fix-ci runs + // record it), and whether apply-patch may push past the human-gated label + // a lane escalation put on. + output("skip-reason", result.decision === "skip" ? result.skipReason : result.decision === "escalate" ? stopSkip(result.escalationReason) : ""); output("lane-escalated", String(result.laneEscalated === true)); + output("model-rung", result.modelRung ?? ""); log( `${result.decision}: ${result.file?.checks.length ?? 0} failing check(s), ${result.file?.review_findings.length ?? 0} review finding(s), ${result.rounds ?? 0} earlier round(s)${result.escalationReason ? `, ${result.escalationReason}` : ""}`, ); diff --git a/.github/scripts/pr-unblock/fix-ci-context.test.mjs b/.github/scripts/pr-unblock/fix-ci-context.test.mjs index fc7db96acc..80a1289f66 100644 --- a/.github/scripts/pr-unblock/fix-ci-context.test.mjs +++ b/.github/scripts/pr-unblock/fix-ci-context.test.mjs @@ -43,6 +43,8 @@ function fakeApi(overrides = {}) { getPull: async () => overrides.pull ?? { state: "open", head: { sha: HEAD } }, listPullCommits: async () => overrides.commits ?? [], listCommitCheckRuns: async () => overrides.checkRuns ?? [run(11, "lint-shell", "failure"), run(13, "ci-status", "failure")], + // The head's pr-unblock / apply-patch checks: a lane stop runs the retry a rung up. + listCheckRuns: async () => overrides.stopRuns ?? [], listAnnotations: async (id) => overrides.annotations?.[id] ?? [ { path: "scripts/x.sh", start_line: 12, annotation_level: "failure", message: "SC2034: foo appears unused" }, @@ -392,3 +394,12 @@ test("main refuses a malformed round cap", async () => { const env = { GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "o/r", GITHUB_OUTPUT: "/dev/null", PR_NUMBER: "42", APP_SENDER_ID: String(APP), CONTEXT_FILE: "/dev/null", TRUSTED_ACTORS }; await assert.rejects(main({ ...env, FIX_CI_ROUNDS: "0" }, { api: fakeApi() }), /FIX_CI_ROUNDS/); }); + +// A retry after the push job stopped on this head runs one model rung up, so +// apply-patch takes no second in-run rung-up. +test("a head whose push job stopped proceeds at model rung 1; any other at its own", async () => { + const stop = { status: "completed", conclusion: "neutral", output: { title: "Skipped: lane-stopped" }, app: { id: 15368 } }; + assert.equal((await selectWith(fakeApi({ stopRuns: [stop] }))).modelRung, "1"); + assert.equal((await selectWith(fakeApi())).modelRung, ""); + assert.equal((await selectWith(fakeApi({ stopRuns: [{ ...stop, app: { id: 2 } }] }))).modelRung, ""); +}); diff --git a/.github/scripts/pr-unblock/fix-ci-dispatch.mjs b/.github/scripts/pr-unblock/fix-ci-dispatch.mjs index a324bd36a3..f81cda0934 100644 --- a/.github/scripts/pr-unblock/fix-ci-dispatch.mjs +++ b/.github/scripts/pr-unblock/fix-ci-dispatch.mjs @@ -38,12 +38,17 @@ import { AGGREGATE_JOBS, FAILED, infraCause, infraOnly, RERUN_DUE_MINUTES, recon import { emptyRecord, isHandled, markHandled } from "./dispatch-record.mjs"; import { runTitle } from "./feedback-dispatch.mjs"; import { prHold } from "./gate-source.mjs"; +import { laneStop } from "./limit-stop.mjs"; export const DEFAULTS = Object.freeze({ base: "main", workflow: "pr-unblock.yml", activity: "fix-ci", checkName: "pr-unblock / fix-ci", + // The push job's check: a lane-stopped skip there (a stop routed to a + // failing status after the model turn) counts the fix-ci run before it as + // a failed one, which the per-head cap and the restarts retry. + stopCheckName: "pr-unblock / apply-patch", // The run whose failure counts: pr-require-checks when its lanes failed, // else the advisory pr-test-windows, which ci-status does not aggregate but // the merge gate holds on. @@ -136,6 +141,22 @@ const fromRest = (run) => ({ completedAt: run.completed_at ?? null, }); +// The head's fix-ci runs (REST) in the rollup's shape, each one that a +// lane-stopped push job check (`stopRuns`, REST) followed before the next +// fix-ci run started read as a failure: the run stopped, so the head may be +// retried under the same caps a failed run is. +export function withStops(runs, stopRuns) { + const startOf = (run) => Date.parse(run.started_at ?? ""); + const stops = stopRuns.filter(laneStop).map(startOf); + const starts = runs.map(startOf).sort((a, b) => a - b); + return runs.map((run) => { + const from = startOf(run); + const until = starts.find((start) => start > from) ?? Infinity; + const stopped = stops.some((at) => at > from && at < until); + return stopped ? { ...fromRest(run), conclusion: "FAILURE" } : fromRest(run); + }); +} + // The fix-ci runs that count: an untrusted-trigger skip ran nothing. const counted = (runs) => runs.filter((run) => !(run.conclusion === "NEUTRAL" && String(run.title ?? "").startsWith("Skipped: untrusted-trigger"))); @@ -341,6 +362,7 @@ export async function dispatchCiFixes({ workflow = DEFAULTS.workflow, activity = DEFAULTS.activity, checkName = DEFAULTS.checkName, + stopCheckName = DEFAULTS.stopCheckName, actionsAppId = DEFAULTS.actionsAppId, holdLabels = DEFAULTS.holdLabels, gateLabel = null, @@ -440,7 +462,10 @@ export async function dispatchCiFixes({ // read is not dispatched. let fixCiRuns; try { - fixCiRuns = (await api.listCheckRuns({ sha: node.headRefOid, checkName, appId: actionsAppId })).map(fromRest); + fixCiRuns = withStops( + await api.listCheckRuns({ sha: node.headRefOid, checkName, appId: actionsAppId }), + await api.listCheckRuns({ sha: node.headRefOid, checkName: stopCheckName, appId: actionsAppId }), + ); } catch (error) { if (error.rateLimited) { log(`::warning::#${node.number}: its ${checkName} run read is rate limited (${error.message}); no fix-ci dispatched, a later run retries`); diff --git a/.github/scripts/pr-unblock/fix-ci-dispatch.test.mjs b/.github/scripts/pr-unblock/fix-ci-dispatch.test.mjs index 0553f7faa1..0d35372e44 100644 --- a/.github/scripts/pr-unblock/fix-ci-dispatch.test.mjs +++ b/.github/scripts/pr-unblock/fix-ci-dispatch.test.mjs @@ -85,13 +85,19 @@ function fakeApi({ comments = new Map(), labelEvents = new Map(), history = new Map(), + stops = new Map(), } = {}) { - const calls = { list: 0, runs: 0, dispatch: [], jobs: [], paths: [], timeline: [], history: [] }; + const calls = { list: 0, runs: 0, dispatch: [], jobs: [], paths: [], timeline: [], history: [], stops: [] }; return { calls, // The head's check runs of one name and app over REST, every one of them: // the test's history for the head, else the rollup's runs in REST's shape. async listCheckRuns({ sha, checkName, appId }) { + // The push job's checks: `stops` for the head. + if (checkName === "pr-unblock / apply-patch") { + calls.stops.push(sha); + return stops.get(sha) ?? []; + } calls.history.push(sha); const rows = history.get(sha); if (rows instanceof Error) { @@ -912,3 +918,28 @@ test("a failed job whose steps never listed is logged as such past the deadline" ); assert.ok(lines.some((line) => line.startsWith("dispatched fix-ci #5 ") && line.endsWith(`(cause: steps unlisted past ${RERUN_DUE_MINUTES} min in lint-repo)`)), lines.join("\n")); }); + +// A fix-ci run whose push job stopped on a reason routed to a failing status +// (a lane-stopped skip on pr-unblock / apply-patch) counts as a failed run: +// the head is retried under fix-ci-restarts and the per-head cap, as a run +// that failed is. +test("a fix-ci run its push job stopped is retried on the same head once, then left at fix-ci-restarts", async () => { + const head = "7".padStart(40, "a"); + const at = (time) => `2026-10-22T${time}:00Z`; + const fixCi = (time) => ({ name: "pr-unblock / fix-ci", status: "completed", conclusion: "success", output: { title: "Fixed" }, started_at: at(time), completed_at: at(time) }); + const stop = (time) => ({ name: "pr-unblock / apply-patch", status: "completed", conclusion: "neutral", output: { title: "Skipped: lane-stopped" }, started_at: at(time) }); + const once = await run({ pages: [[pr(7)]], history: new Map([[head, [fixCi("10:00")]]]), stops: new Map([[head, [stop("10:05")]]]) }); + assert.ok(once.lines.some((line) => line.startsWith("dispatched fix-ci #7 ")), once.lines.join("\n")); + const twice = await run({ + pages: [[pr(7)]], + history: new Map([[head, [fixCi("10:00"), fixCi("10:30")]]]), + stops: new Map([[head, [stop("10:05"), stop("10:35")]]]), + }); + assert.ok(twice.lines.some((line) => line.includes("failed 2 time(s) on this base, past 1 restart(s)")), twice.lines.join("\n")); + // A push job that did not stop leaves the run handling the head. + const pushed = await run({ pages: [[pr(7)]], history: new Map([[head, [fixCi("10:00")]]]), stops: new Map([[head, [{ ...stop("10:05"), output: { title: "Pushed" }, conclusion: "success" }]]]) }); + assert.ok(pushed.lines.some((line) => line.includes("head already has a pr-unblock / fix-ci check run")), pushed.lines.join("\n")); + // A select job's own stop (the round cap) on the fix-ci check is final. + const capped = await run({ pages: [[pr(7)]], history: new Map([[head, [{ ...stop("10:00"), name: "pr-unblock / fix-ci" }]]]) }); + assert.ok(capped.lines.some((line) => line.includes("head already has a pr-unblock / fix-ci check run")), capped.lines.join("\n")); +}); diff --git a/.github/scripts/pr-unblock/gate-source.mjs b/.github/scripts/pr-unblock/gate-source.mjs index 872b35c40f..487c631793 100644 --- a/.github/scripts/pr-unblock/gate-source.mjs +++ b/.github/scripts/pr-unblock/gate-source.mjs @@ -17,7 +17,7 @@ // `label`, `created_at`, `updated_at`. Text written by anyone but those // authors is never read. -import { readFileSync } from "node:fs"; +import { escalationActivity } from "./escalation-routes.mjs"; export const ESCALATION = ""; export const RESUME_KEY = /^$/; @@ -149,17 +149,7 @@ export function gateSource({ timeline, label, head, isLanesApp }) { // The Reason line report-escalation.sh writes into its escalation comment. const REASON_LINE = /^- Reason: `([a-z][a-z-]*)`$/; -// escalation-routes.json, the one routing table: each reason's activity and -// route (`hold` hands the PR to a person with the label; `fail` ends in a -// failing status and no label), as report-escalation.sh reads it. -export const ROUTES = Object.freeze(JSON.parse(readFileSync(new URL("./escalation-routes.json", import.meta.url), "utf8"))); - -// The activity a reason resumes, or null for a reason the table does not -// list, which holds every activity. -export const escalationActivity = (reason) => (Object.hasOwn(ROUTES, reason) ? ROUTES[reason].activity : null); - -// A reason's route, or null for a reason the table does not list. -export const escalationRoute = (reason) => (Object.hasOwn(ROUTES, reason) ? ROUTES[reason].route : null); +export { escalationActivity, escalationRoute, ROUTES } from "./escalation-routes.mjs"; // When a timeline item was written, and by whom, for a comment or a review // (clear-escalations.mjs reads people speaking the same way). diff --git a/.github/scripts/pr-unblock/limit-stop.mjs b/.github/scripts/pr-unblock/limit-stop.mjs index f0f8f301fc..49027a48ee 100644 --- a/.github/scripts/pr-unblock/limit-stop.mjs +++ b/.github/scripts/pr-unblock/limit-stop.mjs @@ -18,6 +18,9 @@ // `limit-stops-per-head` stops the dispatchers stop re-dispatching the head; // the next push makes a new one. import { LIMIT_DEFAULTS, clampReset } from "../work-resolve-issue/limits.mjs"; +import { laneStop } from "./escalation-routes.mjs"; + +export { ACTIONS_APP_ID, LANE_STOP, laneStop, retryRung } from "./escalation-routes.mjs"; const ISO = String.raw`\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?Z`; const NAME = "[a-z][a-z0-9]*(?:-[a-z0-9]+)*"; @@ -124,3 +127,21 @@ export function cappedAt(runs, { limits = LIMIT_DEFAULTS, failuresRetry = false, const { done, retryable } = completedRuns(runs, { limits, failuresRetry, heldSkipsRelease }); return done.length >= limits["limit-stops-per-head"] && done.length > 0 && done.every(retryable) ? done.length : 0; } + +const startOf = (run) => Date.parse(run?.started_at ?? ""); + +// How a head's emitter checks (`stopRuns`, REST shape) release its activity +// checks for a same-head retry: `cut` is the newest emitter check's start +// when that check is a lane stop and fewer than `cap` lane stops stand on the +// head (the dispatcher's own per-head cap), so activity checks that started +// before it stop counting; null otherwise. `stops` counts the lane stops. +export function stopRelease(stopRuns, cap) { + const stops = stopRuns.filter(laneStop).length; + const newest = stopRuns.reduce((a, b) => (startOf(b) > startOf(a) ? b : a), stopRuns[0]); + return { cut: laneStop(newest) && stops < cap ? startOf(newest) : null, stops }; +} + +// `runs` without those started before `cut`, a limit stop kept. +export const releasedBy = (runs, cut, limits = LIMIT_DEFAULTS) => + cut === null ? runs : runs.filter((run) => limitResetsAt(run, limits) !== null || !(startOf(run) < cut)); + diff --git a/.github/scripts/pr-unblock/publish-feedback.mjs b/.github/scripts/pr-unblock/publish-feedback.mjs index 0a0f41f2ff..8e1ae427e9 100755 --- a/.github/scripts/pr-unblock/publish-feedback.mjs +++ b/.github/scripts/pr-unblock/publish-feedback.mjs @@ -94,6 +94,7 @@ import { } from "./address-feedback.mjs"; import { DISPUTED, parseVerdicts, settleDisputes } from "./adjudicate-feedback.mjs"; import { inThreadScope } from "./thread-scope.mjs"; +import { stopSkip } from "./escalation-routes.mjs"; // bytes: replies.jsonl's cap, the read runner's carry, seal and scan cap too; // runBytes: run.json's, the 1 MiB the read runner accepts for it. @@ -745,7 +746,7 @@ export async function main(env, { api, log = (line) => console.log(line) } = {}) }); if (escalationReason) { writeFileSync(env.ESCALATION_REASON_FILE, `${escalationReason}\n`); - writeFileSync(env.SKIP_REASON_FILE, "awaiting-human\n"); + writeFileSync(env.SKIP_REASON_FILE, `${stopSkip(escalationReason)}\n`); } else if (skipReason) { writeFileSync(env.SKIP_REASON_FILE, `${skipReason}\n`); } diff --git a/.github/scripts/pr-unblock/publish-feedback.test.mjs b/.github/scripts/pr-unblock/publish-feedback.test.mjs index e441e39745..4fcf6ca5b0 100644 --- a/.github/scripts/pr-unblock/publish-feedback.test.mjs +++ b/.github/scripts/pr-unblock/publish-feedback.test.mjs @@ -1150,7 +1150,7 @@ test("main writes the escalation and skip reason files and refuses a symlinked a assert.deepEqual(forged.posted(), []); assert.equal(forged.markers().length, 1); } - assert.equal(readFileSync(env.SKIP_REASON_FILE, "utf8"), "awaiting-human\n"); + assert.equal(readFileSync(env.SKIP_REASON_FILE, "utf8"), "lane-stopped\n"); assert.equal(api.posted().length, 1); assert.equal(api.markers()[0][2].split("\n")[1], MODEL_RUN); // No model attempt names no run, so even a marker already on the PR from diff --git a/.github/scripts/pr-unblock/report-escalation.sh b/.github/scripts/pr-unblock/report-escalation.sh index d24f041587..a8b4697a56 100755 --- a/.github/scripts/pr-unblock/report-escalation.sh +++ b/.github/scripts/pr-unblock/report-escalation.sh @@ -39,6 +39,8 @@ # label without its explanation. Fail writes nothing when its comment already # reads as it would. # +# With the argument `clear` (below), it reads ACTIVITY in place of the reason. +# # With the argument `status`, it instead posts one failing commit status on # HEAD_SHA for a fail-routed reason, and nothing for a held one: context # `pr-unblock/conflict` for resolve-conflicts (GitHub starts no pull_request @@ -55,6 +57,28 @@ die() { } routes=$(dirname "${BASH_SOURCE[0]}")/escalation-routes.json + +# `clear`: an attempt of ACTIVITY on HEAD_SHA ended without a stop (a same-head +# retry landed). When the newest status of the activity's context on the +# head, by github-actions[bot], is a failure, it posts success over it, so a +# stop the retry mended no longer fails the head. Needs statuses: write. +if [[ ${1:-} == clear ]]; then + [[ ${HEAD_SHA:-} =~ ^([0-9a-f]{40}|[0-9a-f]{64})$ ]] || die "HEAD_SHA is not a commit id" + [[ ${GITHUB_REPOSITORY:-} =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || die "GITHUB_REPOSITORY is not owner/repo" + jq -e --arg a "${ACTIVITY:-}" 'any(.[]; .activity == $a)' "$routes" >/dev/null || die "ACTIVITY is not an activity the table routes" + if [[ $ACTIVITY == resolve-conflicts ]]; then context=pr-unblock/conflict; else context=pr-unblock/$ACTIVITY; fi + state=$(gh api "repos/$GITHUB_REPOSITORY/commits/$HEAD_SHA/statuses?per_page=100" | jq -r --arg c "$context" \ + '[.[] | select(.context == $c and .creator.login == "github-actions[bot]")][0].state // ""') + if [[ $state != failure ]]; then + echo "no failing $context on $HEAD_SHA to clear" + exit 0 + fi + args=(-f state=success -f "context=$context" -f "description=The lane ran again on this head and did not stop") + [[ -n ${TARGET_URL:-} ]] && args+=(-f "target_url=$TARGET_URL") + gh api -X POST "repos/$GITHUB_REPOSITORY/statuses/$HEAD_SHA" "${args[@]}" >/dev/null + echo "cleared $context on $HEAD_SHA" + exit 0 +fi [[ ${ESCALATION_REASON:-} =~ ^[a-z]+(-[a-z]+)*$ ]] || die "unknown escalation reason [${ESCALATION_REASON:-}]" entry=$(jq -ce --arg r "$ESCALATION_REASON" 'if has($r) then .[$r] else null end' "$routes") || die "unknown escalation reason [$ESCALATION_REASON]" @@ -179,6 +203,13 @@ if [[ $route == fail ]]; then # The rounds already run keep counting for a capped reason, so a new head # meets the cap again; a write-access fix request, or a person resolving # the threads, moves a review that is not converging. + # The run that stopped: each stop names its own, so the comment's last edit + # is the newest stop's time (the monitoring lane's stalled-PR signal reads + # it). + run_line='' + if [[ ${GITHUB_RUN_ID:-} =~ ^[1-9][0-9]*$ ]]; then + run_line="- Run: ${GITHUB_SERVER_URL:-https://github.com}/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"$'\n' + fi more='' if [[ $ESCALATION_REASON == feedback-not-converging ]]; then more=" A reply on a thread, from an account with write access, starting \`PR-fix lane: please fix this finding\` grants one more round on that thread." @@ -189,8 +220,8 @@ The pr-unblock lane stopped \`$activity\` on this pull request: $why. This needs - Reason: \`$ESCALATION_REASON\` - Head: \`$HEAD_SHA\` -$details -The status stays on the head it was posted on: the lane checks again on the next commit to the branch, and edits this comment if it stops again.$more +$run_line$details +When a retry on this head can change the outcome, the lane runs one on its own, up to its per-head cap, and edits this comment if it stops again; a run that does not stop clears the status. A new commit to the branch starts clean.$more To run it again on this head now, dispatch the \`pr-unblock\` workflow: diff --git a/.github/scripts/pr-unblock/report-escalation.test.sh b/.github/scripts/pr-unblock/report-escalation.test.sh index 9c42997900..b63c912dab 100755 --- a/.github/scripts/pr-unblock/report-escalation.test.sh +++ b/.github/scripts/pr-unblock/report-escalation.test.sh @@ -76,6 +76,9 @@ case "$method $endpoint" in "PATCH repos/o/r/issues/comments/"*) echo '{}' ;; +"GET repos/o/r/commits/"*"/statuses?per_page=100") + cat "$STUB/statuses.json" 2>/dev/null || echo '[]' + ;; "POST repos/o/r/statuses/"*) [[ -f $STUB/fail_status ]] && { echo 'stub gh: status POST failed' >&2; exit 1; } echo '{}' @@ -137,7 +140,7 @@ HOLD_REASONS=(protected-path-conflict workflow-staged conflict-patch-protected-p feedback-protected-path feedback-round-cap feedback-uncertain feedback-disputed feedback-question-intent feedback-question-design feedback-question-security dependency-security-close dependency-escalate dependency-fix-refused dependency-workflow-unpinned - ci-fix-protected-path) + ci-fix-protected-path ci-fix-weaken-required) FAIL_REASONS=(unexpected-path-staged repeat-cap unresolved-conflict conflict-patch-invalid conflict-patch-not-applied feedback-not-converging feedback-verifier-open feedback-not-landed feedback-invalid feedback-partial feedback-incomplete feedback-stood-open feedback-unadjudicated @@ -194,7 +197,7 @@ for reason in "${FAIL_REASONS[@]}"; do has "$reason, fail: names the reason" "- Reason: \`$reason\`" "$STUB/bodies" has "$reason, fail: names the head" "- Head: \`$HEAD_A\`" "$STUB/bodies" has "$reason, fail: names the failing status" "failing \`$(context_of "$reason")\` status" "$STUB/bodies" - has "$reason, fail: says a new head is checked again" 'the lane checks again on the next commit' "$STUB/bodies" + has "$reason, fail: says a retry or a new head runs again" 'A new commit to the branch starts clean' "$STUB/bodies" has "$reason, fail: gives the dispatch" "-f pr-number=42 -f activity=$want" "$STUB/bodies" check "$reason, fail: names no label" 0 "$(grep -c 'needs-human' "$STUB/bodies" "$STUB/calls" | awk -F: '{s+=$2} END {print s}')" check "$reason, fail: reads no issue labels" 0 "$(grep -c '^GET repos/o/r/issues/42$' "$STUB/requests" || true)" @@ -467,6 +470,41 @@ run repeat-cap not-a-sha check "rejects a malformed head sha" 1 "$([[ $RC -ne 0 ]] && echo 1 || echo 0)" check "malformed head sha: no gh call" '' "$(cat "$STUB/calls")" +# Each stop names its run, so a later stop on the same head edits the comment. +setup '[]' '[]' +GITHUB_RUN_ID=555 run ci-fix-escalated +has "fail: names the run that stopped" '- Run: https://github.com/o/r/actions/runs/555' "$STUB/bodies" + +# clear: a same-head attempt that did not stop posts success over the +# activity's failing status, and nothing when the newest one is not a failure +# or is another account's. +clear_case() { + setup '[]' '[]' + printf '%s' "$2" >"$STUB/statuses.json" + set +e + (cd "$STUB/ws" && env PATH="$TMP/bin:$PATH" STUB="$STUB" GH_TOKEN=t GITHUB_REPOSITORY=o/r HEAD_SHA="$HEAD_A" \ + ACTIVITY="$1" "$SUT" clear) >"$STUB/out" 2>&1 + RC=$? + set -e +} +GHA='{"login":"github-actions[bot]"}' +clear_case address-feedback "[{\"context\":\"pr-unblock/address-feedback\",\"state\":\"failure\",\"creator\":$GHA}]" +check "clear: exit 0" 0 "$RC" +check "clear: success posted over the failure" "POST repos/o/r/statuses/$HEAD_A" "$(grep '^POST' "$STUB/requests")" +has "clear: state success" 'state=success' "$STUB/calls" +has "clear: the activity's context" 'context=pr-unblock/address-feedback' "$STUB/calls" +clear_case resolve-conflicts "[{\"context\":\"pr-unblock/conflict\",\"state\":\"failure\",\"creator\":$GHA}]" +has "clear: resolve-conflicts clears pr-unblock/conflict" 'context=pr-unblock/conflict' "$STUB/calls" +clear_case fix-ci "[{\"context\":\"pr-unblock/fix-ci\",\"state\":\"success\",\"creator\":$GHA},{\"context\":\"pr-unblock/fix-ci\",\"state\":\"failure\",\"creator\":$GHA}]" +check "clear: the newest is already success, nothing posted" 0 "$(writes)" +clear_case fix-ci "[{\"context\":\"pr-unblock/fix-ci\",\"state\":\"failure\",\"creator\":{\"login\":\"someone\"}}]" +check "clear: another account's status is not the lane's, nothing posted" 0 "$(writes)" +clear_case fix-ci '[]' +check "clear: no status, nothing posted" 0 "$(writes)" +clear_case bogus '[]' +check "clear: an activity the table does not route fails" 1 "$([[ $RC -ne 0 ]] && echo 1 || echo 0)" +check "clear: an unknown activity makes no gh call" '' "$(cat "$STUB/calls")" + # Never touches the branch: no git, no contents API, no write outside the PR's issue. for reason in workflow-staged ci-fix-escalated; do setup '[]' "$(comment ''$'\nold' 'melodic-automation-lanes[bot]' Bot 9)" diff --git a/.github/scripts/pr-unblock/review-dependency.mjs b/.github/scripts/pr-unblock/review-dependency.mjs index e11e713884..732b37fe47 100644 --- a/.github/scripts/pr-unblock/review-dependency.mjs +++ b/.github/scripts/pr-unblock/review-dependency.mjs @@ -30,6 +30,7 @@ import { provenanceReasons, withProvenance, } from "./dependency-classify.mjs"; +import { stopSkip } from "./escalation-routes.mjs"; export const DEFAULT_MAX_ROUNDS = 2; export const MODEL_CHECK = "pr-unblock / review-dependency"; @@ -343,7 +344,7 @@ export async function main(argv, env, { api, log = (line) => console.log(line), } const output = (key, value) => appendFileSync(env.GITHUB_OUTPUT, `${key}=${value}\n`); output("proceed", String(result.decision === "proceed")); - output("skip-reason", result.decision === "skip" ? result.skipReason : result.decision === "escalate" ? "awaiting-human" : ""); + output("skip-reason", result.decision === "skip" ? result.skipReason : result.decision === "escalate" ? stopSkip(result.escalationReason) : ""); output("escalation-reason", result.escalationReason ?? ""); const gaps = classifierGaps(result.classification?.tier === "review" ? result.classification.reasons : []); if (gaps.length) { diff --git a/.github/workflows/pr-run-activity-write.yml b/.github/workflows/pr-run-activity-write.yml index 629f74ac85..2ece600a59 100644 --- a/.github/workflows/pr-run-activity-write.yml +++ b/.github/workflows/pr-run-activity-write.yml @@ -412,7 +412,8 @@ on: dependency-files-truncated, dependency-control-uncited, dependency-workflow-unpinned, dependency-fix-refused, dependency-security-close or dependency-escalate; for apply-patch, before the mint, - ci-fix-empty, ci-fix-invalid or ci-fix-escalated, and from its + ci-fix-empty, ci-fix-invalid, ci-fix-escalated, ci-fix-protected-path + or ci-fix-weaken-required, and from its script ci-fix-invalid, ci-fix-protected-path or ci-fix-not-applied; for issue-mode apply-patch, patch-empty, patch-invalid, patch-not-applied or patch-protected-path; for apply-patch under @@ -591,7 +592,7 @@ jobs: dependency-files-truncated | dependency-control-uncited | dependency-workflow-unpinned) ;; dependency-fix-refused) ;; ci-fix-round-cap | ci-fix-empty | ci-fix-invalid | ci-fix-escalated) ;; - ci-fix-protected-path | ci-fix-not-applied) ;; + ci-fix-protected-path | ci-fix-not-applied | ci-fix-weaken-required) ;; conflict-patch-invalid | conflict-patch-not-applied | conflict-patch-protected-path) ;; patch-empty | patch-invalid | patch-not-applied | patch-protected-path) if [ -z "$ISSUE_NUMBER" ]; then @@ -1214,7 +1215,8 @@ jobs: echo "required=$required" >>"$GITHUB_OUTPUT" # apply-patch: the head checkout replaces .base, so the script's sibling - # imports (thread-scope.mjs and dependency-classify.mjs, beside the + # imports (thread-scope.mjs, dependency-classify.mjs and + # escalation-routes.mjs with its table, beside the # script's base-script copy), the patch-rules.json it reads, and the # dependabot.yml whose pairs the dependency-manifests rule reads are # copied out of the base first. @@ -1228,6 +1230,7 @@ jobs: mkdir -p "$RUNNER_TEMP/pr-unblock" "$DEST/$(dirname "$RULES")" cp .base/.github/scripts/pr-unblock/thread-scope.mjs "$RUNNER_TEMP/pr-unblock/" cp .base/.github/scripts/pr-unblock/dependency-classify.mjs "$RUNNER_TEMP/pr-unblock/" + cp .base/.github/scripts/pr-unblock/escalation-routes.mjs .base/.github/scripts/pr-unblock/escalation-routes.json "$RUNNER_TEMP/pr-unblock/" cp ".base/$RULES" "$DEST/$RULES" # A base without one leaves the rule's pairs unreadable: a row that # needs them fails before any write. diff --git a/.github/workflows/pr-unblock.yml b/.github/workflows/pr-unblock.yml index 8a67ec98c6..b7df9b50fe 100644 --- a/.github/workflows/pr-unblock.yml +++ b/.github/workflows/pr-unblock.yml @@ -252,8 +252,10 @@ jobs: # no model job, so no resolve-conflicts check would land on the head and # dispatch.mjs would dispatch it again on every run. This job posts it # neutral on the head the select job read: the slot's skip reason, else - # clean-merge (the push job commits it), else awaiting-human. A - # kill-switch stop reads no head, and nothing is posted. + # clean-merge (the push job commits it), else the stop's own skip + # (escalation-routes.mjs stopSkip: lane-stopped for a stop routed to a + # failing status, which ends the head; awaiting-human for one held for a + # person). A kill-switch stop reads no head, and nothing is posted. resolve-conflicts-not-run: needs: resolve-conflicts-select if: >- @@ -279,6 +281,12 @@ jobs: node-version: 24 package-manager-cache: false + - name: Read the stop's skip reason + id: stop + env: + REASON: ${{ needs.resolve-conflicts-select.outputs.escalation-reason }} + run: echo "skip=$(node .base/.github/scripts/pr-unblock/escalation-routes.mjs skip)" >>"$GITHUB_OUTPUT" + - name: Post the resolve-conflicts check run uses: ./.base/.github/actions/report-check-run # zizmor: ignore[self-repository] with: @@ -288,7 +296,7 @@ jobs: not-run-reason: >- ${{ needs.resolve-conflicts-select.outputs.skip-reason || needs.resolve-conflicts-select.outputs.merge-state == 'clean' && 'clean-merge' - || 'awaiting-human' }} + || steps.stop.outputs.skip }} head-sha: ${{ needs.resolve-conflicts-select.outputs.head-sha }} can-commit: "false" base-path: .base @@ -531,6 +539,9 @@ jobs: escalation-reason: ${{ steps.select.outputs.escalation-reason }} skip-reason: ${{ steps.slot.outputs.applies == 'false' && steps.slot.outputs.skip-reason || steps.select.outputs.skip-reason }} fix-requested: ${{ steps.select.outputs.fix-requested }} + # "1" for a same-head retry after the reply job stopped there + # (escalation-routes.mjs retryRung), else empty. + model-rung: ${{ steps.select.outputs.model-rung }} # `round of max ` when the round cap stopped the PR, for the # neutral checks' summary. not-run-detail: ${{ steps.select.outputs.not-run-detail }} @@ -726,6 +737,7 @@ jobs: pr-number: ${{ inputs.pr-number }} dispatcher-run-id: ${{ inputs.dispatcher-run-id }} trusted-base: default-tip + model-rung: ${{ needs.address-feedback-select.outputs.model-rung }} feedback-context-attempt: ${{ needs.address-feedback-select.outputs.context-attempt }} feedback-context-sha256: ${{ needs.address-feedback-select.outputs.threads-sha256 }} head-sha: ${{ needs.address-feedback-select.outputs.head-sha }} @@ -792,6 +804,7 @@ jobs: pr-number: ${{ inputs.pr-number }} dispatcher-run-id: ${{ inputs.dispatcher-run-id }} trusted-base: default-tip + model-rung: ${{ needs.address-feedback-select.outputs.model-rung }} feedback-context-attempt: ${{ needs.address-feedback-select.outputs.context-attempt }} feedback-context-sha256: ${{ needs.address-feedback-select.outputs.threads-sha256 }} head-sha: ${{ needs.address-feedback-select.outputs.head-sha }} @@ -1640,11 +1653,14 @@ jobs: head-sha: ${{ steps.context.outputs.head-sha }} escalation-reason: ${{ steps.context.outputs.escalation-reason }} context-sha256: ${{ steps.context.outputs.context-sha256 }} - # awaiting-human when the human-gated label holds fix-ci, which - # fix-ci-not-run posts; lane-escalated when another lane's escalation + # awaiting-human when the human-gated label holds fix-ci, lane-stopped + # on the round cap, which fix-ci-not-run posts; lane-escalated when another lane's escalation # put the label on, so apply-patch pushes past it. skip-reason: ${{ steps.context.outputs.skip-reason }} lane-escalated: ${{ steps.context.outputs.lane-escalated }} + # "1" for a same-head retry after the push job stopped there + # (escalation-routes.mjs retryRung), else empty. + model-rung: ${{ steps.context.outputs.model-rung }} # The artifact name's attempt, so a re-run of the model job alone still # finds it. context-attempt: ${{ steps.upload.outcome == 'success' && github.run_attempt || '' }} @@ -1711,7 +1727,7 @@ jobs: tokens: github-token # A fix-ci-context skip (a closed PR, a human-gated hold, or no failing - # check) runs no fix-ci job, + # check) or stop (the round cap, lane-stopped) runs no fix-ci job, # so no fix-ci check would land on the head and fix-ci-dispatch.mjs would # dispatch it again on every run. This job posts fix-ci neutral on that head, # as publish-feedback-not-run does; a kill-switch stop leaves the head empty. @@ -1720,7 +1736,6 @@ jobs: if: >- ${{ !cancelled() && inputs.activity == 'fix-ci' && needs.fix-ci-context.outputs.proceed == 'false' - && needs.fix-ci-context.outputs.escalation-reason == '' && needs.fix-ci-context.outputs.head-sha != '' }} runs-on: ubuntu-24.04 timeout-minutes: 10 @@ -1780,6 +1795,7 @@ jobs: pr-number: ${{ inputs.pr-number }} dispatcher-run-id: ${{ inputs.dispatcher-run-id }} trusted-base: default-tip + model-rung: ${{ needs.fix-ci-context.outputs.model-rung }} ci-context-attempt: ${{ needs.fix-ci-context.outputs.context-attempt }} ci-context-sha256: ${{ needs.fix-ci-context.outputs.context-sha256 }} head-sha: ${{ needs.fix-ci-context.outputs.head-sha }} @@ -1851,6 +1867,7 @@ jobs: pr-number: ${{ inputs.pr-number }} dispatcher-run-id: ${{ inputs.dispatcher-run-id }} trusted-base: default-tip + model-rung: ${{ needs.fix-ci-context.outputs.model-rung }} ci-context-attempt: ${{ needs.fix-ci-context.outputs.context-attempt }} ci-context-sha256: ${{ needs.fix-ci-context.outputs.context-sha256 }} head-sha: ${{ needs.fix-ci-context.outputs.head-sha }} @@ -1898,6 +1915,8 @@ jobs: patch-attempt: ${{ needs.fix-ci-subscription.result != 'skipped' && needs.fix-ci-subscription.outputs.scan-patch-attempt || needs.fix-ci-subscription.result == 'skipped' && needs.fix-ci.outputs.scan-patch-attempt || '' }} patch-activity: fix-ci patch-prepare: ci-context + # A same-head retry's producer ran a rung up: no second in-run rung-up. + patch-model-rung: ${{ needs.fix-ci-context.outputs.model-rung }} # The head fix-ci-context gated, from a job that ran no head code; the # read runner's own head-sha output is the run job's, which head code # could forge. Both fix-ci calls refuse a context for another head, so a @@ -2105,7 +2124,11 @@ jobs: # its CI is missing), pr-unblock/ otherwise; a new head carries # none. It runs when report-escalation does, on the same reason, except # on a cancelled run, and report-escalation.sh status posts nothing for a - # reason held for a person. This script job, with no model, is the lane's only + # reason held for a person. It also runs after an attempt whose push, reply + # or verdict job ended with no stop: report-escalation.sh clear then posts + # success over a failure an earlier stop left on the same head, so a + # same-head retry that landed leaves nothing red. This script job, with no + # model, is the lane's only # statuses: write, on its own GITHUB_TOKEN: the lanes App token the # escalation comment uses carries no statuses permission. report-stop-status: @@ -2136,7 +2159,12 @@ jobs: || needs.apply-patch.outputs.rung-up == 'true' && !(needs.apply-patch-rung-up.result == 'success' && needs.apply-patch-rung-up.outputs.escalation-reason == '' - && needs.apply-patch-rung-up.outputs.patch-skip-reason == '')) }} + && needs.apply-patch-rung-up.outputs.patch-skip-reason == '') + || needs.apply-conflict-patch.result == 'success' + || needs.publish-feedback.result == 'success' + || needs.apply-dependency-verdict.result == 'success' + || needs.apply-patch.result == 'success' && needs.apply-patch.outputs.rung-up != 'true' + || needs.apply-patch-rung-up.result == 'success') }} runs-on: ubuntu-24.04 timeout-minutes: 10 permissions: @@ -2153,6 +2181,7 @@ jobs: # One activity runs per dispatch, so one select job's head is set; a # reason is a fixed enum value from a base script or this run's jobs, # which report-escalation.sh rejects unless the routing table lists it. + # No reason: the attempt did not stop, and clear runs instead. - name: Post the stop status env: GH_TOKEN: ${{ github.token }} @@ -2171,9 +2200,12 @@ jobs: || needs.fix-ci-context.outputs.escalation-reason || needs.apply-patch.outputs.escalation-reason || needs.apply-patch-rung-up.outputs.escalation-reason - || needs.apply-patch.outputs.rung-up == 'true' && 'ci-fix-escalated' || '' }} + || needs.apply-patch.outputs.rung-up == 'true' + && !(needs.apply-patch-rung-up.result == 'success' && needs.apply-patch-rung-up.outputs.patch-skip-reason == '') + && 'ci-fix-escalated' || '' }} + ACTIVITY: ${{ inputs.activity }} TARGET_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - run: bash .base/.github/scripts/pr-unblock/report-escalation.sh status + run: bash .base/.github/scripts/pr-unblock/report-escalation.sh "$([ -n "$ESCALATION_REASON" ] && echo status || echo clear)" report-escalation: needs: diff --git a/docs/conventions/pr-pipeline/README.md b/docs/conventions/pr-pipeline/README.md index f6398de3ab..6590d00134 100644 --- a/docs/conventions/pr-pipeline/README.md +++ b/docs/conventions/pr-pipeline/README.md @@ -302,7 +302,9 @@ runs `fix-ci` and `apply-patch` once more at `model-rung` 1 (`fix-ci-rung-up`, `fix-ci-rung-up-subscription`, `apply-patch-rung-up`; [Models](#models)). `report-escalation` ends the run unless that retry's patch applied cleanly: the retry's own reason, else `ci-fix-escalated` (a neutral or failed retry), each routed to a failing `pr-unblock/fix-ci` -status but `ci-fix-protected-path`, which goes to a person. A cancelled run applies no label: the write +status but `ci-fix-protected-path` and `ci-fix-weaken-required` (a check the model could fix only +through a path the lane token may not push, or by weakening a required check), which go to a +person. A cancelled run applies no label: the write workflow's `run` job has no status function in its `if:`, so it skips on a cancel, nothing pushes and `ci-status` stays red. In a run that changed nothing, `protected-path`, `weaken-required` and `flaky` go to a person without a retry; beside a pushed patch, the new @@ -523,9 +525,18 @@ security-review or CRITICAL one). Every other reason is a defect the lane's boun `fix-ci` rung-up, the adjudicator, the round, repeat and per-head caps) did not mend, routed `fail`: no label, a failing commit status on the head the activity gated (`pr-unblock/conflict` for `resolve-conflicts`, `pr-unblock/` otherwise; `report-stop-status` posts it), and one -lanes App comment per activity (marked ``) -that the next stop edits in place. A new head carries no status, so the lane's next run there -starts clean. +lanes App comment per activity (marked ``, +naming the run) that the next stop edits in place. The stopping job's check skips `lane-stopped`. +A select job's stop (a cap, or a conflict no model can resolve) is final for its head. A stop +after a model turn (the push, reply or verdict job's) is retried on the same head by the +activity's dispatcher, under the cap it already keeps: `limit-stops-per-head` for +`resolve-conflicts` and `review-dependency`, `fix-ci-restarts` and the per-head cap for `fix-ci`, +`SWEEP_HEAD_CAP` for `address-feedback`. A `fix-ci` or `address-feedback` retry runs one model +rung up (`escalate-to`), read in the run from the head's own stop record, never from a dispatch +input; the others retry as a fresh session. A retry that does not stop posts success over the +failing status (`report-escalation.sh clear`). Once the retries are spent, the monitoring lane's +`stalled lane stop` signal names the PR. A new head carries no status, so the lane's next run +there starts clean. When an activity needs a human, it escalates with a resume key and exits instead of blocking. An activity whose `labels-absent` lists `needs-human` or `do-not-merge` skips with `awaiting-human` @@ -682,7 +693,10 @@ held back for one retry one model rung up, [Lanes](#lanes)), `no-model-turn` (an round whose model never ran, so the scan uploaded no replies), `finding-stands` (an address-feedback round's reply job, when an adjudicator let a disputed finding stand and the head did not move, so no review run starts the round that fixes it), `clean-merge` (a resolve-conflicts -round whose base merge left no conflict, so no model ran and the push job commits the merge), or, for an issue run under +round whose base merge left no conflict, so no model ran and the push job commits the merge), +`lane-stopped` (a pr-unblock stop `escalation-routes.json` routes to a failing status: on a select +job's activity check it ends the head; on a push, reply or verdict job's check the dispatcher +retries the head under its per-head cap), or, for an issue run under `models.issue-dimensions`, `research-first` or `probe-first` (its Uncertainty), or `split-required` (its Size is `split`). Issue mode posts no check, so work-resolve-issue.yml reports those three dimension skips on the issue instead: `needs-human`, no `agent-ready`, and a comment naming the next step @@ -1954,6 +1968,7 @@ findings listed by key. Its scripts and tests are in | 7 platform incident | githubstatus.com has an incident naming Actions that is open or began in the window, or shows Actions not operational; listed in the run summary only, never filed, since nothing in this repository acts on it | `platform:`, `platform:actions` | | 8 throughput | no PR by `merge.agent-identity` merged for 3 hours while pr-merge's newest live sweep, at most 70 minutes old, counts candidates | `throughput-zero` | | 9 lane stop | a comment by `merge.agent-identity` created in the window starts with a `` marker, alone or after the attended queue's escalation marker line, one issue per outcome listing each stopped issue | `lane-stop:funding`, `lane-stop:limited`, `lane-stop:turn-cap` | +| 10 stalled lane stop | an open pull request into main, of any author, carries a pr-unblock lane-stopped comment by `merge.agent-identity` naming its current head, last edited over 2 hours ago (four `pr-unblock-dispatch.yml` ticks), with no `pr-unblock / ` check run started on that head since: its same-head retries are spent; one issue lists each such stop | `stalled-lane-stop` | A rolling issue, titled `Monitoring: `, has the first body line ``; the body names the signal and its state diff --git a/docs/conventions/pr-pipeline/pr-pipeline.schema.json b/docs/conventions/pr-pipeline/pr-pipeline.schema.json index 87ec7f2024..9da1e5a0a0 100644 --- a/docs/conventions/pr-pipeline/pr-pipeline.schema.json +++ b/docs/conventions/pr-pipeline/pr-pipeline.schema.json @@ -455,7 +455,8 @@ "rung-up", "no-model-turn", "finding-stands", - "clean-merge" + "clean-merge", + "lane-stopped" ], "description": "The one reason a skipped activity's neutral check carries. Not used by the config file; the runner and check-run reporter validate reasons against it. Meaning: README.md, Outputs and skips." }, From eb7c342dda75a809205893ed959e0a716c3b66cf Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 26 Oct 2026 01:09:21 -0400 Subject: [PATCH 03/33] fix(pr-pipeline): count only real attempts in the stalled-stop signal, and cover labels clear-escalations lifted The stalled lane stop signal treated any later activity check as an attempt. That hid two kinds of PR: one with a select job's own lane-stopped skip, and one with a no-op feedback retry that skipped not-applicable. A skip is no longer an attempt. The signal also reads clear-escalations' needs-human-cleared comment. A cleared fail-routed reason, such as #10136's ci-fix-escalated, is a stop on that head, so a lifted label is still reported once its retries are spent. The reads now stay within the call budget: at most 20 pull requests and 20 check-run reads, and a full comment page with no marker is recorded as a gap. Co-Authored-By: Claude Opus 5.5 --- .../fix-pr-unblock-route-not-park-5c1e7a20.md | 2 +- .../maintenance-monitor-signals/collect.mjs | 60 ++++++++++++++++--- .../collect.test.mjs | 41 ++++++++++++- docs/conventions/pr-pipeline/README.md | 2 +- 4 files changed, 93 insertions(+), 12 deletions(-) diff --git a/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md b/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md index 1a314347ab..fe439e5097 100644 --- a/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md +++ b/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md @@ -13,4 +13,4 @@ bump: minor - `SWEEP_HEAD_CAP` for address-feedback The fix-ci and address-feedback retries run one model rung up, read in the run from the head's own stop record. A retry that does not stop posts success over the failing status. fix-ci's model escalations `protected-path` and `weaken-required` stay with a person, as `ci-fix-protected-path` and the new `ci-fix-weaken-required`. -- The monitoring lane gains signal 10, `stalled lane stop`. Each open pull request whose pr-unblock stop has had no new attempt on its head for over 2 hours is listed in one rolling issue. +- The monitoring lane gains signal 10, `stalled lane stop`. Each open pull request whose pr-unblock stop has had no new attempt on its head for over 2 hours is listed in one rolling issue. A skip is not an attempt. A stop includes a `needs-human` that `clear-escalations` took off for a reason now routed to a failing status. diff --git a/.github/scripts/maintenance-monitor-signals/collect.mjs b/.github/scripts/maintenance-monitor-signals/collect.mjs index bc70e02047..900621394e 100644 --- a/.github/scripts/maintenance-monitor-signals/collect.mjs +++ b/.github/scripts/maintenance-monitor-signals/collect.mjs @@ -3,6 +3,7 @@ // that fails to read is recorded in `gaps` and the others still run. Every // list is bounded, and `get` spends the run's call budget. import { parseOutcomeMarker } from "../work-resolve-issue/report-outcome.mjs"; +import { escalationActivity, escalationRoute } from "../pr-unblock/escalation-routes.mjs"; import { parseBudget, STALLED_HOURS } from "./signals.mjs"; export const LIMITS = { @@ -15,13 +16,23 @@ export const LIMITS = { sweepArtifacts: 5, commentPages: 5, // Open pull requests the stalled-stop read covers, oldest update first; - // one comment page each, and one check-run read per candidate. - stalledPulls: 40, + // one comment page each, and at most stalledChecks check-run reads in all, + // so the source spends at most 1 + 20 + 20 of CALL_BUDGET. + stalledPulls: 20, + stalledChecks: 20, }; // report-escalation.sh's comment for a stop routed to a failing status, one // per activity, edited at each stop (gate-source.mjs LANE_STOPPED). const LANE_STOPPED = /^$/; const STOP_FIELD = (name) => new RegExp(`^- ${name}: \`([^\`]+)\``, "m"); +// clear-escalations.mjs's comment when it took off a needs-human an older run +// set (gate-source.mjs CLEARED): a reason now routed to a failing status left +// that head with no stop comment of its own. +const CLEARED = /^$/; +// An attempt of the activity on the head: any of its check runs but a skip, +// which a select job posts without running the activity (a stop's own +// lane-stopped, or a not-applicable round that left the thread as it was). +const attempted = (run) => !(run.conclusion === "neutral" && String(run.output?.title ?? "").startsWith("Skipped:")); export const AUDIT_WORKFLOWS = ["maintenance-audit-test-selection", "maintenance-check-upstream-drift"]; // Pull-request and merge-group runs hold no lane token, and a merge-group // failure is counted as an ejection instead. @@ -254,6 +265,7 @@ export async function collect({ get, getStatus, readArtifact, repo, now, windowS await source("stalled stops", async () => { data.stalledStops = []; const cutoff = now.getTime() - STALLED_HOURS * 3600_000; + let checks = 0; const pulls = await get(`/repos/${repo}/pulls?state=open&base=main&sort=updated&direction=asc&per_page=100`); if (pulls.length > LIMITS.stalledPulls) { gaps.push(`stalled stops: ${pulls.length} open pull requests, only the ${LIMITS.stalledPulls} least recently updated read`); @@ -261,16 +273,46 @@ export async function collect({ get, getStatus, readArtifact, repo, now, windowS for (const pull of pulls.slice(0, LIMITS.stalledPulls)) { const head = pull.head?.sha; const comments = await get(`/repos/${repo}/issues/${pull.number}/comments?per_page=100`); + // Each stop on this head: the newest lane-stopped comment per activity, + // or a cleared label whose reason is now routed to a failing status. + const stops = new Map(); + let marked = false; for (const c of comments) { + if (c.user?.login !== agentIdentity) continue; const lines = String(c.body ?? "").split("\n").map((line) => line.replace(/\r$/, "")); - const activity = LANE_STOPPED.exec(lines[0])?.[1]; const body = lines.join("\n"); - if (c.user?.login !== agentIdentity || !activity || STOP_FIELD("Head").exec(body)?.[1] !== head) continue; - const stoppedAt = c.updated_at ?? c.created_at; - if (!(Date.parse(stoppedAt) < cutoff)) continue; - const runs = (await get(`/repos/${repo}/commits/${head}/check-runs?check_name=${encodeURIComponent(`pr-unblock / ${activity}`)}&filter=all&per_page=100`)).check_runs ?? []; - if (runs.some((run) => Date.parse(run.started_at) > Date.parse(stoppedAt))) continue; - data.stalledStops.push({ pr: pull.number, head, activity, reason: STOP_FIELD("Reason").exec(body)?.[1] ?? "unknown", stoppedAt, url: c.html_url }); + const activity = LANE_STOPPED.exec(lines[0])?.[1]; + if (activity) { + marked = true; + if (STOP_FIELD("Head").exec(body)?.[1] === head) { + stops.set(activity, { activity, reason: STOP_FIELD("Reason").exec(body)?.[1] ?? "unknown", stoppedAt: c.updated_at ?? c.created_at, url: c.html_url }); + } + continue; + } + const cleared = CLEARED.exec(lines[0]); + if (cleared && cleared[2] === head) { + marked = true; + for (const reason of cleared[1].split(",")) { + const of = escalationActivity(reason); + if (escalationRoute(reason) === "fail" && !stops.has(of)) { + stops.set(of, { activity: of, reason, stoppedAt: c.created_at, url: c.html_url }); + } + } + } + } + if (comments.length >= 100 && !marked) { + gaps.push(`stalled stops: #${pull.number} has over 100 comments; only the first page was read`); + } + for (const stop of stops.values()) { + if (!(Date.parse(stop.stoppedAt) < cutoff)) continue; + if (checks >= LIMITS.stalledChecks) { + gaps.push(`stalled stops: over ${LIMITS.stalledChecks} candidate stops; #${pull.number} and later not judged`); + return; + } + checks += 1; + const runs = (await get(`/repos/${repo}/commits/${head}/check-runs?check_name=${encodeURIComponent(`pr-unblock / ${stop.activity}`)}&filter=all&per_page=100`)).check_runs ?? []; + if (runs.some((run) => attempted(run) && Date.parse(run.started_at) > Date.parse(stop.stoppedAt))) continue; + data.stalledStops.push({ pr: pull.number, head, ...stop }); } } }); diff --git a/.github/scripts/maintenance-monitor-signals/collect.test.mjs b/.github/scripts/maintenance-monitor-signals/collect.test.mjs index 01a715a99d..07b9cfb3e9 100644 --- a/.github/scripts/maintenance-monitor-signals/collect.test.mjs +++ b/.github/scripts/maintenance-monitor-signals/collect.test.mjs @@ -424,6 +424,45 @@ test("stalled stops: more open PRs than the read covers is a gap, and the least const api = fakeApi([[/\/pulls\?state=open/, many], [/\/issues\/9\d{3}\/comments/, []], ...routes()]); const data = await collect(input(api)); assert.ok(data.gaps.some((g) => g.startsWith("stalled stops: 45 open pull requests"))); - assert.equal(api.calls.filter((p) => /\/issues\/9\d{3}\/comments/.test(p)).length, 40); + assert.equal(api.calls.filter((p) => /\/issues\/9\d{3}\/comments/.test(p)).length, 20); assert.ok(api.calls.some((p) => p.includes("sort=updated&direction=asc"))); }); + +// Shapes a skip-blind attempt rule missed: a select job's lane-stopped check +// posted after the comment's edit, and a feedback retry that found nothing to +// work and skipped not-applicable. Neither ran the activity, so both stay +// stalled. A label clear-escalations took off for a reason now routed to a +// failing status (#10136's shape: no stop comment on that head) is a stop too. +test("stalled stops: a skip is no attempt, and a cleared fail-routed label is a stop", async () => { + const skip = (title, at) => ({ conclusion: "neutral", output: { title }, started_at: at }); + const cleared = (pr, head, reason) => ({ + id: pr, + body: `\nThe pr-unblock lane escalated this pull request.`, + user: { login: "melodic-automation-lanes[bot]" }, + created_at: "2026-10-19T10:00:00Z", + html_url: `https://github.com/o/r/pull/${pr}#c`, + }); + const api = fakeApi([ + [/\/pulls\?state=open/, [8401, 8402, 8403, 8404, 8405].map((number) => ({ number, head: { sha: sha(String(number % 10)) } }))], + [/\/issues\/8401\/comments/, [stopComment(8401, sha("1"), "2026-10-19T12:00:00Z", { activity: "resolve-conflicts", reason: "repeat-cap" })]], + [/\/issues\/8402\/comments/, [stopComment(8402, sha("2"), "2026-10-19T12:00:00Z", { activity: "address-feedback", reason: "feedback-partial" })]], + [/\/issues\/8403\/comments/, [cleared(8403, sha("3"), "ci-fix-escalated")]], + [/\/issues\/8404\/comments/, [cleared(8404, sha("4"), "feedback-uncertain")]], + [/\/issues\/8405\/comments/, Array.from({ length: 100 }, (_, i) => ({ id: i, body: "chatter", user: { login: "someone" } }))], + [/\/commits\/1{40}\/check-runs/, { check_runs: [skip("Skipped: lane-stopped", "2026-10-19T12:00:05Z")] }], + [/\/commits\/2{40}\/check-runs/, { check_runs: [skip("Skipped: not-applicable", "2026-10-19T12:40:00Z")] }], + [/\/commits\/3{40}\/check-runs/, { check_runs: [{ conclusion: "success", output: { title: "Fixed" }, started_at: "2026-10-19T09:00:00Z" }] }], + ...routes(), + ]); + const data = await collect(input(api)); + assert.deepEqual( + data.stalledStops.map((s) => [s.pr, s.activity, s.reason]), + [ + [8401, "resolve-conflicts", "repeat-cap"], + [8402, "address-feedback", "feedback-partial"], + [8403, "fix-ci", "ci-fix-escalated"], + ], + "a held reason's clear is no stop", + ); + assert.ok(data.gaps.includes("stalled stops: #8405 has over 100 comments; only the first page was read")); +}); diff --git a/docs/conventions/pr-pipeline/README.md b/docs/conventions/pr-pipeline/README.md index 6590d00134..055f0fc52c 100644 --- a/docs/conventions/pr-pipeline/README.md +++ b/docs/conventions/pr-pipeline/README.md @@ -1968,7 +1968,7 @@ findings listed by key. Its scripts and tests are in | 7 platform incident | githubstatus.com has an incident naming Actions that is open or began in the window, or shows Actions not operational; listed in the run summary only, never filed, since nothing in this repository acts on it | `platform:`, `platform:actions` | | 8 throughput | no PR by `merge.agent-identity` merged for 3 hours while pr-merge's newest live sweep, at most 70 minutes old, counts candidates | `throughput-zero` | | 9 lane stop | a comment by `merge.agent-identity` created in the window starts with a `` marker, alone or after the attended queue's escalation marker line, one issue per outcome listing each stopped issue | `lane-stop:funding`, `lane-stop:limited`, `lane-stop:turn-cap` | -| 10 stalled lane stop | an open pull request into main, of any author, carries a pr-unblock lane-stopped comment by `merge.agent-identity` naming its current head, last edited over 2 hours ago (four `pr-unblock-dispatch.yml` ticks), with no `pr-unblock / ` check run started on that head since: its same-head retries are spent; one issue lists each such stop | `stalled-lane-stop` | +| 10 stalled lane stop | an open pull request into main, of any author, carries a pr-unblock lane-stopped comment by `merge.agent-identity` naming its current head, or a `needs-human-cleared` comment for that head whose reason now routes to a failing status, older than 2 hours (four `pr-unblock-dispatch.yml` ticks), with no `pr-unblock / ` check run but a skip started on that head since: its same-head retries are spent; one issue lists each such stop | `stalled-lane-stop` | A rolling issue, titled `Monitoring: `, has the first body line ``; the body names the signal and its state From 2237125da1bcf53f91be6050b2425cbb1cb8c2eb Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 26 Oct 2026 07:18:25 -0400 Subject: [PATCH 04/33] fix(pr-pipeline): hold escalation-routes.mjs on the merge path floor clear-escalations.mjs and gate-source.mjs now import escalation-routes.mjs, so the floor's import closure loads it and test_merge_holds_sync requires it on the floor. List it beside its sibling pr-unblock modules in babysit_risk.HIGH_FLOOR, HOLD_PATH_FLOOR, the review lanes' hold-paths and ci-status's hold_patterns, and pin the floor at 38 entries. Co-Authored-By: Claude Opus 5.5 --- .changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md | 1 + .github/scripts/pr-merge/merge-holds.mjs | 1 + .github/scripts/pr-merge/merge-holds.test.mjs | 1 + .github/workflows/pr-require-checks.yml | 2 +- .github/workflows/pr-review-hosted.yml | 1 + .github/workflows/pr-review-security-hosted.yml | 1 + .../source-control/skills/babysit-prs/scripts/babysit_risk.py | 1 + .../skills/babysit-prs/scripts/tests/test_babysit_risk.py | 3 +++ .../skills/babysit-prs/scripts/tests/test_merge_holds_sync.py | 2 +- 9 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md b/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md index fe439e5097..856d95b888 100644 --- a/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md +++ b/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md @@ -14,3 +14,4 @@ bump: minor The fix-ci and address-feedback retries run one model rung up, read in the run from the head's own stop record. A retry that does not stop posts success over the failing status. fix-ci's model escalations `protected-path` and `weaken-required` stay with a person, as `ci-fix-protected-path` and the new `ci-fix-weaken-required`. - The monitoring lane gains signal 10, `stalled lane stop`. Each open pull request whose pr-unblock stop has had no new attempt on its head for over 2 hours is listed in one rolling issue. A skip is not an attempt. A stop includes a `needs-human` that `clear-escalations` took off for a reason now routed to a failing status. +- The merge path floor (`babysit_risk.HIGH_FLOOR`, `merge-holds.mjs` `HOLD_PATH_FLOOR`, and the review lanes' and ci-status's hold lists) also holds `.github/scripts/pr-unblock/escalation-routes.mjs`, which `clear-escalations.mjs` and `gate-source.mjs` now import. diff --git a/.github/scripts/pr-merge/merge-holds.mjs b/.github/scripts/pr-merge/merge-holds.mjs index e7e6bb63f9..7e17b46a14 100755 --- a/.github/scripts/pr-merge/merge-holds.mjs +++ b/.github/scripts/pr-merge/merge-holds.mjs @@ -269,6 +269,7 @@ export const HOLD_PATH_FLOOR = Object.freeze([ ".github/workflows/pr-merge.yml", "docs/conventions/pr-pipeline.yaml", ".github/scripts/pr-unblock/address-feedback.mjs", + ".github/scripts/pr-unblock/escalation-routes.mjs", ".github/scripts/pr-unblock/gate-source.mjs", ".github/scripts/pr-unblock/thread-scope.mjs", ".github/scripts/work-resolve-issue/report-outcome.mjs", diff --git a/.github/scripts/pr-merge/merge-holds.test.mjs b/.github/scripts/pr-merge/merge-holds.test.mjs index 7c3b922234..dae39f9b16 100644 --- a/.github/scripts/pr-merge/merge-holds.test.mjs +++ b/.github/scripts/pr-merge/merge-holds.test.mjs @@ -316,6 +316,7 @@ const FLOOR_FILES = [ ".github/workflows/pr-require-checks.yml", "docs/conventions/pr-pipeline.yaml", ".github/scripts/pr-unblock/address-feedback.mjs", + ".github/scripts/pr-unblock/escalation-routes.mjs", ".github/scripts/pr-unblock/gate-source.mjs", ".github/scripts/pr-unblock/thread-scope.mjs", ".github/scripts/work-resolve-issue/report-outcome.mjs", diff --git a/.github/workflows/pr-require-checks.yml b/.github/workflows/pr-require-checks.yml index 0711ca530f..7657b42341 100644 --- a/.github/workflows/pr-require-checks.yml +++ b/.github/workflows/pr-require-checks.yml @@ -4258,7 +4258,7 @@ jobs: } if [[ $AUTHOR_ID =~ ^[0-9]+$ && " $exempt_author_ids " == *" $AUTHOR_ID "* ]]; then # The trust-gate paths, as bash patterns (`*` crosses `/`). - hold_patterns=('.github/standards/trusted-actors/*' '.github/actions/check-trusted-trigger/*' '.github/actions/resolve-config/*' '.github/actions/request-lane-token/*' '.github/actions/check-kill-switch/*' '.github/actions/log-api-budget/rate-guard.mjs' '.github/scripts/pr-merge/merge-holds.mjs' '.github/scripts/pr-merge/enable-auto-merge.mjs' '.github/scripts/pr-merge/superseded.mjs' '.github/scripts/pr-merge/clear-escalations.mjs' '.github/scripts/pr-unblock/address-feedback.mjs' '.github/scripts/pr-unblock/gate-source.mjs' '.github/scripts/pr-unblock/thread-scope.mjs' '.github/scripts/work-resolve-issue/report-outcome.mjs' '.github/scripts/work-resolve-issue/limits.mjs' '.github/workflows/pr-require-checks.yml' '.github/workflows/pr-merge.yml' 'plugins/source-control/skills/babysit-prs/scripts/babysit_risk.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_check_inputs.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_checks.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_classify.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_dependabot.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_feedback.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_gh.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_merge.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_merge_policy.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_postmerge.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_repo_config.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_review_evidence.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_review_trigger.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_run_cache.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_state.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_util.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_verify_evidence.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_whole_review.py' 'docs/conventions/pr-pipeline.yaml' 'CODEOWNERS' '*/CODEOWNERS' '.gitmodules' '*/.gitmodules') + hold_patterns=('.github/standards/trusted-actors/*' '.github/actions/check-trusted-trigger/*' '.github/actions/resolve-config/*' '.github/actions/request-lane-token/*' '.github/actions/check-kill-switch/*' '.github/actions/log-api-budget/rate-guard.mjs' '.github/scripts/pr-merge/merge-holds.mjs' '.github/scripts/pr-merge/enable-auto-merge.mjs' '.github/scripts/pr-merge/superseded.mjs' '.github/scripts/pr-merge/clear-escalations.mjs' '.github/scripts/pr-unblock/address-feedback.mjs' '.github/scripts/pr-unblock/escalation-routes.mjs' '.github/scripts/pr-unblock/gate-source.mjs' '.github/scripts/pr-unblock/thread-scope.mjs' '.github/scripts/work-resolve-issue/report-outcome.mjs' '.github/scripts/work-resolve-issue/limits.mjs' '.github/workflows/pr-require-checks.yml' '.github/workflows/pr-merge.yml' 'plugins/source-control/skills/babysit-prs/scripts/babysit_risk.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_check_inputs.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_checks.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_classify.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_dependabot.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_feedback.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_gh.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_merge.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_merge_policy.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_postmerge.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_repo_config.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_review_evidence.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_review_trigger.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_run_cache.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_state.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_util.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_verify_evidence.py' 'plugins/source-control/skills/babysit-prs/scripts/babysit_whole_review.py' 'docs/conventions/pr-pipeline.yaml' 'CODEOWNERS' '*/CODEOWNERS' '.gitmodules' '*/.gitmodules') if [[ ! $PR_NUMBER =~ ^[1-9][0-9]*$ ]]; then echo "::error::the pull request number '$PR_NUMBER' is not a number." exit 1 diff --git a/.github/workflows/pr-review-hosted.yml b/.github/workflows/pr-review-hosted.yml index e735e1aaed..b7d677d66a 100644 --- a/.github/workflows/pr-review-hosted.yml +++ b/.github/workflows/pr-review-hosted.yml @@ -211,6 +211,7 @@ jobs: .github/scripts/pr-merge/superseded.mjs .github/scripts/pr-merge/clear-escalations.mjs .github/scripts/pr-unblock/address-feedback.mjs + .github/scripts/pr-unblock/escalation-routes.mjs .github/scripts/pr-unblock/gate-source.mjs .github/scripts/pr-unblock/thread-scope.mjs .github/scripts/work-resolve-issue/report-outcome.mjs diff --git a/.github/workflows/pr-review-security-hosted.yml b/.github/workflows/pr-review-security-hosted.yml index 81fad3b23b..547aeaf1fe 100644 --- a/.github/workflows/pr-review-security-hosted.yml +++ b/.github/workflows/pr-review-security-hosted.yml @@ -211,6 +211,7 @@ jobs: .github/scripts/pr-merge/superseded.mjs .github/scripts/pr-merge/clear-escalations.mjs .github/scripts/pr-unblock/address-feedback.mjs + .github/scripts/pr-unblock/escalation-routes.mjs .github/scripts/pr-unblock/gate-source.mjs .github/scripts/pr-unblock/thread-scope.mjs .github/scripts/work-resolve-issue/report-outcome.mjs diff --git a/plugins/source-control/skills/babysit-prs/scripts/babysit_risk.py b/plugins/source-control/skills/babysit-prs/scripts/babysit_risk.py index 0134912476..5dd61f8b3f 100755 --- a/plugins/source-control/skills/babysit-prs/scripts/babysit_risk.py +++ b/plugins/source-control/skills/babysit-prs/scripts/babysit_risk.py @@ -228,6 +228,7 @@ class RiskResult: ".github/workflows/pr-merge.yml", "docs/conventions/pr-pipeline.yaml", ".github/scripts/pr-unblock/address-feedback.mjs", + ".github/scripts/pr-unblock/escalation-routes.mjs", ".github/scripts/pr-unblock/gate-source.mjs", ".github/scripts/pr-unblock/thread-scope.mjs", ".github/scripts/work-resolve-issue/report-outcome.mjs", diff --git a/plugins/source-control/skills/babysit-prs/scripts/tests/test_babysit_risk.py b/plugins/source-control/skills/babysit-prs/scripts/tests/test_babysit_risk.py index 74fb948ffa..3f34001580 100644 --- a/plugins/source-control/skills/babysit-prs/scripts/tests/test_babysit_risk.py +++ b/plugins/source-control/skills/babysit-prs/scripts/tests/test_babysit_risk.py @@ -309,6 +309,7 @@ def test_floor_paths_are_high_whatever_the_switch(self): ".github/workflows/pr-merge.yml", "docs/conventions/pr-pipeline.yaml", ".github/scripts/pr-unblock/address-feedback.mjs", + ".github/scripts/pr-unblock/escalation-routes.mjs", ".github/scripts/pr-unblock/gate-source.mjs", ".github/scripts/pr-unblock/thread-scope.mjs", ".github/scripts/work-resolve-issue/report-outcome.mjs", @@ -336,6 +337,7 @@ def test_floor_paths_are_high_whatever_the_switch(self): ("docs/conventions/pr-pipeline.yaml", "docs/conventions/pr-pipeline.yaml"), ("Docs/Conventions/PR-Pipeline.yaml", "docs/conventions/pr-pipeline.yaml"), (".github/scripts/pr-unblock/address-feedback.mjs", ".github/scripts/pr-unblock/address-feedback.mjs"), + (".github/scripts/pr-unblock/escalation-routes.mjs", ".github/scripts/pr-unblock/escalation-routes.mjs"), (".github/scripts/pr-unblock/gate-source.mjs", ".github/scripts/pr-unblock/gate-source.mjs"), (".github/scripts/pr-unblock/thread-scope.mjs", ".github/scripts/pr-unblock/thread-scope.mjs"), (".github/scripts/work-resolve-issue/report-outcome.mjs", ".github/scripts/work-resolve-issue/report-outcome.mjs"), @@ -1189,6 +1191,7 @@ def test_the_floor_holds_with_the_switch_off(self): ".github/workflows/pr-merge.yml", "docs/conventions/pr-pipeline.yaml", ".github/scripts/pr-unblock/address-feedback.mjs", + ".github/scripts/pr-unblock/escalation-routes.mjs", ".github/scripts/pr-unblock/gate-source.mjs", ".github/scripts/pr-unblock/thread-scope.mjs", ".github/scripts/work-resolve-issue/report-outcome.mjs", diff --git a/plugins/source-control/skills/babysit-prs/scripts/tests/test_merge_holds_sync.py b/plugins/source-control/skills/babysit-prs/scripts/tests/test_merge_holds_sync.py index be4e813b2c..69be99db32 100644 --- a/plugins/source-control/skills/babysit-prs/scripts/tests/test_merge_holds_sync.py +++ b/plugins/source-control/skills/babysit-prs/scripts/tests/test_merge_holds_sync.py @@ -202,7 +202,7 @@ def test_the_cost_paths_are_the_same(self) -> None: def test_the_floor_is_the_same_on_both_sides(self) -> None: answer = module(FLOOR_PATHS) self.assertEqual(answer["floor"], list(risk.HIGH_FLOOR)) - self.assertEqual(len(risk.HIGH_FLOOR), 37) + self.assertEqual(len(risk.HIGH_FLOOR), 38) for path, held, configured in zip( FLOOR_PATHS, answer["held"], answer["configured"], strict=True ): From d9ac6edcdcd9bdde9ca84f2d49ea2c7f3bc5003d Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 26 Oct 2026 07:19:53 -0400 Subject: [PATCH 05/33] fix(pr-pipeline): read a fix-ci run and its rung-up as one attempt fix-ci and the in-run fix-ci-rung-up both post a `pr-unblock / fix-ci` check. When the rung-up's apply-patch stopped, withStops reclassified only the second check as failed; the first stayed "Passed: no changes", so skipReason held the head at `!runs.every(retryable)` and the same-head restart never ran. withStops now groups the checks by check suite (one workflow run), reads each group from its last check, and marks the group failed when a stop in the same suite (or, with no suite id, by timing) followed any of its checks. Co-Authored-By: Claude Opus 5.5 --- .../fix-pr-unblock-route-not-park-5c1e7a20.md | 2 + .../scripts/pr-unblock/fix-ci-dispatch.mjs | 37 ++++++++++++++----- .../pr-unblock/fix-ci-dispatch.test.mjs | 24 ++++++++++++ 3 files changed, 54 insertions(+), 9 deletions(-) diff --git a/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md b/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md index 856d95b888..3a81892bba 100644 --- a/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md +++ b/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md @@ -12,6 +12,8 @@ bump: minor - `fix-ci-restarts` and the per-head cap for fix-ci - `SWEEP_HEAD_CAP` for address-feedback + The fix-ci dispatcher reads a fix-ci run and its in-run rung-up, which both post a `pr-unblock / fix-ci` check, as one attempt: a stop in either one's push job makes the attempt a failed one, and the pair counts once toward the caps. + The fix-ci and address-feedback retries run one model rung up, read in the run from the head's own stop record. A retry that does not stop posts success over the failing status. fix-ci's model escalations `protected-path` and `weaken-required` stay with a person, as `ci-fix-protected-path` and the new `ci-fix-weaken-required`. - The monitoring lane gains signal 10, `stalled lane stop`. Each open pull request whose pr-unblock stop has had no new attempt on its head for over 2 hours is listed in one rolling issue. A skip is not an attempt. A stop includes a `needs-human` that `clear-escalations` took off for a reason now routed to a failing status. - The merge path floor (`babysit_risk.HIGH_FLOOR`, `merge-holds.mjs` `HOLD_PATH_FLOOR`, and the review lanes' and ci-status's hold lists) also holds `.github/scripts/pr-unblock/escalation-routes.mjs`, which `clear-escalations.mjs` and `gate-source.mjs` now import. diff --git a/.github/scripts/pr-unblock/fix-ci-dispatch.mjs b/.github/scripts/pr-unblock/fix-ci-dispatch.mjs index 57a6650032..b3abd18796 100644 --- a/.github/scripts/pr-unblock/fix-ci-dispatch.mjs +++ b/.github/scripts/pr-unblock/fix-ci-dispatch.mjs @@ -142,20 +142,39 @@ const fromRest = (run) => ({ completedAt: run.completed_at ?? null, }); -// The head's fix-ci runs (REST) in the rollup's shape, each one that a -// lane-stopped push job check (`stopRuns`, REST) followed before the next -// fix-ci run started read as a failure: the run stopped, so the head may be -// retried under the same caps a failed run is. +// The head's fix-ci runs (REST) in the rollup's shape, one per workflow run +// (check suite): fix-ci and the in-run fix-ci-rung-up both post a fix-ci +// check, and the pair is one attempt, read from its last check. An attempt a +// lane-stopped push job check (`stopRuns`, REST) followed reads as a failure: +// the run stopped, so the head may be retried under the same caps a failed +// run is. A stop marks the attempt of its own check suite; a check with no +// suite id falls back to timing, a stop after it and before the next fix-ci +// check started. export function withStops(runs, stopRuns) { const startOf = (run) => Date.parse(run.started_at ?? ""); - const stops = stopRuns.filter(laneStop).map(startOf); + const suiteOf = (run) => run.check_suite?.id ?? null; + const stops = stopRuns.filter(laneStop); const starts = runs.map(startOf).sort((a, b) => a - b); - return runs.map((run) => { + const stoppedAt = (run) => { const from = startOf(run); const until = starts.find((start) => start > from) ?? Infinity; - const stopped = stops.some((at) => at > from && at < until); - return stopped ? { ...fromRest(run), conclusion: "FAILURE" } : fromRest(run); - }); + return stops.some((stop) => + suiteOf(stop) !== null && suiteOf(run) !== null + ? suiteOf(stop) === suiteOf(run) + : startOf(stop) > from && startOf(stop) < until, + ); + }; + const attempts = new Map(); + for (const run of runs) { + const key = suiteOf(run) ?? Symbol(); + const attempt = attempts.get(key) ?? { last: run, stopped: false }; + if (startOf(run) >= startOf(attempt.last)) { + attempt.last = run; + } + attempt.stopped ||= stoppedAt(run); + attempts.set(key, attempt); + } + return [...attempts.values()].map(({ last, stopped }) => (stopped ? { ...fromRest(last), conclusion: "FAILURE" } : fromRest(last))); } // The fix-ci runs that count: an untrusted-trigger skip ran nothing. diff --git a/.github/scripts/pr-unblock/fix-ci-dispatch.test.mjs b/.github/scripts/pr-unblock/fix-ci-dispatch.test.mjs index 578f366fd5..f824e2bcc5 100644 --- a/.github/scripts/pr-unblock/fix-ci-dispatch.test.mjs +++ b/.github/scripts/pr-unblock/fix-ci-dispatch.test.mjs @@ -967,3 +967,27 @@ test("a fix-ci run its push job stopped is retried on the same head once, then l const capped = await run({ pages: [[pr(7)]], history: new Map([[head, [{ ...stop("10:00"), name: "pr-unblock / fix-ci" }]]]) }); assert.ok(capped.lines.some((line) => line.includes("head already has a pr-unblock / fix-ci check run")), capped.lines.join("\n")); }); + +// fix-ci and the in-run fix-ci-rung-up each post a fix-ci check in one +// workflow run (one check suite). When the rung-up's push job stops, the pair +// is one stopped attempt: the first check's "Passed: no changes" does not +// leave the head handled, and the pair spends one restart, not two. +test("a fix-ci run whose rung-up push job stopped is one failed attempt, retried on the same head", async () => { + const head = "7".padStart(40, "a"); + const at = (time) => `2026-10-22T${time}:00Z`; + const fixCi = (time, suite, title) => ({ name: "pr-unblock / fix-ci", status: "completed", conclusion: "success", output: { title }, started_at: at(time), completed_at: at(time), check_suite: { id: suite } }); + const stop = (time, suite) => ({ name: "pr-unblock / apply-patch", status: "completed", conclusion: "neutral", output: { title: "Skipped: lane-stopped" }, started_at: at(time), check_suite: { id: suite } }); + const pair = [fixCi("10:00", 1, "Passed: no changes"), fixCi("10:10", 1, "Fixed")]; + const once = await run({ pages: [[pr(7)]], history: new Map([[head, pair]]), stops: new Map([[head, [stop("10:15", 1)]]]) }); + assert.ok(once.lines.some((line) => line.startsWith("dispatched fix-ci #7 ")), once.lines.join("\n")); + // A stop in another run's suite marks only that run. + const other = await run({ pages: [[pr(7)]], history: new Map([[head, pair]]), stops: new Map([[head, [stop("10:15", 2)]]]) }); + assert.ok(other.lines.some((line) => line.includes("head already has a pr-unblock / fix-ci check run")), other.lines.join("\n")); + // A second stopped attempt is past the one restart. + const twice = await run({ + pages: [[pr(7)]], + history: new Map([[head, [...pair, fixCi("11:00", 3, "Passed: no changes"), fixCi("11:10", 3, "Fixed")]]]), + stops: new Map([[head, [stop("10:15", 1), stop("11:15", 3)]]]), + }); + assert.ok(twice.lines.some((line) => line.includes("failed 2 time(s) on this base, past 1 restart(s)")), twice.lines.join("\n")); +}); From 4890f952e5515406d07d8d1b27e07acc9e336ba1 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 26 Oct 2026 07:23:48 -0400 Subject: [PATCH 06/33] fix(pr-pipeline): key a fix-ci attempt on its workflow run, not its suite report-check-run creates the fix-ci and apply-patch checks over the API, and GitHub attaches such a check to a check suite of its choosing on the head (ci-status lands in pr-require-checks' suite), so two runs' checks can share a suite id and collapse into one attempt, which would keep the per-head cap from ever tripping. Read the posting run from the check's details_url, which report-check-run sets to that run's page. Co-Authored-By: Claude Opus 5.5 --- .../scripts/pr-unblock/fix-ci-dispatch.mjs | 25 +++++++++++-------- .../pr-unblock/fix-ci-dispatch.test.mjs | 9 ++++--- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/.github/scripts/pr-unblock/fix-ci-dispatch.mjs b/.github/scripts/pr-unblock/fix-ci-dispatch.mjs index b3abd18796..486a835c99 100644 --- a/.github/scripts/pr-unblock/fix-ci-dispatch.mjs +++ b/.github/scripts/pr-unblock/fix-ci-dispatch.mjs @@ -142,31 +142,34 @@ const fromRest = (run) => ({ completedAt: run.completed_at ?? null, }); -// The head's fix-ci runs (REST) in the rollup's shape, one per workflow run -// (check suite): fix-ci and the in-run fix-ci-rung-up both post a fix-ci -// check, and the pair is one attempt, read from its last check. An attempt a -// lane-stopped push job check (`stopRuns`, REST) followed reads as a failure: -// the run stopped, so the head may be retried under the same caps a failed -// run is. A stop marks the attempt of its own check suite; a check with no -// suite id falls back to timing, a stop after it and before the next fix-ci +// The head's fix-ci runs (REST) in the rollup's shape, one per workflow run: +// fix-ci and the in-run fix-ci-rung-up both post a fix-ci check, and the pair +// is one attempt, read from its last check. An attempt a lane-stopped push job +// check (`stopRuns`, REST) followed reads as a failure: the run stopped, so +// the head may be retried under the same caps a failed run is. A check's +// workflow run is the id in its details_url (report-check-run sets it to the +// posting run's page); not its check suite, which GitHub picks for a check +// created over the API. A stop marks the attempt of its own run; a check with +// no run id falls back to timing, a stop after it and before the next fix-ci // check started. +const RUN_PAGE = /\/actions\/runs\/([1-9][0-9]{0,19})$/; export function withStops(runs, stopRuns) { const startOf = (run) => Date.parse(run.started_at ?? ""); - const suiteOf = (run) => run.check_suite?.id ?? null; + const runOf = (run) => RUN_PAGE.exec(String(run.details_url ?? ""))?.[1] ?? null; const stops = stopRuns.filter(laneStop); const starts = runs.map(startOf).sort((a, b) => a - b); const stoppedAt = (run) => { const from = startOf(run); const until = starts.find((start) => start > from) ?? Infinity; return stops.some((stop) => - suiteOf(stop) !== null && suiteOf(run) !== null - ? suiteOf(stop) === suiteOf(run) + runOf(stop) !== null && runOf(run) !== null + ? runOf(stop) === runOf(run) : startOf(stop) > from && startOf(stop) < until, ); }; const attempts = new Map(); for (const run of runs) { - const key = suiteOf(run) ?? Symbol(); + const key = runOf(run) ?? Symbol(); const attempt = attempts.get(key) ?? { last: run, stopped: false }; if (startOf(run) >= startOf(attempt.last)) { attempt.last = run; diff --git a/.github/scripts/pr-unblock/fix-ci-dispatch.test.mjs b/.github/scripts/pr-unblock/fix-ci-dispatch.test.mjs index f824e2bcc5..0cc761ee53 100644 --- a/.github/scripts/pr-unblock/fix-ci-dispatch.test.mjs +++ b/.github/scripts/pr-unblock/fix-ci-dispatch.test.mjs @@ -969,18 +969,19 @@ test("a fix-ci run its push job stopped is retried on the same head once, then l }); // fix-ci and the in-run fix-ci-rung-up each post a fix-ci check in one -// workflow run (one check suite). When the rung-up's push job stops, the pair +// workflow run (the run id in each check's details_url). When the rung-up's +// push job stops, the pair // is one stopped attempt: the first check's "Passed: no changes" does not // leave the head handled, and the pair spends one restart, not two. test("a fix-ci run whose rung-up push job stopped is one failed attempt, retried on the same head", async () => { const head = "7".padStart(40, "a"); const at = (time) => `2026-10-22T${time}:00Z`; - const fixCi = (time, suite, title) => ({ name: "pr-unblock / fix-ci", status: "completed", conclusion: "success", output: { title }, started_at: at(time), completed_at: at(time), check_suite: { id: suite } }); - const stop = (time, suite) => ({ name: "pr-unblock / apply-patch", status: "completed", conclusion: "neutral", output: { title: "Skipped: lane-stopped" }, started_at: at(time), check_suite: { id: suite } }); + const fixCi = (time, suite, title) => ({ name: "pr-unblock / fix-ci", status: "completed", conclusion: "success", output: { title }, started_at: at(time), completed_at: at(time), details_url: `https://github.com/o/r/actions/runs/${suite}` }); + const stop = (time, suite) => ({ name: "pr-unblock / apply-patch", status: "completed", conclusion: "neutral", output: { title: "Skipped: lane-stopped" }, started_at: at(time), details_url: `https://github.com/o/r/actions/runs/${suite}` }); const pair = [fixCi("10:00", 1, "Passed: no changes"), fixCi("10:10", 1, "Fixed")]; const once = await run({ pages: [[pr(7)]], history: new Map([[head, pair]]), stops: new Map([[head, [stop("10:15", 1)]]]) }); assert.ok(once.lines.some((line) => line.startsWith("dispatched fix-ci #7 ")), once.lines.join("\n")); - // A stop in another run's suite marks only that run. + // A stop in another workflow run marks only that run. const other = await run({ pages: [[pr(7)]], history: new Map([[head, pair]]), stops: new Map([[head, [stop("10:15", 2)]]]) }); assert.ok(other.lines.some((line) => line.includes("head already has a pr-unblock / fix-ci check run")), other.lines.join("\n")); // A second stopped attempt is past the one restart. From f38371ef10b6fe3a3b10060b4933418358ae2745 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 26 Oct 2026 07:25:36 -0400 Subject: [PATCH 07/33] fix(pr-pipeline): read the newest open pull requests for stalled stops The stalled-stop signal listed open pull requests oldest update first and read only the first 20, so a pull request with a fresh lane-stopped comment sorted behind idle ones and was never read once more than 20 were older; with 37 open today, the permanent gap also left signal 10 unread on every run. It now pages the open list most recently updated first (at most 3 pages), reads comments for the first 40, and names the true open count in the gap instead of a single page's 100. Co-Authored-By: Claude Opus 5.5 --- .../fix-pr-unblock-route-not-park-5c1e7a20.md | 2 +- .../maintenance-monitor-signals/collect.mjs | 24 ++++++++--- .../collect.test.mjs | 42 ++++++++++++++++--- 3 files changed, 55 insertions(+), 13 deletions(-) diff --git a/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md b/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md index 3a81892bba..960a011389 100644 --- a/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md +++ b/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md @@ -15,5 +15,5 @@ bump: minor The fix-ci dispatcher reads a fix-ci run and its in-run rung-up, which both post a `pr-unblock / fix-ci` check, as one attempt: a stop in either one's push job makes the attempt a failed one, and the pair counts once toward the caps. The fix-ci and address-feedback retries run one model rung up, read in the run from the head's own stop record. A retry that does not stop posts success over the failing status. fix-ci's model escalations `protected-path` and `weaken-required` stay with a person, as `ci-fix-protected-path` and the new `ci-fix-weaken-required`. -- The monitoring lane gains signal 10, `stalled lane stop`. Each open pull request whose pr-unblock stop has had no new attempt on its head for over 2 hours is listed in one rolling issue. A skip is not an attempt. A stop includes a `needs-human` that `clear-escalations` took off for a reason now routed to a failing status. +- The monitoring lane gains signal 10, `stalled lane stop`. Each open pull request whose pr-unblock stop has had no new attempt on its head for over 2 hours is listed in one rolling issue. A skip is not an attempt. A stop includes a `needs-human` that `clear-escalations` took off for a reason now routed to a failing status. It reads the 40 most recently updated open pull requests, and past 40 it names the open count in the run's gaps. - The merge path floor (`babysit_risk.HIGH_FLOOR`, `merge-holds.mjs` `HOLD_PATH_FLOOR`, and the review lanes' and ci-status's hold lists) also holds `.github/scripts/pr-unblock/escalation-routes.mjs`, which `clear-escalations.mjs` and `gate-source.mjs` now import. diff --git a/.github/scripts/maintenance-monitor-signals/collect.mjs b/.github/scripts/maintenance-monitor-signals/collect.mjs index 900621394e..e7ad6452dd 100644 --- a/.github/scripts/maintenance-monitor-signals/collect.mjs +++ b/.github/scripts/maintenance-monitor-signals/collect.mjs @@ -15,10 +15,15 @@ export const LIMITS = { mergeGroupRuns: 10, sweepArtifacts: 5, commentPages: 5, - // Open pull requests the stalled-stop read covers, oldest update first; - // one comment page each, and at most stalledChecks check-run reads in all, - // so the source spends at most 1 + 20 + 20 of CALL_BUDGET. - stalledPulls: 20, + // Open pull requests the stalled-stop read covers, most recently updated + // first: the open list in pages of 100 (at most stalledPages, so the gap + // names the true count), one comment page for each of the first + // stalledPulls, and at most stalledChecks check-run reads in all, so the + // source spends at most 3 + 40 + 20 of CALL_BUDGET. stalledPulls covers + // every open pull request at today's count; past it, a gap leaves signal 10 + // unread rather than missing a stop. + stalledPulls: 40, + stalledPages: 3, stalledChecks: 20, }; // report-escalation.sh's comment for a stop routed to a failing status, one @@ -266,9 +271,16 @@ export async function collect({ get, getStatus, readArtifact, repo, now, windowS data.stalledStops = []; const cutoff = now.getTime() - STALLED_HOURS * 3600_000; let checks = 0; - const pulls = await get(`/repos/${repo}/pulls?state=open&base=main&sort=updated&direction=asc&per_page=100`); + const pulls = []; + let more = true; + for (let page = 1; more && page <= LIMITS.stalledPages; page += 1) { + const batch = await get(`/repos/${repo}/pulls?state=open&base=main&sort=updated&direction=desc&per_page=100&page=${page}`); + pulls.push(...batch); + more = batch.length === 100; + } if (pulls.length > LIMITS.stalledPulls) { - gaps.push(`stalled stops: ${pulls.length} open pull requests, only the ${LIMITS.stalledPulls} least recently updated read`); + const total = more ? `over ${pulls.length}` : String(pulls.length); + gaps.push(`stalled stops: ${total} open pull requests, only the ${LIMITS.stalledPulls} most recently updated read`); } for (const pull of pulls.slice(0, LIMITS.stalledPulls)) { const head = pull.head?.sha; diff --git a/.github/scripts/maintenance-monitor-signals/collect.test.mjs b/.github/scripts/maintenance-monitor-signals/collect.test.mjs index 07b9cfb3e9..07cb3f758e 100644 --- a/.github/scripts/maintenance-monitor-signals/collect.test.mjs +++ b/.github/scripts/maintenance-monitor-signals/collect.test.mjs @@ -419,13 +419,43 @@ test("listFiled lists only github-actions[bot]'s issues", async () => { assert.deepEqual(api.calls.map((path) => new URL(path, "https://x").searchParams.get("creator")), ["github-actions[bot]"]); }); -test("stalled stops: more open PRs than the read covers is a gap, and the least recently updated are read", async () => { - const many = Array.from({ length: 45 }, (_, i) => ({ number: 9000 + i, head: { sha: sha("e") } })); - const api = fakeApi([[/\/pulls\?state=open/, many], [/\/issues\/9\d{3}\/comments/, []], ...routes()]); +// GitHub's open pull request list, honoring sort direction, page and per_page. +const openPulls = (pulls) => (path) => { + const q = new URL(path, "https://x").searchParams; + const sorted = [...pulls].sort((a, b) => a.updated_at.localeCompare(b.updated_at) * (q.get("direction") === "desc" ? -1 : 1)); + const per = Number(q.get("per_page") ?? 30); + const page = Number(q.get("page") ?? 1); + return sorted.slice((page - 1) * per, page * per); +}; +const idle = (number, day) => ({ number, head: { sha: sha("e") }, updated_at: `2026-10-${String(day).padStart(2, "0")}T00:00:00Z` }); + +test("stalled stops: more open PRs than the read covers is a gap naming the true count, and the most recently updated are read", async () => { + const many = Array.from({ length: 145 }, (_, i) => idle(9000 + i, 1 + (i % 17))); + const api = fakeApi([[/\/pulls\?state=open/, openPulls(many)], [/\/issues\/9\d{3}\/comments/, []], ...routes()]); + const data = await collect(input(api)); + assert.ok(data.gaps.some((g) => g === "stalled stops: 145 open pull requests, only the 40 most recently updated read"), data.gaps.join("\n")); + assert.equal(api.calls.filter((p) => /\/pulls\?state=open/.test(p)).length, 2); + const read = api.calls.flatMap((p) => /\/issues\/(9\d{3})\/comments/.exec(p)?.[1] ?? []).map(Number); + assert.equal(read.length, 40); + const newest = [...many].sort((a, b) => b.updated_at.localeCompare(a.updated_at)).slice(0, 40); + assert.ok(read.every((n) => many.find((p) => p.number === n).updated_at >= newest.at(-1).updated_at)); +}); + +// A pull request stopped three hours ago sorts ahead of older idle ones, so +// more than 20 idle pull requests no longer hide it. +test("stalled stops: a freshly stopped PR behind more than 20 idle ones is read and listed", async () => { + const idlePulls = Array.from({ length: 25 }, (_, i) => idle(9100 + i, 1 + i)); + const fresh = { number: 9200, head: { sha: sha("f") }, updated_at: "2026-10-19T13:30:00Z" }; + const api = fakeApi([ + [/\/pulls\?state=open/, openPulls([...idlePulls, fresh])], + [/\/issues\/9200\/comments/, [stopComment(9200, sha("f"), "2026-10-19T13:30:00Z")]], + [/\/issues\/91\d{2}\/comments/, []], + [/\/commits\/f+\/check-runs/, { check_runs: [] }], + ...routes(), + ]); const data = await collect(input(api)); - assert.ok(data.gaps.some((g) => g.startsWith("stalled stops: 45 open pull requests"))); - assert.equal(api.calls.filter((p) => /\/issues\/9\d{3}\/comments/.test(p)).length, 20); - assert.ok(api.calls.some((p) => p.includes("sort=updated&direction=asc"))); + assert.deepEqual(data.stalledStops.map((s) => s.pr), [9200]); + assert.ok(!data.gaps.some((g) => g.startsWith("stalled stops:")), data.gaps.join("\n")); }); // Shapes a skip-blind attempt rule missed: a select job's lane-stopped check From 2820d5c67658503adf7ec40b3c0889043d90ee96 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 26 Oct 2026 07:25:58 -0400 Subject: [PATCH 08/33] chore(source-control): add the changelog fragment for the floor entry Co-Authored-By: Claude Opus 5.5 --- .../fix-pr-unblock-route-not-park-f8205d45.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changes/source-control/fix-pr-unblock-route-not-park-f8205d45.md diff --git a/.changes/source-control/fix-pr-unblock-route-not-park-f8205d45.md b/.changes/source-control/fix-pr-unblock-route-not-park-f8205d45.md new file mode 100644 index 0000000000..c900109e78 --- /dev/null +++ b/.changes/source-control/fix-pr-unblock-route-not-park-f8205d45.md @@ -0,0 +1,7 @@ +--- +bump: patch +--- + +### Changed + +- `babysit_risk.HIGH_FLOOR` also holds `.github/scripts/pr-unblock/escalation-routes.mjs`, which `clear-escalations.mjs` and `gate-source.mjs` import, so the floor still holds every module its scripts load. From 50a2161a950e982eb1cac2a7af71f1f8c649a9c5 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 26 Oct 2026 08:00:11 -0400 Subject: [PATCH 09/33] fix(pr-pipeline): clear a fix-ci needs-human only on a failed ci-lanes read clear-escalations took needs-human off any ci-fix-* reason routed to a failing status without reading a CI result. It now reads the current head's ci-lanes commit status (one GraphQL read per PR) and clears only when the GitHub Actions bot recorded it failed. Green, pending, missing, another creator, or an unreadable read keeps the label. ci-status itself is not read: it fails on the label, so it proves nothing. The ci-lanes reader moves from fix-ci-dispatch.mjs into gate-source.mjs, already on the merge path floor, so clear-escalations imports no new file. Co-Authored-By: Claude Opus 5.5 --- .../fix-pr-unblock-route-not-park-5c1e7a20.md | 2 +- .../scripts/pr-merge/clear-escalations.mjs | 48 +++++++++++++++---- .../pr-merge/clear-escalations.test.mjs | 38 +++++++++++++-- .../scripts/pr-unblock/fix-ci-dispatch.mjs | 19 ++------ .github/scripts/pr-unblock/gate-source.mjs | 11 +++++ docs/conventions/pr-pipeline/README.md | 2 +- 6 files changed, 92 insertions(+), 28 deletions(-) diff --git a/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md b/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md index 960a011389..71280338c2 100644 --- a/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md +++ b/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md @@ -6,7 +6,7 @@ bump: minor - A `pr-unblock` run that stops now ends on the route its reason has in [`escalation-routes.json`](https://github.com/melodic-software/claude-code-plugins/blob/main/.github/scripts/pr-unblock/escalation-routes.json), the one routing table that `report-escalation.sh`, `gate-source.mjs` and `clear-escalations.mjs` read. Only a reason a person must settle keeps the `needs-human` label: a path the lane token may not push or a supply-chain refusal, an intent, design or security question, a person's own thread or hold, or a finding no adjudicator may judge (security-review or CRITICAL). Every other reason (for example `ci-fix-escalated`, `feedback-not-landed`, the round and repeat caps) ends with no label, a failing commit status on the head the activity gated (`pr-unblock/conflict` for `resolve-conflicts`, `pr-unblock/` otherwise, posted by the `report-stop-status` job that replaces `report-conflict-status`), and one lanes App comment per activity that the next stop edits in place. A new head carries no status. - An `UNCERTAIN` or `VALID-defer` review reply now goes to the adjudicator as an `INCORRECT` one does, instead of to a person. A pr-verify note left open ends `feedback-verifier-open`, and a finding that stood at adjudication and was left open again ends `feedback-stood-open`; both route to a failing status. A round's reason held for a person now outranks any reason routed to a failing status. -- `clear-escalations` takes off a `needs-human` an older run set for a reason now routed to a failing status: a `ci-fix-*` reason at once, any other once the head moved past the one the lane stopped on. +- `clear-escalations` takes off a `needs-human` an older run set for a reason now routed to a failing status: a `ci-fix-*` reason once the GitHub Actions bot's `ci-lanes` status on the current head reads failed, any other once the head moved past the one the lane stopped on. - A pr-unblock stop routed to a failing status now skips `lane-stopped` (a new `skip-reason`) instead of `awaiting-human`. On a select job's own check (a cap, or a conflict no model can resolve) it ends the head, so the dispatchers stop re-running a deterministic stop. On the push, reply or verdict job's check, the dispatcher retries the same head under the cap it already keeps: - `limit-stops-per-head` for resolve-conflicts and review-dependency - `fix-ci-restarts` and the per-head cap for fix-ci diff --git a/.github/scripts/pr-merge/clear-escalations.mjs b/.github/scripts/pr-merge/clear-escalations.mjs index 6041b32649..b96e60135a 100755 --- a/.github/scripts/pr-merge/clear-escalations.mjs +++ b/.github/scripts/pr-merge/clear-escalations.mjs @@ -33,9 +33,11 @@ // protected-path-conflict is left to resolve // feedback-protected-path the PR changes nothing under .github/workflows/ // a reason escalation-routes.json routes `fail` (a lane run before that -// route set the label): a fix-ci reason at once, the head's own CI -// checks staying red while it fails; any other once the head is not one -// its escalations name as their resume key +// route set the label): a fix-ci reason once the GitHub Actions bot's +// `ci-lanes` status on the current head reads failed (ci-status itself +// fails on the label, so it proves nothing), so the head's own CI stays +// red without the label; any other once the head is not one its +// escalations name as their resume key // A listing that may be cut short, or a mergeable state not yet computed, // counts as the reason still holding. // Any other reason is decided by a person and is never cleared here. @@ -72,11 +74,13 @@ import { pathToFileURL } from "node:url"; import { createGitHub, paginate } from "../../actions/check-trusted-trigger/check-trusted-trigger.mjs"; import { humanGatedLabelFrom } from "../pr-unblock/address-feedback.mjs"; import { + actionsLanesState, CLEARED, ESCALATION, escalationsSince, escalationActivity, escalationRoute, + failedState, foreignMarker, isApp, labelHistory, @@ -118,6 +122,27 @@ const FILES_CAP = 3000; // Each clear lets the lanes run on the PR again (judgment). export const CLEARS_PER_RUN = 5; +// The head commit's `ci-lanes` status, the lanes' verdict ci-status records. +export const LANES_QUERY = `query($owner: String!, $name: String!, $oid: GitObjectID!) { + repository(owner: $owner, name: $name) { + object(oid: $oid) { + ... on Commit { status { context(name: "ci-lanes") { state creator { __typename ... on Bot { databaseId } } } } } + } + } +}`; + +// Whether the GitHub Actions bot recorded the head's lanes failed. One read +// per PR; a read that fails, or any other state, reads as not failing. +async function lanesFailedAt({ github, repo, pull }) { + const [owner, name] = repo.split("/"); + try { + const data = await github("POST", "/graphql", { query: LANES_QUERY, variables: { owner, name, oid: pull.head.sha } }); + return failedState(actionsLanesState(data?.data?.repository?.object?.status?.context)); + } catch { + return false; + } +} + const escapeCommand = (text) => String(text).replaceAll(/[\r\n%]/g, " "); const reportError = (number, error) => console.log(`::error title=clear-escalations::#${number}: ${escapeCommand(error?.message ?? error)}`); @@ -143,17 +168,20 @@ const CHECKS = Object.freeze({ : "the pull request changes nothing under .github/workflows/"; }, // A reason escalation-routes.json now routes to a failing status rather - // than a person, on a label an older lane run set: fix-ci's at once, since - // the head's own CI checks stay red while it fails; any other once the head - // is not one the lane stopped on, since the lane checks a new head itself. + // than a person, on a label an older lane run set: fix-ci's once the head's + // lanes read failed, so CI stays red without the label (green, missing or + // unreadable keeps it); any other once the head is not one the lane stopped + // on, since the lane checks a new head itself. ...Object.fromEntries( Object.keys(ROUTES) .filter((reason) => escalationRoute(reason) === "fail" && reason !== "unresolved-conflict") .map((reason) => [ reason, - async ({ pull, keys }) => + async ({ pull, keys, lanesFailed }) => escalationActivity(reason) === "fix-ci" - ? "the lane now ends it in a failing status and the head's own red checks, not a person" + ? (await lanesFailed()) + ? "the lane now ends it in a failing status, not a person, and the head's ci-lanes status reads failed" + : null : keys.length > 0 && !keys.includes(pull.head.sha) ? "the lane now ends it in a failing status, not a person, and checks the new head itself" : null, @@ -274,12 +302,14 @@ async function verdict({ github, repo, number, label, bot, mainSha }) { reasons = [HEAD_MOVED]; why.push(`its escalation names head \`${[...new Set(merged.map(({ head }) => head))].join("`, `")}\`, and the head is now \`${pull.head.sha}\``); } else { + let lanes; + const lanesFailed = () => (lanes ??= lanesFailedAt({ github, repo, pull })); for (const reason of reasons) { // The resume keys of this reason's escalations: the heads it stopped on. const keys = escalations .filter(({ body }) => REASON.exec(body.join("\n"))?.[1] === reason) .map(({ body }) => RESUME_KEY.exec(body[1])?.[1]); - const cleared = await CHECKS[reason]({ github, repo, pull, mainSha, keys }); + const cleared = await CHECKS[reason]({ github, repo, pull, mainSha, keys, lanesFailed }); if (cleared === null) { return { reason: `${reason} still holds at ${pull.head.sha}` }; } diff --git a/.github/scripts/pr-merge/clear-escalations.test.mjs b/.github/scripts/pr-merge/clear-escalations.test.mjs index df4c5a0ce3..309cf6244b 100644 --- a/.github/scripts/pr-merge/clear-escalations.test.mjs +++ b/.github/scripts/pr-merge/clear-escalations.test.mjs @@ -30,10 +30,20 @@ const escalated = (reason = "workflow-staged", { user = BOT, at = "2026-10-18T10 // One open pull request (number 7) labeled needs-human; `timeline` is its timeline and // `mainFiles` the files main changed since its merge base with the head. -function world({ timeline, mainFiles = [{ filename: "README.md" }], mergeable = false, pullFiles = [], author }) { +// `lanes` is the head's ci-lanes status context (GraphQL shape), or an Error +// its read throws. +const ACTIONS_LANES = (state) => ({ state, creator: { __typename: "Bot", databaseId: 41898282 } }); +function world({ timeline, mainFiles = [{ filename: "README.md" }], mergeable = false, pullFiles = [], author, lanes = null }) { const calls = []; const github = async (method, path, body) => { calls.push({ method, path, body }); + if (method === "POST" && path === "/graphql") { + if (lanes instanceof Error) { + throw lanes; + } + assert.equal(body.variables.oid, HEAD); + return { data: { repository: { object: { status: lanes === null ? null : { context: lanes } } } } }; + } const route = path.replace(/[?&]per_page=100&page=\d+$/, ""); if (method === "GET" && route === `/repos/${REPO}/git/ref/heads/main`) { return { ref: "refs/heads/main", object: { sha: MAIN } }; @@ -58,7 +68,7 @@ function world({ timeline, mainFiles = [{ filename: "README.md" }], mergeable = } throw new Error(`unexpected ${method} ${path}`); }; - return { github, calls, writes: () => calls.filter((call) => call.method !== "GET") }; + return { github, calls, writes: () => calls.filter((call) => call.method !== "GET" && call.path !== "/graphql") }; } const run = async (w, env = {}) => { @@ -207,7 +217,7 @@ test("a reason a person decides is never cleared", async () => { test("a label for a reason now routed to a failing status comes off: fix-ci at once, others once the head moved", async () => { for (const reason of ["ci-fix-escalated", "ci-fix-round-cap"]) { const atHead = commented(BOT, escalationBody(reason).replaceAll(KEY, HEAD), "2026-10-18T10:00:05Z"); - const w = world({ timeline: [labeled(BOT, "2026-10-18T10:00:00Z"), atHead] }); + const w = world({ timeline: [labeled(BOT, "2026-10-18T10:00:00Z"), atHead], lanes: ACTIONS_LANES("FAILURE") }); await run(w); assert.equal(w.writes().at(-1)?.method, "DELETE", reason); assert.match(w.writes()[0].body.body, new RegExp(`\`${reason}\``)); @@ -223,6 +233,28 @@ test("a label for a reason now routed to a failing status comes off: fix-ci at o } }); +// The security finding on #10252: a fix-ci label came off with no CI result +// read. Only the Actions bot's failed ci-lanes on the current head clears it; +// ci-status itself fails on the label, so it is not read. +test("a fix-ci label comes off only while the head's ci-lanes reads failed", async () => { + const timeline = [labeled(BOT, "2026-10-18T10:00:00Z"), escalated("ci-fix-escalated")]; + for (const [lanes, cleared] of [ + [ACTIONS_LANES("FAILURE"), true], + [ACTIONS_LANES("ERROR"), true], + [ACTIONS_LANES("SUCCESS"), false], + [ACTIONS_LANES("PENDING"), false], + [null, false], + [{ state: "FAILURE", creator: { __typename: "User", databaseId: 1 } }, false], + [new Error("GraphQL down"), false], + ]) { + const w = world({ timeline, lanes }); + const { code } = await run(w); + assert.equal(code, 0); + assert.equal(w.writes().at(-1)?.method === "DELETE", cleared, JSON.stringify(lanes?.message ?? lanes)); + assert.equal(w.calls.filter((call) => call.path === "/graphql").length, 1); + } +}); + test("a reason still held for a person keeps the label beside a fail-routed one", async () => { for (const reason of ["feedback-uncertain", "feedback-question-security", "dependency-escalate", "dependency-security-close", "ci-fix-protected-path"]) { const w = world({ timeline: [labeled(BOT, "2026-10-18T10:00:00Z"), escalated(reason), escalated("ci-fix-escalated", { at: "2026-10-18T10:05:00Z" })] }); diff --git a/.github/scripts/pr-unblock/fix-ci-dispatch.mjs b/.github/scripts/pr-unblock/fix-ci-dispatch.mjs index e81ee85b30..b6610f9156 100644 --- a/.github/scripts/pr-unblock/fix-ci-dispatch.mjs +++ b/.github/scripts/pr-unblock/fix-ci-dispatch.mjs @@ -42,7 +42,7 @@ import { AGGREGATE_JOBS, FAILED, infraCause, infraOnly, RERUN_DUE_MINUTES, recon import { emptyRecord, isHandled, markHandled } from "./dispatch-record.mjs"; import { runTitle } from "./feedback-dispatch.mjs"; import { failingChecks } from "./fix-ci-context.mjs"; -import { prHold } from "./gate-source.mjs"; +import { actionsLanesState, failedState, prHold } from "./gate-source.mjs"; import { laneStop } from "./limit-stop.mjs"; import { serverFailure } from "./request-budget.mjs"; @@ -213,25 +213,16 @@ function ownerIdle(node, idleHours, now) { return commit?.oid === node.headRefOid && Date.parse(node.updatedAt) <= since && Date.parse(commit.committedDate) <= since; } -// The GitHub Actions bot's user id. GraphQL reports its login as -// `github-actions`, REST as `github-actions[bot]`; the id is the same in both. -const ACTIONS_BOT_ID = 41898282; - // The head's `ci-lanes` commit status, whoever set it. function lanesStatus(node) { const contexts = node.commits?.nodes?.[0]?.commit?.statusCheckRollup?.contexts?.nodes ?? []; return contexts.find((context) => context?.__typename === "StatusContext" && context.context === "ci-lanes"); } -// The ci-lanes state when the GitHub Actions bot set it, else null: the -// lanes' verdict alone. ci-status itself also fails on a hold label, a -// pending or missing review and the PR contract, none of which a CI fix mends. -function lanesState(node) { - const status = lanesStatus(node); - return status?.creator?.__typename === "Bot" && status.creator.databaseId === ACTIONS_BOT_ID ? status.state : null; -} - -const failedState = (state) => ["FAILURE", "ERROR"].includes(state); +// The lanes' verdict alone (gate-source.mjs lanesState): ci-status itself +// also fails on a hold label, a pending or missing review and the PR +// contract, none of which a CI fix mends. +const lanesState = (node) => actionsLanesState(lanesStatus(node)); const lanesFailed = (node) => failedState(lanesState(node)); const cancelled = (required) => required?.run?.status === "completed" && required.run.conclusion === "cancelled"; diff --git a/.github/scripts/pr-unblock/gate-source.mjs b/.github/scripts/pr-unblock/gate-source.mjs index 487c631793..47071cf608 100644 --- a/.github/scripts/pr-unblock/gate-source.mjs +++ b/.github/scripts/pr-unblock/gate-source.mjs @@ -151,6 +151,17 @@ const REASON_LINE = /^- Reason: `([a-z][a-z-]*)`$/; export { escalationActivity, escalationRoute, ROUTES } from "./escalation-routes.mjs"; +// The GitHub Actions bot's user id. GraphQL reports its login as +// `github-actions`, REST as `github-actions[bot]`; the id is the same in both. +const ACTIONS_BOT_ID = 41898282; + +// A `ci-lanes` status context's state (GraphQL shape) when the GitHub Actions +// bot set it, else null: the lanes' verdict ci-status records. fix-ci-dispatch +// and clear-escalations read it. +export const actionsLanesState = (status) => + status?.creator?.__typename === "Bot" && status.creator.databaseId === ACTIONS_BOT_ID ? status.state : null; +export const failedState = (state) => ["FAILURE", "ERROR"].includes(state); + // When a timeline item was written, and by whom, for a comment or a review // (clear-escalations.mjs reads people speaking the same way). export function spoken(item) { diff --git a/docs/conventions/pr-pipeline/README.md b/docs/conventions/pr-pipeline/README.md index 25d6170a20..462acf3455 100644 --- a/docs/conventions/pr-pipeline/README.md +++ b/docs/conventions/pr-pipeline/README.md @@ -561,7 +561,7 @@ head: | `workflow-staged` | main changes nothing under `.github/workflows/` since its merge base with the head | | `unresolved-conflict`, `protected-path-conflict` | GitHub reports the PR mergeable | | `feedback-protected-path` | the PR changes nothing under `.github/workflows/` | -| a reason `escalation-routes.json` routes `fail`, on a label set before it did | a `ci-fix-*` reason at once (the head's own CI checks stay red while it fails); any other once the head is not one its escalations name as the resume key | +| a reason `escalation-routes.json` routes `fail`, on a label set before it did | a `ci-fix-*` reason once the GitHub Actions bot's `ci-lanes` status on the current head reads failed (ci-status itself fails on the label, so it is not read; green, missing or unreadable keeps the label); any other once the head is not one its escalations name as the resume key | | `head-moved` (pr-merge's `babysit-merge-verify-escalation`, `babysit-merge-risk` and `babysit-merge-shared-copy-escalation`, with no pr-unblock escalation since the label, and the marked App comment written last before the label is pr-merge's and within a minute of it) | the newest App comment carrying each marker names, as ``at head `` ``, a 40-hex head other than the current one; pr-merge then judges the new head and escalates again if its refusal still holds | It removes only a label whose newest `labeled` event has the lanes App as its actor, and keeps it From 3c08904ac1abb48c8c8b5ade2330e0f715bcec07 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 26 Oct 2026 08:04:20 -0400 Subject: [PATCH 10/33] fix(pr-pipeline): leave a stop's failing status on a retry that skipped report-stop-status posted report-escalation.sh clear after any write job that succeeded with no stop, so a same-head address-feedback retry whose reply script skipped no-model-turn posted success over the earlier feedback-not-landed or feedback-invalid failure. The write runner now hands up its skip reason (the gate's reason or the check run's skip), and clear runs only when the push, reply or verdict job acted. The sweep already counts a no-model-turn skip toward SWEEP_HEAD_CAP, so a stuck head stays red and the stalled-stop signal lists it. Co-Authored-By: Claude Opus 5.5 --- .../fix-pr-unblock-route-not-park-5c1e7a20.md | 2 +- .../actions/resolve-config/callers.test.mjs | 25 ++++++++++++++++++- .github/workflows/pr-run-activity-write.yml | 16 ++++++++++++ .github/workflows/pr-unblock.yml | 20 +++++++++------ docs/conventions/pr-pipeline/README.md | 5 ++-- 5 files changed, 56 insertions(+), 12 deletions(-) diff --git a/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md b/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md index 71280338c2..13ea8cabab 100644 --- a/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md +++ b/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md @@ -14,6 +14,6 @@ bump: minor The fix-ci dispatcher reads a fix-ci run and its in-run rung-up, which both post a `pr-unblock / fix-ci` check, as one attempt: a stop in either one's push job makes the attempt a failed one, and the pair counts once toward the caps. - The fix-ci and address-feedback retries run one model rung up, read in the run from the head's own stop record. A retry that does not stop posts success over the failing status. fix-ci's model escalations `protected-path` and `weaken-required` stay with a person, as `ci-fix-protected-path` and the new `ci-fix-weaken-required`. + The fix-ci and address-feedback retries run one model rung up, read in the run from the head's own stop record. A retry that acts and does not stop posts success over the failing status; one that ends on a skip (a reply job with no model turn) leaves it. fix-ci's model escalations `protected-path` and `weaken-required` stay with a person, as `ci-fix-protected-path` and the new `ci-fix-weaken-required`. - The monitoring lane gains signal 10, `stalled lane stop`. Each open pull request whose pr-unblock stop has had no new attempt on its head for over 2 hours is listed in one rolling issue. A skip is not an attempt. A stop includes a `needs-human` that `clear-escalations` took off for a reason now routed to a failing status. It reads the 40 most recently updated open pull requests, and past 40 it names the open count in the run's gaps. - The merge path floor (`babysit_risk.HIGH_FLOOR`, `merge-holds.mjs` `HOLD_PATH_FLOOR`, and the review lanes' and ci-status's hold lists) also holds `.github/scripts/pr-unblock/escalation-routes.mjs`, which `clear-escalations.mjs` and `gate-source.mjs` now import. diff --git a/.github/actions/resolve-config/callers.test.mjs b/.github/actions/resolve-config/callers.test.mjs index 8659fd84f8..47dae51c29 100644 --- a/.github/actions/resolve-config/callers.test.mjs +++ b/.github/actions/resolve-config/callers.test.mjs @@ -450,9 +450,10 @@ test("pr-unblock.yml: report-stop-status runs when report-escalation does, or af ]; const HEADS = ["resolve-conflicts-select", "address-feedback-select", "review-dependency-select", "fix-ci-context"]; const EMITTERS = ["apply-conflict-patch", "publish-feedback", "apply-dependency-verdict", "apply-patch"]; - const context = ({ from = "", reason = "", headFrom = "resolve-conflicts-select", sha = head, rungUp = "", rungUpResult = "skipped", rungUpReason = "", rungUpSkip = "", landed = "" } = {}) => ({ + const context = ({ from = "", reason = "", headFrom = "resolve-conflicts-select", sha = head, rungUp = "", rungUpResult = "skipped", rungUpReason = "", rungUpSkip = "", landed = "", skip = "" } = {}) => ({ ...Object.fromEntries(SOURCES.map((id) => [`needs.${id}.outputs.escalation-reason`, id === from ? reason : ""])), ...Object.fromEntries(EMITTERS.map((id) => [`needs.${id}.result`, id === landed ? "success" : "skipped"])), + ...Object.fromEntries([...EMITTERS, "apply-patch-rung-up"].map((id) => [`needs.${id}.outputs.skip-reason`, id === landed ? skip : ""])), ...Object.fromEntries(HEADS.map((id) => [`needs.${id}.outputs.head-sha`, id === headFrom ? sha : ""])), "needs.apply-patch.outputs.rung-up": rungUp, "needs.apply-patch-rung-up.result": rungUpResult, @@ -476,7 +477,29 @@ test("pr-unblock.yml: report-stop-status runs when report-escalation does, or af assert.equal(evaluate(job.if, context({ headFrom: "fix-ci-context", rungUp: "true", rungUpResult: "success" })), true, "the retry's patch applied: clear"); for (const landed of EMITTERS) { assert.equal(evaluate(job.if, context({ landed })), true, `${landed} ended with no stop: clear`); + // Review finding on #10252: a retry that succeeded on a skip (a reply + // job with no model turn) landed nothing, so it must not clear the + // failure an earlier stop left on the head. + for (const skip of ["no-model-turn", "superseded-sha", "rung-up"]) { + assert.equal(evaluate(job.if, context({ landed, skip })), false, `${landed} skipped ${skip}: no clear`); + } } + assert.equal(evaluate(job.if, context({ landed: "apply-patch-rung-up", rungUpResult: "success", skip: "no-model-turn" })), false, "the rung-up push job's skip: no clear"); + // The write runner hands its skip up: the reply script's no-model-turn, or a gate stop. + const write = workflow("pr-run-activity-write.yml"); + assert.equal(write.on.workflow_call.outputs["skip-reason"].value, "${{ jobs.run.outputs.skip-reason }}"); + const steps = (extra = {}) => ({ + "steps.kill.outputs.proceed": "true", + "steps.kill.outputs.reason": "", + "steps.gate.outputs.proceed": "true", + "steps.gate.outputs.reason": "", + ...Object.fromEntries(["config", "head-recheck", "lane-token", "prepare", "promote", "act-script", "patch-select"].map((id) => [`steps.${id}.outputs.skip-reason`, ""])), + ...extra, + }); + const skipOf = (extra) => evaluate(write.jobs.run.outputs["skip-reason"], steps(extra)); + assert.equal(skipOf(), ""); + assert.equal(skipOf({ "steps.act-script.outputs.skip-reason": "no-model-turn" }), "no-model-turn"); + assert.equal(skipOf({ "steps.gate.outputs.proceed": "false", "steps.gate.outputs.reason": "untrusted-trigger" }), "untrusted-trigger"); // The head is a select job's, never a model call's; the reason is the one // report-escalation reads. const post = job.steps.find((step) => step.name === "Post the stop status"); diff --git a/.github/workflows/pr-run-activity-write.yml b/.github/workflows/pr-run-activity-write.yml index a45534e0fe..2bf2395529 100644 --- a/.github/workflows/pr-run-activity-write.yml +++ b/.github/workflows/pr-run-activity-write.yml @@ -433,6 +433,13 @@ on: apply-patch: the skip reason the patch's select gave, empty when it proceeded. value: ${{ jobs.run.outputs.patch-skip-reason }} + skip-reason: + description: >- + Why the run did no work: the kill switch's or trigger gate's reason, + or the skip reason its check run posts (a script's own, such as the + reply script's no-model-turn); empty when it acted. A caller that + reads a success result as a landed attempt checks this is empty. + value: ${{ jobs.run.outputs.skip-reason }} head-sha: description: The PR head the gate passed, before any push of this run. Set only by runner steps. value: ${{ jobs.run.outputs.head-sha }} @@ -482,6 +489,15 @@ jobs: ${{ steps.act-script.outputs.escalation-reason || steps.patch-select.outputs.escalation-reason }} rung-up: ${{ steps.patch-select.outputs.skip-reason == 'rung-up' }} patch-skip-reason: ${{ steps.patch-select.outputs.skip-reason }} + # The verdict step's skip reason, after the gate's reason. + skip-reason: >- + ${{ steps.kill.outputs.proceed != 'true' && steps.kill.outputs.reason + || steps.gate.outputs.proceed != 'true' && steps.gate.outputs.reason + || steps.config.outputs.skip-reason || steps.head-recheck.outputs.skip-reason + || steps.lane-token.outputs.skip-reason + || steps.prepare.outputs.skip-reason || steps.promote.outputs.skip-reason + || steps.act-script.outputs.skip-reason + || steps.patch-select.outputs.skip-reason }} # The base action's own failure text, so the report names why the run # job failed before its act step. run-failure: ${{ steps.lane-token.outputs.failure-reason }} diff --git a/.github/workflows/pr-unblock.yml b/.github/workflows/pr-unblock.yml index 0addd41193..dafa20b10d 100644 --- a/.github/workflows/pr-unblock.yml +++ b/.github/workflows/pr-unblock.yml @@ -2508,9 +2508,13 @@ jobs: # none. It runs when report-escalation does, on the same reason, except # on a cancelled run, and report-escalation.sh status posts nothing for a # reason held for a person. It also runs after an attempt whose push, reply - # or verdict job ended with no stop: report-escalation.sh clear then posts - # success over a failure an earlier stop left on the same head, so a - # same-head retry that landed leaves nothing red. This script job, with no + # or verdict job acted and ended with no stop: report-escalation.sh clear + # then posts success over a failure an earlier stop left on the same head, + # so a same-head retry that landed leaves nothing red. A job that succeeded + # on a skip (the reply script's no-model-turn, a gate or head-recheck skip) + # (a rung-up skip included) landed nothing and leaves the failure in place; + # the address-feedback sweep counts a no-model-turn skip toward + # SWEEP_HEAD_CAP (feedback-dispatch.mjs RELEASES). This script job, with no # model, is the lane's only # statuses: write, on its own GITHUB_TOKEN: the lanes App token the # escalation comment uses carries no statuses permission. @@ -2543,11 +2547,11 @@ jobs: && !(needs.apply-patch-rung-up.result == 'success' && needs.apply-patch-rung-up.outputs.escalation-reason == '' && needs.apply-patch-rung-up.outputs.patch-skip-reason == '') - || needs.apply-conflict-patch.result == 'success' - || needs.publish-feedback.result == 'success' - || needs.apply-dependency-verdict.result == 'success' - || needs.apply-patch.result == 'success' && needs.apply-patch.outputs.rung-up != 'true' - || needs.apply-patch-rung-up.result == 'success') }} + || needs.apply-conflict-patch.result == 'success' && needs.apply-conflict-patch.outputs.skip-reason == '' + || needs.publish-feedback.result == 'success' && needs.publish-feedback.outputs.skip-reason == '' + || needs.apply-dependency-verdict.result == 'success' && needs.apply-dependency-verdict.outputs.skip-reason == '' + || needs.apply-patch.result == 'success' && needs.apply-patch.outputs.skip-reason == '' + || needs.apply-patch-rung-up.result == 'success' && needs.apply-patch-rung-up.outputs.skip-reason == '') }} runs-on: ubuntu-24.04 timeout-minutes: 10 permissions: diff --git a/docs/conventions/pr-pipeline/README.md b/docs/conventions/pr-pipeline/README.md index 462acf3455..5217bfa324 100644 --- a/docs/conventions/pr-pipeline/README.md +++ b/docs/conventions/pr-pipeline/README.md @@ -535,8 +535,9 @@ activity's dispatcher, under the cap it already keeps: `limit-stops-per-head` fo `resolve-conflicts` and `review-dependency`, `fix-ci-restarts` and the per-head cap for `fix-ci`, `SWEEP_HEAD_CAP` for `address-feedback`. A `fix-ci` or `address-feedback` retry runs one model rung up (`escalate-to`), read in the run from the head's own stop record, never from a dispatch -input; the others retry as a fresh session. A retry that does not stop posts success over the -failing status (`report-escalation.sh clear`). Once the retries are spent, the monitoring lane's +input; the others retry as a fresh session. A retry that acts and does not stop posts success over the +failing status (`report-escalation.sh clear`); one that ends on a skip, such as a reply job with no +model turn, leaves it. Once the retries are spent, the monitoring lane's `stalled lane stop` signal names the PR. A new head carries no status, so the lane's next run there starts clean. From 9656b9638b7d9ccd387179c9809b54e3b8764b9f Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 26 Oct 2026 08:10:09 -0400 Subject: [PATCH 11/33] fix(pr-pipeline): read a fix-ci stop's head checks with the job token The ci-lanes commit status needs statuses: read, which neither the lane App token nor the merge job's token holds, so that read always failed closed and a fix-ci label could never clear. clear-escalations now reads the head's check runs with the job's own token (checks: read, passed as CHECKS_TOKEN) and clears only when a GitHub Actions check other than ci-status failed, through fix-ci-context's failingChecks, which moves to gate-source.mjs on the merge path floor. Green, skipped, another app's, a listing past three pages, or a failed read keeps the label. Co-Authored-By: Claude Opus 5.5 --- .../fix-pr-unblock-route-not-park-5c1e7a20.md | 2 +- .../scripts/pr-merge/clear-escalations.mjs | 70 +++++++++++-------- .../pr-merge/clear-escalations.test.mjs | 61 ++++++++-------- .github/scripts/pr-unblock/fix-ci-context.mjs | 29 ++------ .../scripts/pr-unblock/fix-ci-dispatch.mjs | 19 +++-- .github/scripts/pr-unblock/gate-source.mjs | 33 ++++++--- .github/workflows/pr-merge.yml | 3 + docs/conventions/pr-pipeline/README.md | 2 +- 8 files changed, 119 insertions(+), 100 deletions(-) diff --git a/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md b/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md index 13ea8cabab..dc2b29f2cf 100644 --- a/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md +++ b/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md @@ -6,7 +6,7 @@ bump: minor - A `pr-unblock` run that stops now ends on the route its reason has in [`escalation-routes.json`](https://github.com/melodic-software/claude-code-plugins/blob/main/.github/scripts/pr-unblock/escalation-routes.json), the one routing table that `report-escalation.sh`, `gate-source.mjs` and `clear-escalations.mjs` read. Only a reason a person must settle keeps the `needs-human` label: a path the lane token may not push or a supply-chain refusal, an intent, design or security question, a person's own thread or hold, or a finding no adjudicator may judge (security-review or CRITICAL). Every other reason (for example `ci-fix-escalated`, `feedback-not-landed`, the round and repeat caps) ends with no label, a failing commit status on the head the activity gated (`pr-unblock/conflict` for `resolve-conflicts`, `pr-unblock/` otherwise, posted by the `report-stop-status` job that replaces `report-conflict-status`), and one lanes App comment per activity that the next stop edits in place. A new head carries no status. - An `UNCERTAIN` or `VALID-defer` review reply now goes to the adjudicator as an `INCORRECT` one does, instead of to a person. A pr-verify note left open ends `feedback-verifier-open`, and a finding that stood at adjudication and was left open again ends `feedback-stood-open`; both route to a failing status. A round's reason held for a person now outranks any reason routed to a failing status. -- `clear-escalations` takes off a `needs-human` an older run set for a reason now routed to a failing status: a `ci-fix-*` reason once the GitHub Actions bot's `ci-lanes` status on the current head reads failed, any other once the head moved past the one the lane stopped on. +- `clear-escalations` takes off a `needs-human` an older run set for a reason now routed to a failing status: a `ci-fix-*` reason once a GitHub Actions check on the current head other than `ci-status` reads failed, any other once the head moved past the one the lane stopped on. - A pr-unblock stop routed to a failing status now skips `lane-stopped` (a new `skip-reason`) instead of `awaiting-human`. On a select job's own check (a cap, or a conflict no model can resolve) it ends the head, so the dispatchers stop re-running a deterministic stop. On the push, reply or verdict job's check, the dispatcher retries the same head under the cap it already keeps: - `limit-stops-per-head` for resolve-conflicts and review-dependency - `fix-ci-restarts` and the per-head cap for fix-ci diff --git a/.github/scripts/pr-merge/clear-escalations.mjs b/.github/scripts/pr-merge/clear-escalations.mjs index b96e60135a..63ae9eb011 100755 --- a/.github/scripts/pr-merge/clear-escalations.mjs +++ b/.github/scripts/pr-merge/clear-escalations.mjs @@ -33,11 +33,11 @@ // protected-path-conflict is left to resolve // feedback-protected-path the PR changes nothing under .github/workflows/ // a reason escalation-routes.json routes `fail` (a lane run before that -// route set the label): a fix-ci reason once the GitHub Actions bot's -// `ci-lanes` status on the current head reads failed (ci-status itself -// fails on the label, so it proves nothing), so the head's own CI stays -// red without the label; any other once the head is not one its -// escalations name as their resume key +// route set the label): a fix-ci reason once a GitHub Actions check on +// the current head other than ci-status reads failed (fix-ci-context's +// failingChecks; ci-status itself fails on the label, so it proves +// nothing), so the head stays red without the label; any other once the +// head is not one its escalations name as their resume key // A listing that may be cut short, or a mergeable state not yet computed, // counts as the reason still holding. // Any other reason is decided by a person and is never cleared here. @@ -61,6 +61,8 @@ // // Environment: // GH_TOKEN the lane App token (pull-requests and issues write) +// CHECKS_TOKEN the job's own token (checks: read), for the head's +// check runs, which the lane token cannot read // GITHUB_REPOSITORY owner/repo // KILL_SWITCH `proceed` from the kill-switch step; anything else stops // TRACKER_CONFIG the base .work-item-tracker.json, for the human-gated @@ -74,13 +76,12 @@ import { pathToFileURL } from "node:url"; import { createGitHub, paginate } from "../../actions/check-trusted-trigger/check-trusted-trigger.mjs"; import { humanGatedLabelFrom } from "../pr-unblock/address-feedback.mjs"; import { - actionsLanesState, CLEARED, ESCALATION, escalationsSince, escalationActivity, escalationRoute, - failedState, + failingChecks, foreignMarker, isApp, labelHistory, @@ -122,22 +123,28 @@ const FILES_CAP = 3000; // Each clear lets the lanes run on the PR again (judgment). export const CLEARS_PER_RUN = 5; -// The head commit's `ci-lanes` status, the lanes' verdict ci-status records. -export const LANES_QUERY = `query($owner: String!, $name: String!, $oid: GitObjectID!) { - repository(owner: $owner, name: $name) { - object(oid: $oid) { - ... on Commit { status { context(name: "ci-lanes") { state creator { __typename ... on Bot { databaseId } } } } } - } - } -}`; +// The check-run pages read per head, as fix-ci-context reads them. +const CHECK_PAGES = 3; -// Whether the GitHub Actions bot recorded the head's lanes failed. One read -// per PR; a read that fails, or any other state, reads as not failing. -async function lanesFailedAt({ github, repo, pull }) { - const [owner, name] = repo.split("/"); +// Whether a GitHub Actions check on the head other than ci-status failed. One +// read per PR; a listing that fails, or may be cut short, reads as not failing. +async function headFailing({ checks, repo, pull }) { try { - const data = await github("POST", "/graphql", { query: LANES_QUERY, variables: { owner, name, oid: pull.head.sha } }); - return failedState(actionsLanesState(data?.data?.repository?.object?.status?.context)); + const runs = []; + for (let page = 1; ; page += 1) { + const listed = (await checks("GET", `/repos/${repo}/commits/${pull.head.sha}/check-runs?filter=latest&per_page=100&page=${page}`))?.check_runs; + if (!Array.isArray(listed)) { + return false; + } + runs.push(...listed); + if (listed.length < 100) { + break; + } + if (page === CHECK_PAGES) { + return false; + } + } + return failingChecks(runs.filter((run) => run.app?.slug === "github-actions")).length > 0; } catch { return false; } @@ -169,7 +176,7 @@ const CHECKS = Object.freeze({ }, // A reason escalation-routes.json now routes to a failing status rather // than a person, on a label an older lane run set: fix-ci's once the head's - // lanes read failed, so CI stays red without the label (green, missing or + // checks read failed, so it stays red without the label (green, missing or // unreadable keeps it); any other once the head is not one the lane stopped // on, since the lane checks a new head itself. ...Object.fromEntries( @@ -177,10 +184,10 @@ const CHECKS = Object.freeze({ .filter((reason) => escalationRoute(reason) === "fail" && reason !== "unresolved-conflict") .map((reason) => [ reason, - async ({ pull, keys, lanesFailed }) => + async ({ pull, keys, failing }) => escalationActivity(reason) === "fix-ci" - ? (await lanesFailed()) - ? "the lane now ends it in a failing status, not a person, and the head's ci-lanes status reads failed" + ? (await failing()) + ? "the lane now ends it in a failing status, not a person, and the head's own checks read failed" : null : keys.length > 0 && !keys.includes(pull.head.sha) ? "the lane now ends it in a failing status, not a person, and checks the new head itself" @@ -239,7 +246,7 @@ function pairedWithPrMerge(appComments, since) { } // What to clear on `number`, or { reason } saying why nothing is. -async function verdict({ github, repo, number, label, bot, mainSha }) { +async function verdict({ github, checks, repo, number, label, bot, mainSha }) { const timeline = await paginate(github, `/repos/${repo}/issues/${number}/timeline`); const { labeled, since, lifted } = labelHistory(timeline, label); if (!isApp(labeled?.actor, bot)) { @@ -302,14 +309,14 @@ async function verdict({ github, repo, number, label, bot, mainSha }) { reasons = [HEAD_MOVED]; why.push(`its escalation names head \`${[...new Set(merged.map(({ head }) => head))].join("`, `")}\`, and the head is now \`${pull.head.sha}\``); } else { - let lanes; - const lanesFailed = () => (lanes ??= lanesFailedAt({ github, repo, pull })); + let red; + const failing = () => (red ??= headFailing({ checks, repo, pull })); for (const reason of reasons) { // The resume keys of this reason's escalations: the heads it stopped on. const keys = escalations .filter(({ body }) => REASON.exec(body.join("\n"))?.[1] === reason) .map(({ body }) => RESUME_KEY.exec(body[1])?.[1]); - const cleared = await CHECKS[reason]({ github, repo, pull, mainSha, keys, lanesFailed }); + const cleared = await CHECKS[reason]({ github, repo, pull, mainSha, keys, failing }); if (cleared === null) { return { reason: `${reason} still holds at ${pull.head.sha}` }; } @@ -324,7 +331,7 @@ async function verdict({ github, repo, number, label, bot, mainSha }) { return { reason: null, recorded, head: pull.head.sha, why: why.join("; "), explained }; } -export async function main({ env = process.env, github } = {}) { +export async function main({ env = process.env, github, checks } = {}) { try { if (!REPO.test(env.GITHUB_REPOSITORY ?? "")) { throw new Error("GITHUB_REPOSITORY is not owner/repo"); @@ -337,6 +344,7 @@ export async function main({ env = process.env, github } = {}) { const bot = env.LANES_BOT_LOGIN || DEFAULT_BOT; const label = humanGatedLabelFrom(env.TRACKER_CONFIG || ".base/.work-item-tracker.json"); github ??= createGitHub({ token: env.GH_TOKEN, apiUrl: env.GITHUB_API_URL || undefined }); + checks ??= createGitHub({ token: env.CHECKS_TOKEN, apiUrl: env.GITHUB_API_URL || undefined }); const tip = await github("GET", `/repos/${repo}/git/ref/heads/main`); const mainSha = tip?.ref === "refs/heads/main" ? tip.object?.sha : undefined; if (!SHA.test(String(mainSha))) { @@ -353,7 +361,7 @@ export async function main({ env = process.env, github } = {}) { continue; } try { - const result = await verdict({ github, repo, number, label, bot, mainSha }); + const result = await verdict({ github, checks, repo, number, label, bot, mainSha }); if (result.reason !== null) { console.log(`clear-escalations: #${number} kept: ${result.reason}`); continue; diff --git a/.github/scripts/pr-merge/clear-escalations.test.mjs b/.github/scripts/pr-merge/clear-escalations.test.mjs index 309cf6244b..a2984e252b 100644 --- a/.github/scripts/pr-merge/clear-escalations.test.mjs +++ b/.github/scripts/pr-merge/clear-escalations.test.mjs @@ -30,20 +30,22 @@ const escalated = (reason = "workflow-staged", { user = BOT, at = "2026-10-18T10 // One open pull request (number 7) labeled needs-human; `timeline` is its timeline and // `mainFiles` the files main changed since its merge base with the head. -// `lanes` is the head's ci-lanes status context (GraphQL shape), or an Error -// its read throws. -const ACTIONS_LANES = (state) => ({ state, creator: { __typename: "Bot", databaseId: 41898282 } }); -function world({ timeline, mainFiles = [{ filename: "README.md" }], mergeable = false, pullFiles = [], author, lanes = null }) { +// `headChecks` is the head's check runs, as the job token lists them, or an +// Error its read throws. +const check = (name, conclusion, { id = 1, slug = "github-actions" } = {}) => ({ id, name, status: "completed", conclusion, app: { slug } }); +const RED = [check("test-node", "failure")]; +function world({ timeline, mainFiles = [{ filename: "README.md" }], mergeable = false, pullFiles = [], author, headChecks = [] }) { const calls = []; + const checks = async (method, path) => { + calls.push({ method, path, token: "checks" }); + if (headChecks instanceof Error) { + throw headChecks; + } + assert.equal(path, `/repos/${REPO}/commits/${HEAD}/check-runs?filter=latest&per_page=100&page=1`); + return { check_runs: headChecks }; + }; const github = async (method, path, body) => { calls.push({ method, path, body }); - if (method === "POST" && path === "/graphql") { - if (lanes instanceof Error) { - throw lanes; - } - assert.equal(body.variables.oid, HEAD); - return { data: { repository: { object: { status: lanes === null ? null : { context: lanes } } } } }; - } const route = path.replace(/[?&]per_page=100&page=\d+$/, ""); if (method === "GET" && route === `/repos/${REPO}/git/ref/heads/main`) { return { ref: "refs/heads/main", object: { sha: MAIN } }; @@ -68,7 +70,7 @@ function world({ timeline, mainFiles = [{ filename: "README.md" }], mergeable = } throw new Error(`unexpected ${method} ${path}`); }; - return { github, calls, writes: () => calls.filter((call) => call.method !== "GET" && call.path !== "/graphql") }; + return { github, checks, calls, writes: () => calls.filter((call) => call.method !== "GET") }; } const run = async (w, env = {}) => { @@ -76,7 +78,7 @@ const run = async (w, env = {}) => { const original = console.log; console.log = (line) => logs.push(line); try { - return { code: await main({ env: { ...ENV, ...env }, github: w.github }), logs }; + return { code: await main({ env: { ...ENV, ...env }, github: w.github, checks: w.checks }), logs }; } finally { console.log = original; } @@ -217,7 +219,7 @@ test("a reason a person decides is never cleared", async () => { test("a label for a reason now routed to a failing status comes off: fix-ci at once, others once the head moved", async () => { for (const reason of ["ci-fix-escalated", "ci-fix-round-cap"]) { const atHead = commented(BOT, escalationBody(reason).replaceAll(KEY, HEAD), "2026-10-18T10:00:05Z"); - const w = world({ timeline: [labeled(BOT, "2026-10-18T10:00:00Z"), atHead], lanes: ACTIONS_LANES("FAILURE") }); + const w = world({ timeline: [labeled(BOT, "2026-10-18T10:00:00Z"), atHead], headChecks: RED }); await run(w); assert.equal(w.writes().at(-1)?.method, "DELETE", reason); assert.match(w.writes()[0].body.body, new RegExp(`\`${reason}\``)); @@ -234,24 +236,25 @@ test("a label for a reason now routed to a failing status comes off: fix-ci at o }); // The security finding on #10252: a fix-ci label came off with no CI result -// read. Only the Actions bot's failed ci-lanes on the current head clears it; -// ci-status itself fails on the label, so it is not read. -test("a fix-ci label comes off only while the head's ci-lanes reads failed", async () => { +// read. Only a failed GitHub Actions check on the current head other than +// ci-status clears it; ci-status itself fails on the label, so it is skipped. +test("a fix-ci label comes off only while the head's own checks read failed", async () => { const timeline = [labeled(BOT, "2026-10-18T10:00:00Z"), escalated("ci-fix-escalated")]; - for (const [lanes, cleared] of [ - [ACTIONS_LANES("FAILURE"), true], - [ACTIONS_LANES("ERROR"), true], - [ACTIONS_LANES("SUCCESS"), false], - [ACTIONS_LANES("PENDING"), false], - [null, false], - [{ state: "FAILURE", creator: { __typename: "User", databaseId: 1 } }, false], - [new Error("GraphQL down"), false], + for (const [label, headChecks, cleared] of [ + ["red", RED, true], + ["timed out", [check("test-bash (1/4)", "timed_out")], true], + ["green", [check("test-node", "success"), check("lint", "success", { id: 2 })], false], + ["only ci-status red", [check("ci-status", "failure"), check("test-node", "success", { id: 2 })], false], + ["a skip", [check("test-node", "skipped")], false], + ["no checks", [], false], + ["another app's failure", [check("test-node", "failure", { slug: "someone-else" })], false], + ["read fails", new Error("HTTP 502"), false], ]) { - const w = world({ timeline, lanes }); + const w = world({ timeline, headChecks }); const { code } = await run(w); - assert.equal(code, 0); - assert.equal(w.writes().at(-1)?.method === "DELETE", cleared, JSON.stringify(lanes?.message ?? lanes)); - assert.equal(w.calls.filter((call) => call.path === "/graphql").length, 1); + assert.equal(code, 0, label); + assert.equal(w.writes().at(-1)?.method === "DELETE", cleared, label); + assert.equal(w.calls.filter((call) => call.token === "checks").length, 1, label); } }); diff --git a/.github/scripts/pr-unblock/fix-ci-context.mjs b/.github/scripts/pr-unblock/fix-ci-context.mjs index d5645739cc..c5ab51ba67 100644 --- a/.github/scripts/pr-unblock/fix-ci-context.mjs +++ b/.github/scripts/pr-unblock/fix-ci-context.mjs @@ -21,15 +21,16 @@ import { appendFileSync, writeFileSync } from "node:fs"; import { pathToFileURL } from "node:url"; import { createGitHubApi, humanGatedLabelFrom, keepThreads, need, positive, reviewedHeads, reviewerIdFrom, SHA } from "./address-feedback.mjs"; -import { prHold } from "./gate-source.mjs"; +import { failingChecks, prHold } from "./gate-source.mjs"; + +// clear-escalations reads the head the same way, so the reader lives on the +// merge path floor in gate-source.mjs. +export { failingChecks }; import { retryRung, stopSkip } from "./escalation-routes.mjs"; // The trailer apply-patch.mjs writes on every fix commit, from the ci-context // row of patch-rules.json; a round is one. export const FIX_TRAILER = "Lane-Activity: pr-unblock/fix-ci"; -const FAILING = new Set(["failure", "timed_out", "startup_failure"]); -// The aggregate of the others: fixing them fixes it. -const AGGREGATE = "ci-status"; const WORKFLOW_PATH = /^\.github\/workflows\/[A-Za-z0-9._-]+\.ya?ml$/; const MAX_CHECKS = 10; const MAX_ANNOTATIONS = 20; @@ -64,26 +65,6 @@ export function logExcerpt(text) { return excerpt; } -// A contract-only run skips every lane, and ci-status then carries the last -// full run's recorded verdict forward, so a skip says nothing about a check. -const SILENT = new Set(["skipped", "neutral"]); -const silent = (run) => run.status === "completed" && SILENT.has(run.conclusion); - -// The latest run of each check name that is not a skip, when it failed. The -// fix-ci dispatcher reads a cancelled head through it too, so the two agree. -export function failingChecks(checkRuns) { - const latest = new Map(); - for (const run of checkRuns) { - const kept = latest.get(run.name); - if (!kept || (silent(kept) && !silent(run)) || (silent(kept) === silent(run) && run.id > kept.id)) { - latest.set(run.name, run); - } - } - return [...latest.values()] - .filter((run) => run.name !== AGGREGATE && run.status === "completed" && FAILING.has(run.conclusion)) - .sort((a, b) => a.id - b.id); -} - const annotationOf = (row) => ({ path: String(row.path ?? ""), line: Number.isSafeInteger(row.start_line) ? row.start_line : null, diff --git a/.github/scripts/pr-unblock/fix-ci-dispatch.mjs b/.github/scripts/pr-unblock/fix-ci-dispatch.mjs index b6610f9156..e81ee85b30 100644 --- a/.github/scripts/pr-unblock/fix-ci-dispatch.mjs +++ b/.github/scripts/pr-unblock/fix-ci-dispatch.mjs @@ -42,7 +42,7 @@ import { AGGREGATE_JOBS, FAILED, infraCause, infraOnly, RERUN_DUE_MINUTES, recon import { emptyRecord, isHandled, markHandled } from "./dispatch-record.mjs"; import { runTitle } from "./feedback-dispatch.mjs"; import { failingChecks } from "./fix-ci-context.mjs"; -import { actionsLanesState, failedState, prHold } from "./gate-source.mjs"; +import { prHold } from "./gate-source.mjs"; import { laneStop } from "./limit-stop.mjs"; import { serverFailure } from "./request-budget.mjs"; @@ -213,16 +213,25 @@ function ownerIdle(node, idleHours, now) { return commit?.oid === node.headRefOid && Date.parse(node.updatedAt) <= since && Date.parse(commit.committedDate) <= since; } +// The GitHub Actions bot's user id. GraphQL reports its login as +// `github-actions`, REST as `github-actions[bot]`; the id is the same in both. +const ACTIONS_BOT_ID = 41898282; + // The head's `ci-lanes` commit status, whoever set it. function lanesStatus(node) { const contexts = node.commits?.nodes?.[0]?.commit?.statusCheckRollup?.contexts?.nodes ?? []; return contexts.find((context) => context?.__typename === "StatusContext" && context.context === "ci-lanes"); } -// The lanes' verdict alone (gate-source.mjs lanesState): ci-status itself -// also fails on a hold label, a pending or missing review and the PR -// contract, none of which a CI fix mends. -const lanesState = (node) => actionsLanesState(lanesStatus(node)); +// The ci-lanes state when the GitHub Actions bot set it, else null: the +// lanes' verdict alone. ci-status itself also fails on a hold label, a +// pending or missing review and the PR contract, none of which a CI fix mends. +function lanesState(node) { + const status = lanesStatus(node); + return status?.creator?.__typename === "Bot" && status.creator.databaseId === ACTIONS_BOT_ID ? status.state : null; +} + +const failedState = (state) => ["FAILURE", "ERROR"].includes(state); const lanesFailed = (node) => failedState(lanesState(node)); const cancelled = (required) => required?.run?.status === "completed" && required.run.conclusion === "cancelled"; diff --git a/.github/scripts/pr-unblock/gate-source.mjs b/.github/scripts/pr-unblock/gate-source.mjs index 47071cf608..5f3916bba3 100644 --- a/.github/scripts/pr-unblock/gate-source.mjs +++ b/.github/scripts/pr-unblock/gate-source.mjs @@ -151,16 +151,31 @@ const REASON_LINE = /^- Reason: `([a-z][a-z-]*)`$/; export { escalationActivity, escalationRoute, ROUTES } from "./escalation-routes.mjs"; -// The GitHub Actions bot's user id. GraphQL reports its login as -// `github-actions`, REST as `github-actions[bot]`; the id is the same in both. -const ACTIONS_BOT_ID = 41898282; +const FAILING = new Set(["failure", "timed_out", "startup_failure"]); +// The aggregate of the others: fixing them fixes it. It also fails on a hold +// label, so it never stands for the head's CI. +const AGGREGATE = "ci-status"; +// A contract-only run skips every lane, and ci-status then carries the last +// full run's recorded verdict forward, so a skip says nothing about a check. +const SILENT = new Set(["skipped", "neutral"]); +const silent = (run) => run.status === "completed" && SILENT.has(run.conclusion); + +// The latest run of each check name that is not a skip, when it failed. The +// fix-ci dispatcher reads a cancelled head through it too, so the two agree, +// and clear-escalations reads a fix-ci stop's head through it. +export function failingChecks(checkRuns) { + const latest = new Map(); + for (const run of checkRuns) { + const kept = latest.get(run.name); + if (!kept || (silent(kept) && !silent(run)) || (silent(kept) === silent(run) && run.id > kept.id)) { + latest.set(run.name, run); + } + } + return [...latest.values()] + .filter((run) => run.name !== AGGREGATE && run.status === "completed" && FAILING.has(run.conclusion)) + .sort((a, b) => a.id - b.id); +} -// A `ci-lanes` status context's state (GraphQL shape) when the GitHub Actions -// bot set it, else null: the lanes' verdict ci-status records. fix-ci-dispatch -// and clear-escalations read it. -export const actionsLanesState = (status) => - status?.creator?.__typename === "Bot" && status.creator.databaseId === ACTIONS_BOT_ID ? status.state : null; -export const failedState = (state) => ["FAILURE", "ERROR"].includes(state); // When a timeline item was written, and by whom, for a comment or a review // (clear-escalations.mjs reads people speaking the same way). diff --git a/.github/workflows/pr-merge.yml b/.github/workflows/pr-merge.yml index 2438697ca4..02b26385d9 100644 --- a/.github/workflows/pr-merge.yml +++ b/.github/workflows/pr-merge.yml @@ -1433,6 +1433,9 @@ jobs: timeout-minutes: 3 env: GH_TOKEN: ${{ steps.lane-token.outputs.token }} + # The job's own token reads a fix-ci stop's head checks (checks: + # read); the lane token holds no Checks permission. + CHECKS_TOKEN: ${{ github.token }} KILL_SWITCH: ${{ steps.kill.outputs.proceed == 'true' && 'proceed' || '' }} LANE_PRS: ${{ steps.mode.outputs.lane-prs }} run: node .base/.github/scripts/pr-merge/clear-escalations.mjs diff --git a/docs/conventions/pr-pipeline/README.md b/docs/conventions/pr-pipeline/README.md index 5217bfa324..2233092f63 100644 --- a/docs/conventions/pr-pipeline/README.md +++ b/docs/conventions/pr-pipeline/README.md @@ -562,7 +562,7 @@ head: | `workflow-staged` | main changes nothing under `.github/workflows/` since its merge base with the head | | `unresolved-conflict`, `protected-path-conflict` | GitHub reports the PR mergeable | | `feedback-protected-path` | the PR changes nothing under `.github/workflows/` | -| a reason `escalation-routes.json` routes `fail`, on a label set before it did | a `ci-fix-*` reason once the GitHub Actions bot's `ci-lanes` status on the current head reads failed (ci-status itself fails on the label, so it is not read; green, missing or unreadable keeps the label); any other once the head is not one its escalations name as the resume key | +| a reason `escalation-routes.json` routes `fail`, on a label set before it did | a `ci-fix-*` reason once a GitHub Actions check on the current head other than `ci-status` reads failed (`ci-status` itself fails on the label, so it is not read; green, missing or unreadable keeps the label); any other once the head is not one its escalations name as the resume key | | `head-moved` (pr-merge's `babysit-merge-verify-escalation`, `babysit-merge-risk` and `babysit-merge-shared-copy-escalation`, with no pr-unblock escalation since the label, and the marked App comment written last before the label is pr-merge's and within a minute of it) | the newest App comment carrying each marker names, as ``at head `` ``, a 40-hex head other than the current one; pr-merge then judges the new head and escalates again if its refusal still holds | It removes only a label whose newest `labeled` event has the lanes App as its actor, and keeps it From 0a9ebb0cbd3d2f1fe9d905f1df23b1db66c35d5b Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 26 Oct 2026 08:13:04 -0400 Subject: [PATCH 12/33] perf(pr-pipeline): read stalled stops in one GraphQL query The stalled-stop source listed open pull requests over REST and read one comment page per pull request, up to 3 + 40 + 20 of the run's 250-call budget. One GraphQL query now lists the 40 most recently updated open pull requests with each one's newest 100 comments and the open count; at most two more queries read older comments, batched one alias per pull request (batchQuery, the stacked branch's pattern). Worst case: 1 + 2 + 20 calls. A lane-stopped comment for an older head also set `marked`, so it hid the current head's stop in an older comment and recorded no gap. Only a marker naming the current head now ends the read; a pull request whose older comments stay unread past the rounds is a gap. Co-Authored-By: Claude Opus 5.5 --- .../fix-pr-unblock-route-not-park-5c1e7a20.md | 2 +- .../maintenance-monitor-signals/collect.mjs | 96 ++++++++++++----- .../collect.test.mjs | 100 ++++++++++++++++-- .../maintenance-monitor-signals/monitor.mjs | 2 + 4 files changed, 164 insertions(+), 36 deletions(-) diff --git a/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md b/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md index dc2b29f2cf..0d2ab39fa2 100644 --- a/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md +++ b/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md @@ -15,5 +15,5 @@ bump: minor The fix-ci dispatcher reads a fix-ci run and its in-run rung-up, which both post a `pr-unblock / fix-ci` check, as one attempt: a stop in either one's push job makes the attempt a failed one, and the pair counts once toward the caps. The fix-ci and address-feedback retries run one model rung up, read in the run from the head's own stop record. A retry that acts and does not stop posts success over the failing status; one that ends on a skip (a reply job with no model turn) leaves it. fix-ci's model escalations `protected-path` and `weaken-required` stay with a person, as `ci-fix-protected-path` and the new `ci-fix-weaken-required`. -- The monitoring lane gains signal 10, `stalled lane stop`. Each open pull request whose pr-unblock stop has had no new attempt on its head for over 2 hours is listed in one rolling issue. A skip is not an attempt. A stop includes a `needs-human` that `clear-escalations` took off for a reason now routed to a failing status. It reads the 40 most recently updated open pull requests, and past 40 it names the open count in the run's gaps. +- The monitoring lane gains signal 10, `stalled lane stop`. Each open pull request whose pr-unblock stop has had no new attempt on its head for over 2 hours is listed in one rolling issue. A skip is not an attempt. A stop includes a `needs-human` that `clear-escalations` took off for a reason now routed to a failing status. It reads the 40 most recently updated open pull requests and their newest comments in one GraphQL query, reading up to 200 older comments only for a pull request whose newest name no stop on its head; past 40 pull requests, or past those comments, it names what it left unread in the run's gaps. - The merge path floor (`babysit_risk.HIGH_FLOOR`, `merge-holds.mjs` `HOLD_PATH_FLOOR`, and the review lanes' and ci-status's hold lists) also holds `.github/scripts/pr-unblock/escalation-routes.mjs`, which `clear-escalations.mjs` and `gate-source.mjs` now import. diff --git a/.github/scripts/maintenance-monitor-signals/collect.mjs b/.github/scripts/maintenance-monitor-signals/collect.mjs index e7ad6452dd..c7b0537367 100644 --- a/.github/scripts/maintenance-monitor-signals/collect.mjs +++ b/.github/scripts/maintenance-monitor-signals/collect.mjs @@ -16,14 +16,15 @@ export const LIMITS = { sweepArtifacts: 5, commentPages: 5, // Open pull requests the stalled-stop read covers, most recently updated - // first: the open list in pages of 100 (at most stalledPages, so the gap - // names the true count), one comment page for each of the first - // stalledPulls, and at most stalledChecks check-run reads in all, so the - // source spends at most 3 + 40 + 20 of CALL_BUDGET. stalledPulls covers - // every open pull request at today's count; past it, a gap leaves signal 10 - // unread rather than missing a stop. + // first: one GraphQL query lists the first stalledPulls with each one's + // newest 100 comments and the open count; at most stalledCommentRounds more + // queries read 100 older comments for every PR whose newest name no stop on + // its head; at most stalledChecks check-run reads in all. So the source + // spends at most 1 + 2 + 20 of CALL_BUDGET. stalledPulls covers every open + // pull request at today's count; past it, a gap leaves signal 10 unread + // rather than missing a stop. stalledPulls: 40, - stalledPages: 3, + stalledCommentRounds: 2, stalledChecks: 20, }; // report-escalation.sh's comment for a stop routed to a failing status, one @@ -37,6 +38,27 @@ const CLEARED = /^`, })), listReviewThreads: async () => [ - { isResolved: false, comments: { nodes: [{ author: CLAUDE }], pageInfo: { hasNextPage: false } } }, + // A reviewer-bot finding on a review of the marked head. + { isResolved: false, comments: { nodes: [{ author: CLAUDE, pullRequestReview: { author: CLAUDE, commit: { oid: HEAD } } }], pageInfo: { hasNextPage: false } } }, ], listCheckRuns: async () => [], listWorkflowRuns: async () => [], diff --git a/.github/scripts/pr-unblock/address-feedback.mjs b/.github/scripts/pr-unblock/address-feedback.mjs index 92863f6f8c..cee5f7260c 100644 --- a/.github/scripts/pr-unblock/address-feedback.mjs +++ b/.github/scripts/pr-unblock/address-feedback.mjs @@ -389,6 +389,19 @@ export function threadAuthors(comments, { reviewerId, appId, trustedHumans = new return authors; } +// Whether a thread threadAuthors accepted (`authors`) sits on a review its +// finder submitted on a marker-confirmed head: the reviewer bot's, or +// github-actions[bot]'s for a verifier note or a lane comment. A trusted +// person's direction (a fix request, or a reply to the lane's escalation) +// asks for the thread, so its review may be on an earlier head. `review` is +// { user: { id }, commit_id }. keepThreads and the dispatcher +// (feedback-dispatch.mjs hasWorkableThread) both judge a thread with this. +export function onReviewedHead({ authors, firstAuthorId, review, heads, reviewerId }) { + const directed = authors.some((a) => a.direction === true); + const reviewAuthor = authors[0].author === "verifier" || firstAuthorId === MARKER_AUTHOR_ID ? MARKER_AUTHOR_ID : reviewerId; + return review?.user?.id === reviewAuthor && (directed || heads.has(review.commit_id)); +} + // A thread is kept when it is unresolved, its first comment belongs to a // review its author (the reviewer bot, or github-actions[bot] for a verifier // note) submitted on a marker-confirmed head (any head for a thread carrying @@ -420,13 +433,8 @@ export function keepThreads({ threads, reviewComments, reviews, heads, reviewerI if (authors === null) { continue; } - // A trusted person's direction (a fix request, or a reply to the lane's - // escalation) asks for the thread, so its review may be on an earlier head. - const directed = authors.some((a) => a.direction === true); const review = reviewsById.get(rows[0].pull_request_review_id); - // A verifier note's or a lane comment's review is github-actions[bot]'s. - const reviewAuthor = authors[0].author === "verifier" || rows[0].user?.id === MARKER_AUTHOR_ID ? MARKER_AUTHOR_ID : reviewerId; - if (review?.user?.id !== reviewAuthor || (!directed && !heads.has(review.commit_id))) { + if (!onReviewedHead({ authors, firstAuthorId: rows[0].user?.id, review, heads, reviewerId })) { continue; } kept.push({ @@ -490,8 +498,13 @@ export async function select({ requesters: gated ? fixers : null, fixers, }); + // An unresolved thread the lane does not take (a reviewer it does not + // trust, a review on an older head, a reply it already gave) would block + // the merge with no stop on record, so it ends in a failing status. if (kept.length === 0) { - return { decision: "skip", skipReason: gated ? "awaiting-human" : "not-applicable", threads: kept }; + return gated || !threads.some((thread) => thread.isResolved === false) + ? { decision: "skip", skipReason: gated ? "awaiting-human" : "not-applicable", threads: kept } + : { decision: "escalate", escalationReason: "feedback-untaken-threads", threads: kept }; } // The files listing stops at FILES_CAP, so a longer diff could hide a // .github/workflows/ path and is treated as touching one. diff --git a/.github/scripts/pr-unblock/address-feedback.test.mjs b/.github/scripts/pr-unblock/address-feedback.test.mjs index b8403ba2c0..1b52b7bfe8 100644 --- a/.github/scripts/pr-unblock/address-feedback.test.mjs +++ b/.github/scripts/pr-unblock/address-feedback.test.mjs @@ -134,8 +134,8 @@ test("a thread with a human reply is never kept", async () => { const result = await run( fakeApi({ reviewComments: [comment(100, REVIEWER), comment(101, HUMAN)], threads: [thread("PRRT_1", [100, 101])] }), ); - assert.equal(result.decision, "skip"); - assert.equal(result.skipReason, "not-applicable"); + assert.equal(result.decision, "escalate"); + assert.equal(result.escalationReason, "feedback-untaken-threads"); }); // The thread on #7794: the reviewer bot's finding, the lanes App's UNCERTAIN @@ -185,14 +185,14 @@ test("a reply the lane did not leave open, or one not from the lanes App, is no [101, HUMAN, ESCALATION], [101, REVIEWER, ESCALATION], ]) { - assert.equal((await run(fakeApi(escalated({ escalation })))).decision, "skip", escalation[2]); + assert.equal((await run(fakeApi(escalated({ escalation })))).decision, "escalate", escalation[2]); } }); test("an escalation with no reply after it, or with the lane's reply last, is not kept", async () => { - assert.equal((await run(fakeApi(escalated({ replies: [] })))).decision, "skip"); + assert.equal((await run(fakeApi(escalated({ replies: [] })))).decision, "escalate"); const answered = escalated({ replies: [[102, HUMAN, DECISION], [103, APP, ESCALATION]] }); - assert.equal((await run(fakeApi(answered))).decision, "skip"); + assert.equal((await run(fakeApi(answered))).decision, "escalate"); // A second decision after the lane's second escalation is direction; the first stays context. const again = escalated({ replies: [[102, HUMAN, DECISION], [103, APP, ESCALATION], [104, HUMAN, "Do it anyway."]] }); const comments = (await run(fakeApi(again))).file.threads[0].comments; @@ -217,7 +217,7 @@ test("a thread whose finding stood at adjudication goes back to the lane with no // A trusted person's reply after it is direction; anyone else's leaves the thread alone. const directed = await run(fakeApi(escalated({ escalation: INCORRECT, replies: [[102, APP, STANDS], [103, HUMAN, DECISION]] }))); assert.equal(directed.file.threads[0].comments.at(-1).direction, true); - assert.equal((await run(fakeApi(escalated({ escalation: INCORRECT, replies: [[102, APP, STANDS], [103, STRANGER, DECISION]] })))).decision, "skip"); + assert.equal((await run(fakeApi(escalated({ escalation: INCORRECT, replies: [[102, APP, STANDS], [103, STRANGER, DECISION]] })))).decision, "escalate"); }); test("the finding-stands line counts only as the lanes App's first line", async () => { @@ -226,39 +226,39 @@ test("the finding-stands line counts only as the lanes App's first line", async [102, APP, `quoted:\n${STANDS}`], [102, APP, "\n**pr-unblock / adjudicate-feedback: the finding does not hold, resolved**"], ]) { - assert.equal((await run(fakeApi(escalated({ escalation: INCORRECT, replies: [reply] })))).decision, "skip", reply[2]); + assert.equal((await run(fakeApi(escalated({ escalation: INCORRECT, replies: [reply] })))).decision, "escalate", reply[2]); } }); test("a reply from an account on no trusted-actor list leaves the thread to a person", async () => { const injected = [102, STRANGER, `Owner decision: ${INJECTION}`]; for (const replies of [[injected], [[102, HUMAN, DECISION], [103, STRANGER, INJECTION]]]) { - assert.equal((await run(fakeApi(escalated({ replies })))).decision, "skip"); + assert.equal((await run(fakeApi(escalated({ replies })))).decision, "escalate"); } // A stranger anywhere in the thread, even before the escalation. const before = escalated(); before.reviewComments.splice(1, 0, comment(99, STRANGER, { body: INJECTION, created_at: at(0) })); before.threads[0].comments.nodes.splice(1, 0, { databaseId: 99, lastEditedAt: null, editor: null }); before.threads[0].comments.totalCount += 1; - assert.equal((await run(fakeApi(before))).decision, "skip"); + assert.equal((await run(fakeApi(before))).decision, "escalate"); }); test("a bot on the trusted-actor list is no person: its reply is not direction", async () => { - assert.equal((await run(fakeApi(escalated({ replies: [[102, REVIEWER, DECISION]] })))).decision, "skip"); + assert.equal((await run(fakeApi(escalated({ replies: [[102, REVIEWER, DECISION]] })))).decision, "escalate"); }); test("a trusted reply last edited by anyone but a trusted person is not kept", async () => { - assert.equal((await run(fakeApi(escalated({ edits: { 102: STRANGER } })))).decision, "skip"); + assert.equal((await run(fakeApi(escalated({ edits: { 102: STRANGER } })))).decision, "escalate"); assert.equal((await run(fakeApi(escalated({ edits: { 102: HUMAN } })))).decision, "proceed"); }); test("a reply not created after the escalation, or with no readable time, is not kept", async () => { const early = escalated(); early.reviewComments[2].created_at = at(1); - assert.equal((await run(fakeApi(early))).decision, "skip"); + assert.equal((await run(fakeApi(early))).decision, "escalate"); const unreadable = escalated(); unreadable.reviewComments[2].created_at = "yesterday"; - assert.equal((await run(fakeApi(unreadable))).decision, "skip"); + assert.equal((await run(fakeApi(unreadable))).decision, "escalate"); }); // The phrase the owner's replies on #8044, #8065 and eleven other PRs use. @@ -406,8 +406,8 @@ for (const [label, overrides] of [ ]) { test(`with no escalation, ${label} is not kept`, async () => { const result = await run(fakeApi(overrides)); - assert.equal(result.decision, "skip"); - assert.equal(result.skipReason, "not-applicable"); + assert.equal(result.decision, "escalate"); + assert.equal(result.escalationReason, "feedback-untaken-threads"); }); } @@ -448,8 +448,8 @@ for (const [label, overrides] of [ ]) { test(`on an earlier head, ${label} is not kept`, async () => { const result = await run(fakeApi({ ...OLD_REVIEW, ...overrides })); - assert.equal(result.decision, "skip"); - assert.equal(result.skipReason, "not-applicable"); + assert.equal(result.decision, "escalate"); + assert.equal(result.escalationReason, "feedback-untaken-threads"); }); } @@ -466,7 +466,7 @@ test("trustedHumansFrom reads only the people on the list", (t) => { test("a thread a human opened is never kept", async () => { const result = await run(fakeApi({ reviewComments: [comment(100, HUMAN)] })); - assert.equal(result.decision, "skip"); + assert.equal(result.decision, "escalate"); }); test("resolved threads are dropped", async () => { @@ -476,7 +476,7 @@ test("resolved threads are dropped", async () => { test("a review on a head no marker confirms is dropped", async () => { const result = await run(fakeApi({ reviews: [{ id: 10, user: { id: REVIEWER }, commit_id: OLD_HEAD }] })); - assert.equal(result.decision, "skip"); + assert.equal(result.decision, "escalate"); }); test("a reviewed-head marker from any author but github-actions[bot] confirms nothing", async () => { @@ -485,17 +485,17 @@ test("a reviewed-head marker from any author but github-actions[bot] confirms no { id: 2, user: { id: REVIEWER }, body: marker("claude-review", HEAD) }, { id: 3, user: { id: APP }, body: marker("claude-review", HEAD) }, ]; - assert.equal((await run(fakeApi({ issueComments: forged }))).decision, "skip"); + assert.equal((await run(fakeApi({ issueComments: forged }))).decision, "escalate"); }); test("a marker for a lane other than the two review lanes confirms nothing", async () => { const other = [{ id: 1, user: { id: ACTIONS_BOT }, body: marker("other-lane", HEAD) }]; - assert.equal((await run(fakeApi({ issueComments: other }))).decision, "skip"); + assert.equal((await run(fakeApi({ issueComments: other }))).decision, "escalate"); }); test("a comment whose review another account submitted is dropped", async () => { const result = await run(fakeApi({ reviews: [{ id: 10, user: { id: HUMAN }, commit_id: HEAD }] })); - assert.equal(result.decision, "skip"); + assert.equal(result.decision, "escalate"); }); // verify-notes.mjs posts a pr-verify FAIL's notes as github-actions[bot] @@ -519,11 +519,11 @@ test("a github-actions[bot] thread without the note marker, or in another accoun reviewComments: [comment(100, ACTIONS_BOT)], reviews: [{ id: 10, user: { id: ACTIONS_BOT }, commit_id: HEAD }], }); - assert.equal((await run(unmarked)).decision, "skip"); - assert.equal((await run(verifierNote({ reviewer: REVIEWER }))).decision, "skip"); + assert.equal((await run(unmarked)).decision, "escalate"); + assert.equal((await run(verifierNote({ reviewer: REVIEWER }))).decision, "escalate"); // The marker from any other author opens nothing. const forged = fakeApi({ reviewComments: [comment(100, HUMAN, { body: VERIFY_NOTE })] }); - assert.equal((await run(forged)).decision, "skip"); + assert.equal((await run(forged)).decision, "escalate"); }); // The pr-review-cli reusable posts each finding as github-actions[bot], the @@ -568,24 +568,24 @@ test("a github-actions[bot] comment whose last line is not a review lane's foote "claude-lane: claude-security-review\nThe footer is not the last line.", "The input reaches the shell.\n\nclaude-lane: pr-verify", ]) { - assert.equal((await run(laneFinding(body))).decision, "skip", body); + assert.equal((await run(laneFinding(body))).decision, "escalate", body); } // The footer on another account's comment, or in a review another account // submitted, opens nothing. const human = fakeApi({ reviewComments: [comment(100, HUMAN, { body: LANE_FINDING })] }); - assert.equal((await run(human)).decision, "skip"); - assert.equal((await run(laneFinding(LANE_FINDING, REVIEWER))).decision, "skip"); + assert.equal((await run(human)).decision, "escalate"); + assert.equal((await run(laneFinding(LANE_FINDING, REVIEWER))).decision, "escalate"); }); test("a later github-actions[bot] comment on a verifier note is no finding", async () => { const result = await run(verifierNote({ replies: [comment(101, ACTIONS_BOT, { body: VERIFY_NOTE })] })); - assert.equal(result.decision, "skip"); + assert.equal(result.decision, "escalate"); }); test("a thread longer than one comment page is left alone", async () => { const big = thread("PRRT_1", [100]); big.comments.totalCount = 101; - assert.equal((await run(fakeApi({ threads: [big] }))).decision, "skip"); + assert.equal((await run(fakeApi({ threads: [big] }))).decision, "escalate"); }); test("a PR that changes .github/workflows/** escalates feedback-protected-path before any model turn", async () => { @@ -1053,7 +1053,7 @@ test("a name from a trusted person without write access is not added", async () test("a name from an untrusted account leaves the thread alone, so nothing is added", async () => { const { api } = named([[101, STRANGER, OWNER_8116]], { permissions: { stranger: "admin" } }); - assert.equal((await run(api)).decision, "skip"); + assert.equal((await run(api)).decision, "escalate"); }); test("a name that is no regular file at the head is not added: missing, a symbolic link, a gitlink, or another letter case", async () => { @@ -1128,3 +1128,29 @@ test("select asks for model rung 1 only after a lane stop on the head's reply ch assert.equal(await rungOf([{ ...stop, app: { id: 1 } }]), ""); assert.equal(await rungOf([{ ...stop, output: { title: "Skipped: no-model-turn" } }]), ""); }); + +// #10092's silent stall: an unresolved thread the lane does not take left +// the head with a neutral not-applicable skip, no stop and no signal. The +// select job now ends it in feedback-untaken-threads (routed to a failing +// status); only a PR with no unresolved thread still skips. +const CODEX = 199175422; +test("an unresolved thread the lane does not take escalates feedback-untaken-threads", async () => { + const codex = await run( + fakeApi({ + reviewComments: [comment(100, CODEX)], + reviews: [{ id: 10, user: { id: CODEX }, commit_id: HEAD }], + }), + ); + assert.deepEqual([codex.decision, codex.escalationReason], ["escalate", "feedback-untaken-threads"], "a Codex-root thread"); + const stale = await run( + fakeApi({ + reviewComments: [comment(100, ACTIONS_BOT, { body: "The loop skips the last item.\n\nclaude-lane: claude-review" })], + reviews: [{ id: 10, user: { id: ACTIONS_BOT }, commit_id: OLD_HEAD }], + }), + ); + assert.deepEqual([stale.decision, stale.escalationReason], ["escalate", "feedback-untaken-threads"], "a lane thread on an older head"); + const none = await run(fakeApi({ threads: [thread("PRRT_1", [100], { isResolved: true })] })); + assert.deepEqual([none.decision, none.skipReason], ["skip", "not-applicable"], "no unresolved thread"); + const held = await run(fakeApi({ labels: ["needs-human"], reviewComments: [comment(100, CODEX)], reviews: [{ id: 10, user: { id: CODEX }, commit_id: HEAD }] })); + assert.deepEqual([held.decision, held.skipReason], ["skip", "awaiting-human"], "a person holds the PR"); +}); diff --git a/.github/scripts/pr-unblock/escalation-routes.json b/.github/scripts/pr-unblock/escalation-routes.json index a8ba8fe220..0d80a409b2 100644 --- a/.github/scripts/pr-unblock/escalation-routes.json +++ b/.github/scripts/pr-unblock/escalation-routes.json @@ -15,6 +15,8 @@ "feedback-not-landed": { "activity": "address-feedback", "route": "fail", "why": "a fix the lane recorded did not reach the branch, so none of its review replies was posted" }, "feedback-invalid": { "activity": "address-feedback", "route": "fail", "why": "the review replies the lane recorded did not pass validation, so none was posted" }, "feedback-partial": { "activity": "address-feedback", "route": "fail", "why": "posting the review replies stopped partway, or a fixed thread could not be resolved" }, + "feedback-untaken-threads": { "activity": "address-feedback", "route": "fail", "why": "the pull request has an unresolved review thread the lane does not take (a reviewer it does not act on, a review of an older head, or a thread it already answered), so nothing will resolve it" }, + "feedback-publish-failed": { "activity": "address-feedback", "route": "fail", "why": "the reply job failed before it decided anything (a lane token it could not get, or a script error), so no reply was posted" }, "feedback-incomplete": { "activity": "address-feedback", "route": "fail", "why": "the lane did not record a disposition for every review thread it kept, so none was posted" }, "feedback-disputed": { "activity": "address-feedback", "route": "hold", "why": "a disputed review finding is one only a person decides (a security-review or CRITICAL finding, one no known review lane wrote, or a thread a person directed)" }, "feedback-stood-open": { "activity": "address-feedback", "route": "fail", "why": "an adjudicator found a review finding stands, and the next round left it open again" }, diff --git a/.github/scripts/pr-unblock/feedback-dispatch.mjs b/.github/scripts/pr-unblock/feedback-dispatch.mjs index 627e104b6f..06c0c3e2da 100644 --- a/.github/scripts/pr-unblock/feedback-dispatch.mjs +++ b/.github/scripts/pr-unblock/feedback-dispatch.mjs @@ -74,7 +74,7 @@ import { appendFileSync, readFileSync } from "node:fs"; import { pathToFileURL } from "node:url"; import { commitPulls } from "../../actions/check-trusted-trigger/check-trusted-trigger.mjs"; -import { fixRequesters, humanGatedLabelFrom, threadAuthors, trustedHumansFrom } from "./address-feedback.mjs"; +import { fixRequesters, humanGatedLabelFrom, onReviewedHead, reviewedHeads, threadAuthors, trustedHumansFrom } from "./address-feedback.mjs"; import { commentItems, gateSource, listLabelEvents } from "./gate-source.mjs"; import { emptyRecord, isHandled, markHandled, readRecord, writeRecord } from "./dispatch-record.mjs"; import { anyHandles, LANE_STOP, limitResetsAt, permanentlyHandles } from "./limit-stop.mjs"; @@ -137,23 +137,37 @@ export function markedHead(comments, lane, markerAuthorId = DEFAULTS.markerAutho return SHA.test(sha) ? sha : ""; } -// An unresolved thread, read in full, that threadAuthors accepts. -export function hasWorkableThread(threads, { reviewerId, appId, trustedHumans, requesters = null, fixers = requesters ?? new Set() }) { - return threads.some( - (thread) => - thread.isResolved === false && - thread.comments.pageInfo?.hasNextPage !== true && - threadAuthors( - thread.comments.nodes.map((node) => ({ - authorId: node.author?.databaseId, - createdAt: node.createdAt, - body: node.body, - lastEditedAt: node.lastEditedAt, - editorId: node.editor?.databaseId, - })), - { reviewerId, appId, trustedHumans, requesters, fixers }, - ) !== null, - ); +// An unresolved thread, read in full, that the select job keeps: threadAuthors +// accepts it and its review is on a marker-confirmed head (onReviewedHead). +// `heads` is reviewedHeads of the PR's issue comments. +export function hasWorkableThread(threads, { reviewerId, appId, trustedHumans, heads, requesters = null, fixers = requesters ?? new Set() }) { + return threads.some((thread) => { + if (thread.isResolved !== false || thread.comments.pageInfo?.hasNextPage === true) { + return false; + } + const nodes = thread.comments.nodes; + const authors = threadAuthors( + nodes.map((node) => ({ + authorId: node.author?.databaseId, + createdAt: node.createdAt, + body: node.body, + lastEditedAt: node.lastEditedAt, + editorId: node.editor?.databaseId, + })), + { reviewerId, appId, trustedHumans, requesters, fixers }, + ); + const review = nodes[0]?.pullRequestReview; + return ( + authors !== null && + onReviewedHead({ + authors, + firstAuthorId: nodes[0].author?.databaseId, + review: review && { user: { id: review.author?.databaseId }, commit_id: review.commit?.oid }, + heads, + reviewerId, + }) + ); + }); } export const VERIFY_WORKFLOW = ".github/workflows/pr-verify.yml"; @@ -293,7 +307,7 @@ export async function dispatchFeedback({ permission: api.getPermission, }); const requesters = gated ? fixers : null; - if (!hasWorkableThread(threads, { reviewerId: settings.reviewerId, appId: appSenderId, trustedHumans, requesters, fixers })) { + if (!hasWorkableThread(threads, { reviewerId: settings.reviewerId, appId: appSenderId, trustedHumans, heads: reviewedHeads(comments), requesters, fixers })) { return skip( gated ? `#${number} is labeled ${humanGatedLabel} (${gate.source}: ${gate.reason}) and has no thread a trusted person with write access asked the lane to fix` @@ -432,6 +446,7 @@ const THREADS_QUERY = `query($owner: String!, $name: String!, $number: Int!, $af author { login ... on Bot { databaseId } ... on User { databaseId } } createdAt body lastEditedAt editor { ... on Bot { databaseId } ... on User { databaseId } } + pullRequestReview { author { ... on Bot { databaseId } ... on User { databaseId } } commit { oid } } } pageInfo { hasNextPage } } diff --git a/.github/scripts/pr-unblock/feedback-dispatch.test.mjs b/.github/scripts/pr-unblock/feedback-dispatch.test.mjs index 009d86a63c..764fd9b3eb 100644 --- a/.github/scripts/pr-unblock/feedback-dispatch.test.mjs +++ b/.github/scripts/pr-unblock/feedback-dispatch.test.mjs @@ -46,9 +46,12 @@ const failedRun = (line, completed = "2026-10-17T10:00:00Z") => ({ completed_at: completed, output: { title: "Failed", summary: line === null ? "The activity step failed." : `${line}\n\nThe activity step failed.` }, }); -const thread = (authors, isResolved = false) => ({ +// The first comment's review, as THREADS_QUERY reads it: its author's +// review of `reviewed`. +const onReview = (author, reviewed = HEAD) => ({ pullRequestReview: { author, commit: { oid: reviewed } } }); +const thread = (authors, isResolved = false, reviewed = HEAD) => ({ isResolved, - comments: { nodes: authors.map((author) => ({ author })), pageInfo: { hasNextPage: false } }, + comments: { nodes: authors.map((author, i) => ({ author, ...(i === 0 ? onReview(author, reviewed) : {}) })), pageInfo: { hasNextPage: false } }, }); // The #7794 shape: the reviewer bot's finding, the lanes App's UNCERTAIN reply // (publish-feedback.mjs renderReply), then `reply`, each a minute apart. @@ -58,7 +61,7 @@ const decidedThread = (reply = { author: KYLE }) => ({ isResolved: false, comments: { nodes: [ - { author: CLAUDE, body: "finding" }, + { author: CLAUDE, body: "finding", ...onReview(CLAUDE) }, { author: LANE, body: "**pr-unblock / address-feedback: UNCERTAIN, left open for a person**\n\nnote" }, { body: "Owner decision: drop the reference.", ...reply }, ].map((node, i) => ({ createdAt: `2026-10-17T20:0${i}:00Z`, lastEditedAt: null, editor: null, ...node })), @@ -240,7 +243,7 @@ test("the security review workflow is an upstream too", async () => { // footer (ci-workflows publish-review publish.cjs commentBody at v0.63.0). const laneThread = (body) => ({ isResolved: false, - comments: { nodes: [{ author: ACTIONS_BOT_NODE, body }], pageInfo: { hasNextPage: false } }, + comments: { nodes: [{ author: ACTIONS_BOT_NODE, body, ...onReview(ACTIONS_BOT_NODE) }], pageInfo: { hasNextPage: false } }, }); test("a github-actions[bot] thread ending in the lane footer dispatches like the reviewer bot's", async () => { @@ -517,7 +520,7 @@ const VERIFY_RUN = upstreamRun({ id: 99, path: ".github/workflows/pr-verify.yml" // run at the head. const VERIFY_CREATED = "2026-10-17T12:00:00Z"; function verifyApi(options = {}) { - const note = { isResolved: false, comments: { nodes: [{ author: ACTIONS_BOT_NODE, body: "\nnotes" }], pageInfo: { hasNextPage: false } } }; + const note = { isResolved: false, comments: { nodes: [{ author: ACTIONS_BOT_NODE, body: "\nnotes", ...onReview(ACTIONS_BOT_NODE) }], pageInfo: { hasNextPage: false } } }; const api = fakeApi({ threads: [note], ...options }); api.getWorkflowRun = async (id) => (id === VERIFY_RUN.id ? { ...VERIFY_RUN, created_at: VERIFY_CREATED } : null); api.listCompletedRuns = async () => [upstreamRun({ id: 13, created_at: "2026-10-17T10:30:00Z" })]; @@ -896,7 +899,7 @@ test("without needs-human a trusted reply dispatches with no permission read", a const askedThread = (reply) => ({ isResolved: false, comments: { - nodes: [{ author: CLAUDE, body: "finding" }, reply].map((node, i) => ({ + nodes: [{ author: CLAUDE, body: "finding", ...onReview(CLAUDE, OLD_HEAD) }, reply].map((node, i) => ({ createdAt: `2026-10-17T20:0${i}:00Z`, lastEditedAt: null, editor: null, @@ -972,6 +975,24 @@ test("the API client reads review threads across pages through GraphQL", async ( bodies.map((body) => body.variables.after), [null, "c1"], ); + // The first comment's review and its commit, which the select job reads too. + assert.match(bodies[0].query, /pullRequestReview \{ author \{ \.\.\. on Bot \{ databaseId \} \.\.\. on User \{ databaseId \} \} commit \{ oid \} \}/); +}); + +// #10092: the dispatcher fired for a thread whose review is on an older head, +// which the select job never keeps; its neutral skip then handled the head +// for good. Both now judge the review's head with onReviewedHead. +test("a thread reviewed on an older head dispatches nothing, as select would keep none", async () => { + for (const author of [CLAUDE, ACTIONS_BOT_NODE]) { + const body = author === CLAUDE ? "finding" : "The loop skips the last item.\n\nclaude-lane: claude-review"; + const stale = { isResolved: false, comments: { nodes: [{ author, body, ...onReview(author, OLD_HEAD) }], pageInfo: { hasNextPage: false } } }; + const api = fakeApi({ threads: [stale] }); + assert.equal((await decide(api)).result, null); + assert.deepEqual(api.calls.dispatch, []); + } + const otherReviewer = { isResolved: false, comments: { nodes: [{ author: CLAUDE, body: "finding", ...onReview(KYLE) }], pageInfo: { hasNextPage: false } } }; + assert.equal((await decide(fakeApi({ threads: [otherReviewer] }))).result, null, "a review by someone else"); + assert.equal((await decide(fakeApi({ threads: [thread([CLAUDE])] }))).result.number, 42, "the current head dispatches"); }); // The record one run leaves for the next at the same head. `counting` wraps @@ -1135,6 +1156,7 @@ function sweepApi({ openPulls = pull(), checksAt = { [HEAD]: [failedRound, noMod // Both review lanes marked each PR's head reviewed. const headOf = (number) => openPulls.find((p) => p.number === number).head.sha; api.listIssueComments = async (number) => [marker("claude-review", headOf(number)), marker("claude-security-review", headOf(number))]; + api.listReviewThreads = async (number) => [thread([CLAUDE], false, headOf(number))]; api.listCheckRuns = async ({ sha, checkName }) => (checksAt[sha] ?? []).filter((run) => (run.name ?? "pr-unblock / address-feedback") === checkName); api.listCompletedRuns = async (path, head) => (path.endsWith("pr-review-hosted.yml") ? (reviewRuns[head] ?? []) : []); return api; diff --git a/.github/scripts/pr-unblock/report-escalation.test.sh b/.github/scripts/pr-unblock/report-escalation.test.sh index b63c912dab..a2ef8928f8 100755 --- a/.github/scripts/pr-unblock/report-escalation.test.sh +++ b/.github/scripts/pr-unblock/report-escalation.test.sh @@ -143,7 +143,7 @@ HOLD_REASONS=(protected-path-conflict workflow-staged conflict-patch-protected-p ci-fix-protected-path ci-fix-weaken-required) FAIL_REASONS=(unexpected-path-staged repeat-cap unresolved-conflict conflict-patch-invalid conflict-patch-not-applied feedback-not-converging feedback-verifier-open feedback-not-landed feedback-invalid feedback-partial - feedback-incomplete feedback-stood-open feedback-unadjudicated + feedback-incomplete feedback-stood-open feedback-unadjudicated feedback-untaken-threads feedback-publish-failed dependency-round-cap dependency-invalid dependency-fix-not-landed dependency-config-unreadable dependency-files-truncated dependency-control-uncited ci-fix-round-cap ci-fix-empty ci-fix-invalid ci-fix-not-applied ci-fix-escalated) diff --git a/.github/workflows/pr-run-activity-write.yml b/.github/workflows/pr-run-activity-write.yml index 2bf2395529..1e2868988b 100644 --- a/.github/workflows/pr-run-activity-write.yml +++ b/.github/workflows/pr-run-activity-write.yml @@ -404,6 +404,7 @@ on: or to a failing status, escalation-routes.json): for the review-replies script feedback-uncertain, feedback-not-landed or feedback-invalid, feedback-partial, feedback-incomplete, + feedback-publish-failed (the reply job failed before deciding), feedback-disputed, feedback-stood-open, feedback-verifier-open, feedback-unadjudicated, feedback-question-intent, feedback-question-design or feedback-question-security; for the @@ -486,7 +487,8 @@ jobs: || '' }} can-commit: ${{ steps.config.outputs.can-commit }} escalation-reason: >- - ${{ steps.act-script.outputs.escalation-reason || steps.patch-select.outputs.escalation-reason }} + ${{ steps.act-script.outputs.escalation-reason || steps.patch-select.outputs.escalation-reason + || steps.publish-failed.outputs.reason }} rung-up: ${{ steps.patch-select.outputs.skip-reason == 'rung-up' }} patch-skip-reason: ${{ steps.patch-select.outputs.skip-reason }} # The verdict step's skip reason, after the gate's reason. @@ -601,6 +603,7 @@ jobs: "" | protected-path-conflict | workflow-staged | unexpected-path-staged | repeat-cap | unresolved-conflict) ;; feedback-protected-path | feedback-round-cap | feedback-not-converging | feedback-uncertain | feedback-not-landed | feedback-invalid) ;; feedback-partial | feedback-incomplete | feedback-disputed) ;; + feedback-untaken-threads | feedback-publish-failed) ;; feedback-unadjudicated | feedback-question-intent | feedback-question-design) ;; feedback-question-security | feedback-stood-open | feedback-verifier-open) ;; dependency-round-cap | dependency-invalid | dependency-fix-not-landed) ;; @@ -2018,6 +2021,14 @@ jobs: overwrite: true # Written after head code ran, so the report job only cross-checks it. + # A reply job that failed before its script decided anything (the lane + # token, a script error) names a stop, so the caller posts a failing + # status and the lane-stopped comment instead of nothing. + - name: Name a failed reply job's stop + id: publish-failed + if: failure() && inputs.activity == 'publish-feedback' && steps.act-script.outcome != 'success' + run: echo "reason=feedback-publish-failed" >>"$GITHUB_OUTPUT" + - name: Write the verdict if: always() env: diff --git a/docs/conventions/pr-pipeline/pr-run-activity.md b/docs/conventions/pr-pipeline/pr-run-activity.md index 023116445a..71fbeac328 100644 --- a/docs/conventions/pr-pipeline/pr-run-activity.md +++ b/docs/conventions/pr-pipeline/pr-run-activity.md @@ -901,7 +901,12 @@ the base's `.github/scripts/pr-unblock/address-feedback.mjs`, `apply-patch.mjs` same way. On a PR carrying the base's human-gated role label it keeps a thread only when one of those replies is such a fix request by a write-access human, and reports `fix-requested=true`, which lifts only the human-gated hold of the push job. No kept - thread skips `not-applicable`, or `awaiting-human` on such a PR. A PR diff touching + thread skips `awaiting-human` on such a PR; elsewhere it escalates `feedback-untaken-threads` + (routed to a failing status) while any review thread is unresolved, and skips `not-applicable` + only when none is. The dispatcher (`feedback-dispatch.mjs` `hasWorkableThread`) judges a + thread's review head with the same `onReviewedHead`, so it dispatches no thread this step would + not keep. A reply job that fails before its script decides (its lane token, a script error) + names `feedback-publish-failed`, routed to a failing status. A PR diff touching `.github/workflows/**` escalates `feedback-protected-path`. Rounds count as the larger of the round markers the lanes App posted and the PR commits carrying a `github-actions` `pr-unblock / address-feedback` check run that ended `success` or `failure` (a round whose From a431c0c76b15ee320420f15816382957634a1491 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 26 Oct 2026 08:32:18 -0400 Subject: [PATCH 14/33] fix(pr-pipeline): count only pr-require-checks lanes for a fix-ci clear The fix-ci clear counted a failure from any GitHub Actions check, so a red advisory pr-test-windows run kept clearing the label while ci-status, which does not aggregate it, went green with the label gone. The read now lists the head's pr-require-checks.yml runs (actions: read on the same job token) and counts only check runs in their check suites. It reads every check run on the head (filter=all), so a contract-only run's later skip of a lane does not hide that lane's failure. Co-Authored-By: Claude Opus 5.5 --- .../fix-pr-unblock-route-not-park-5c1e7a20.md | 2 +- .../scripts/pr-merge/clear-escalations.mjs | 34 +++++++++++++------ .../pr-merge/clear-escalations.test.mjs | 15 ++++++-- .github/workflows/pr-merge.yml | 4 +-- docs/conventions/pr-pipeline/README.md | 2 +- 5 files changed, 40 insertions(+), 17 deletions(-) diff --git a/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md b/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md index 17e0cce840..9c86619881 100644 --- a/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md +++ b/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md @@ -6,7 +6,7 @@ bump: minor - A `pr-unblock` run that stops now ends on the route its reason has in [`escalation-routes.json`](https://github.com/melodic-software/claude-code-plugins/blob/main/.github/scripts/pr-unblock/escalation-routes.json), the one routing table that `report-escalation.sh`, `gate-source.mjs` and `clear-escalations.mjs` read. Only a reason a person must settle keeps the `needs-human` label: a path the lane token may not push or a supply-chain refusal, an intent, design or security question, a person's own thread or hold, or a finding no adjudicator may judge (security-review or CRITICAL). Every other reason (for example `ci-fix-escalated`, `feedback-not-landed`, the round and repeat caps) ends with no label, a failing commit status on the head the activity gated (`pr-unblock/conflict` for `resolve-conflicts`, `pr-unblock/` otherwise, posted by the `report-stop-status` job that replaces `report-conflict-status`), and one lanes App comment per activity that the next stop edits in place. A new head carries no status. - An `UNCERTAIN` or `VALID-defer` review reply now goes to the adjudicator as an `INCORRECT` one does, instead of to a person. A pr-verify note left open ends `feedback-verifier-open`, and a finding that stood at adjudication and was left open again ends `feedback-stood-open`; both route to a failing status. A round's reason held for a person now outranks any reason routed to a failing status. -- `clear-escalations` takes off a `needs-human` an older run set for a reason now routed to a failing status: a `ci-fix-*` reason once a GitHub Actions check on the current head other than `ci-status` reads failed, any other once the head moved past the one the lane stopped on. +- `clear-escalations` takes off a `needs-human` an older run set for a reason now routed to a failing status: a `ci-fix-*` reason once a lane check of a `pr-require-checks.yml` run on the current head reads failed, any other once the head moved past the one the lane stopped on. - A pr-unblock stop routed to a failing status now skips `lane-stopped` (a new `skip-reason`) instead of `awaiting-human`. On a select job's own check (a cap, or a conflict no model can resolve) it ends the head, so the dispatchers stop re-running a deterministic stop. On the push, reply or verdict job's check, the dispatcher retries the same head under the cap it already keeps: - `limit-stops-per-head` for resolve-conflicts and review-dependency - `fix-ci-restarts` and the per-head cap for fix-ci diff --git a/.github/scripts/pr-merge/clear-escalations.mjs b/.github/scripts/pr-merge/clear-escalations.mjs index 63ae9eb011..7f66d544d1 100755 --- a/.github/scripts/pr-merge/clear-escalations.mjs +++ b/.github/scripts/pr-merge/clear-escalations.mjs @@ -33,11 +33,12 @@ // protected-path-conflict is left to resolve // feedback-protected-path the PR changes nothing under .github/workflows/ // a reason escalation-routes.json routes `fail` (a lane run before that -// route set the label): a fix-ci reason once a GitHub Actions check on -// the current head other than ci-status reads failed (fix-ci-context's +// route set the label): a fix-ci reason once a lane check of a +// pr-require-checks run on the current head reads failed (fix-ci-context's // failingChecks; ci-status itself fails on the label, so it proves -// nothing), so the head stays red without the label; any other once the -// head is not one its escalations name as their resume key +// nothing, and an advisory lane's failure leaves ci-status green), so the +// head stays red without the label; any other once the head is not one +// its escalations name as their resume key // A listing that may be cut short, or a mergeable state not yet computed, // counts as the reason still holding. // Any other reason is decided by a person and is never cleared here. @@ -61,8 +62,9 @@ // // Environment: // GH_TOKEN the lane App token (pull-requests and issues write) -// CHECKS_TOKEN the job's own token (checks: read), for the head's -// check runs, which the lane token cannot read +// CHECKS_TOKEN the job's own token (checks: read, actions: read), for +// the head's pr-require-checks runs and check runs, +// which the lane token cannot read // GITHUB_REPOSITORY owner/repo // KILL_SWITCH `proceed` from the kill-switch step; anything else stops // TRACKER_CONFIG the base .work-item-tracker.json, for the human-gated @@ -126,13 +128,25 @@ export const CLEARS_PER_RUN = 5; // The check-run pages read per head, as fix-ci-context reads them. const CHECK_PAGES = 3; -// Whether a GitHub Actions check on the head other than ci-status failed. One -// read per PR; a listing that fails, or may be cut short, reads as not failing. +// The workflow whose lanes ci-status aggregates. An advisory lane's failure +// (pr-test-windows) leaves ci-status green once the label is off, so only +// this workflow's checks count. +const REQUIRED_WORKFLOW = "pr-require-checks.yml"; + +// Whether a lane check of a pr-require-checks run on the head (not ci-status) +// failed. Every such run counts: a contract-only run posts its lanes skipped, +// and failingChecks prefers a lane's real result over that skip. One read per +// PR; a listing that fails, or may be cut short, reads as not failing. async function headFailing({ checks, repo, pull }) { try { + const required = (await checks("GET", `/repos/${repo}/actions/workflows/${REQUIRED_WORKFLOW}/runs?head_sha=${pull.head.sha}&per_page=100`))?.workflow_runs; + if (!Array.isArray(required) || required.length >= 100) { + return false; + } + const suites = new Set(required.map((run) => run.check_suite_id)); const runs = []; for (let page = 1; ; page += 1) { - const listed = (await checks("GET", `/repos/${repo}/commits/${pull.head.sha}/check-runs?filter=latest&per_page=100&page=${page}`))?.check_runs; + const listed = (await checks("GET", `/repos/${repo}/commits/${pull.head.sha}/check-runs?filter=all&per_page=100&page=${page}`))?.check_runs; if (!Array.isArray(listed)) { return false; } @@ -144,7 +158,7 @@ async function headFailing({ checks, repo, pull }) { return false; } } - return failingChecks(runs.filter((run) => run.app?.slug === "github-actions")).length > 0; + return failingChecks(runs.filter((run) => run.app?.slug === "github-actions" && suites.has(run.check_suite?.id))).length > 0; } catch { return false; } diff --git a/.github/scripts/pr-merge/clear-escalations.test.mjs b/.github/scripts/pr-merge/clear-escalations.test.mjs index a2984e252b..ae1ecd4b59 100644 --- a/.github/scripts/pr-merge/clear-escalations.test.mjs +++ b/.github/scripts/pr-merge/clear-escalations.test.mjs @@ -32,7 +32,9 @@ const escalated = (reason = "workflow-staged", { user = BOT, at = "2026-10-18T10 // `mainFiles` the files main changed since its merge base with the head. // `headChecks` is the head's check runs, as the job token lists them, or an // Error its read throws. -const check = (name, conclusion, { id = 1, slug = "github-actions" } = {}) => ({ id, name, status: "completed", conclusion, app: { slug } }); +// The head's pr-require-checks runs are suites 500 and 501. +const REQUIRED_SUITES = [500, 501]; +const check = (name, conclusion, { id = 1, slug = "github-actions", suite = 500 } = {}) => ({ id, name, status: "completed", conclusion, app: { slug }, check_suite: { id: suite } }); const RED = [check("test-node", "failure")]; function world({ timeline, mainFiles = [{ filename: "README.md" }], mergeable = false, pullFiles = [], author, headChecks = [] }) { const calls = []; @@ -41,7 +43,10 @@ function world({ timeline, mainFiles = [{ filename: "README.md" }], mergeable = if (headChecks instanceof Error) { throw headChecks; } - assert.equal(path, `/repos/${REPO}/commits/${HEAD}/check-runs?filter=latest&per_page=100&page=1`); + if (path === `/repos/${REPO}/actions/workflows/pr-require-checks.yml/runs?head_sha=${HEAD}&per_page=100`) { + return { workflow_runs: REQUIRED_SUITES.map((id) => ({ id: id * 10, check_suite_id: id })) }; + } + assert.equal(path, `/repos/${REPO}/commits/${HEAD}/check-runs?filter=all&per_page=100&page=1`); return { check_runs: headChecks }; }; const github = async (method, path, body) => { @@ -249,12 +254,16 @@ test("a fix-ci label comes off only while the head's own checks read failed", as ["no checks", [], false], ["another app's failure", [check("test-node", "failure", { slug: "someone-else" })], false], ["read fails", new Error("HTTP 502"), false], + // An advisory lane is no pr-require-checks lane: ci-status would go green. + ["only pr-test-windows red", [check("test-windows", "failure", { suite: 900 }), check("test-node", "success", { id: 2 })], false], + // A contract-only run's later skip does not hide the real run's failure. + ["a failed lane with a newer skipped duplicate", [check("test-node", "failure"), check("test-node", "skipped", { id: 2, suite: 501 })], true], ]) { const w = world({ timeline, headChecks }); const { code } = await run(w); assert.equal(code, 0, label); assert.equal(w.writes().at(-1)?.method === "DELETE", cleared, label); - assert.equal(w.calls.filter((call) => call.token === "checks").length, 1, label); + assert.ok(w.calls.filter((call) => call.token === "checks").length <= 2, label); } }); diff --git a/.github/workflows/pr-merge.yml b/.github/workflows/pr-merge.yml index 02b26385d9..3e9248e512 100644 --- a/.github/workflows/pr-merge.yml +++ b/.github/workflows/pr-merge.yml @@ -1433,8 +1433,8 @@ jobs: timeout-minutes: 3 env: GH_TOKEN: ${{ steps.lane-token.outputs.token }} - # The job's own token reads a fix-ci stop's head checks (checks: - # read); the lane token holds no Checks permission. + # The job's own token reads a fix-ci stop's head runs and checks + # (actions: read, checks: read); the lane token holds neither. CHECKS_TOKEN: ${{ github.token }} KILL_SWITCH: ${{ steps.kill.outputs.proceed == 'true' && 'proceed' || '' }} LANE_PRS: ${{ steps.mode.outputs.lane-prs }} diff --git a/docs/conventions/pr-pipeline/README.md b/docs/conventions/pr-pipeline/README.md index 2233092f63..a68a42a4c0 100644 --- a/docs/conventions/pr-pipeline/README.md +++ b/docs/conventions/pr-pipeline/README.md @@ -562,7 +562,7 @@ head: | `workflow-staged` | main changes nothing under `.github/workflows/` since its merge base with the head | | `unresolved-conflict`, `protected-path-conflict` | GitHub reports the PR mergeable | | `feedback-protected-path` | the PR changes nothing under `.github/workflows/` | -| a reason `escalation-routes.json` routes `fail`, on a label set before it did | a `ci-fix-*` reason once a GitHub Actions check on the current head other than `ci-status` reads failed (`ci-status` itself fails on the label, so it is not read; green, missing or unreadable keeps the label); any other once the head is not one its escalations name as the resume key | +| a reason `escalation-routes.json` routes `fail`, on a label set before it did | a `ci-fix-*` reason once a lane check of a `pr-require-checks.yml` run on the current head reads failed (`ci-status` itself fails on the label, so it is not read, and an advisory lane such as `pr-test-windows` does not count; green, missing or unreadable keeps the label); any other once the head is not one its escalations name as the resume key | | `head-moved` (pr-merge's `babysit-merge-verify-escalation`, `babysit-merge-risk` and `babysit-merge-shared-copy-escalation`, with no pr-unblock escalation since the label, and the marked App comment written last before the label is pr-merge's and within a minute of it) | the newest App comment carrying each marker names, as ``at head `` ``, a 40-hex head other than the current one; pr-merge then judges the new head and escalates again if its refusal still holds | It removes only a label whose newest `labeled` event has the lanes App as its actor, and keeps it From abb345eed734965d6e562fbd265f81e16fc4a8ae Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 26 Oct 2026 08:39:26 -0400 Subject: [PATCH 15/33] fix(pr-pipeline): dispatch untaken review threads once and derive a failed reply's stop The previous commit let select escalate feedback-untaken-threads, but the dispatcher still refused a PR whose only unresolved threads select will not keep, so the escalation could not fire for #10092's shape. On an ungated head the dispatcher now fires for any unresolved thread. Select escalates, and its not-run job's lane-stopped check ends the head: a reply check's lane-stopped skip no longer releases a head whose address-feedback check is a select stop, so the next tick does not fire again. A gated head still dispatches only for a workable thread, and onReviewedHead still decides what select keeps. feedback-publish-failed is now derived in pr-unblock.yml from needs.publish-feedback.result == 'failure' with no escalation reason, instead of a failure() step's output in the write runner, which relied on a failed job's outputs reaching the caller. Co-Authored-By: Claude Opus 5.5 --- .../fix-pr-unblock-route-not-park-5c1e7a20.md | 2 +- .../actions/resolve-config/callers.test.mjs | 30 +++--- .../scripts/pr-unblock/feedback-dispatch.mjs | 23 ++-- .../pr-unblock/feedback-dispatch.test.mjs | 102 ++++++++++++------ .github/workflows/pr-run-activity-write.yml | 13 +-- .github/workflows/pr-unblock.yml | 6 ++ .../pr-pipeline/pr-run-activity.md | 9 +- 7 files changed, 112 insertions(+), 73 deletions(-) diff --git a/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md b/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md index 9c86619881..228a9a4096 100644 --- a/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md +++ b/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md @@ -15,6 +15,6 @@ bump: minor The fix-ci dispatcher reads a fix-ci run and its in-run rung-up, which both post a `pr-unblock / fix-ci` check, as one attempt: a stop in either one's push job makes the attempt a failed one, and the pair counts once toward the caps. The fix-ci and address-feedback retries run one model rung up, read in the run from the head's own stop record. A retry that acts and does not stop posts success over the failing status; one that ends on a skip (a reply job with no model turn) leaves it. fix-ci's model escalations `protected-path` and `weaken-required` stay with a person, as `ci-fix-protected-path` and the new `ci-fix-weaken-required`. -- An address-feedback round that keeps no thread while a review thread is still unresolved (a reviewer the lane does not act on, such as Codex, a review of an older head, a thread it already answered) now ends `feedback-untaken-threads`, routed to a failing status, instead of a silent `not-applicable` skip. The dispatcher judges a thread's review head as the select job does, so it no longer dispatches for a thread the select job would skip, whose neutral check then marked the head handled for good. A reply job that fails before its script decides (a lane token it could not get) ends `feedback-publish-failed`, also routed to a failing status, instead of posting no stop. +- An address-feedback round that keeps no thread while a review thread is still unresolved (a reviewer the lane does not act on, such as Codex, a review of an older head, a thread it already answered) now ends `feedback-untaken-threads`, routed to a failing status, instead of a silent `not-applicable` skip. The dispatcher now dispatches an ungated head once for any unresolved thread, so that stop is reached; the select job's lane-stopped check ends the head, and the next tick does not dispatch it again. A reply job that fails with no escalation reason (a lane token it could not get) ends `feedback-publish-failed`, also routed to a failing status, instead of posting no stop. - The monitoring lane gains signal 10, `stalled lane stop`. Each open pull request whose pr-unblock stop has had no new attempt on its head for over 2 hours is listed in one rolling issue. A skip is not an attempt. A stop includes a `needs-human` that `clear-escalations` took off for a reason now routed to a failing status. It reads the 40 most recently updated open pull requests and their newest comments in one GraphQL query, reading up to 200 older comments only for a pull request whose newest name no stop on its head; past 40 pull requests, or past those comments, it names what it left unread in the run's gaps. - The merge path floor (`babysit_risk.HIGH_FLOOR`, `merge-holds.mjs` `HOLD_PATH_FLOOR`, and the review lanes' and ci-status's hold lists) also holds `.github/scripts/pr-unblock/escalation-routes.mjs`, which `clear-escalations.mjs` and `gate-source.mjs` now import. diff --git a/.github/actions/resolve-config/callers.test.mjs b/.github/actions/resolve-config/callers.test.mjs index b7f6760c86..5e230451f6 100644 --- a/.github/actions/resolve-config/callers.test.mjs +++ b/.github/actions/resolve-config/callers.test.mjs @@ -501,23 +501,18 @@ test("pr-unblock.yml: report-stop-status runs when report-escalation does, or af assert.equal(skipOf({ "steps.act-script.outputs.skip-reason": "no-model-turn" }), "no-model-turn"); assert.equal(skipOf({ "steps.gate.outputs.proceed": "false", "steps.gate.outputs.reason": "untrusted-trigger" }), "untrusted-trigger"); // #10092's 20:57 run: the reply job failed at its lane token with no - // escalation reason, so no stop was posted. A reply job that fails before - // its script decided names feedback-publish-failed, routed to a failing - // status, and report-stop-status and report-escalation then run on it. - const named = write.jobs.run.steps.find((step) => step.id === "publish-failed"); - assert.equal(named.if, "failure() && inputs.activity == 'publish-feedback' && steps.act-script.outcome != 'success'"); - assert.equal(named.run.trim(), 'echo "reason=feedback-publish-failed" >>"$GITHUB_OUTPUT"'); - const reasonOf = (extra) => - evaluate(write.jobs.run.outputs["escalation-reason"], { - "steps.act-script.outputs.escalation-reason": "", - "steps.patch-select.outputs.escalation-reason": "", - "steps.publish-failed.outputs.reason": "", - ...extra, - }); - assert.equal(reasonOf({ "steps.publish-failed.outputs.reason": "feedback-publish-failed" }), "feedback-publish-failed"); - assert.equal(reasonOf({ "steps.act-script.outputs.escalation-reason": "feedback-partial", "steps.publish-failed.outputs.reason": "feedback-publish-failed" }), "feedback-partial"); - assert.equal(reasonOf(), ""); - assert.equal(evaluate(job.if, context({ from: "publish-feedback", reason: "feedback-publish-failed", headFrom: "address-feedback-select" })), true); + // escalation reason, so no stop was posted. The caller derives + // feedback-publish-failed from the failed result, and report-stop-status + // and report-escalation then run on it; a recorded reason still wins. + const failed = (extra = {}) => ({ ...context({ headFrom: "address-feedback-select", ...extra }), "needs.publish-feedback.result": "failure" }); + assert.equal(evaluate(job.if, failed()), true, "a failed reply job posts a stop"); + assert.equal(evaluate(job.steps.find((step) => step.name === "Post the stop status").env.ESCALATION_REASON, failed()), "feedback-publish-failed"); + assert.equal( + evaluate(job.steps.find((step) => step.name === "Post the stop status").env.ESCALATION_REASON, failed({ from: "publish-feedback", reason: "feedback-partial" })), + "feedback-partial", + ); + assert.equal(evaluate(escalation.if, failed()), true); + assert.equal(evaluate(escalation.with["escalation-reason"], failed()), "feedback-publish-failed"); assert.equal(stopSkip("feedback-publish-failed"), "lane-stopped", "routed to a failing status"); // The head is a select job's, never a model call's; the reason is the one // report-escalation reads. @@ -613,6 +608,7 @@ const rungUpContext = ({ rungUp = "true", result = "success", escalation = "", s "needs.apply-patch-rung-up.result": result, "needs.apply-patch-rung-up.outputs.escalation-reason": escalation, "needs.apply-patch-rung-up.outputs.patch-skip-reason": skip, + "needs.publish-feedback.result": "skipped", }); test("pr-unblock.yml: a rung-up stop runs fix-ci and apply-patch again at model-rung 1 in the same run", () => { diff --git a/.github/scripts/pr-unblock/feedback-dispatch.mjs b/.github/scripts/pr-unblock/feedback-dispatch.mjs index 06c0c3e2da..e507e2006f 100644 --- a/.github/scripts/pr-unblock/feedback-dispatch.mjs +++ b/.github/scripts/pr-unblock/feedback-dispatch.mjs @@ -77,7 +77,7 @@ import { commitPulls } from "../../actions/check-trusted-trigger/check-trusted-t import { fixRequesters, humanGatedLabelFrom, onReviewedHead, reviewedHeads, threadAuthors, trustedHumansFrom } from "./address-feedback.mjs"; import { commentItems, gateSource, listLabelEvents } from "./gate-source.mjs"; import { emptyRecord, isHandled, markHandled, readRecord, writeRecord } from "./dispatch-record.mjs"; -import { anyHandles, LANE_STOP, limitResetsAt, permanentlyHandles } from "./limit-stop.mjs"; +import { anyHandles, LANE_STOP, laneStop, limitResetsAt, permanentlyHandles } from "./limit-stop.mjs"; import { budgetedFetch } from "./request-budget.mjs"; export const DEFAULTS = Object.freeze({ @@ -307,12 +307,16 @@ export async function dispatchFeedback({ permission: api.getPermission, }); const requesters = gated ? fixers : null; - if (!hasWorkableThread(threads, { reviewerId: settings.reviewerId, appId: appSenderId, trustedHumans, heads: reviewedHeads(comments), requesters, fixers })) { - return skip( - gated - ? `#${number} is labeled ${humanGatedLabel} (${gate.source}: ${gate.reason}) and has no thread a trusted person with write access asked the lane to fix` - : `#${number} has no unresolved thread left by the reviewer bot alone, asked to be fixed, or answered by a trusted person`, - ); + // On an ungated head, any unresolved thread dispatches, even one the select + // job will not keep (a reviewer the lane does not act on, a review of an + // older head): select then escalates feedback-untaken-threads, and its + // lane-stopped check handles the head, so the stall gets a stop once. + const workable = hasWorkableThread(threads, { reviewerId: settings.reviewerId, appId: appSenderId, trustedHumans, heads: reviewedHeads(comments), requesters, fixers }); + if (gated && !workable) { + return skip(`#${number} is labeled ${humanGatedLabel} (${gate.source}: ${gate.reason}) and has no thread a trusted person with write access asked the lane to fix`); + } + if (!workable && !threads.some((thread) => thread.isResolved === false)) { + return skip(`#${number} has no unresolved review thread`); } // Both review lanes finish on one head, and their dispatchers queue in one // concurrency group. The first one's run posts no check until it ends, so @@ -353,10 +357,13 @@ export async function dispatchFeedback({ // limit stop is never released: limit-stop.mjs still holds the head until // its reset and counts it toward limits.limit-stops-per-head. let replies = []; + // A select job's own stop (lane-stopped on this check) ends the head; only + // a reply job's skip releases it. + const selectStopped = existing.some(laneStop); if (anyHandles(existing, { now, limits: resolved.limits })) { replies = (await api.listCheckRuns({ sha: head, checkName: settings.replyCheckName, appId: settings.appId })).filter(sinceNotes); const newest = replies.reduce((a, b) => (startOf(b) > startOf(a) ? b : a), replies[0]); - if (releases(newest)) { + if (releases(newest) && !selectStopped) { const cut = startOf(newest); existing = existing.filter((run) => limitResetsAt(run, resolved.limits) !== null || !(startOf(run) < cut)); } diff --git a/.github/scripts/pr-unblock/feedback-dispatch.test.mjs b/.github/scripts/pr-unblock/feedback-dispatch.test.mjs index 764fd9b3eb..b21a3e96eb 100644 --- a/.github/scripts/pr-unblock/feedback-dispatch.test.mjs +++ b/.github/scripts/pr-unblock/feedback-dispatch.test.mjs @@ -256,13 +256,23 @@ test("the code lane's github-actions[bot] thread dispatches like the reviewer bo assert.equal((await decide(api)).result.number, 42); }); -test("a github-actions[bot] thread without a review lane's footer last dispatches nothing", async () => { - for (const body of ["Unquoted input.", "claude-lane: claude-security-review\nUnquoted input."]) { - const api = fakeApi({ threads: [laneThread(body)] }); - assert.equal((await decide(api)).result, null, body); - assert.deepEqual(api.calls.dispatch, []); - } -}); +// An unresolved thread the select job will not keep still dispatches once on +// an ungated head: select escalates feedback-untaken-threads, so the stall +// ends in a stop instead of silence (#10092). +for (const [label, threads] of [ + ["a github-actions[bot] thread without a review lane's footer last", [laneThread("Unquoted input.")]], + ["a github-actions[bot] thread with the footer first", [laneThread("claude-lane: claude-security-review\nUnquoted input.")]], + ["the only open thread has a human reply", [thread([CLAUDE, KYLE])]], + ["the only open thread is a human's", [thread([KYLE])]], + ["the reply after the lane's escalation is a stranger's", [decidedThread({ author: STRANGER })]], + ["the trusted reply was last edited by a stranger", [decidedThread({ author: KYLE, lastEditedAt: "2026-10-17T20:30:00Z", editor: STRANGER })]], +]) { + test(`${label} dispatches once, for select to escalate feedback-untaken-threads`, async () => { + const api = fakeApi({ threads }); + assert.equal((await decide(api)).result.number, 42); + assert.equal(api.calls.dispatch.length, 1); + }); +} for (const [label, options] of [ ["the upstream is another workflow", { run: { path: ".github/workflows/ci.yml" } }], @@ -290,12 +300,6 @@ for (const [label, options] of [ comments: [marker("claude-review", HEAD, { id: 9999001 }), marker("claude-security-review")], }], ["every thread is resolved", { threads: [thread([CLAUDE], true)] }], - ["the only open thread has a human reply", { threads: [thread([CLAUDE, KYLE])] }], - ["the only open thread is a human's", { threads: [thread([KYLE])] }], - ["the reply after the lane's escalation is a stranger's", { threads: [decidedThread({ author: STRANGER })] }], - ["the trusted reply was last edited by a stranger", { - threads: [decidedThread({ author: KYLE, lastEditedAt: "2026-10-17T20:30:00Z", editor: STRANGER })], - }], ["the head already has an address-feedback check run", { checkRuns: [{ status: "completed", conclusion: "success", output: { title: "Done" } }], }], @@ -924,10 +928,10 @@ for (const [label, options] of [ }], ["a trusted reply that asks no fix", { threads: [askedThread({ author: KYLE_LOGIN, body: "Owner decision: drop it." })] }], ]) { - test(`a thread the lane never answered dispatches nothing for ${label}`, async () => { + test(`a thread the lane never answered dispatches once for select to escalate, for ${label}`, async () => { const api = fakeApi(options); - assert.equal((await decide(api)).result, null); - assert.deepEqual(api.calls.dispatch, []); + assert.equal((await decide(api)).result.number, 42); + assert.equal(api.calls.dispatch.length, 1); }); } @@ -975,24 +979,58 @@ test("the API client reads review threads across pages through GraphQL", async ( bodies.map((body) => body.variables.after), [null, "c1"], ); - // The first comment's review and its commit, which the select job reads too. - assert.match(bodies[0].query, /pullRequestReview \{ author \{ \.\.\. on Bot \{ databaseId \} \.\.\. on User \{ databaseId \} \} commit \{ oid \} \}/); }); -// #10092: the dispatcher fired for a thread whose review is on an older head, -// which the select job never keeps; its neutral skip then handled the head -// for good. Both now judge the review's head with onReviewedHead. -test("a thread reviewed on an older head dispatches nothing, as select would keep none", async () => { - for (const author of [CLAUDE, ACTIONS_BOT_NODE]) { - const body = author === CLAUDE ? "finding" : "The loop skips the last item.\n\nclaude-lane: claude-review"; - const stale = { isResolved: false, comments: { nodes: [{ author, body, ...onReview(author, OLD_HEAD) }], pageInfo: { hasNextPage: false } } }; - const api = fakeApi({ threads: [stale] }); - assert.equal((await decide(api)).result, null); - assert.deepEqual(api.calls.dispatch, []); - } - const otherReviewer = { isResolved: false, comments: { nodes: [{ author: CLAUDE, body: "finding", ...onReview(KYLE) }], pageInfo: { hasNextPage: false } } }; - assert.equal((await decide(fakeApi({ threads: [otherReviewer] }))).result, null, "a review by someone else"); - assert.equal((await decide(fakeApi({ threads: [thread([CLAUDE])] }))).result.number, 42, "the current head dispatches"); +// #10092: a PR whose only unresolved threads the select job will not keep (a +// Codex-root thread, a review of an older head) stalled with no stop. On an +// ungated head the dispatcher fires once; select escalates +// feedback-untaken-threads, and the not-run job's lane-stopped checks (on +// address-feedback and publish-feedback) handle the head, so the next tick +// does not fire again. +const CODEX = { databaseId: 199175422 }; +const selectStopped = [ + roundRun("address-feedback", "neutral", "Skipped: lane-stopped", "2026-10-17T10:00:00Z"), + roundRun("publish-feedback", "neutral", "Skipped: lane-stopped", "2026-10-17T10:00:01Z"), +]; +for (const [label, author, reviewed] of [ + ["a Codex-only PR", CODEX, HEAD], + ["a stale-head-only PR", CLAUDE, OLD_HEAD], +]) { + test(`${label} dispatches once, then not again on the same head`, async () => { + const open = { isResolved: false, comments: { nodes: [{ author, body: "finding", ...onReview(author, reviewed) }], pageInfo: { hasNextPage: false } } }; + const first = fakeApi({ threads: [open] }); + assert.equal((await decide(first)).result.number, 42); + assert.equal(first.calls.dispatch.length, 1); + const second = fakeApi({ threads: [open], checkRuns: selectStopped }); + assert.equal((await decide(second)).result, null); + assert.deepEqual(second.calls.dispatch, []); + }); +} + +test("a PR with no unresolved thread does not dispatch", async () => { + const api = fakeApi({ threads: [thread([CODEX], true), thread([CLAUDE], true)] }); + const { result, lines } = await decide(api); + assert.equal(result, null); + assert.deepEqual(api.calls.dispatch, []); + assert.match(lines.at(-1), /has no unresolved review thread/); +}); + +test("on a gated head only a workable thread dispatches; an untaken one does not", async () => { + const gated = { pulls: pull({ labels: [{ name: "needs-human" }] }) }; + const api = fakeApi({ ...gated, threads: [thread([CODEX])] }); + assert.equal((await decide(api)).result, null); + assert.deepEqual(api.calls.dispatch, []); +}); + +test("the API client reads each thread's review and its commit", async () => { + const bodies = []; + const fetch = async (url, init) => { + bodies.push(JSON.parse(init.body)); + return new Response(JSON.stringify({ data: { repository: { pullRequest: { reviewThreads: { nodes: [], pageInfo: { hasNextPage: false } } } } } })); + }; + const api = createGitHubApi({ token: "t", repository: REPOSITORY, apiUrl: "https://api.example", graphqlUrl: "https://api.example/graphql", fetch }); + await api.listReviewThreads(42); + assert.match(bodies[0].query, /pullRequestReview \{ author \{ \.\.\. on Bot \{ databaseId \} \.\.\. on User \{ databaseId \} \} commit \{ oid \} \}/); }); // The record one run leaves for the next at the same head. `counting` wraps diff --git a/.github/workflows/pr-run-activity-write.yml b/.github/workflows/pr-run-activity-write.yml index 1e2868988b..24f05720d1 100644 --- a/.github/workflows/pr-run-activity-write.yml +++ b/.github/workflows/pr-run-activity-write.yml @@ -404,7 +404,7 @@ on: or to a failing status, escalation-routes.json): for the review-replies script feedback-uncertain, feedback-not-landed or feedback-invalid, feedback-partial, feedback-incomplete, - feedback-publish-failed (the reply job failed before deciding), + (feedback-publish-failed is the caller's, for a reply job that failed), feedback-disputed, feedback-stood-open, feedback-verifier-open, feedback-unadjudicated, feedback-question-intent, feedback-question-design or feedback-question-security; for the @@ -487,8 +487,7 @@ jobs: || '' }} can-commit: ${{ steps.config.outputs.can-commit }} escalation-reason: >- - ${{ steps.act-script.outputs.escalation-reason || steps.patch-select.outputs.escalation-reason - || steps.publish-failed.outputs.reason }} + ${{ steps.act-script.outputs.escalation-reason || steps.patch-select.outputs.escalation-reason }} rung-up: ${{ steps.patch-select.outputs.skip-reason == 'rung-up' }} patch-skip-reason: ${{ steps.patch-select.outputs.skip-reason }} # The verdict step's skip reason, after the gate's reason. @@ -2021,14 +2020,6 @@ jobs: overwrite: true # Written after head code ran, so the report job only cross-checks it. - # A reply job that failed before its script decided anything (the lane - # token, a script error) names a stop, so the caller posts a failing - # status and the lane-stopped comment instead of nothing. - - name: Name a failed reply job's stop - id: publish-failed - if: failure() && inputs.activity == 'publish-feedback' && steps.act-script.outcome != 'success' - run: echo "reason=feedback-publish-failed" >>"$GITHUB_OUTPUT" - - name: Write the verdict if: always() env: diff --git a/.github/workflows/pr-unblock.yml b/.github/workflows/pr-unblock.yml index dafa20b10d..29cd53d392 100644 --- a/.github/workflows/pr-unblock.yml +++ b/.github/workflows/pr-unblock.yml @@ -2539,6 +2539,7 @@ jobs: || needs.conflict-scan-refusal.outputs.escalation-reason != '' || needs.address-feedback-select.outputs.escalation-reason != '' || needs.publish-feedback.outputs.escalation-reason != '' + || needs.publish-feedback.result == 'failure' || needs.review-dependency-select.outputs.escalation-reason != '' || needs.apply-dependency-verdict.outputs.escalation-reason != '' || needs.fix-ci-context.outputs.escalation-reason != '' @@ -2582,6 +2583,7 @@ jobs: || needs.conflict-scan-refusal.outputs.escalation-reason || needs.address-feedback-select.outputs.escalation-reason || needs.publish-feedback.outputs.escalation-reason + || needs.publish-feedback.result == 'failure' && 'feedback-publish-failed' || needs.review-dependency-select.outputs.escalation-reason || needs.apply-dependency-verdict.outputs.escalation-reason || needs.fix-ci-context.outputs.escalation-reason @@ -2606,6 +2608,8 @@ jobs: - fix-ci-context - apply-patch - apply-patch-rung-up + # A reply job that failed with no reason (its lane token, a script error) + # stops as feedback-publish-failed, derived here from its result. # escalation-reason is set only by the write runner's own steps, the # base's reply, verdict and apply-patch scripts, and the base scripts of # resolve-conflicts-select, conflict-scan-refusal, address-feedback-select, @@ -2623,6 +2627,7 @@ jobs: || needs.conflict-scan-refusal.outputs.escalation-reason != '' || needs.address-feedback-select.outputs.escalation-reason != '' || needs.publish-feedback.outputs.escalation-reason != '' + || needs.publish-feedback.result == 'failure' || needs.review-dependency-select.outputs.escalation-reason != '' || needs.apply-dependency-verdict.outputs.escalation-reason != '' || needs.fix-ci-context.outputs.escalation-reason != '' @@ -2652,6 +2657,7 @@ jobs: || needs.conflict-scan-refusal.outputs.escalation-reason || needs.address-feedback-select.outputs.escalation-reason || needs.publish-feedback.outputs.escalation-reason + || needs.publish-feedback.result == 'failure' && 'feedback-publish-failed' || needs.review-dependency-select.outputs.escalation-reason || needs.apply-dependency-verdict.outputs.escalation-reason || needs.fix-ci-context.outputs.escalation-reason diff --git a/docs/conventions/pr-pipeline/pr-run-activity.md b/docs/conventions/pr-pipeline/pr-run-activity.md index 71fbeac328..f8392f72dc 100644 --- a/docs/conventions/pr-pipeline/pr-run-activity.md +++ b/docs/conventions/pr-pipeline/pr-run-activity.md @@ -903,10 +903,11 @@ the base's `.github/scripts/pr-unblock/address-feedback.mjs`, `apply-patch.mjs` reports `fix-requested=true`, which lifts only the human-gated hold of the push job. No kept thread skips `awaiting-human` on such a PR; elsewhere it escalates `feedback-untaken-threads` (routed to a failing status) while any review thread is unresolved, and skips `not-applicable` - only when none is. The dispatcher (`feedback-dispatch.mjs` `hasWorkableThread`) judges a - thread's review head with the same `onReviewedHead`, so it dispatches no thread this step would - not keep. A reply job that fails before its script decides (its lane token, a script error) - names `feedback-publish-failed`, routed to a failing status. A PR diff touching + only when none is. The dispatcher (`feedback-dispatch.mjs`) dispatches an ungated head for any + unresolved thread, so such a head reaches this step once: its lane-stopped check then ends the + head, which a reply job's skip does not release. A failed reply job with no escalation reason + (its lane token, a script error) stops as `feedback-publish-failed`, which `pr-unblock.yml` + derives from the job's result. A PR diff touching `.github/workflows/**` escalates `feedback-protected-path`. Rounds count as the larger of the round markers the lanes App posted and the PR commits carrying a `github-actions` `pr-unblock / address-feedback` check run that ended `success` or `failure` (a round whose From 764ec0b3e66ae7dd9dff8e55aa950dee5e1c6c04 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 26 Oct 2026 10:48:47 -0400 Subject: [PATCH 16/33] fix(pr-pipeline): fail ci-status on a failing pr-unblock status A fail-routed pr-unblock stop ends in a failing pr-unblock/ commit status, but only ci-status is required, so the merge queue and a hand merge ignored it. ci-status now fails while the newest pr-unblock/* status on the head by github-actions[bot] is failure or error, naming each context; a later success on the same head supersedes it, as report-escalation.sh clear reads it. In the merge queue it reads the queued pull request's head. Unreadable reads fail closed. Co-Authored-By: Claude Opus 5.5 --- .../fix-pr-unblock-route-not-park-5c1e7a20.md | 2 +- .github/workflows/pr-require-checks.yml | 66 ++++++- docs/conventions/pr-pipeline/README.md | 5 + scripts/ci-hold-pr-unblock-status.test.sh | 171 ++++++++++++++++++ 4 files changed, 240 insertions(+), 4 deletions(-) create mode 100755 scripts/ci-hold-pr-unblock-status.test.sh diff --git a/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md b/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md index 228a9a4096..7e91cafe2c 100644 --- a/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md +++ b/.changes/pr-pipeline/fix-pr-unblock-route-not-park-5c1e7a20.md @@ -4,7 +4,7 @@ bump: minor ### Changed -- A `pr-unblock` run that stops now ends on the route its reason has in [`escalation-routes.json`](https://github.com/melodic-software/claude-code-plugins/blob/main/.github/scripts/pr-unblock/escalation-routes.json), the one routing table that `report-escalation.sh`, `gate-source.mjs` and `clear-escalations.mjs` read. Only a reason a person must settle keeps the `needs-human` label: a path the lane token may not push or a supply-chain refusal, an intent, design or security question, a person's own thread or hold, or a finding no adjudicator may judge (security-review or CRITICAL). Every other reason (for example `ci-fix-escalated`, `feedback-not-landed`, the round and repeat caps) ends with no label, a failing commit status on the head the activity gated (`pr-unblock/conflict` for `resolve-conflicts`, `pr-unblock/` otherwise, posted by the `report-stop-status` job that replaces `report-conflict-status`), and one lanes App comment per activity that the next stop edits in place. A new head carries no status. +- A `pr-unblock` run that stops now ends on the route its reason has in [`escalation-routes.json`](https://github.com/melodic-software/claude-code-plugins/blob/main/.github/scripts/pr-unblock/escalation-routes.json), the one routing table that `report-escalation.sh`, `gate-source.mjs` and `clear-escalations.mjs` read. Only a reason a person must settle keeps the `needs-human` label: a path the lane token may not push or a supply-chain refusal, an intent, design or security question, a person's own thread or hold, or a finding no adjudicator may judge (security-review or CRITICAL). Every other reason (for example `ci-fix-escalated`, `feedback-not-landed`, the round and repeat caps) ends with no label, a failing commit status on the head the activity gated (`pr-unblock/conflict` for `resolve-conflicts`, `pr-unblock/` otherwise, posted by the `report-stop-status` job that replaces `report-conflict-status`), and one lanes App comment per activity that the next stop edits in place. A new head carries no status. ci-status, the one required check, fails while the newest `pr-unblock/*` status on the head by `github-actions[bot]` is `failure` or `error`, naming each such context, on pull request runs and in the merge queue (the queued pull request's head); a later `success` on the same head supersedes it. - An `UNCERTAIN` or `VALID-defer` review reply now goes to the adjudicator as an `INCORRECT` one does, instead of to a person. A pr-verify note left open ends `feedback-verifier-open`, and a finding that stood at adjudication and was left open again ends `feedback-stood-open`; both route to a failing status. A round's reason held for a person now outranks any reason routed to a failing status. - `clear-escalations` takes off a `needs-human` an older run set for a reason now routed to a failing status: a `ci-fix-*` reason once a lane check of a `pr-require-checks.yml` run on the current head reads failed, any other once the head moved past the one the lane stopped on. - A pr-unblock stop routed to a failing status now skips `lane-stopped` (a new `skip-reason`) instead of `awaiting-human`. On a select job's own check (a cap, or a conflict no model can resolve) it ends the head, so the dispatchers stop re-running a deterministic stop. On the push, reply or verdict job's check, the dispatcher retries the same head under the cap it already keeps: diff --git a/.github/workflows/pr-require-checks.yml b/.github/workflows/pr-require-checks.yml index fb9cfa9f6c..11016628a9 100644 --- a/.github/workflows/pr-require-checks.yml +++ b/.github/workflows/pr-require-checks.yml @@ -3079,9 +3079,10 @@ jobs: # of own-gate retries, 7 s of status-write retries and the 90 s # `rerun-wait-seconds` wait, about four and a half minutes, plus up to # 15 s of retry sleeps for each of the hold-lift step's two reads, the - # review-lane step's six and the lanes-hold step's eight (in the merge - # queue up to four verdict lookups and their run reads more), about four - # minutes, plus the cost-hold step's reads: one for most pull requests, + # review-lane step's six, the lanes-hold step's eight (in the merge + # queue up to four verdict lookups and their run reads more) and the + # pr-unblock status step's one (two in the merge queue), about four and a + # half minutes, plus the cost-hold step's reads: one for most pull requests, # which stop at the label events; for a lane-labeled one whose body cites a # decision, about eight (nine on a Refs issue), and two more per review # run since the window opened, plus the base-config checkout's fetch @@ -4648,6 +4649,65 @@ jobs: # reader holds it (verify:spec-changed). A record or link that does not # read holds. # Pinned by scripts/ci-lanes-pr-hold.test.sh, which runs this body. + # A HEAD A PR-UNBLOCK LANE STOPPED IS HELD. A lane stop that + # escalation-routes.json routes `fail` ends in a failing + # `pr-unblock/` commit status on the head (pr-unblock.yml + # report-stop-status), and only ci-status is required, so this step + # fails while the newest status of any `pr-unblock/*` context on the + # head is `failure` or `error`, naming each. Newest per context, by + # github-actions[bot] only, as report-escalation.sh `clear` reads it: + # its later `success` on the same head supersedes the failure, and + # another account's status neither holds nor clears. On a pull request + # event the head is the event's; in the merge queue it is the head of + # the pull request named in the group's ref, read live (the ref's SHA is + # the group's base). A status event re-runs nothing, so a stop posted + # after this step passed holds at the merge queue's run. An unparsable + # ref, a head that is not a commit SHA, or an unreadable pull request or + # status list fails closed. + # Pinned by scripts/ci-hold-pr-unblock-status.test.sh, which runs this body. + - name: Fail a head a pr-unblock lane stopped + id: pr_unblock_status + if: ${{ !cancelled() && (github.event_name == 'pull_request' || github.event_name == 'merge_group') }} + env: + GH_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + . "$RUNNER_TEMP/gh-read.sh" + read3() { + local attempt + for attempt in 1 2 3; do + gh_read "${@:2}" && return 0 + [ "$attempt" -lt 3 ] || { echo "::error::could not read $1; re-run this run." >&2; return 1; } + sleep $((attempt * 5)) + done + } + head=$HEAD_SHA + if [ "$GITHUB_EVENT_NAME" = merge_group ]; then + pr=$(sed -nE 's#^(refs/heads/)?gh-readonly-queue/.+/pr-([0-9]+)-[0-9a-f]{40}$#\2#p' <<<"$GITHUB_REF") + if [ -z "$pr" ]; then + echo "::error::cannot read a pull request number from the merge-group ref '$GITHUB_REF'." + exit 1 + fi + pull=$(read3 "#$pr" "repos/$GITHUB_REPOSITORY/pulls/$pr") || exit 1 + head=$(jq -r '.head.sha // ""' <<<"$pull") || head= + fi + if [[ ! $head =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::the head '$head' is not a commit SHA." + exit 1 + fi + # Newest first across pages, so the first row of a context is its newest. + rows=$(read3 "the statuses of $head" "repos/$GITHUB_REPOSITORY/commits/$head/statuses?per_page=100" --paginate \ + --jq '.[] | select((.context | startswith("pr-unblock/")) and .creator.login == "github-actions[bot]") + | [.context, .state, (.description // "" | gsub("[\t\r\n]"; " "))] | @tsv') || exit 1 + failing=$(awk -F'\t' '!seen[$1]++ && ($2 == "failure" || $2 == "error")' <<<"$rows") + if [ -z "$failing" ]; then + echo "No pr-unblock/* status fails on $head." + exit 0 + fi + while IFS=$'\t' read -r context state description; do + echo "::error::'$context' is $state on $head: ${description//%/%25}. The lane that stopped clears it by running again on this head, or a new head carries none." + done <<<"$failing" + exit 1 - name: Fail a lanes App pull request its body or a member holds if: ${{ !cancelled() && (github.event_name == 'pull_request' || github.event_name == 'merge_group') }} env: diff --git a/docs/conventions/pr-pipeline/README.md b/docs/conventions/pr-pipeline/README.md index 43530def89..662dacaa93 100644 --- a/docs/conventions/pr-pipeline/README.md +++ b/docs/conventions/pr-pipeline/README.md @@ -1834,6 +1834,11 @@ them on every `pull_request` run, contract-only runs included: author id, except that their PR fails when a changed file (or a renamed file's old path) is a trust-gate path (the hold-paths above less the security review's own files), since no lane reviews it; a person merges it. A draft is exempt. +- **A stopped lane holds its head.** ci-status fails while the newest `pr-unblock/*` commit status + on the head by `github-actions[bot]` is `failure` or `error`, naming each such context; a later + `success` on the same head (`report-escalation.sh clear`) supersedes it, and another account's + status is not read. The merge-group run checks the queued pull request's head too. A status post + re-runs nothing, so a stop posted after ci-status passed holds at the merge queue's run. A review usually finishes after ci-status has read it, so `pr-feedback-dispatch.yml`'s `rerun-ci-status` job re-runs the ci-status job of the newest `pr-require-checks` run on the head diff --git a/scripts/ci-hold-pr-unblock-status.test.sh b/scripts/ci-hold-pr-unblock-status.test.sh new file mode 100755 index 0000000000..34a1085b77 --- /dev/null +++ b/scripts/ci-hold-pr-unblock-status.test.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash +# Runs the `Fail a head a pr-unblock lane stopped` step body of +# .github/workflows/pr-require-checks.yml's ci-status under the shell Actions +# gives a step with no `shell:` (bash -e), against stub `gh` and `sleep`. +# +# ci-status fails while the newest `pr-unblock/*` status on the head, by +# github-actions[bot], is a failure or error, and names each such context. A +# later success on the same head supersedes a failure. In the merge queue the +# head is the queued pull request's, read live. It fails closed when it cannot +# tell. +# test-scope: .github/workflows/pr-require-checks.yml +set -uo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +WORKFLOW="$ROOT/.github/workflows/pr-require-checks.yml" + +# shellcheck source=lib/test-harness.sh +. "$ROOT/scripts/lib/test-harness.sh" + +TMP_ROOT="$(mktemp -d)" +trap 'rm -rf "$TMP_ROOT"' EXIT + +# shellcheck source=lib/ci-gh-read.sh +. "$ROOT/scripts/lib/ci-gh-read.sh" +export RUNNER_TEMP="$TMP_ROOT" +ci_gh_read::install "$WORKFLOW" "$TMP_ROOT" budget || fail "the ci-status gh_read helper step is missing or fails" + +NAME='Fail a head a pr-unblock lane stopped' +step="$(awk -v name=" - name: $NAME" '$0 == name { on = 1; print; next } + on && /^ - / { exit } + on { print }' "$WORKFLOW")" +awk '/^ run: \|$/ { on = 1; next } + on && /^ / { print substr($0, 11); next } + on && /^[[:blank:]]*$/ { print ""; next } + on { exit }' <<<"$step" >"$TMP_ROOT/body.sh" + +if [[ ! -s "$TMP_ROOT/body.sh" ]]; then + fail "no '$NAME' step with a run: | body in $WORKFLOW" + test_harness::report + exit 1 +fi +if [[ "$step" == *"if: \${{ !cancelled() && (github.event_name == 'pull_request' || github.event_name == 'merge_group') }}"* ]]; then + ok "step runs on every pull_request and merge_group run that is not cancelled" +else + fail "step lost its pull_request and merge_group condition" +fi +if [[ "$step" == *"continue-on-error"* ]]; then + fail "step must not continue on error: its exit code is the hold" +else + ok "step fails the job on its exit code" +fi +order="$(grep -nE "^ ci-status:$|^ - name: ($NAME|Aggregate lane results)$" "$WORKFLOW" | cut -d: -f1 | paste -sd' ')" +read -r job_line hold_line agg_line <<<"$order" +if [[ -n "${agg_line:-}" && "$job_line" -lt "$hold_line" && "$hold_line" -lt "$agg_line" ]]; then + ok "step sits in ci-status before the aggregate" +else + fail "step is not inside ci-status before the aggregate (lines: $order)" +fi + +# The gh stub logs its arguments. A pulls read answers STUB_PULL, after the +# api-budget wrapper's header block when it added -i. A statuses read fails +# while STUB_STATUS_FAILS is set, else applies the step's --jq filter to the +# STUB_STATUSES array (newest first, as the API lists them), as gh does per +# page. +mkdir -p "$TMP_ROOT/bin" +cat >"$TMP_ROOT/bin/gh" <<'EOF' +#!/usr/bin/env bash +printf '%s\n' "$*" >>"$STUB_LOG" +case "$*" in +*/pulls/*) + [[ -z "${STUB_PULL_FAILS:-}" ]] || exit 1 + [[ "$2" != -i ]] || printf 'HTTP/2.0 200 OK\r\nX-Ratelimit-Remaining: 4000\r\nX-Ratelimit-Resource: core\r\n\r\n' + printf '%s\n' "$STUB_PULL" + ;; +*/statuses*) + [[ -z "${STUB_STATUS_FAILS:-}" ]] || exit 1 + while [[ $# -gt 0 && "$1" != --jq ]]; do shift; done + jq -r "$2" <<<"${STUB_STATUSES:-[]}" + ;; +*) exit 1 ;; +esac +EOF +cat >"$TMP_ROOT/bin/sleep" <<'EOF' +#!/usr/bin/env bash +printf '%s\n' "$*" >>"$STUB_SLEEPS" +EOF +chmod +x "$TMP_ROOT/bin/gh" "$TMP_ROOT/bin/sleep" + +HEAD=0123456789abcdef0123456789abcdef01234567 +QUEUED=fedcba9876543210fedcba9876543210fedcba98 +GROUP_BASE=89abcdef0123456789abcdef0123456789abcdef +LANE='{"login":"github-actions[bot]"}' +# status [creator] [description] +status() { jq -cn --arg c "$1" --arg s "$2" --argjson u "${3:-$LANE}" --arg d "${4:-The lane stopped}" \ + '{context: $c, state: $s, creator: $u, description: $d}'; } + +# expect