fix(ci): re-run ci-status on lanes App pull requests once the kill switch is lifted

.changes/pr-pipeline/fix-rerun-ci-status-after-kill-switch-09f65a12.md

@@ -0,0 +1,7 @@
+---
+bump: patch
+---
+
+### Fixed
+
+- Lifting the lanes kill switch no longer leaves open lanes App pull requests red until a push or a manual re-run. The next `maintenance-reconcile-lost-runs` run re-runs the failed `ci-status` of each lanes App pull request whose newest checks run failed, on its first attempt, only because the switch was on, recorded so `pr-verify` trusts the re-run, at most three per run. While the switch is on, the reconciler still reads nothing.

.github/scripts/maintenance-reconcile-lost-runs/reconcile.mjs

@@ -41,6 +41,14 @@
 //   their comments' times (`prefilterSpec`) keeps that full read off heads
 //   whose spec nobody touched since the pass. Once per passed checks run, as attempt 2 is never
 //   touched again; an edit or label later starts a new attempt-1 run.
+// - Switch: a lanes App PR whose newest pr-require-checks run failed, at
+//   attempt 1, only in ci-status, with ci-status's note that the lanes kill
+//   switch was on (KILL_SWITCH_NOTE). ci-status fails every lanes App PR while
+//   the switch is on, and nothing else re-runs it once the switch is lifted.
+//   This script reads nothing while the switch is on, so the first run after
+//   it is lifted re-runs that ci-status, recorded as a CI re-run, with no
+//   lookback: the switch may have been on for days. A run at attempt 2 or
+//   later is left to a person, since pr-verify trusts only a recorded attempt 2.
 // Other lost lane runs (the review lanes, refine) are listed in the summary for
 // a person: their gates or the lanes token broker refuse a re-run
 // github-actions[bot] starts. A merge-group run is not covered: a re-run puts
@@ -61,8 +69,10 @@
 //   RECORD_PATH          the file plan writes for the record uploads
 //   GH_TOKEN             the job's GITHUB_TOKEN (actions: write, pull-requests: read)
 //   GITHUB_REPOSITORY    owner/repo
-//   TRUSTED_ACTORS_PATH  the default branch's trusted-actor list; unset checks no spec
-//   VERIFY_AUTHORS, LANE_IDENTITY, SPEC_FROM_BODY_APPS  as pr-verify.yml passes resolve-pr
+//   TRUSTED_ACTORS_PATH  the default branch's trusted-actor list; unset checks no
+//                        spec and no kill-switch stop
+//   VERIFY_AUTHORS, LANE_IDENTITY, SPEC_FROM_BODY_APPS  as pr-verify.yml passes
+//                        resolve-pr; LANE_IDENTITY also names the lanes App
 //   KILL_SWITCH         `proceed` from the kill-switch step; anything else stops
 //   STATUS_URL           the status page's components endpoint (default githubstatus.com)
 //   GITHUB_OUTPUT        plan writes record-1 to record-3 there (empty when unused)
@@ -78,7 +88,19 @@ import { createGitHub, loadTrustedList, paginate, rerunRecordName } from "../../
 import { refsIssue, resolvePull, specFromBodyApps } from "../pr-verify/resolve-pr.mjs";
 import { headRefDigest } from "../pr-verify/write-verdict.mjs";
 import { ACTIONS, componentStatus, fetchStatus } from "./component-status.mjs";
-import { firstFailure, INFRA_BACKOFF_MINUTES, infraOnly, LOOKBACK_HOURS, LOST, lostRunnerNoted, reconcilerRetries, stepsKnown, testedRun } from "./infra-failure.mjs";
+import {
+  AGGREGATE_JOBS,
+  FAILED,
+  firstFailure,
+  INFRA_BACKOFF_MINUTES,
+  infraOnly,
+  LOOKBACK_HOURS,
+  LOST,
+  lostRunnerNoted,
+  reconcilerRetries,
+  stepsKnown,
+  testedRun,
+} from "./infra-failure.mjs";
 
 const STATUS_URL = ACTIONS.url;
 // A failed run's failed jobs are re-run only while both pages show indicator none.
@@ -192,13 +214,28 @@ const isLostLane = (r, since) =>
 // pull_request run that passed on its first attempt.
 const isPassedChecks = (r) =>
   RECORDED_UPSTREAMS.has(r?.path) && r.event === "pull_request" && r.status === "completed" && r.conclusion === "success" && r.run_attempt === 1;
+// The error ci-status's lanes-hold step writes for a lanes App PR while the
+// kill switch is on (pr-require-checks.yml; reconcile.test.mjs pins it there).
+export const KILL_SWITCH_NOTE = "the lanes kill switch is on";
+// The checks run a lifted kill switch leaves red: failed on its first attempt.
+const isFailedChecks = (r) =>
+  RECORDED_UPSTREAMS.has(r?.path) && r.event === "pull_request" && r.status === "completed" && r.conclusion === "failure" && r.run_attempt === 1;
+// The failed ci-status job when it is the run's only failed job, else undefined.
+const onlyAggregateFailed = (jobs) =>
+  Array.isArray(jobs) && jobs.every((job) => !FAILED.has(job?.conclusion) || AGGREGATE_JOBS.has(job.name))
+    ? jobs.find((job) => AGGREGATE_JOBS.has(job?.name) && job.conclusion === "failure" && Number.isSafeInteger(job.id))
+    : undefined;
+// True when one of the ci-status job's check-run annotations is the kill-switch
+// error: a re-run with the switch off passes that step.
+export const killSwitchNoted = (annotations) =>
+  Array.isArray(annotations) && annotations.some((a) => a?.annotation_level === "failure" && String(a.message ?? "").includes(KILL_SWITCH_NOTE));
 // A failed run whose failed jobs may be re-run now, if its failures are infra only.
 const isRetryable = (r, now) => reconcilerRetries(r, now) && now - Date.parse(r.updated_at) >= INFRA_BACKOFF_MINUTES * 60_000;
 const lostNow = (item, r, since) => {
   if (r?.head_sha !== item.head) {
     return false;
   }
-  if (item.kind === "infra" || item.kind === "flake") {
+  if (item.kind === "infra" || item.kind === "flake" || item.kind === "switch") {
     return r.status === "completed" && r.conclusion === "failure" && r.run_attempt === item.attempt;
   }
   if (item.kind === "intake") {
@@ -398,7 +435,7 @@ export async function prefilterSpec(call, repo, candidates) {
 // Reads every open PR's head and the recorded lanes' runs, and returns the
 // re-runs to make in order, with a record name on each lane re-run and each
 // recorded CI re-run.
-async function plan({ repo, call, say, since, now, readStatus, judgeSpec }) {
+async function plan({ repo, call, say, since, now, readStatus, judgeSpec, laneApp }) {
   const pulls = (await paginate(call, `/repos/${repo}/pulls?state=open&base=main`, 5))
     .filter((p) => p.state === "open" && p.draft !== true && p.base?.ref === "main" && p.head?.repo?.full_name === repo && SHA.test(p.head?.sha ?? ""))
     .sort(byMergeCriticality);
@@ -452,18 +489,18 @@ async function plan({ repo, call, say, since, now, readStatus, judgeSpec }) {
         continue;
       }
       const set = rerun.map((r) => ({ kind: "ci", pr: pull.number, id: r.id, path: r.path, conclusion: r.conclusion, head: pull.head.sha }));
+      // A run's latest jobs, with the annotations of a failed job whose steps
+      // did not list in full (infraCause reads them).
+      const readJobs = async (r) => {
+        const listed = await call("GET", `/repos/${repo}/actions/runs/${r.id}/jobs?filter=latest&per_page=100`);
+        if (!Array.isArray(listed?.jobs) || listed.total_count > 100) {
+          throw new Error(`run ${r.id}'s job listing is not one complete page`);
+        }
+        return withAnnotations(call, repo, listed.jobs, (job) => job?.conclusion === "failure");
+      };
       const tested = [];
       for (const runs of ciRunsByWorkflow(listing.workflow_runs)) {
-        tested.push(
-          await testedRun(runs, async (r) => {
-            const listed = await call("GET", `/repos/${repo}/actions/runs/${r.id}/jobs?filter=latest&per_page=100`);
-            if (!Array.isArray(listed?.jobs) || listed.total_count > 100) {
-              throw new Error(`run ${r.id}'s job listing is not one complete page`);
-            }
-            // infraCause reads a failed job's annotations.
-            return withAnnotations(call, repo, listed.jobs, (job) => job?.conclusion === "failure");
-          }),
-        );
+        tested.push(await testedRun(runs, readJobs));
       }
       // A re-run still going under a newer failed contract-only run is in
       // flight too, though classify sees only the newest run.
@@ -491,11 +528,19 @@ async function plan({ repo, call, say, since, now, readStatus, judgeSpec }) {
         }
         set.push({ kind: "infra", pr: pull.number, id: r.id, path: r.path, conclusion: "failure (infra only)", attempt: r.run_attempt, head: pull.head.sha });
       }
+      const checks = newestByWorkflow(listing.workflow_runs).find((r) => RECORDED_UPSTREAMS.has(r.path));
+      // A lanes App PR whose newest checks run the kill switch failed: its
+      // failed ci-status is re-run now that the switch is off.
+      if (set.length === 0 && laneApp !== undefined && pull.user?.id === laneApp && pull.user?.type === "Bot" && isFailedChecks(checks)) {
+        const status = onlyAggregateFailed(await readJobs(checks));
+        if (status && killSwitchNoted(status.annotations ?? (await call("GET", `/repos/${repo}/check-runs/${status.id}/annotations?per_page=100`)))) {
+          set.push({ kind: "switch", pr: pull.number, id: checks.id, path: checks.path, conclusion: "failure (kill switch)", attempt: 1, head: pull.head.sha });
+        }
+      }
       // A pass the spec has since changed under: re-running the head's
       // passed checks starts pr-verify, which judges it again (resolve-pr's
       // `spec-changed`). Attempt 1 only, so each passed checks run is re-run
       // at most once; a later edit or label starts a new attempt-1 run.
-      const checks = newestByWorkflow(listing.workflow_runs).find((r) => RECORDED_UPSTREAMS.has(r.path));
       // A read that fails here re-runs nothing and is no listing failure: the
       // next tick asks again. A rate limit still stops the run.
       if (set.length === 0 && judgeSpec && isPassedChecks(checks)) {
@@ -662,9 +707,14 @@ async function act({ repo, call, say, since, sleep, items }) {
         await sleep(WRITE_GAP_MS);
       }
       wrote = true;
-      if (item.kind === "intake" || item.kind === "infra" || item.kind === "flake") {
+      if (item.kind === "intake" || item.kind === "infra" || item.kind === "flake" || item.kind === "switch") {
         await call("POST", `/repos/${repo}/actions/runs/${item.id}/rerun-failed-jobs`);
-        const why = { intake: "", infra: ", which failed on infra only", flake: ", its one re-run before fix-ci takes it" }[item.kind];
+        const why = {
+          intake: "",
+          infra: ", which failed on infra only",
+          flake: ", its one re-run before fix-ci takes it",
+          switch: ": ci-status failed while the lanes kill switch was on",
+        }[item.kind];
         say(
           item.kind === "intake"
             ? `${where}: re-ran the failed jobs of floor-label run ${item.id}, whose first attempt failed`
@@ -780,8 +830,11 @@ export async function main({ env = process.env, github, fetchStatus: readStatus
     // exactly a head pr-verify would judge again. Without the trusted list
     // nothing is checked.
     let judgeSpec;
+    // The lanes App's account id, from the trusted list by merge.agent-identity.
+    let laneApp;
     if (env.TRUSTED_ACTORS_PATH) {
       const trusted = loadTrustedList(env.TRUSTED_ACTORS_PATH);
+      laneApp = env.LANE_IDENTITY ? trusted.logins.get(env.LANE_IDENTITY.toLowerCase()) : undefined;
       // The Refs read needs GraphQL errors back to tell a typo from a failure.
       const readRefs = injected ? github : createGitHub({ token: env.GH_TOKEN, apiUrl: env.GITHUB_API_URL || undefined, keepGraphQLErrors: true });
       judgeSpec = (upstreamRunId) =>
@@ -797,7 +850,7 @@ export async function main({ env = process.env, github, fetchStatus: readStatus
           specFromBodyApps: specFromBodyApps(env.SPEC_FROM_BODY_APPS),
         });
     }
-    result = await plan({ repo, call, say, since, now, readStatus, judgeSpec });
+    result = await plan({ repo, call, say, since, now, readStatus, judgeSpec, laneApp });
     if (result.items.some(recorded) && !env.RECORD_PATH) {
       throw new Error("RECORD_PATH is not set");
     }

.github/scripts/maintenance-reconcile-lost-runs/reconcile.test.mjs

@@ -20,7 +20,7 @@ import { test } from "node:test";
 
 import { createGitHub, GitHubError } from "../../actions/check-trusted-trigger/check-trusted-trigger.mjs";
 import { createRateLimitLatch } from "../../actions/log-api-budget/rate-guard.mjs";
-import { GITHUB_STATUS_URL, JUDGE_PREDECESSOR, JUDGE_STEP, main, NPM_STATUS_URL, prefilterSpec, WRITE_STEP } from "./reconcile.mjs";
+import { GITHUB_STATUS_URL, JUDGE_PREDECESSOR, JUDGE_STEP, KILL_SWITCH_NOTE, main, NPM_STATUS_URL, prefilterSpec, WRITE_STEP } from "./reconcile.mjs";
 
 const REPO = "melodic-software/claude-code-plugins";
 const NOW = Date.parse("2026-10-19T20:00:00Z");
@@ -1280,3 +1280,112 @@ test("act re-runs no CI run whose plan names another record", async (t) => {
     rmSync(dir, { recursive: true, force: true });
   }
 });
+
+// A lanes App PR whose newest pr-require-checks run failed only in ci-status,
+// whose lanes-hold step noted the kill switch: once the switch is off, that
+// ci-status is re-run, recorded as attempt 2, whatever the run's age. The
+// annotation text is the one pr-require-checks.yml writes (pinned below).
+const LANE_ID = 9;
+const lanePr = (number, opts) => ({ ...pr(number, opts), user: { id: LANE_ID, login: LANE, type: "Bot" } });
+const switchJobs = ({ message = `#1: ${KILL_SWITCH_NOTE} (CLAUDE_LANES_DISABLED or CLAUDE_LANES_REPO_DISABLED is not 'false')`, lane = "success" } = {}) => [
+  { id: 501, name: "lint-repo", status: "completed", conclusion: lane, steps: [] },
+  { id: 502, name: "ci-status", status: "completed", conclusion: "failure", steps: [], notes: [{ annotation_level: "failure", message }] },
+];
+async function withLaneList(fn) {
+  const dir = mkdtempSync(path.join(tmpdir(), "reconcile-actors-"));
+  try {
+    const actors = path.join(dir, "trusted-actors.json");
+    writeFileSync(
+      actors,
+      JSON.stringify({
+        version: 1,
+        actors: [
+          { id: 100, login: "maintainer", kind: "human" },
+          { id: LANE_ID, login: LANE, kind: "bot" },
+        ],
+      }),
+    );
+    return await fn({ TRUSTED_ACTORS_PATH: actors, LANE_IDENTITY: LANE });
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+test("a lanes App PR whose ci-status the kill switch failed has it re-run once the switch is off, recorded, past the lookback", async () => {
+  await withLaneList(async (env) => {
+    for (const created of ["2026-10-19T19:00:00Z", "2026-10-12T19:00:00Z"]) {
+      const runs = [failed(101, sha(1), { jobs: switchJobs(), created })];
+      const { code, outputs, reruns, failedJobReruns } = await reconcile({ pulls: [lanePr(1)], runs, env });
+      assert.equal(code, 0);
+      assert.deepEqual(failedJobReruns, [101]);
+      assert.deepEqual(reruns, []);
+      assert.deepEqual(records(outputs), [`reconcile-rerun-101-2-${sha(1)}`, "", ""]);
+    }
+  });
+});
+
+test("while the kill switch is on, no lanes App PR is read", async () => {
+  await withLaneList(async (env) => {
+    const { code, calls } = await reconcile({ pulls: [lanePr(1)], runs: [failed(101, sha(1), { jobs: switchJobs() })], env: { ...env, KILL_SWITCH: "" } });
+    assert.equal(code, 0);
+    assert.deepEqual(calls, []);
+  });
+});
+
+test("a ci-status failure with no kill-switch note, another author's PR, a second attempt or no lane identity is not re-run", async () => {
+  await withLaneList(async (env) => {
+    const cases = [
+      { pulls: [lanePr(1)], runs: [failed(101, sha(1), { jobs: switchJobs({ message: "#1: the PR body holds it" }) })], env },
+      { pulls: [pr(1)], runs: [failed(101, sha(1), { jobs: switchJobs() })], env },
+      { pulls: [{ ...lanePr(1), user: { id: LANE_ID, login: LANE, type: "User" } }], runs: [failed(101, sha(1), { jobs: switchJobs() })], env },
+      { pulls: [lanePr(1)], runs: [failed(101, sha(1), { jobs: switchJobs(), attempt: 2 })], env },
+      { pulls: [lanePr(1)], runs: [failed(101, sha(1), { jobs: switchJobs() })], env: { TRUSTED_ACTORS_PATH: env.TRUSTED_ACTORS_PATH } },
+      { pulls: [lanePr(1)], runs: [failed(101, sha(1), { jobs: switchJobs() })] },
+    ];
+    for (const setup of cases) {
+      const { code, calls } = await reconcile(setup);
+      assert.equal(code, 0);
+      assert.deepEqual(calls.filter((c) => c.startsWith("POST ")), []);
+    }
+  });
+});
+
+test("a kill-switch stop on a newer failed contract-only run is re-run, not hidden by the tested run below it", async () => {
+  await withLaneList(async (env) => {
+    const contract = [
+      { id: 511, name: "lint-repo", status: "completed", conclusion: "skipped", steps: [] },
+      { ...switchJobs()[1], id: 512 },
+    ];
+    const runs = [run(100, sha(1), { conclusion: "success", created: "2026-10-19T18:00:00Z" }), failed(101, sha(1), { jobs: contract })];
+    const { failedJobReruns } = await reconcile({ pulls: [lanePr(1)], runs, env });
+    assert.deepEqual(failedJobReruns, [101]);
+  });
+});
+
+test("a kill-switch stop with a lane that also failed takes the one flake re-run, not a second", async () => {
+  await withLaneList(async (env) => {
+    const jobs = [...MIXED_JOBS, switchJobs()[1]];
+    const { failedJobReruns, outputs } = await reconcile({ pulls: [lanePr(1)], runs: [failed(101, sha(1), { jobs, updated: "2026-10-19T19:59:00Z" })], env });
+    assert.deepEqual(failedJobReruns, [101]);
+    assert.deepEqual(records(outputs), [`reconcile-rerun-101-2-${sha(1)}`, "", ""]);
+  });
+});
+
+test("a kill-switch re-run shares the cap of 3, and one re-run since the listing is left alone", async () => {
+  await withLaneList(async (env) => {
+    const runs = [1, 2, 3, 4].map((n) => failed(100 + n, sha(n), { jobs: switchJobs().map((j) => ({ ...j, id: j.id * 10 + n })) }));
+    const capped = await reconcile({ pulls: [1, 2, 3, 4].map((n) => lanePr(n)), runs, env });
+    assert.deepEqual(capped.failedJobReruns, [101, 102, 103]);
+    const moved = await reconcile({ pulls: [lanePr(1)], runs: [failed(101, sha(1), { jobs: switchJobs() })], env, github: rerunSinceListing(101) });
+    assert.deepEqual(moved.failedJobReruns, []);
+  });
+});
+
+test("ci-status's lanes-hold step still fails a lanes App PR with the kill-switch note the reconciler matches", () => {
+  const workflow = readFileSync(new URL("../../workflows/pr-require-checks.yml", import.meta.url), "utf8");
+  const step = /- name: Fail a lanes App pull request its body or a member holds\n(?:(?:[ ]{8,}.*)?\n)+/.exec(workflow)?.[0] ?? "";
+  assert.match(step, /LANES_DISABLED: \$\{\{ vars\.CLAUDE_LANES_DISABLED \}\}/);
+  const fail = step.split("\n").findIndex((line) => line.includes(`echo "::error::#$pr: ${KILL_SWITCH_NOTE} (`));
+  assert.ok(fail > 0, "the kill-switch error line is in the step");
+  assert.match(step.split("\n")[fail + 1], /^[ ]+exit 1$/);
+});

.github/workflows/maintenance-reconcile-lost-runs.yml

@@ -31,6 +31,12 @@ name: maintenance-reconcile-lost-runs
 # issue edit or a new spec comment), so pr-verify judges the head again
 # instead of leaving it at verify:spec-changed.
 #
+# A lanes App pull request whose newest pr-require-checks run failed, on its
+# first attempt, only in ci-status, which noted that the lanes kill switch was
+# on, has that ci-status re-run, recorded as above, by the first run after the
+# switch is lifted, whatever the run's age; it takes a place under the cap. A
+# run at a later attempt is left to a person.
+#
 # An intake-label-needs-triage run an issue event started whose first attempt
 # failed (a rate limit, most often) left the issue without its floor label;
 # its failed jobs are re-run once, taking no place under the cap.