diff --git a/.changes/pr-pipeline/fix-reconcile-paginate-lane-runs-ac2fefa2.md b/.changes/pr-pipeline/fix-reconcile-paginate-lane-runs-ac2fefa2.md new file mode 100644 index 0000000000..5169888a62 --- /dev/null +++ b/.changes/pr-pipeline/fix-reconcile-paginate-lane-runs-ac2fefa2.md @@ -0,0 +1,7 @@ +--- +bump: patch +--- + +### Fixed + +- The lost-run reconciler pages through a day of pr-verify runs instead of failing when there are more than 100, says how many listings failed instead of "nothing lost", and explains an exit 1 in the act step. diff --git a/.github/scripts/maintenance-reconcile-lost-runs/reconcile.mjs b/.github/scripts/maintenance-reconcile-lost-runs/reconcile.mjs index b35dc86759..6251d9dd4a 100755 --- a/.github/scripts/maintenance-reconcile-lost-runs/reconcile.mjs +++ b/.github/scripts/maintenance-reconcile-lost-runs/reconcile.mjs @@ -72,6 +72,9 @@ export const MAX_IN_FLIGHT = 3; export const WRITE_GAP_MS = 1000; // An outage loss is re-run within a day; an older one is left to a person (judgment). export const LOOKBACK_HOURS = 24; +// Pages of 100 read from a lane's run listing; a day of pr-verify runs fits in +// them, and a listing that does not is a failure, never a partial read. +export const LANE_RUN_PAGES = 5; class RateLimited extends Error {} @@ -176,11 +179,18 @@ async function plan({ repo, call, say, since }) { for (const lanePath of RECORDED_LANES) { try { const file = lanePath.slice(lanePath.lastIndexOf("/") + 1); - const listing = await call("GET", `/repos/${repo}/actions/workflows/${file}/runs?created=${created}&per_page=100`); - if (!Array.isArray(listing?.workflow_runs) || listing.total_count > 100) { - throw new Error("its run listing is not one complete page"); - } - const runs = listing.workflow_runs.filter((r) => r.path === lanePath); + const listed = await paginate( + async (...args) => { + const page = await call(...args); + if (!Array.isArray(page?.workflow_runs)) { + throw new Error("its run listing is not a page of runs"); + } + return page.workflow_runs; + }, + `/repos/${repo}/actions/workflows/${file}/runs?created=${created}`, + LANE_RUN_PAGES, + ); + const runs = listed.filter((r) => r.path === lanePath); inFlight += runs.filter((r) => r.status !== "completed" && r.run_attempt > 1).length; const unpublished = []; for (const r of runs.filter((run) => isLostLane(run, since))) { @@ -323,7 +333,11 @@ export async function main({ env = process.env, github, fetchStatus: readStatus throw new Error("the plan file does not hold a plan"); } const failed = await act({ repo, call, say, since, sleep, items: planned.items }); - return finish(planned.failed + failed === 0 ? 0 : 1); + if (planned.failed + failed > 0) { + console.log(`::error title=reconcile::the plan step recorded ${planned.failed} listing failures and ${failed} re-runs failed; see the Find lost runs step`); + return finish(1); + } + return finish(0); } let result = { items: [], failed: 0 }; @@ -359,7 +373,7 @@ export async function main({ env = process.env, github, fetchStatus: readStatus } write(); if (lines.length === 0 && result.items.length === 0) { - say("nothing lost"); + say(result.failed > 0 ? `${result.failed} listings failed; what was lost is unknown` : "nothing lost"); } return finish(0); } catch (error) { diff --git a/.github/scripts/maintenance-reconcile-lost-runs/reconcile.test.mjs b/.github/scripts/maintenance-reconcile-lost-runs/reconcile.test.mjs index 1b7a7d5549..f88730e714 100644 --- a/.github/scripts/maintenance-reconcile-lost-runs/reconcile.test.mjs +++ b/.github/scripts/maintenance-reconcile-lost-runs/reconcile.test.mjs @@ -76,10 +76,11 @@ function fakeGitHub({ pulls, runs = [], fail = {} }) { const found = runs.filter((r) => r.head_sha === listing[1]); return { total_count: found.length, workflow_runs: found }; } - const byWorkflow = /^\/repos\/[^/]+\/[^/]+\/actions\/workflows\/([a-z0-9-]+\.yml)\/runs\?created=%3E%3D2026-10-18T20%3A00%3A00Z&per_page=100$/.exec(path); + const byWorkflow = /^\/repos\/[^/]+\/[^/]+\/actions\/workflows\/([a-z0-9-]+\.yml)\/runs\?created=%3E%3D2026-10-18T20%3A00%3A00Z&per_page=100(?:&page=([0-9]+))?$/.exec(path); if (method === "GET" && byWorkflow) { const found = runs.filter((r) => r.path === `.github/workflows/${byWorkflow[1]}`); - return { total_count: found.length, workflow_runs: found }; + const page = Number(byWorkflow[2] ?? 1); + return { total_count: found.length, workflow_runs: found.slice((page - 1) * 100, page * 100) }; } const jobs = /^\/repos\/[^/]+\/[^/]+\/actions\/runs\/([0-9]+)\/attempts\/1\/jobs\?per_page=100$/.exec(path); if (method === "GET" && jobs) { @@ -448,3 +449,35 @@ test("a non-operational Actions status records nothing", async () => { assert.deepEqual(calls, []); assert.deepEqual(records(outputs), ["", "", ""]); }); + +// A day of pr-verify runs: succeeded ones, with the given runs among them. +const verifyDay = (count, lost) => + Array.from({ length: count }, (_, i) => lost.get(i) ?? { ...run(10_000 + i, MAIN_TIP, { path: VERIFY, event: "workflow_run", conclusion: "success" }), head_branch: "main" }); + +test("a lane listing of 150 runs over 2 pages finds a lost run on page 2", async () => { + const lost = verifyRun(201); + const { code, reruns, calls, outputs } = await reconcile({ pulls: [], runs: verifyDay(150, new Map([[120, lost]])) }); + assert.equal(code, 0); + assert.deepEqual(reruns, [201]); + assert.deepEqual(records(outputs).slice(0, 1), [`reconcile-rerun-201-2-${MAIN_TIP}`]); + assert.equal(calls.filter((c) => c.includes("/workflows/pr-verify.yml/runs?")).length, 2); +}); + +test("a lane listing past the page cap is a failure, and nothing is called lost", async (t) => { + const log = t.mock.method(console, "log", () => {}); + const { code, reruns } = await reconcile({ pulls: [], runs: verifyDay(501, new Map([[0, verifyRun(201)]])) }); + const lines = log.mock.calls.map((c) => String(c.arguments[0])); + assert.equal(code, 1); + assert.deepEqual(reruns, []); + assert.ok(lines.some((l) => l.startsWith("::error title=reconcile::.github/workflows/pr-verify.yml:") && l.includes("has more than 5 pages"))); + assert.ok(!lines.includes("reconcile: nothing lost")); + assert.ok(lines.includes("reconcile: 1 listings failed; what was lost is unknown")); +}); + +test("act says why it exits 1 when the plan recorded listing failures", async (t) => { + const log = t.mock.method(console, "log", () => {}); + const { code } = await reconcile({ pulls: [], runs: verifyDay(501, new Map()) }); + const errors = log.mock.calls.map((c) => String(c.arguments[0])).filter((l) => l.startsWith("::error title=reconcile::the plan step")); + assert.equal(code, 1); + assert.deepEqual(errors, ["::error title=reconcile::the plan step recorded 1 listing failures and 0 re-runs failed; see the Find lost runs step"]); +});