diff --git a/.changes/pr-pipeline/perf-full-run-cap-bypass-4e8a1c90.md b/.changes/pr-pipeline/perf-full-run-cap-bypass-4e8a1c90.md new file mode 100644 index 00000000000..d4acb8c5462 --- /dev/null +++ b/.changes/pr-pipeline/perf-full-run-cap-bypass-4e8a1c90.md @@ -0,0 +1,7 @@ +--- +bump: minor +--- + +### Added + +- `merge.full-run-cap-bypass` (default `true`): pr-merge's `post-merge-dispatch` starts a full run of main inside its 30-minute cap when the post-merge check that judged (exit 0 or 5) lists an unverified lane merge with no dispatch run under way and main's run listing holds no non-cancelled run created at or after the merge on a commit that contains it. One run of main's tip covers the whole batch, and the next tick finds it in the listing, so the same batch never starts a second run. A run at a commit the job's checkout lacks counts as cover; a merge the checkout lacks, or one older than every listed run, is skipped. The lane arms nothing past `merge.max-in-flight` until a full run verifies the merges in flight, so a batch that landed inside the cap no longer waits it out. `false` keeps the cap for every merge, as before. `dispatch-full-run` (the push path) keeps the cap: it has no post-merge report and no history to judge coverage. The step reads files and the checkout only, so it makes no API call. diff --git a/.github/scripts/pr-merge/merge-job.test.mjs b/.github/scripts/pr-merge/merge-job.test.mjs index 1559f528bf6..2fc33d2e3bd 100644 --- a/.github/scripts/pr-merge/merge-job.test.mjs +++ b/.github/scripts/pr-merge/merge-job.test.mjs @@ -55,6 +55,7 @@ test("the re-dispatch for earlier lane merges waits on neither the listing nor t "Check earlier lane merges", "Keep the post-merge report for enable-auto-merge", "Read main's full-run cover", + "Find lane merges no full run covers", "Start the full CI run on main", "Record the dispatched run", "Re-run a red lane-dispatched full run", diff --git a/.github/scripts/pr-merge/workflow.test.mjs b/.github/scripts/pr-merge/workflow.test.mjs index a381587e4bf..ebfa401276d 100644 --- a/.github/scripts/pr-merge/workflow.test.mjs +++ b/.github/scripts/pr-merge/workflow.test.mjs @@ -8,7 +8,7 @@ // cancelled full run counts as red, unless it never started, so no running // pr-require-checks run may be cancelled by concurrency. import assert from "node:assert/strict"; -import { spawn, spawnSync } from "node:child_process"; +import { execFileSync, spawn, spawnSync } from "node:child_process"; import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { createServer } from "node:http"; import { join } from "node:path"; @@ -219,11 +219,12 @@ const stepIf = (id, name) => { }; const evalStep = (text, steps) => new Function("steps", `return (${text.replace(/\.([a-z]+(?:-[a-z]+)+)/g, '["$1"]')});`)(steps); -const outs = ({ capped = "", uncovered = "", needs = "false", limited = "", proceed = "true", live = "true" } = {}) => ({ +const outs = ({ capped = "", uncovered = "", needs = "false", limited = "", proceed = "true", live = "true", bypass = "" } = {}) => ({ kill: { outputs: { proceed } }, mode: { outputs: { live } }, cover: { outputs: { capped, uncovered, "rate-limited": limited } }, "post-merge": { outputs: { "needs-dispatch": needs } }, + bypass: { outputs: { bypass } }, }); test("dispatch-full-run starts nothing within the cap, and anything outside it", () => { @@ -245,6 +246,10 @@ test("post-merge-dispatch starts main's uncovered tip for any merge, outside the assert.equal(evalStep(expr, outs({ uncovered: "true", limited: "true" })), false, "a rate-limited cover read"); assert.equal(evalStep(expr, outs({ uncovered: "true", proceed: "false", live: "" })), false, "the kill switch on"); assert.equal(evalStep(expr, outs({ uncovered: "true", live: "" })), false, "report-only on"); + // merge.full-run-cap-bypass: an uncovered lane batch starts one run inside the cap. + assert.equal(evalStep(expr, outs({ capped: "true", bypass: "true" })), true, "an uncovered lane batch inside the cap"); + assert.equal(evalStep(expr, outs({ capped: "true", bypass: "true", limited: "true" })), false, "a rate-limited cover read"); + assert.equal(evalStep(expr, outs({ capped: "true", bypass: "true", live: "" })), false, "report-only on"); }); // Review decision on #10491: the scheduled job reads the cover only while the @@ -255,7 +260,8 @@ test("post-merge-dispatch reads the cover only while the lane is live", () => { const block = dispatch.split("\n - name: Read main's full-run cover\n")[1].split("\n - name: ")[0]; assert.match(block, /\n {8}if: steps\.kill\.outputs\.proceed == 'true' && steps\.mode\.outputs\.live == 'true'\n/); const names = [...dispatch.matchAll(/\n {6}- name: (.*)/g)].map((m) => m[1]); - assert.equal(names.indexOf("Read main's full-run cover") + 1, names.indexOf("Start the full CI run on main")); + assert.equal(names.indexOf("Read main's full-run cover") + 1, names.indexOf("Find lane merges no full run covers")); + assert.equal(names.indexOf("Find lane merges no full run covers") + 1, names.indexOf("Start the full CI run on main")); assert.ok(names.indexOf("Check earlier lane merges") < names.indexOf("Read main's full-run cover")); const cover = job("dispatch-full-run").split("\n - name: Read main's full-run cover\n")[1].split("\n - name: ")[0]; assert.doesNotMatch(cover, /\n {8}if:/); @@ -1376,3 +1382,105 @@ test("enable-auto-merge reads the verifier answers only from a trusted run and u assert.doesNotMatch(merge, /uses: actions\/cache[^\n]*\n(?:[^\n]*\n){0,6}[^\n]*verify-answers/); assert.doesNotMatch(merge, /uses: actions\/download-artifact[^\n]*\n(?:[^\n]*\n){0,3}[^\n]*verify-answers/); }); + +// merge.full-run-cap-bypass (owner standing direction, relayed 2026-10-29: +// runner minutes on this repository are not a cost, and the cap now limits +// lane throughput). Inside the cap, post-merge-dispatch starts one run of +// main's tip when the post-merge report lists an unverified lane merge with no +// dispatch under way and no listed run created at or after the merge on a +// commit that contains it. The checkout is a real history: BEFORE, then the +// merge MERGED, then TIP; LATER is a commit the checkout lacks. +const LATER = "f".repeat(40); +function runBypass({ merges, runs, bypass = true, mergedAt = "2026-10-29T01:00:00Z" }) { + const dir = tempDir("pr-merge-bypass-"); + const git = (...args) => + execFileSync("git", ["-C", join(dir, ".base"), ...args], { + encoding: "utf8", + env: { ...process.env, GIT_CONFIG_GLOBAL: "/dev/null", GIT_CONFIG_NOSYSTEM: "1" }, + }).trim(); + mkdirSync(join(dir, ".base/docs/conventions"), { recursive: true }); + git("init", "-q", "-b", "main"); + const commit = (message) => { + git("-c", "user.name=t", "-c", "user.email=t@example.com", "commit", "-q", "--allow-empty", "-m", message); + return git("rev-parse", "HEAD"); + }; + const shas = { BEFORE: commit("before"), MERGED: commit("the lane merge"), TIP: commit("tip"), LATER }; + const config = bypass === undefined ? "merge: {}\n" : `merge:\n full-run-cap-bypass: ${bypass}\n`; + writeFileSync(join(dir, ".base/docs/conventions/pr-pipeline.yaml"), config); + const autoMerges = merges.map((m, i) => ({ + number: 100 + i, + mergeCommit: shas.MERGED, + mergedAt, + verified: m.verified ?? false, + pending: m.pending ?? ["dispatch-run-pending", "canary-run-pending"], + })); + writeFileSync(join(dir, "post-merge.json"), JSON.stringify({ ok: true, autoMerges })); + writeFileSync(join(dir, "cover-runs.json"), JSON.stringify(runs.map((r) => ({ sha: shas[r.at], created: r.created })))); + writeFileSync(join(dir, "output"), ""); + const block = job("post-merge-dispatch").split("\n - name: Find lane merges no full run covers\n")[1].split("\n - name: ")[0]; + writeFileSync(join(dir, "body.sh"), runBody(block)); + const result = spawnSync("bash", ["-e", join(dir, "body.sh")], { + cwd: dir, + env: { ...process.env, GITHUB_OUTPUT: join(dir, "output"), RUNNER_TEMP: dir, GIT_CONFIG_GLOBAL: "/dev/null", GIT_CONFIG_NOSYSTEM: "1" }, + encoding: "utf8", + }); + const output = readFileSync(join(dir, "output"), "utf8"); + rmSync(dir, { recursive: true, force: true }); + return { status: result.status, out: result.stdout + result.stderr, bypass: /^bypass=true$/m.test(output) }; +} +// The run that started the window: before the merge, at the commit before it. +const STARTED = { at: "BEFORE", created: "2026-10-29T00:50:00Z" }; + +test("inside the cap, an unverified lane merge no started run covers starts one run", () => { + const r = runBypass({ merges: [{}, {}], runs: [STARTED] }); + assert.equal(r.status, 0, r.out); + assert.equal(r.bypass, true); + assert.match(r.out, /::notice::no full run covers lane merge\(s\) #100 #101; one run of main's tip starts inside the cap/); +}); + +test("a run created after the merge on a commit that contains it covers the batch: the cap holds", () => { + for (const at of ["MERGED", "TIP", "LATER"]) { + const r = runBypass({ merges: [{}], runs: [{ at, created: "2026-10-29T01:05:00Z" }, STARTED] }); + assert.equal(r.status, 0, r.out); + assert.equal(r.bypass, false, `a run at ${at}`); + } +}); + +test("a run after the merge on a commit without it, or one before the merge at the tip, covers nothing", () => { + assert.equal(runBypass({ merges: [{}], runs: [{ at: "BEFORE", created: "2026-10-29T01:05:00Z" }, STARTED] }).bypass, true); + assert.equal(runBypass({ merges: [{}], runs: [{ at: "TIP", created: "2026-10-29T00:59:59Z" }, STARTED] }).bypass, true); +}); + +test("the second tick in the window finds the run the first tick started and starts none", () => { + // The first tick's dispatch, at main's tip, is in the next listing. + const r = runBypass({ merges: [{}], runs: [{ at: "TIP", created: "2026-10-29T01:11:30Z" }, STARTED] }); + assert.equal(r.bypass, false, r.out); +}); + +test("no lane merge, a verified one, or one with a run under way keeps the cap", () => { + assert.equal(runBypass({ merges: [], runs: [STARTED] }).bypass, false); + assert.equal(runBypass({ merges: [{ verified: true, pending: [] }], runs: [STARTED] }).bypass, false); + for (const pending of [["dispatch-run-pending", "dispatch-run-in-progress:77"], ["dispatch-run-pending", "attribution-pending:77"], ["canary-run-pending"]]) { + assert.equal(runBypass({ merges: [{ pending }], runs: [STARTED] }).bypass, false, pending.join(",")); + } +}); + +test("merge.full-run-cap-bypass false keeps the cap; absent is true", () => { + const off = runBypass({ merges: [{}], runs: [STARTED], bypass: false }); + assert.equal(off.bypass, false); + assert.match(off.out, /::notice::merge\.full-run-cap-bypass is false: the cap holds/); + assert.equal(runBypass({ merges: [{}], runs: [STARTED], bypass: undefined }).bypass, true); +}); + +test("a merge older than every listed run is skipped, not read as uncovered", () => { + const r = runBypass({ merges: [{}], runs: [STARTED], mergedAt: "2026-10-29T00:40:00Z" }); + assert.equal(r.bypass, false, r.out); +}); + +test("the bypass step runs only inside the cap after a check that judged, and the yaml default is on", () => { + const block = job("post-merge-dispatch").split("\n - name: Find lane merges no full run covers\n")[1].split("\n - name: ")[0]; + assert.match(block, /\n {8}if: >-\n {10}steps\.cover\.outputs\.capped == 'true'\n {10}&& contains\(fromJSON\('\["0","5"\]'\), steps\.post-merge\.outputs\.code\)\n/); + const config = readFileSync(new URL("../../../docs/conventions/pr-pipeline.yaml", import.meta.url), "utf8"); + assert.match(config, /\n {2}full-run-cap-bypass: true\n/); + assert.doesNotMatch(job("dispatch-full-run"), /steps\.bypass/); +}); diff --git a/.github/workflows/pr-merge.yml b/.github/workflows/pr-merge.yml index 33659073933..015706b62a0 100644 --- a/.github/workflows/pr-merge.yml +++ b/.github/workflows/pr-merge.yml @@ -336,6 +336,8 @@ jobs: echo "::warning::the read of main's full runs failed; no cap applies to this run" exit 0 fi + # The listing, for post-merge-dispatch's cap bypass. + printf '%s\n' "$runs" >"$RUNNER_TEMP/cover-runs.json" since=$(date -u -d "@$(($(date +%s) - CAP_MINUTES * 60))" +%Y-%m-%dT%H:%M:%SZ) recent=$(jq -r --arg since "$since" '[.[] | select(.created >= $since) | .created] | max // ""' <<<"$runs") if [ -n "$recent" ]; then @@ -346,9 +348,61 @@ jobs: echo "uncovered=true" >>"$GITHUB_OUTPUT" fi + # The cap gives way for the lane's own merges (merge.full-run-cap-bypass, + # default true): the lane arms nothing past merge.max-in-flight until a + # full run verifies the merges in flight, so a batch that lands inside + # the cap would wait it out before its one run. bypass=true when the + # post-merge check that judged (exit 0 or 5) lists an unverified lane + # merge with no dispatch run under way for it, and the cover listing + # holds no non-cancelled run created at or after the merge on a commit + # that contains it. A run at a commit this checkout lacks is newer than + # the checkout, so it counts as cover; a merge the checkout lacks, or + # older than the oldest listed run, is skipped. The dispatch then starts + # one run of main's tip for the whole batch, and the next tick finds it + # in the listing. pr-require-checks.yml's main-dispatch concurrency + # group keeps a burst to one run going and one waiting. + - name: Find lane merges no full run covers + id: bypass + if: >- + steps.cover.outputs.capped == 'true' + && contains(fromJSON('["0","5"]'), steps.post-merge.outputs.code) + run: | + set -euo pipefail + if [ "$(yq '.merge."full-run-cap-bypass"' .base/docs/conventions/pr-pipeline.yaml)" = false ]; then + echo "::notice::merge.full-run-cap-bypass is false: the cap holds" + exit 0 + fi + report=$RUNNER_TEMP/post-merge.json runs=$RUNNER_TEMP/cover-runs.json + [ -f "$report" ] && [ -f "$runs" ] || exit 0 + oldest=$(jq -r 'map(.created) | min // ""' "$runs") + uncovered=() + while IFS=$'\t' read -r number sha at; do + [[ $number =~ ^[1-9][0-9]*$ && $sha =~ ^[0-9a-f]{40}$ && $at =~ ^[0-9T:-]+Z$ ]] || continue + git -C .base cat-file -e "$sha^{commit}" 2>/dev/null || continue + [[ -n $oldest && ! $at < $oldest ]] || continue + covered='' + while IFS=$'\t' read -r run_sha created; do + [[ $created < $at ]] && continue + if ! git -C .base cat-file -e "$run_sha^{commit}" 2>/dev/null || + git -C .base merge-base --is-ancestor "$sha" "$run_sha"; then + covered=1 + break + fi + done < <(jq -r '.[] | [.sha, .created] | @tsv' "$runs") + [ -n "$covered" ] || uncovered+=("#$number") + done < <(jq -r '.autoMerges[]? | select(.verified != true) + | select((.pending // []) | any(.[]; . == "dispatch-run-pending") + and all(.[]; (startswith("dispatch-run-in-progress:") or startswith("attribution-pending:")) | not)) + | [.number, .mergeCommit, .mergedAt] | @tsv' "$report") + if [ "${#uncovered[@]}" -gt 0 ]; then + echo "::notice::no full run covers lane merge(s) ${uncovered[*]}; one run of main's tip starts inside the cap (merge.full-run-cap-bypass)" + echo "bypass=true" >>"$GITHUB_OUTPUT" + fi + # The same body as dispatch-full-run's step of this name; see there. # It also starts main's tip when no full run covers it and none started - # within the cap, and starts nothing within the cap. + # within the cap, and starts nothing within the cap unless the step + # above found lane merges no full run covers. # WAIT_CAP: of the 15 minutes, the whole job took 97-127 s on the last # ten scheduled runs read on 2026-10-23 (the post-merge check 81-111 s # of it). Add 180 s here, the re-run step's POST waits (up to 30 s) and @@ -362,8 +416,8 @@ jobs: steps.kill.outputs.proceed == 'true' && steps.mode.outputs.live == 'true' && steps.cover.outputs.rate-limited != 'true' - && steps.cover.outputs.capped != 'true' - && (steps.post-merge.outputs.needs-dispatch == 'true' || steps.cover.outputs.uncovered == 'true') + && (steps.cover.outputs.capped != 'true' || steps.bypass.outputs.bypass == 'true') + && (steps.post-merge.outputs.needs-dispatch == 'true' || steps.cover.outputs.uncovered == 'true' || steps.bypass.outputs.bypass == 'true') env: GH_TOKEN: ${{ github.token }} REPO: ${{ github.repository }} @@ -2392,6 +2446,8 @@ jobs: echo "::warning::the read of main's full runs failed; no cap applies to this run" exit 0 fi + # The listing, for post-merge-dispatch's cap bypass. + printf '%s\n' "$runs" >"$RUNNER_TEMP/cover-runs.json" since=$(date -u -d "@$(($(date +%s) - CAP_MINUTES * 60))" +%Y-%m-%dT%H:%M:%SZ) recent=$(jq -r --arg since "$since" '[.[] | select(.created >= $since) | .created] | max // ""' <<<"$runs") if [ -n "$recent" ]; then diff --git a/docs/conventions/pr-pipeline.yaml b/docs/conventions/pr-pipeline.yaml index f3582159c3e..6a363e9b25e 100644 --- a/docs/conventions/pr-pipeline.yaml +++ b/docs/conventions/pr-pipeline.yaml @@ -945,6 +945,12 @@ merge: agent-identity: melodic-automation-lanes[bot] # true: report each decision and write nothing. report-only: false + # true: post-merge-dispatch starts a full run of main inside its 30-minute + # cap when an unverified lane merge has no started full run covering it, + # one run for the whole batch, so the lane's in-flight room frees after one + # verify instead of waiting out the cap. false: the cap holds for every + # merge. Absent is true. + full-run-cap-bypass: true # native: the lanes App's pull requests merge through GitHub's auto-merge # (the enable-auto-merge activity arms them, ci-status gates them), and the # gate below neither lists, scores nor enqueues them. gate: the gate also diff --git a/docs/conventions/pr-pipeline/pr-pipeline.schema.json b/docs/conventions/pr-pipeline/pr-pipeline.schema.json index 1d675c3bc08..9a973911fc6 100644 --- a/docs/conventions/pr-pipeline/pr-pipeline.schema.json +++ b/docs/conventions/pr-pipeline/pr-pipeline.schema.json @@ -1024,6 +1024,11 @@ "default": false, "description": "true: the merge authority reports each decision and writes nothing (no enqueue, dequeue or comment). The pr-merge lane starts true; the owner sets false to let it merge (README.md, Merge authority)." }, + "full-run-cap-bypass": { + "type": "boolean", + "default": true, + "description": "true: pr-merge's post-merge-dispatch starts a full run of main inside its 30-minute cap when the post-merge report lists an unverified lane merge that no started full run covers, one run for the whole uncovered batch. false: the cap holds for every merge. Absent is true." + }, "lane-prs": { "enum": ["native", "gate"], "default": "native",