Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/actions/log-api-budget/api-budget.bash
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,7 @@ api_budget_wrap_gh() {
out=$(mktemp 2>/dev/null) || { api_budget_pass "$@"; return; }
command gh "$@" >"$out" || rc=$?
api_budget_observe "$out" "$pool" "gh $*" || rc=1
api_budget_read "$API_BUDGET_GH_TOKEN" "$out" >/dev/null
cat "$out"
rm -f "$out"
return "$rc"
Expand Down
14 changes: 14 additions & 0 deletions .github/actions/log-api-budget/log-api-budget.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -356,6 +356,20 @@ test("gh guard: a refused answer still returns 1 on both paths", () =>
}
}));

// A caller that reads the headers itself (-i) is still counted, and a
// conditional GET answered 304 counts as not-modified, which GitHub does not
// charge.
test("gh guard: a -i call is counted in api-calls, and a 304 as not-modified", () =>
withDir((dir) => {
const out = guarded(
dir,
["200 4000 - {}", "304 4000 - "],
'gh api -i repos/o/r >/dev/null\ngh api -i -H "If-None-Match: x" repos/o/r >/dev/null || true\napi_budget_emit_calls github-token',
);
assert.equal(out.calls.length, 2);
assert.equal(out.stdout, "::notice title=api-calls::github-token=1 not-modified=1\n");
}));

test("gh guard: a stop-file line whose reset is not a number stops nothing", () =>
withDir((dir) => {
mkdirSync(join(dir, "api-budget"), { recursive: true });
Expand Down
24 changes: 18 additions & 6 deletions .github/scripts/pr-merge/enable-auto-merge.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -715,20 +715,28 @@ export class ReaderRateLimited extends Error {
// the Actions token (`override`, the job's GITHUB_TOKEN), and those paginated,
// each of which made one request per page. Null when there is no such line.
export const GH_CALLS_PREFIX = "babysit-gh-calls ";
export function ghCalls(stderr) {
function ghCallsPayload(stderr) {
const line = String(stderr ?? "")
.split("\n")
.map((text) => text.replace(/\r$/, ""))
.findLast((text) => text.startsWith(GH_CALLS_PREFIX));
if (line === undefined) {
return null;
}
let payload;
try {
payload = JSON.parse(line.slice(GH_CALLS_PREFIX.length));
return line === undefined ? null : JSON.parse(line.slice(GH_CALLS_PREFIX.length));
} catch {
return null;
}
}

// That line as the log carries it, as the merge job's tee does, so one grep
// sums the calls per endpoint across both jobs: re-serialized from the parsed
// JSON, so it is one line whatever the reader printed. Null without one.
export function ghCallsLine(stderr) {
const payload = ghCallsPayload(stderr);
return payload === null || typeof payload !== "object" ? null : `${GH_CALLS_PREFIX}${JSON.stringify(payload)}`;
}

export function ghCalls(stderr) {
const payload = ghCallsPayload(stderr);
const calls = payload?.calls;
if (!Number.isSafeInteger(payload?.total) || calls === null || typeof calls !== "object") {
return null;
Expand Down Expand Up @@ -815,6 +823,10 @@ export function pythonVerify(env, { spawn = spawnSync, clock = Date.now } = {})
}
verify.costs.push(cost);
console.log(`enable-auto-merge: ${costLine(cost)}`);
const calls = ghCallsLine(done.stderr);
if (calls !== null) {
console.log(calls);
}
const line = cacheLine(number, evidence?.cache);
if (line !== null) {
console.log(`enable-auto-merge: ${line}`);
Expand Down
9 changes: 8 additions & 1 deletion .github/scripts/pr-merge/enable-auto-merge.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import {
CACHE_ONLY_TIMEOUT_MS,
CACHE_ONLY_WINDOW_MS,
ghCalls,
ghCallsLine,
LANE_ROOM_TIMEOUT_MS,
VERIFY_TIMEOUT_MS,
VERIFY_BUDGET_MS,
Expand Down Expand Up @@ -2150,6 +2151,8 @@ test("ghCalls reads babysit_gh.py's calls line: in all, under GITHUB_TOKEN, and
assert.deepEqual(ghCalls(`warning: something\n${line}\n`), { total: 4, githubToken: 3, paginated: 1 });
assert.equal(ghCalls("no calls line\n"), null);
assert.equal(ghCalls("babysit-gh-calls {not json\n"), null);
assert.equal(ghCallsLine("babysit-gh-calls {not json\n"), null);
assert.equal(ghCallsLine("no calls line\n"), null);
assert.equal(costLine({ number: 7, calls: { total: 4, githubToken: 3, paginated: 1 }, ms: 1500 }), "#7 calls=4 github-token=3 paginated=1 ms=1500");
assert.equal(costLine({ number: 7, calls: null, ms: 20 }), "#7 calls=unknown ms=20");
});
Expand All @@ -2173,7 +2176,11 @@ test("pythonVerify logs each read's calls and time, keeps them, and names the er
} finally {
console.log = log;
}
assert.deepEqual(lines, ["enable-auto-merge: #1 calls=4 github-token=3 paginated=1 ms=250", "enable-auto-merge: #2 calls=4 github-token=3 paginated=1 ms=250"]);
// The reader's per-endpoint calls line reaches the log after each cost
// line, in the shape the merge job's tee logs, as one line of JSON.
const logged = `babysit-gh-calls ${JSON.stringify(JSON.parse(line.slice("babysit-gh-calls ".length)))}`;
assert.deepEqual(lines, ["enable-auto-merge: #1 calls=4 github-token=3 paginated=1 ms=250", logged, "enable-auto-merge: #2 calls=4 github-token=3 paginated=1 ms=250", logged]);
assert.deepEqual(JSON.parse(logged.slice("babysit-gh-calls ".length)).calls, JSON.parse(line.slice("babysit-gh-calls ".length)).calls);
assert.deepEqual(
verify.costs.map(({ number, ms }) => [number, ms]),
[
Expand Down
2 changes: 1 addition & 1 deletion .github/scripts/pr-merge/merge-job.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -1305,7 +1305,7 @@ test("each candidate's check run is written by its own job, the only one holding
const writerSteps = stepsOf(writer);
assert.deepEqual(
writerSteps.map((s) => s.name),
["Check out the default branch tip", "Set up Node", "Read the gate's rows", "Show each candidate's refusal as a check run"],
["Check out the default branch tip", "Set up Node", "Read the gate's rows", "Show each candidate's refusal as a check run", "Log the API budget at the end"],
);
assert.match(writerSteps[0].block, /\n {10}ref: refs\/heads\/main\n {10}path: \.base\n {10}persist-credentials: false\n/);
for (const { name, block } of writerSteps.slice(2)) {
Expand Down
69 changes: 65 additions & 4 deletions .github/scripts/pr-merge/workflow.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ test("dispatch-full-run ends once it dispatches: no wait on the run it started",
const dispatch = job("dispatch-full-run");
assert.deepEqual(
[...dispatch.matchAll(/\n {6}- name: (.*)/g)].map((m) => m[1]),
["Read main's full-run cover", "Start the full CI run on main", "Record the dispatched run"],
["Check out the api-budget library", "Read main's full-run cover", "Start the full CI run on main", "Record the dispatched run", "Log the API budget at the end"],
);
assert.doesNotMatch(dispatch, /sleep 180|actions\/runs\/\$RUN_ID|post-merge-demotion-alert\.yml\/dispatches/);
const timeout = Number(/\n {4}timeout-minutes: (\d+)\n/.exec(dispatch)[1]);
Expand Down Expand Up @@ -262,12 +262,51 @@ test("post-merge-dispatch reads the cover only while the lane is live", () => {
assert.doesNotMatch(job("dispatch-full-run"), /steps\.(kill|mode)\./);
});

test("the dispatch job's token is actions: write and nothing else", () => {
const block = /\n {4}permissions:\n((?: {6}[a-z-]+: [a-z]+.*\n)+)/.exec(job("dispatch-full-run"))[1];
// contents: read only checks out the api-budget library, sparse, from main.
test("the dispatch job's token is actions: write and contents: read, and nothing else", () => {
const dispatch = job("dispatch-full-run");
const block = /\n {4}permissions:\n((?: {6}[a-z-]+: [a-z]+.*\n)+)/.exec(dispatch)[1];
assert.deepEqual(
[...block.matchAll(/ {6}([a-z-]+): ([a-z]+)/g)].map((m) => `${m[1]}: ${m[2]}`),
["actions: write"],
["actions: write", "contents: read"],
);
const checkout = dispatch.split("\n - name: Check out the api-budget library\n")[1].split("\n - name: ")[0];
assert.match(checkout, /^ {8}continue-on-error: true\n/);
assert.match(checkout, /\n {10}ref: refs\/heads\/main\n {10}path: \.base\n {10}persist-credentials: false\n {10}sparse-checkout: \|\n {12}\/\.github\/actions\/log-api-budget\/\n {10}sparse-checkout-cone-mode: false\n/);
});

// The cover read and each dispatch POST leave a rate-limit reading, so the
// job-end api-calls notice counts them; the POST is recorded, never guarded.
test("dispatch-full-run's cover read and dispatch POST each leave one reading", () => {
const dir = tempDir("pr-merge-reading-");
mkdirSync(join(dir, "bin"));
symlinkSync(fileURLToPath(new URL("../../..", import.meta.url)), join(dir, ".base"));
const headers = "X-Ratelimit-Limit: 5000\r\nX-Ratelimit-Remaining: 4000\r\nX-Ratelimit-Used: 1000\r\nX-Ratelimit-Reset: 9999999999\r\nX-Ratelimit-Resource: core\r\n\r\n";
writeFileSync(
join(dir, "bin", "gh"),
`#!/usr/bin/env bash
if [[ " $* " == *" -X POST "* ]]; then printf 'HTTP/2.0 200 OK\\r\\n${headers.replaceAll("\r\n", "\\r\\n")}{"workflow_run_id":4242}'; exit 0; fi
printf 'HTTP/2.0 200 OK\\r\\n${headers.replaceAll("\r\n", "\\r\\n")}'
while [ $# -gt 0 ]; do if [ "$1" = --jq ]; then echo '[]' | jq -c "$2"; exit; fi; shift; done
`,
{ mode: 0o755 },
);
writeFileSync(join(dir, "cover.sh"), coverBody("dispatch-full-run").body);
writeFileSync(join(dir, "dispatch.sh"), dispatchStep("dispatch-full-run").body);
writeFileSync(join(dir, "output"), "");
const env = { ...process.env, PATH: `${join(dir, "bin")}:${process.env.PATH}`, GH_TOKEN: "x", REPO: "o/r", TIP: HEAD, CAP_MINUTES: "30", WAIT_CAP: "180", GITHUB_EVENT_NAME: "push", GITHUB_SHA: HEAD, GITHUB_RUN_ID: "77", GITHUB_OUTPUT: join(dir, "output"), RUNNER_TEMP: dir };
try {
for (const body of ["cover.sh", "dispatch.sh"]) {
const r = spawnSync("bash", ["-e", join(dir, body)], { cwd: dir, env, encoding: "utf8" });
assert.equal(r.status, 0, r.stdout + r.stderr);
}
assert.match(readFileSync(join(dir, "output"), "utf8"), /^run-id=4242$/m);
assert.equal(readFileSync(join(dir, "api-budget", "github-token"), "utf8"), "200 5000 4000 1000 9999999999 core\n".repeat(2));
const calls = spawnSync("bash", ["-c", ". .base/.github/actions/log-api-budget/api-budget.bash; api_budget_emit_calls github-token"], { cwd: dir, env, encoding: "utf8" });
assert.equal(calls.stdout, "::notice title=api-calls::github-token=2 not-modified=0\n");
} finally {
rmSync(dir, { recursive: true, force: true });
}
});

// pr-require-checks.yml's concurrency, evaluated per event. Main's dispatches
Expand Down Expand Up @@ -456,6 +495,28 @@ test("enable-auto-merge logs both tokens' readings at job end, before the token
assert.ok(names.indexOf("Log the API budget at job end") < names.indexOf("Revoke the lane token"));
});

// The hub ranks GITHUB_TOKEN spend by each job's `api-calls` notice, so a
// job that calls GitHub with the job token and logs nothing is spend no one
// sees. Every such job carries the log-api-budget step, run whatever failed
// before it, naming github-token.
test("every job that calls GitHub with the job token logs its api-calls", () => {
const ids = [...jobs.matchAll(/\n {2}([a-z-]+):\n/g)].map((m) => m[1]);
const callers = ids.filter((id) => job(id).includes("${{ github.token }}"));
for (const id of ["post-merge-dispatch", "merge", "enable-auto-merge", "unarmed-status", "dispatch-full-run", "gate-check"]) {
assert.ok(callers.includes(id), `${id} calls GitHub with the job token`);
}
const unlogged = callers.filter((id) => {
const step = `\n${/- name: Log the API budget[^\n]*\n([\s\S]*?)(?=\n {6}- name: |$)/.exec(job(id))?.[1] ?? ""}`;
return !(
/\n {8}if: always\(\)/.test(step) &&
/\n {8}continue-on-error: true\n/.test(step) &&
/\n {8}uses: \.\/\.base\/\.github\/actions\/log-api-budget /.test(step) &&
/\n {10}tokens: (?:[a-z-]+ )*github-token\b/.test(step)
);
});
assert.deepEqual(unlogged, []);
});

// The lane's room (lane-room) reads this run's post-merge report, which
// post-merge-dispatch keeps, rather than running the check a third time per
// scheduled run, and the rung cap as the merge job passes it: an unset cap
Expand Down
74 changes: 67 additions & 7 deletions .github/workflows/pr-merge.yml
Original file line number Diff line number Diff line change
Expand Up @@ -134,8 +134,7 @@ jobs:
# if one did: the merge and enable-auto-merge jobs then send nothing.
rate-limited: ${{ steps.post-merge.outputs.rate-limited || steps.cover.outputs.stop || steps.dispatch.outputs.stop || steps.rerun.outputs.stop || steps.alert.outputs.stop }}
# Each guarded API call's rate-limit reading, for the job-end api-budget
# notice (.github/actions/log-api-budget). The full-run dispatch's own
# `gh api -i` POST leaves none.
# notice (.github/actions/log-api-budget).
env:
API_BUDGET_TOKEN: github-token
steps:
Expand Down Expand Up @@ -370,13 +369,18 @@ jobs:
else
what="main (this run started at $GITHUB_SHA)"
fi
# Each POST's rate-limit reading, for the job-end api-calls notice;
# recorded only, never guarded, so the POST goes as it would
# without the library, and without it nothing is recorded.
. .base/.github/actions/log-api-budget/api-budget.bash 2>/dev/null || true
# A header of the last response, by case-insensitive name.
header() { awk -F': *' -v n="$1" 'tolower($1) == n { sub(/\r$/, "", $2); print $2; exit }' <<<"$hdr"; }
id='' why='' waited=0
for try in 1 2 3; do
code=0
out=$(gh api -i -X POST "repos/$REPO/actions/workflows/pr-require-checks.yml/dispatches" \
-f ref=main -F return_run_details=true 2>"$RUNNER_TEMP/dispatch-stderr") || code=$?
! declare -F api_budget_read >/dev/null || api_budget_read github-token /dev/stdin <<<"$out" >/dev/null
cat "$RUNNER_TEMP/dispatch-stderr" >&2
hdr=$(sed '/^\r\?$/q' <<<"$out")
body=$(sed '1,/^\r\?$/d' <<<"$out")
Expand Down Expand Up @@ -2234,6 +2238,8 @@ jobs:
permissions:
contents: read # check out main
statuses: write # post pr-merge/unarmed on lanes App pull request heads
env:
API_BUDGET_TOKEN: github-token
steps:
- name: Check out the default branch tip
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
Expand All @@ -2255,6 +2261,14 @@ jobs:
STATUSES: ${{ needs.enable-auto-merge.outputs.unarmed-statuses }}
run: node .base/.github/scripts/pr-merge/post-unarmed-status.mjs

- name: Log the API budget at the end
if: always()
continue-on-error: true
timeout-minutes: 1
uses: ./.base/.github/actions/log-api-budget # zizmor: ignore[self-repository]
with:
tokens: github-token

dispatch-full-run:
needs: [merge]
if: >-
Expand All @@ -2270,7 +2284,26 @@ jobs:
timeout-minutes: 10
permissions:
actions: write # read main's full runs and start pr-require-checks.yml on main
contents: read # check out the api-budget library
# Each API call's rate-limit reading, for the job-end api-calls notice
# (.github/actions/log-api-budget).
env:
API_BUDGET_TOKEN: github-token
steps:
# The api-budget library only, from the default branch. A failed
# checkout leaves no library: the steps below call GitHub as they would
# without it, and no reading is logged.
- name: Check out the api-budget library
continue-on-error: true
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: refs/heads/main
path: .base
persist-credentials: false
sparse-checkout: |
/.github/actions/log-api-budget/
sparse-checkout-cone-mode: false

# At most one full run of main per CAP_MINUTES, each of main's tip,
# covering every merge since the last. The step reads the newest
# pr-require-checks.yml dispatches on main that github-actions[bot]
Expand All @@ -2279,11 +2312,12 @@ jobs:
# within CAP_MINUTES, uncovered=true when none is at TIP. A failed read
# sets neither, so the run dispatches as it would with no cap, except a
# read refused for a rate limit, which stops the step red with
# rate-limited=true and sends nothing more. This job has no checkout of
# the shared guard, so the match is the pattern rate-stop.bash's
# rate_stop_from reads; post-merge-dispatch's copy of this step sources
# that guard. A merge the cap skips is covered by the next push after
# the window or, while the lane is live, by post-merge-dispatch on the
# rate-limited=true and sends nothing more. This job checks out only
# the api-budget library, not rate-stop.bash, so the match is the
# pattern rate-stop.bash's rate_stop_from reads; post-merge-dispatch's
# copy of this step sources that guard. A merge the cap skips is
# covered by the next push after the window or, while the lane is
# live, by post-merge-dispatch on the
# next scheduled run, whose dispatch starts main's tip once nothing
# covers it, whoever merged. CAP_MINUTES: the efficiency plan of 2026-10-26
# (item 5, owner decision): 116 runs a day at a median 84.1 runner-min
Expand All @@ -2297,6 +2331,9 @@ jobs:
CAP_MINUTES: 30
run: |
set -euo pipefail
# The read keeps its rate-limit reading for the job-end api-calls
# notice; none when the checkout of the library failed.
{ . .base/.github/actions/log-api-budget/api-budget.bash 2>/dev/null && api_budget_wrap_gh github-token; } || true
if ! runs=$(gh api "repos/$REPO/actions/workflows/pr-require-checks.yml/runs?event=workflow_dispatch&branch=main&per_page=20" \
--jq '[.workflow_runs[] | select(.actor.login == "github-actions[bot]" and .head_branch == "main" and .conclusion != "cancelled") | {sha: .head_sha, created: .created_at}]' 2>"$RUNNER_TEMP/cover-stderr"); then
cat "$RUNNER_TEMP/cover-stderr" >&2
Expand Down Expand Up @@ -2351,13 +2388,18 @@ jobs:
else
what="main (this run started at $GITHUB_SHA)"
fi
# Each POST's rate-limit reading, for the job-end api-calls notice;
# recorded only, never guarded, so the POST goes as it would
# without the library, and without it nothing is recorded.
. .base/.github/actions/log-api-budget/api-budget.bash 2>/dev/null || true
# A header of the last response, by case-insensitive name.
header() { awk -F': *' -v n="$1" 'tolower($1) == n { sub(/\r$/, "", $2); print $2; exit }' <<<"$hdr"; }
id='' why='' waited=0
for try in 1 2 3; do
code=0
out=$(gh api -i -X POST "repos/$REPO/actions/workflows/pr-require-checks.yml/dispatches" \
-f ref=main -F return_run_details=true 2>"$RUNNER_TEMP/dispatch-stderr") || code=$?
! declare -F api_budget_read >/dev/null || api_budget_read github-token /dev/stdin <<<"$out" >/dev/null
cat "$RUNNER_TEMP/dispatch-stderr" >&2
hdr=$(sed '/^\r\?$/q' <<<"$out")
body=$(sed '1,/^\r\?$/d' <<<"$out")
Expand Down Expand Up @@ -2441,6 +2483,14 @@ jobs:
retention-days: 90
if-no-files-found: error

- name: Log the API budget at the end
if: always()
continue-on-error: true
timeout-minutes: 1
uses: ./.base/.github/actions/log-api-budget # zizmor: ignore[self-repository]
with:
tokens: github-token

# Each candidate the gate judged shows its verdict on its head as one
# `pr-merge / gate` check run: neutral, titled with the top refusal, or
# success over a refusal once enqueued or merged. Not `ci-status`, so never
Expand All @@ -2465,6 +2515,8 @@ jobs:
contents: read # check out main
checks: write
pull-requests: read # write only at each candidate's current head
env:
API_BUDGET_TOKEN: github-token
steps:
- name: Check out the default branch tip
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
Expand Down Expand Up @@ -2496,3 +2548,11 @@ jobs:
GH_TOKEN: ${{ github.token }}
SUMMARY_PATH: ${{ runner.temp }}/rows/pr-merge-summary.json
run: node .base/.github/scripts/pr-merge/gate-check.mjs

- name: Log the API budget at the end
if: always()
continue-on-error: true
timeout-minutes: 1
uses: ./.base/.github/actions/log-api-budget # zizmor: ignore[self-repository]
with:
tokens: github-token
Loading
Loading