ci: count every GitHub request the Node clients send, GraphQL split out

.changes/pr-pipeline/ci-count-every-node-github-request-4b9e21c7.md

@@ -0,0 +1,8 @@
+---
+bump: patch
+---
+
+### Fixed
+
+- The job-end `api-calls` notice counts every GitHub request the lane scripts' Node clients send. Before, the clients built by address-feedback (its select and context mains, fix-ci-context and review-dependency), pr-verify's job-log read, and three steps that named no bucket sent requests the notice never saw. A client on a second token now counts under `API_BUDGET_OVERRIDE_TOKEN` when the request's token is not the step's own `GH_TOKEN` (else `GITHUB_TOKEN`), the rule babysit's gh calls already follow. Before, the job token's calls in a lanes App step were counted as lanes-app. No request changes.
+- The notice adds `graphql=K`, the GraphQL requests among the count, which spend GraphQL's own pool: `<token>=N not-modified=M graphql=K`.

.github/actions/log-api-budget/action.yml

@@ -5,9 +5,10 @@ description: >
   in $RUNNER_TEMP/api-budget (api-budget.bash): one per refused (403/429)
   response and one for the last. GET /rate_limit is never read: it reports a
   different bucket from the one the job spends. Then an `api-calls` notice,
-  `<name>=N not-modified=M`: N is this job's own requests, which cost (the rise in a
-  counter every concurrent job shares) is not, and M its 304 answers, which
-  GitHub does not count. Then one line for the job's content-creating writes
+  `<name>=N not-modified=M graphql=K`: N is this job's own requests, which cost (the rise in a
+  counter every concurrent job shares) is not, M its 304 answers, which
+  GitHub does not count, and K the GraphQL requests among N, which spend
+  GraphQL's own pool. Then one line for the job's content-creating writes
   (count and most in any 60 s). Also writes each line and a `<name>-cost=N`
   line to the step summary. A token with no reading gets only its api-calls
   line, at 0. Never fails the job.

.github/actions/log-api-budget/api-budget.bash

@@ -6,7 +6,8 @@
 #   <status> <limit> <remaining> <used> <reset epoch> <resource>
 # (`na` for a missing value). api-budget.mjs and babysit_gh.py append the same
 # lines. A request the gh wrapper counts with no header block to read (each
-# page of --paginate, a call that is not `gh api`) is a line `request`. Never
+# page of --paginate, a call that is not `gh api`) is a line `request`, or
+# `request graphql` for a GraphQL request. Never
 # fails the caller, except that a stop the rate guard below cannot record ends
 # the step.
 #
@@ -75,9 +76,11 @@
 #                               writes: count and most in any 60 s
 #   api_budget_emit_calls <token>...
 #                               the hub's api-calls notice for each token,
-#                               `<token>=N not-modified=M`: N is the job's own
-#                               requests, M its 304 answers, which GitHub does
-#                               not count; 0 for a token with no reading
+#                               `<token>=N not-modified=M graphql=K`: N is the
+#                               job's own requests, M its 304 answers, which
+#                               GitHub does not count, and K the GraphQL ones
+#                               among N, which spend their own pool; 0 for a
+#                               token with no reading
 #
 # pr-require-checks.yml's ci-status job carries this file verbatim in its
 # API_BUDGET_SH env (it has no checkout of this directory);
@@ -209,7 +212,7 @@ api_budget_traced() {
     /^\* Request at / { open++; next }
     /^\* Request took / { if (open) open--; next }
     /^\* Request to / {
-      if ($4 ~ /^https:\/\/((api|uploads)\.github\.com|[^\/]+\/api)\// && ++n > skip) print "request" >>state
+      if ($4 ~ /^https:\/\/((api|uploads)\.github\.com|[^\/]+\/api)\// && ++n > skip) print "request" ($4 ~ /\/graphql([?]|$)/ ? " graphql" : "") >>state
       next
     }
     /^\* (Using proxy|Skipping TLS verification): / || (open && /^\* /) || /^\[git[] ]/ { next }
@@ -426,7 +429,7 @@ api_budget_emit_calls() {
     file=${RUNNER_TEMP:-}/api-budget/$token
     [[ -n ${RUNNER_TEMP:-} && $token =~ ^[a-z][a-z0-9-]{0,39}$ ]] || continue
     [[ -s $file ]] || file=/dev/null
-    awk '$1 != 304 { n++ } $1 == 304 { m++ } END { printf "::notice title=api-calls::%s=%d not-modified=%d\n", token, n, m }' token="$token" "$file"
+    awk '$1 != 304 { n++ } $1 == 304 { m++ } $6 == "graphql" || $2 == "graphql" { g++ } END { printf "::notice title=api-calls::%s=%d not-modified=%d graphql=%d\n", token, n, m, g }' token="$token" "$file"
   done
   return 0
 }

.github/actions/log-api-budget/count-requests.test.mjs

@@ -0,0 +1,185 @@
+// Every GitHub client factory a lane script builds counts each request it
+// sends in $RUNNER_TEMP/api-budget/<token>, one reading line per answer, for
+// the job-end api-calls notice (api-budget.bash api_budget_emit_calls).
+import assert from "node:assert/strict";
+import { existsSync, readFileSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { test } from "node:test";
+import { createGitHub } from "../check-trusted-trigger/check-trusted-trigger.mjs";
+import { countRequests, createRateLimitLatch } from "./rate-guard.mjs";
+import { countedFetch, createGitHubApi as addressFeedbackApi } from "../../scripts/pr-unblock/address-feedback.mjs";
+import { createGitHubApi as dispatchApi } from "../../scripts/pr-unblock/dispatch.mjs";
+import { createGitHubApi as feedbackDispatchApi } from "../../scripts/pr-unblock/feedback-dispatch.mjs";
+import { budgetedFetch } from "../../scripts/pr-unblock/request-budget.mjs";
+import { tempDir } from "../../../lib/test-tmp.mjs";
+
+const API = "https://api.github.com";
+const READING = { "x-ratelimit-limit": "5000", "x-ratelimit-remaining": "4990", "x-ratelimit-used": "10", "x-ratelimit-reset": "1792751600", "x-ratelimit-resource": "core" };
+const LINE = "200 5000 4990 10 1792751600 core\n";
+
+// A fetch that answers each request 200 with body and the core reading above.
+const fake = (body = {}) => {
+  const sent = [];
+  const fetch = async (url, init = {}) => {
+    sent.push(`${init.method ?? "GET"} ${url}`);
+    return new Response(JSON.stringify(body), { status: 200, headers: READING });
+  };
+  return { sent, fetch };
+};
+
+// Runs fn with RUNNER_TEMP at a fresh directory and the given variables set
+// (undefined unsets one), then restores them; lines(name) reads a token's file.
+async function withBudget(vars, fn) {
+  const dir = tempDir("count-requests-");
+  const keys = ["RUNNER_TEMP", "API_BUDGET_TOKEN", "API_BUDGET_OVERRIDE_TOKEN", "GH_TOKEN", "GITHUB_TOKEN", ...Object.keys(vars)];
+  const saved = Object.fromEntries(keys.map((key) => [key, process.env[key]]));
+  for (const key of keys) delete process.env[key];
+  Object.assign(process.env, { RUNNER_TEMP: dir }, Object.fromEntries(Object.entries(vars).filter(([, value]) => value !== undefined)));
+  const lines = (name) => {
+    const file = join(dir, "api-budget", name);
+    return existsSync(file) ? readFileSync(file, "utf8") : "";
+  };
+  try {
+    return await fn(lines);
+  } finally {
+    for (const [key, value] of Object.entries(saved)) {
+      if (value === undefined) delete process.env[key];
+      else process.env[key] = value;
+    }
+    rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+const quietLatch = () => createRateLimitLatch({ log: () => {}, fail: () => {}, limits: () => ({}), stopFile: null });
+
+test("createGitHub counts each request under API_BUDGET_TOKEN", () =>
+  withBudget({ API_BUDGET_TOKEN: "github-token" }, async (lines) => {
+    const github = createGitHub({ token: "t", fetchImpl: fake().fetch, latch: quietLatch(), log: () => {} });
+    await github("GET", "/repos/o/r/pulls/1");
+    await github("GET", "/repos/o/r/pulls/2");
+    assert.equal(lines("github-token"), LINE + LINE);
+  }));
+
+test("a client on another token than the step's GH_TOKEN counts under API_BUDGET_OVERRIDE_TOKEN", () =>
+  withBudget({ API_BUDGET_TOKEN: "lanes-app", API_BUDGET_OVERRIDE_TOKEN: "github-token", GH_TOKEN: "lane" }, async (lines) => {
+    const latch = quietLatch();
+    const own = createGitHub({ token: "lane", fetchImpl: fake().fetch, latch, log: () => {} });
+    const job = createGitHub({ token: "job", fetchImpl: fake().fetch, latch, log: () => {} });
+    await own("GET", "/repos/o/r/pulls/1");
+    await job("GET", "/repos/o/r/actions/runs");
+    await job("GET", "/repos/o/r/actions/runs");
+    assert.equal(lines("lanes-app"), LINE);
+    assert.equal(lines("github-token"), LINE + LINE);
+  }));
+
+test("without API_BUDGET_OVERRIDE_TOKEN, or with no GH_TOKEN or GITHUB_TOKEN to compare, every client counts under API_BUDGET_TOKEN", async () => {
+  for (const vars of [{ GH_TOKEN: "lane" }, { API_BUDGET_OVERRIDE_TOKEN: "github-token" }]) {
+    await withBudget({ API_BUDGET_TOKEN: "lanes-app", ...vars }, async (lines) => {
+      await createGitHub({ token: "job", fetchImpl: fake().fetch, latch: quietLatch(), log: () => {} })("GET", "/repos/o/r/actions/runs");
+      assert.equal(lines("lanes-app"), LINE, JSON.stringify(vars));
+      assert.equal(lines("github-token"), "", JSON.stringify(vars));
+    });
+  }
+});
+
+test("GITHUB_TOKEN is the step's own token when GH_TOKEN is unset", () =>
+  withBudget({ API_BUDGET_TOKEN: "github-token", API_BUDGET_OVERRIDE_TOKEN: "lanes-app", GITHUB_TOKEN: "job" }, async (lines) => {
+    const latch = quietLatch();
+    await createGitHub({ token: "job", fetchImpl: fake().fetch, latch, log: () => {} })("GET", "/repos/o/r/pulls/1");
+    await createGitHub({ token: "app", fetchImpl: fake().fetch, latch, log: () => {} })("GET", "/repos/o/r/pulls/1");
+    assert.equal(lines("github-token"), LINE);
+    assert.equal(lines("lanes-app"), LINE);
+  }));
+
+test("a fetch that already counts is counted once, however often it is wrapped", () =>
+  withBudget({ API_BUDGET_TOKEN: "github-token" }, async (lines) => {
+    const latch = quietLatch();
+    const twice = countRequests(latch.guard(countRequests(fake().fetch)));
+    await twice(`${API}/repos/o/r/pulls/1`);
+    assert.equal(lines("github-token"), LINE);
+  }));
+
+test("address-feedback's client counts each attempt, a resent read included, through countedFetch", () =>
+  withBudget({ API_BUDGET_TOKEN: "github-token" }, async (lines) => {
+    const answers = [500, 200];
+    const fetch = async () => new Response(JSON.stringify({ number: 1 }), { status: answers.shift(), headers: READING });
+    const api = addressFeedbackApi({ token: "t", repository: "o/r", apiUrl: API, graphqlUrl: `${API}/graphql`, fetch: await countedFetch(fetch), sleep: async () => {} });
+    assert.deepEqual(await api.getPull(1), { number: 1 });
+    assert.equal(lines("github-token"), `500 5000 4990 10 1792751600 core\n${LINE}`);
+  }));
+
+test("dispatch's client counts each request, and once when handed budgetedFetch", () =>
+  withBudget({ API_BUDGET_TOKEN: "github-token" }, async (lines) => {
+    const pulls = { data: { repository: { pullRequests: { nodes: [] } } } };
+    const raw = dispatchApi({ token: "t", repository: "o/r", apiUrl: API, graphqlUrl: `${API}/graphql`, fetch: fake(pulls).fetch });
+    assert.deepEqual(await raw.listOpenPulls({ base: "main" }), { nodes: [] });
+    assert.equal(lines("github-token"), LINE);
+    const budgeted = dispatchApi({ token: "t", repository: "o/r", apiUrl: API, graphqlUrl: `${API}/graphql`, fetch: budgetedFetch({ fetch: fake(pulls).fetch, latch: quietLatch(), log: () => {} }).fetch });
+    await budgeted.listOpenPulls({ base: "main" });
+    assert.equal(lines("github-token"), LINE + LINE);
+  }));
+
+test("feedback-dispatch's client counts each request, and once when handed budgetedFetch", () =>
+  withBudget({ API_BUDGET_TOKEN: "github-token" }, async (lines) => {
+    const raw = feedbackDispatchApi({ token: "t", repository: "o/r", apiUrl: API, graphqlUrl: `${API}/graphql`, fetch: fake({ permission: "write" }).fetch });
+    assert.equal(await raw.getPermission("someone"), "write");
+    assert.equal(lines("github-token"), LINE);
+    const budgeted = feedbackDispatchApi({
+      token: "t",
+      repository: "o/r",
+      apiUrl: API,
+      graphqlUrl: `${API}/graphql`,
+      fetch: budgetedFetch({ fetch: fake({ permission: "read" }).fetch, latch: quietLatch(), log: () => {} }).fetch,
+    });
+    assert.equal(await budgeted.getPermission("someone"), "read");
+    assert.equal(lines("github-token"), LINE + LINE);
+  }));
+
+// The mains run from the base checkout build their address-feedback client
+// with countedFetch, over the global fetch. Each answers 404 here, so a main
+// stops at its first request; every request it sent is one reading.
+const ROOT = new URL("../../../", import.meta.url).pathname;
+const SHA = "a".repeat(40);
+for (const [name, run] of [
+  ["address-feedback select", async (env) => (await import("../../scripts/pr-unblock/address-feedback.mjs")).main(["select"], env, { log: () => {} })],
+  ["fix-ci-context", async (env) => (await import("../../scripts/pr-unblock/fix-ci-context.mjs")).main({ ...env, FIX_CI_ROUNDS: "3" }, { log: () => {} })],
+  ["review-dependency select", async (env) => (await import("../../scripts/pr-unblock/review-dependency.mjs")).main(["select"], env, { log: () => {} })],
+]) {
+  test(`${name}'s main counts each request its client sends`, () =>
+    withBudget({ API_BUDGET_TOKEN: "github-token" }, async (lines) => {
+      const dir = tempDir("count-requests-main-");
+      const saved = globalThis.fetch;
+      let sent = 0;
+      globalThis.fetch = async () => {
+        sent += 1;
+        return new Response("{}", { status: 404, headers: READING });
+      };
+      try {
+        const env = {
+          GITHUB_TOKEN: "t",
+          GITHUB_REPOSITORY: "o/r",
+          GITHUB_API_URL: API,
+          GITHUB_OUTPUT: join(dir, "out"),
+          PR_NUMBER: "7",
+          HEAD_SHA: SHA,
+          APP_SENDER_ID: "1",
+          CONTEXT_FILE: join(dir, "context.json"),
+          THREAD_FILE: join(dir, "threads.json"),
+          RUN_FILE: join(dir, "run.json"),
+          TRUSTED_ACTORS: join(ROOT, ".github/standards/trusted-actors/trusted-actors.json"),
+          TRACKER_CONFIG: join(ROOT, ".work-item-tracker.json"),
+          DEPENDENCY_FILE: join(dir, "dependency.json"),
+          CONTEXT_RUN_FILE: join(dir, "run.json"),
+          DEPENDABOT_CONFIG: join(ROOT, ".github/dependabot.yml"),
+          DEPENDABOT_POLICY: join(ROOT, ".github/standards/dependabot-policy/policy.json"),
+          BASE_PATH: ROOT,
+        };
+        await assert.rejects(run(env), /returned 404/);
+        assert.ok(sent > 0, "the main sent a request");
+        assert.equal(lines("github-token"), "404 5000 4990 10 1792751600 core\n".repeat(sent));
+      } finally {
+        globalThis.fetch = saved;
+        rmSync(dir, { recursive: true, force: true });
+      }
+    }));
+}