From 5daaf70d25b429429c2dcb8653bf81fe1e8b1abf Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:37:01 -0400 Subject: [PATCH] feat(report-lane-outcome): record why a fallback credential ran Add an optional fallback-reason input, credit-stop or no-key, printed as fallback= right after credential= on the lane record. Any other non-empty value fails the step before it reports. intake-triage's subscription job passes it from the claude job's lane-class and credential-present outputs, and the leg parity test allows that one difference. Closes #806 Co-Authored-By: Claude Opus 5.5 --- .../actions/report-lane-outcome/action.yml | 28 ++++++-- .../actions/report-lane-outcome/classify.cjs | 15 +++- .../report-lane-outcome/classify.test.cjs | 28 ++++++++ .../report-lane-outcome/outcome-step.test.cjs | 72 +++++++++++++++++++ .github/scripts/intake-triage-legs.test.cjs | 30 +++++++- .github/workflows/intake-triage.yml | 2 + README.md | 10 ++- 7 files changed, 175 insertions(+), 10 deletions(-) diff --git a/.github/actions/report-lane-outcome/action.yml b/.github/actions/report-lane-outcome/action.yml index 79b96989..b588fb17 100644 --- a/.github/actions/report-lane-outcome/action.yml +++ b/.github/actions/report-lane-outcome/action.yml @@ -20,8 +20,8 @@ description: >- `class=` ::error annotation, and records the failure class and detail. Every annotation it writes, a success notice included, carries the run record as bare tokens: - `turns=`, `max-turns=`, `credential=`, `models=` (the result message's - modelUsage keys) and, on class rate-limit, `resets=` and + `turns=`, `max-turns=`, `credential=`, `fallback=`, `models=` (the result + message's modelUsage keys) and, on class rate-limit, `resets=` and `resets-window=` from the newest rejected rate_limit_event; an unknown value is omitted. Runs entirely on the github-script Node runtime — no jq or other runner-image @@ -79,6 +79,14 @@ inputs: omitted. required: false default: "" + fallback-reason: + description: >- + Why this run used a fallback credential: `credit-stop` (the key job + credit-stopped) or `no-key` (the key was empty). Printed as + `fallback=` right after `credential=`. Empty prints nothing; + any other value fails the step before it reports. + required: false + default: "" install-outcome: description: >- "failure" when the Claude action's Bun install did not put bun on PATH. @@ -152,6 +160,7 @@ runs: REQUIRE_COMPLETION: ${{ inputs.require-completion }} COMPLETION: ${{ inputs.completion }} CREDENTIAL: ${{ inputs.credential }} + FALLBACK_REASON: ${{ inputs.fallback-reason }} MAX_TURNS: ${{ inputs.max-turns }} INSTALL_OUTCOME: ${{ inputs.install-outcome }} KNOWN_FAILURE_CLASS: ${{ inputs.known-failure-class }} @@ -160,9 +169,17 @@ runs: script: | const fs = require("node:fs"); const path = require("node:path"); - const { classifyExecutionFile, readRunRecord, recordTokens } = require( - path.join(process.env.ACTION_PATH, "classify.cjs"), - ); + const { classifyExecutionFile, FALLBACK_REASONS, readRunRecord, recordTokens } = + require(path.join(process.env.ACTION_PATH, "classify.cjs")); + + // A caller wiring error, not a lane outcome: fail before reporting. + const fallbackReason = process.env.FALLBACK_REASON ?? ""; + if (fallbackReason !== "" && !FALLBACK_REASONS.includes(fallbackReason)) { + core.setFailed( + `fallback-reason must be empty or one of: ${FALLBACK_REASONS.join(", ")}.`, + ); + return; + } const outcome = process.env.LANE_OUTCOME ?? ""; const lane = process.env.LANE_NAME ?? "Claude review"; @@ -175,6 +192,7 @@ runs: failureClass, maxTurns: process.env.MAX_TURNS ?? "", credential: process.env.CREDENTIAL ?? "", + fallbackReason, }); return tokens === "" ? "" : ` ${tokens}`; }; diff --git a/.github/actions/report-lane-outcome/classify.cjs b/.github/actions/report-lane-outcome/classify.cjs index 547b5a1e..bf2951f2 100644 --- a/.github/actions/report-lane-outcome/classify.cjs +++ b/.github/actions/report-lane-outcome/classify.cjs @@ -261,6 +261,9 @@ const CREDENTIAL_PATTERN = /^[A-Z][A-Z0-9_]{0,99}$/u; const MODEL_PATTERN = /^[\w.:@[\]-]{1,100}$/u; const COUNT_PATTERN = /^\d{1,6}$/u; const WINDOW_PATTERN = /^[a-z0-9_]{1,40}$/u; +// Why a subscription job ran in the two-job pattern: the key job's credit +// stop, or no key at all (ci-workflows#806). +const FALLBACK_REASONS = ["credit-stop", "no-key"]; // SDK `resetsAt` is epoch time; a value past 1e12 can only be milliseconds. function isoFromEpoch(value) { @@ -311,9 +314,15 @@ function readRunRecord(executionFilePath) { // The record tokens for one annotation line, unknown values omitted. // `resets=` and `resets-window=` are printed only on a limit stop: that // stop's reset time and the window (rateLimitType) that stopped it. +// `fallback=` follows `credential=`, only from FALLBACK_REASONS. function recordTokens( { turns = null, models = [], resetsAt = null, resetsWindow = null }, - { failureClass = "", maxTurns = "", credential = "" } = {}, + { + failureClass = "", + maxTurns = "", + credential = "", + fallbackReason = "", + } = {}, ) { const tokens = []; if (failureClass === "rate-limit" && resetsAt) { @@ -323,6 +332,9 @@ function recordTokens( if (CREDENTIAL_PATTERN.test(credential)) { tokens.push(`credential=${credential}`); } + if (FALLBACK_REASONS.includes(fallbackReason)) { + tokens.push(`fallback=${fallbackReason}`); + } if (turns !== null) tokens.push(`turns=${turns}`); if (COUNT_PATTERN.test(maxTurns)) { tokens.push(`max-turns=${Number(maxTurns)}`); @@ -334,6 +346,7 @@ function recordTokens( module.exports = { classifyExecutionFile, creditStopResetsAt, + FALLBACK_REASONS, isCreditStop, readMessages, readRunRecord, diff --git a/.github/actions/report-lane-outcome/classify.test.cjs b/.github/actions/report-lane-outcome/classify.test.cjs index a6f615e1..3bd05b50 100644 --- a/.github/actions/report-lane-outcome/classify.test.cjs +++ b/.github/actions/report-lane-outcome/classify.test.cjs @@ -473,6 +473,34 @@ test("record tokens: resets only on a limit stop, unknown and malformed values o ); }); +// ci-workflows#806: a subscription job that ran because the key job +// credit-stopped, or had no key, says so right after the credential. +test("record tokens: a fallback reason prints right after the credential, only from its vocabulary", () => { + const run = { turns: 4, models: [], resetsAt: null }; + const credential = "AUTOMATIONS_CLAUDE_CODE_OAUTH_TOKEN"; + assert.equal( + recordTokens(run, { credential, fallbackReason: "credit-stop" }), + "credential=AUTOMATIONS_CLAUDE_CODE_OAUTH_TOKEN fallback=credit-stop turns=4", + ); + assert.equal( + recordTokens(run, { credential, fallbackReason: "no-key" }), + "credential=AUTOMATIONS_CLAUDE_CODE_OAUTH_TOKEN fallback=no-key turns=4", + ); + for (const forged of [ + "", + "auth", + "Credit-Stop", + " no-key", + "credit-stop\n::error::x", + ]) { + assert.equal( + recordTokens(run, { credential, fallbackReason: forged }), + "credential=AUTOMATIONS_CLAUDE_CODE_OAUTH_TOKEN turns=4", + JSON.stringify(forged), + ); + } +}); + // credit-stop. The stop texts come from the published pages: // "Credit balance is too low" (code.claude.com/docs/en/errors), the self-set // spend limit "You have reached your specified API usage limits" (400) and the diff --git a/.github/actions/report-lane-outcome/outcome-step.test.cjs b/.github/actions/report-lane-outcome/outcome-step.test.cjs index 3b17dd3f..e253240f 100644 --- a/.github/actions/report-lane-outcome/outcome-step.test.cjs +++ b/.github/actions/report-lane-outcome/outcome-step.test.cjs @@ -62,6 +62,7 @@ async function runOutcome({ credential = "", maxTurns = "", knownFailureClass = "", + fallbackReason = "", }) { const keys = [ "LANE_OUTCOME", @@ -72,6 +73,7 @@ async function runOutcome({ "CREDENTIAL", "MAX_TURNS", "KNOWN_FAILURE_CLASS", + "FALLBACK_REASON", "ACTION_PATH", ]; const originalValues = Object.fromEntries( @@ -86,6 +88,7 @@ async function runOutcome({ CREDENTIAL: credential, MAX_TURNS: maxTurns, KNOWN_FAILURE_CLASS: knownFailureClass, + FALLBACK_REASON: fallbackReason, ACTION_PATH: __dirname, }); const outputs = {}; @@ -418,3 +421,72 @@ test("a run that finished past its turn cap reads as max-turns with its turns an assert.equal(errors.length, 1); assert.match(errors[0], /\bclass=max-turns turns=16 max-turns=15\b/u); }); + +// ci-workflows#806: the subscription job of the two-job pattern writes the +// run's one lane record, so it names why it ran beside the credential. +test("a fallback reason rides beside the credential on the notice and the class line", async () => { + const succeeded = writeExecutionFile([ + { type: "result", subtype: "success", is_error: false, num_turns: 5 }, + ]); + const ok = await runOutcome({ + outcome: "success", + executionFile: succeeded, + credential: "AUTOMATIONS_CLAUDE_CODE_OAUTH_TOKEN", + fallbackReason: "credit-stop", + }); + assert.deepEqual(ok.failed, []); + assert.equal(ok.notices.length, 1); + assert.match( + ok.notices[0], + / credential=AUTOMATIONS_CLAUDE_CODE_OAUTH_TOKEN fallback=credit-stop turns=5\b/u, + ); + const authFailed = writeExecutionFile([ + { + type: "result", + subtype: "success", + is_error: true, + num_turns: 1, + api_error_status: 401, + }, + ]); + const failed = await runOutcome({ + outcome: "failure", + executionFile: authFailed, + credential: "AUTOMATIONS_CLAUDE_CODE_OAUTH_TOKEN", + fallbackReason: "no-key", + }); + assert.equal(failed.errors.length, 1); + assert.match( + failed.errors[0], + /\bclass=auth credential=AUTOMATIONS_CLAUDE_CODE_OAUTH_TOKEN fallback=no-key turns=1\b/u, + ); + const none = await runOutcome({ + outcome: "success", + executionFile: succeeded, + credential: "AUTOMATIONS_CLAUDE_CODE_OAUTH_TOKEN", + }); + assert.doesNotMatch(none.notices[0], /fallback=/u); +}); + +test("a fallback reason outside credit-stop and no-key fails the step and reports nothing", async () => { + const file = writeExecutionFile([ + { type: "result", subtype: "success", is_error: false, num_turns: 5 }, + ]); + for (const outcome of ["success", "failure"]) { + for (const fallbackReason of ["auth", "Credit-Stop", " credit-stop"]) { + const run = await runOutcome({ + outcome, + executionFile: file, + fallbackReason, + }); + const label = `${outcome}/${JSON.stringify(fallbackReason)}`; + assert.equal(run.failed.length, 1, label); + assert.match(run.failed[0], /credit-stop/u, label); + assert.match(run.failed[0], /no-key/u, label); + assert.deepEqual(run.notices, [], label); + assert.deepEqual(run.errors, [], label); + assert.deepEqual(run.warnings, [], label); + assert.deepEqual(run.outputs, {}, label); + } + } +}); diff --git a/.github/scripts/intake-triage-legs.test.cjs b/.github/scripts/intake-triage-legs.test.cjs index a1c87d98..b92c7ff0 100644 --- a/.github/scripts/intake-triage-legs.test.cjs +++ b/.github/scripts/intake-triage-legs.test.cjs @@ -102,7 +102,12 @@ test("the subscription job downloads nothing from the claude job and never write } }); -test("the subscription job repeats the claude job's steps but for the credential", () => { +// The subscription job's outcome step names why it ran (ci-workflows#806). +const FALLBACK_REASON = + "${{ needs.claude.outputs.lane-class == 'credit-stop' && 'credit-stop' || " + + "needs.claude.outputs.credential-present == 'false' && 'no-key' || '' }}"; + +test("the subscription job repeats the claude job's steps but for the credential and the fallback reason", () => { const strip = (steps, secret, input) => steps.map((step) => { const copy = structuredClone(step); @@ -116,6 +121,10 @@ test("the subscription job repeats the claude job's steps but for the credential assert.equal(copy.with[input], `\${{ secrets.${secret} }}`); delete copy.with[input]; } + if (copy.id === "review-outcome" && input === "subscription-token") { + assert.equal(copy.with["fallback-reason"], FALLBACK_REASON); + delete copy.with["fallback-reason"]; + } return copy; }); assert.deepEqual( @@ -139,6 +148,25 @@ test("the subscription job repeats the claude job's steps but for the credential ); }); +test("the subscription job's lane record names the credit stop or the empty key it ran for", () => { + const value = subscription.steps.find((step) => step.id === "review-outcome") + .with["fallback-reason"]; + const reason = (outputs) => + evaluate(value, { needs: { claude: { outputs } } }); + assert.equal( + reason({ "lane-class": "credit-stop", "credential-present": "true" }), + "credit-stop", + ); + assert.equal( + reason({ "lane-class": "", "credential-present": "false" }), + "no-key", + ); + assert.equal( + reason({ "lane-class": "auth", "credential-present": "true" }), + "", + ); +}); + test("a key credit stop skips the claude job's outcome and extract steps", () => { const byId = (id) => claude.steps.find((step) => step.id === id); const extract = byId("extract"); diff --git a/.github/workflows/intake-triage.yml b/.github/workflows/intake-triage.yml index 7c85fc45..d79b7e9e 100644 --- a/.github/workflows/intake-triage.yml +++ b/.github/workflows/intake-triage.yml @@ -1285,6 +1285,8 @@ jobs: credential: ${{ steps.claude.outputs.credential }} max-turns: ${{ steps.compose.outputs.max-turns }} known-failure-class: ${{ steps.claude.outputs.lane-class }} + # Why this job ran, for the run's one lane record (ci-workflows#806). + fallback-reason: ${{ needs.claude.outputs.lane-class == 'credit-stop' && 'credit-stop' || needs.claude.outputs.credential-present == 'false' && 'no-key' || '' }} # No model runs here. It reduces the run to the job outputs the apply # job reads: the result subtype, the error flag and the denial count, diff --git a/README.md b/README.md index 8171ddeb..284b5479 100644 --- a/README.md +++ b/README.md @@ -643,8 +643,10 @@ consumer to audit it. cache restore. Job A skips its outcome step on a credit stop, so whichever job triaged - reports; a status job that needs both reads job B's outcome when it ran, - else job A's: one lane record per run. A lane that writes does it in a + reports. Job B passes report-lane-outcome's `fallback-reason`, + `credit-stop` or `no-key` from job A's `lane-class` and key presence, so + its record says why it ran. A status job that needs both reads job B's + outcome when it ran, else job A's: one lane record per run. A lane that writes does it in a separate job with no model that needs both and reads job B's outputs when it ran, else job A's, so the write happens once. `intake-triage.yml` is the reference implementation. @@ -1238,7 +1240,9 @@ reached; re-run after it resets" (the `class=rate-limit` token is unchanged). Every outcome annotation, and a notice on a run that completed, also carries the run record as bare tokens, each omitted when unknown: `turns=`, `max-turns=` (the `max-turns` input), `credential=` (the -`credential` input), `models=` and, on `class=rate-limit`, +`credential` input), `fallback=credit-stop` or `fallback=no-key` (the +`fallback-reason` input, which rejects any other value), +`models=` and, on `class=rate-limit`, `resets=` and `resets-window=` from the run's newest `rejected` `rate_limit_event` (none without one), the time also exposed as the composite's `resets-at` output.