diff --git a/.changes/pr-pipeline/feat-pr-verify-two-job-e1205961.md b/.changes/pr-pipeline/feat-pr-verify-two-job-e1205961.md new file mode 100644 index 0000000000..4234d3f380 --- /dev/null +++ b/.changes/pr-pipeline/feat-pr-verify-two-job-e1205961.md @@ -0,0 +1,7 @@ +--- +bump: minor +--- + +### Changed + +- pr-verify runs its model on the automations API key first and on the subscription token only after a key credit stop or when no key is set, as two jobs on run-claude-cli v0.54.0. A permission denial in either job yields no verdict. A key credit stop warns that the automations key needs rotating and leaves one lane record: the subscription job's, named for the stop, or the key job's as rate-limit when a denial on the key leg ends the run. Every ci-workflows action is pinned to v0.54.0. diff --git a/.github/scripts/pr-verify/workflow.test.mjs b/.github/scripts/pr-verify/workflow.test.mjs index d7fa56dacf..c4dcbb8fc4 100644 --- a/.github/scripts/pr-verify/workflow.test.mjs +++ b/.github/scripts/pr-verify/workflow.test.mjs @@ -1,30 +1,70 @@ // pr-verify.yml's ordering rules, from its concurrency comment and the // README's "Verifier verdict" section: GitHub cancels an older pending run in the same concurrency group, so a cancelled -// run must leave no verdict. The upload is therefore the job's only artifact +// run must leave no verdict. The upload is therefore each job's only artifact // step, it runs after the verdict file is written, and only when writing it // succeeded; the group is per PR and head, never cancelling a running judgment. +// The key job judges first; the subscription job repeats it on the other +// credential only when the key job hands the judgment on, so one job per run +// writes a verdict. import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; import { test } from "node:test"; const text = readFileSync(new URL("../../workflows/pr-verify.yml", import.meta.url), "utf8"); -// Each step starts at a ` - name:` line of the one job. -const steps = text - .split(/\n(?= - name: )/) - .slice(1) - .map((block) => ({ name: /^ - name: (.*)$/m.exec(block)[1], block })); -const index = (name) => steps.findIndex((step) => step.name === name); - -test("the upload is the only artifact step and follows the written verdict", () => { - const uploads = steps.filter((step) => /uses: actions\/upload-artifact@/.test(step.block)); - assert.deepEqual( - uploads.map((step) => step.name), - ["Upload the verdict"], - ); - assert.ok(index("Judge the pull request") >= 0 && index("Write the verdict") > index("Judge the pull request")); - assert.ok(index("Upload the verdict") > index("Write the verdict")); - assert.match(uploads[0].block, /if: steps\.verdict\.outcome == 'success'/); - assert.doesNotMatch(text, /actions\/upload-artifact\/merge|gh api .*artifacts.* -X (POST|PUT)/); +const KEY = "check-intent"; +const SUB = "check-intent-subscription"; +const JOBS = [KEY, SUB]; +const jobText = (id) => { + const found = new RegExp(`\\n ${id}:\\n([\\s\\S]*?)(?=\\n [a-z-]+:\\n|$)`).exec(text); + assert.ok(found, `a job ${id}`); + return found[1]; +}; +// Each step starts at a ` - name:` line of its job. +const stepsOf = (id) => + jobText(id) + .split(/\n(?= - name: )/) + .slice(1) + .map((block) => ({ name: /^ - name: (.*)$/m.exec(block)[1], block })); +const STEPS = Object.fromEntries(JOBS.map((id) => [id, stepsOf(id)])); +const index = (name, job = KEY) => STEPS[job].findIndex((step) => step.name === name); +const step = (name, job = KEY) => { + const found = STEPS[job].find((s) => s.name === name); + assert.ok(found, `a step named ${name} in ${job}`); + return found.block; +}; +const input = (block, key) => new RegExp(`^ {10}${key}: (.*)$`, "m").exec(block)?.[1]; +const list = (block, key) => { + const body = new RegExp(`^ {10}${key}: \\|\\n((?: {12}.*\\n?)+)`, "m").exec(block)?.[1] ?? ""; + return body.split("\n").map((line) => line.trim()).filter(Boolean); +}; +const PIN = /^melodic-software\/ci-workflows\/\.github\/actions\/([a-z-]+)@[0-9a-f]{40} # v\d+\.\d+\.\d+$/; +// The subscription job's one lane record names why the key did not judge. +const SUB_LANE = + "${{ needs.check-intent.outputs.lane-class == 'credit-stop' && 'pr-verify (key AUTOMATIONS_CLAUDE_API_KEY hit a credit stop)' || 'pr-verify (key AUTOMATIONS_CLAUDE_API_KEY is empty)' }}"; +const KEY_CLASS = "${{ steps.judge.outputs.lane-class == 'credit-stop' && 'rate-limit' || steps.judge.outputs.lane-class }}"; +const KEY_REPORT_IF = + "(steps.judge.outcome == 'success' || steps.judge.outcome == 'failure') && (steps.judge.outputs.lane-class != 'credit-stop' || steps.key-leg.outcome == 'failure')"; +const CREDENTIAL = { + [KEY]: ["api-key", "AUTOMATIONS_CLAUDE_API_KEY"], + [SUB]: ["subscription-token", "AUTOMATIONS_CLAUDE_CODE_OAUTH_TOKEN"], +}; + +test("the upload is each job's only artifact step and follows the written verdict", () => { + for (const job of JOBS) { + const uploads = STEPS[job].filter((s) => /uses: actions\/upload-artifact@/.test(s.block)); + assert.deepEqual( + uploads.map((s) => s.name), + ["Upload the verdict"], + job, + ); + assert.ok(index("Judge the pull request", job) >= 0 && index("Write the verdict", job) > index("Judge the pull request", job), job); + assert.ok(index("Upload the verdict", job) > index("Write the verdict", job), job); + assert.match(uploads[0].block, /if: steps\.verdict\.outcome == 'success'/, job); + // A verdict needs a judge step that succeeded, which run-claude-cli never + // reports with a lane-class, so a credit stop writes none. + assert.match(step("Write the verdict", job), /^ {8}if: steps\.judge\.outcome == 'success'$/m, job); + } + assert.doesNotMatch(text, /actions\/upload-artifact\/merge|gh api .*artifacts.* -X (POST|PUT)|actions\/download-artifact|actions\/cache/); }); test("pr-merge.yml gives the gate an Actions token for the verdict reads only", () => { @@ -50,82 +90,214 @@ test("the concurrency group is per PR and head and never cancels a running judgm // Funding: pr-verify takes the automations role), under the launcher's input // contract (its action.yml: managed settings while sudo works, then runner // isolation) and the deny rules that keep the model on its data files. -const step = (name) => { - const found = steps.find((s) => s.name === name); - assert.ok(found, `a step named ${name}`); - return found.block; -}; -const input = (block, key) => new RegExp(`^ {10}${key}: (.*)$`, "m").exec(block)?.[1]; -const list = (block, key) => { - const body = new RegExp(`^ {10}${key}: \\|\\n((?: {12}.*\\n?)+)`, "m").exec(block)?.[1] ?? ""; - return body.split("\n").map((line) => line.trim()).filter(Boolean); -}; -const PIN = /^melodic-software\/ci-workflows\/\.github\/actions\/([a-z-]+)@[0-9a-f]{40} # v\d+\.\d+\.\d+$/; - -test("the judge runs on run-claude-cli as the automations role with both of its secrets", () => { - const judge = step("Judge the pull request"); - assert.equal(PIN.exec(/^ {8}uses: (.*)$/m.exec(judge)[1])?.[1], "run-claude-cli"); +test("each job judges on run-claude-cli as the automations role with its one secret", () => { + const pins = new Set(); + for (const job of JOBS) { + const judge = step("Judge the pull request", job); + const uses = /^ {8}uses: (.*)$/m.exec(judge)[1]; + assert.equal(PIN.exec(uses)?.[1], "run-claude-cli", job); + pins.add(uses); + assert.equal(input(judge, "lane"), "pr-verify", job); + assert.equal(input(judge, "role"), "automations", job); + const [own, secret] = CREDENTIAL[job]; + const [other] = CREDENTIAL[job === KEY ? SUB : KEY]; + assert.equal(input(judge, own), `\${{ secrets.${secret} }}`, job); + assert.equal(input(judge, other), undefined, job); + assert.match(judge, /^ {8}if: steps\.credential\.outputs\.present == 'true'$/m, job); + for (const key of ["model", "effort", "max-turns"]) { + assert.equal(input(judge, key), `\${{ steps.models.outputs.${key} }}`, key); + } + assert.equal(input(judge, "permission-mode"), "dontAsk", job); + assert.equal(input(judge, "leg-timeout-minutes"), '"10"', job); + assert.match(judge, /^ {8}continue-on-error: true$/m, job); + assert.match(judge, /^ {8}timeout-minutes: 22$/m, job); + assert.match(judge, /^ {10}CLAUDE_CODE_DISABLE_BACKGROUND_TASKS: "1"$/m, job); + assert.match(input(judge, "prompt-file"), /^\$\{\{ runner\.temp \}\}\//, job); + assert.match(input(judge, "json-schema-file"), /^\$\{\{ runner\.temp \}\}\//, job); + } + assert.equal(pins.size, 1, "both jobs pin the same run-claude-cli"); assert.doesNotMatch(text, /anthropics\/claude-code-action/); - assert.equal(input(judge, "lane"), "pr-verify"); - assert.equal(input(judge, "role"), "automations"); - assert.equal(input(judge, "api-key"), "${{ secrets.AUTOMATIONS_CLAUDE_API_KEY }}"); - assert.equal(input(judge, "subscription-token"), "${{ secrets.AUTOMATIONS_CLAUDE_CODE_OAUTH_TOKEN }}"); - for (const key of ["model", "effort", "max-turns"]) { - assert.equal(input(judge, key), `\${{ steps.models.outputs.${key} }}`, key); +}); + +test("each secret appears only in its own job", () => { + for (const job of JOBS) { + const [, own] = CREDENTIAL[job]; + const [, other] = CREDENTIAL[job === KEY ? SUB : KEY]; + assert.ok(jobText(job).includes(`secrets.${own}`), job); + assert.ok(!jobText(job).includes(`secrets.${other}`), job); + } + assert.doesNotMatch(text, /secrets\.(?!AUTOMATIONS_CLAUDE_API_KEY\b|AUTOMATIONS_CLAUDE_CODE_OAUTH_TOKEN\b)/); + assert.doesNotMatch(text, /secrets: inherit|toJSON\(secrets\)/); +}); + +test("the subscription job runs only on a clean credit stop or an empty key, on a fixed runner", () => { + const job = jobText(SUB); + assert.match(job, /^ {4}needs: check-intent$/m); + const gate = /^ {4}if: >-\n((?: {6}.*\n)+)/m.exec(job)?.[1].replace(/\s+/g, " ").trim(); + assert.equal( + gate, + [ + "github.ref == 'refs/heads/main'", + "&& github.event.workflow_run.event == 'pull_request'", + "&& github.event.workflow_run.conclusion == 'success'", + "&& github.event.workflow_run.head_repository.full_name == github.repository", + "&& ((needs.check-intent.outputs.lane-class == 'credit-stop'", + "&& needs.check-intent.outputs.key-leg-clean == 'true')", + "|| needs.check-intent.outputs.credential-present == 'false')", + ].join(" "), + ); + // No status function, so GitHub adds success(): a failed key job skips it. + assert.doesNotMatch(gate, /always\(\)|failure\(\)|cancelled\(\)|success\(\)/); + assert.match(job, /^ {4}runs-on: ubuntu-24\.04$/m); + // The key job's outputs reach only that if and the record's lane name, + // which they pick between two literals; never a run: script. + const steps = job.slice(job.indexOf("\n steps:\n")); + assert.deepEqual( + steps.split("\n").filter((line) => line.includes("needs.")), + [` lane: ${SUB_LANE}`], + ); + const key = jobText(KEY); + const outputs = /^ {4}outputs:\n((?: {6}.*\n)+)/m.exec(key)?.[1]; + assert.equal( + outputs, + [ + " lane-class: ${{ steps.judge.outputs.lane-class }}", + " credential-present: ${{ steps.credential.outputs.present }}", + " key-leg-clean: ${{ steps.key-leg.outputs.clean }}", + "", + ].join("\n"), + ); + assert.doesNotMatch(job, /^ {4}outputs:/m); + for (const id of JOBS) { + assert.match(jobText(id), /^ {4}permissions:\n {6}contents: read #.*\n {6}pull-requests: read #.*\n {6}actions: read #.*\n {4}[a-z]/m, id); } - assert.equal(input(judge, "permission-mode"), "dontAsk"); - assert.equal(input(judge, "leg-timeout-minutes"), '"10"'); - assert.match(judge, /^ {8}continue-on-error: true$/m); - assert.match(judge, /^ {8}timeout-minutes: 22$/m); - assert.match(judge, /^ {10}CLAUDE_CODE_DISABLE_BACKGROUND_TASKS: "1"$/m); - assert.match(input(judge, "prompt-file"), /^\$\{\{ runner\.temp \}\}\//); - assert.match(input(judge, "json-schema-file"), /^\$\{\{ runner\.temp \}\}\//); +}); + +test("the key job hands the judgment on only after a clean credit stop or with no key", () => { + const leg = step("Check the key leg"); + assert.match(leg, /^ {8}if: steps\.judge\.outputs\.lane-class == 'credit-stop'$/m); + assert.match(leg, /^ {10}LOG_FILE: \$\{\{ steps\.judge\.outputs\.log-file \}\}$/m); + assert.match(leg, /write-verdict\.mjs check-key-leg\n/); + // It runs before the report, which reads its outcome, and its failure + // leaves the red to "Fail without a verdict" after the report. + assert.match(leg, /^ {8}continue-on-error: true$/m); + assert.ok(index("Check the key leg") === index("Judge the pull request") + 1); + assert.ok(index("Report the judgment's outcome") === index("Check the key leg") + 1); + // The owner's signal to rotate a key, printed before the denial check can stop the step. + const run = /^ {8}run: \|\n((?: {10}.*\n?)+)/m.exec(leg)[1]; + assert.match(run.split("\n")[0], /^ {10}echo "::warning::the automations role's API key \(AUTOMATIONS_CLAUDE_API_KEY\) hit a credit stop/); + assert.ok(run.indexOf("::warning::") < run.indexOf("check-key-leg")); + assert.equal(index("Check the key leg", SUB), -1); + for (const job of JOBS) { + const credential = step("Check the credential", job); + assert.match(credential, /^ {8}if: steps\.agree\.outcome == 'success'$/m, job); + assert.match(credential, new RegExp(`^ {10}PRESENT: \\$\\{\\{ secrets\\.${CREDENTIAL[job][1]} != '' \\}\\}$`, "m"), job); + assert.ok(index("Check the credential", job) < index("Judge the pull request", job), job); + } +}); + +test("the subscription job repeats the key job's steps but for the credential", () => { + // Each difference the two-job pattern needs, normalized away on the key + // job's side; everything else, the gates and hardening included, matches. + const normalize = (blocks) => + blocks + .filter((s) => s.name !== "Check the key leg") + .map((s) => + s.block + .split("\n") + .filter((line) => line.trim() !== "" && !line.trim().startsWith("#")) + .join("\n") + .replace(/AUTOMATIONS_CLAUDE_CODE_OAUTH_TOKEN/g, "AUTOMATIONS_CLAUDE_API_KEY") + .replace(/^( {10})subscription-token: /m, "$1api-key: ") + .replace(`if: ${KEY_REPORT_IF}`, "if: steps.judge.outcome == 'success' || steps.judge.outcome == 'failure'") + .replace(`known-failure-class: ${KEY_CLASS}`, "known-failure-class: ${{ steps.judge.outputs.lane-class }}") + .replace(`lane: ${SUB_LANE}`, "lane: pr-verify") + .replace(/ && steps\.credential\.outputs\.present != 'false' && steps\.key-leg\.outcome != 'success'$/m, ""), + ); + assert.deepEqual(normalize(STEPS[SUB]), normalize(STEPS[KEY])); }); test("the model reads only the data directory, with Read alone, and is told nothing is posted", () => { - const judge = step("Judge the pull request"); - assert.equal(input(judge, "tools"), "Read"); - assert.equal(input(judge, "allowed-tools"), "Read(./pr-verify-data/**)"); - const denied = list(judge, "disallowed-tools"); - for (const rule of ["Read(./.base/**)", "Glob", "Grep", "Bash", "Edit", "Write", "NotebookEdit", "WebFetch", "WebSearch", "Agent", "mcp__*"]) { - assert.ok(denied.includes(rule), `disallowed-tools holds ${rule}`); + for (const job of JOBS) { + const judge = step("Judge the pull request", job); + assert.equal(input(judge, "tools"), "Read", job); + assert.equal(input(judge, "allowed-tools"), "Read(./pr-verify-data/**)", job); + const denied = list(judge, "disallowed-tools"); + for (const rule of ["Read(./.base/**)", "Glob", "Grep", "Bash", "Edit", "Write", "NotebookEdit", "WebFetch", "WebSearch", "Agent", "mcp__*"]) { + assert.ok(denied.includes(rule), `${job}: disallowed-tools holds ${rule}`); + } + const preamble = /^ {10}preamble: >-\n((?: {12}.*\n)+)/m.exec(judge)?.[1].replace(/\s+/g, " ").trim(); + assert.match(preamble ?? "", /unattended/, job); + assert.match(preamble ?? "", /nothing you write is posted/, job); } - const preamble = /^ {10}preamble: >-\n((?: {12}.*\n)+)/m.exec(judge)?.[1].replace(/\s+/g, " ").trim(); - assert.match(preamble ?? "", /unattended/); - assert.match(preamble ?? "", /nothing you write is posted/); }); test("every data path the model reads sits under the working directory", () => { assert.doesNotMatch(text, /runner\.temp \}\}\/pr-verify-data/); - for (const name of ["Resolve the pull request", "Select the trusted PR text", "Write the diff", "Write the prompt and schema", "Write the verdict"]) { - assert.match(step(name), /\$\{\{ github\.workspace \}\}\/pr-verify-data/, name); + for (const job of JOBS) { + for (const name of ["Resolve the pull request", "Select the trusted PR text", "Write the diff", "Write the prompt and schema", "Write the verdict"]) { + assert.match(step(name, job), /\$\{\{ github\.workspace \}\}\/pr-verify-data/, `${job}: ${name}`); + } + } +}); + +test("each job checks out main fresh and writes the managed settings before isolating the runner", () => { + for (const job of JOBS) { + assert.equal(STEPS[job][0].name, "Check out the default branch tip", job); + assert.match(STEPS[job][0].block, /^ {10}ref: refs\/heads\/main$/m, job); + assert.match(STEPS[job][0].block, /^ {10}persist-credentials: false$/m, job); + const settings = step("Apply the Claude managed settings", job); + assert.equal(PIN.exec(/^ {8}uses: (.*)$/m.exec(settings)[1])?.[1], "apply-claude-managed-settings", job); + assert.match(settings, /^ {8}if: steps\.agree\.outcome == 'success'$/m, job); + assert.ok(index("Apply the Claude managed settings", job) < index("Isolate the runner", job), job); + assert.ok(index("Isolate the runner", job) < index("Judge the pull request", job), job); } }); -test("the managed settings are written while sudo works, before the runner is isolated", () => { - const settings = step("Apply the Claude managed settings"); - assert.equal(PIN.exec(/^ {8}uses: (.*)$/m.exec(settings)[1])?.[1], "apply-claude-managed-settings"); - assert.match(settings, /^ {8}if: steps\.agree\.outcome == 'success'$/m); - assert.ok(index("Apply the Claude managed settings") < index("Isolate the runner")); - assert.ok(index("Isolate the runner") < index("Judge the pull request")); +test("the outcome is reported whenever a judge ran to a final class, and a missing verdict fails the job last", () => { + for (const job of JOBS) { + const report = step("Report the judgment's outcome", job); + assert.equal(PIN.exec(/^ {8}uses: (.*)$/m.exec(report)[1])?.[1], "report-lane-outcome", job); + assert.ok(index("Report the judgment's outcome", job) === index("Judge the pull request", job) + (job === KEY ? 2 : 1), job); + assert.equal(input(report, "credential"), "${{ steps.judge.outputs.credential }}", job); + + assert.equal(input(report, "step-timeout-minutes"), "22", job); + // The start time is taken right before the judge, so a timeout is told from other failures. + assert.equal(input(report, "step-started-at"), "${{ steps.prompt.outputs.started-at }}", job); + assert.ok(index("Write the prompt and schema", job) === index("Judge the pull request", job) - 1, job); + assert.match(step("Write the verdict", job), /^ {10}EXECUTION_FILE: \$\{\{ steps\.judge\.outputs\.execution-file \}\}$/m, job); + assert.match(step("Write the verdict", job), /^ {10}LOG_FILE: \$\{\{ steps\.judge\.outputs\.log-file \}\}$/m, job); + assert.equal(STEPS[job].at(-1).name, "Fail without a verdict", job); + assert.match(STEPS[job].at(-1).block, /exit 1/, job); + } + assert.match(step("Report the judgment's outcome", SUB), /^ {8}if: steps\.judge\.outcome == 'success' \|\| steps\.judge\.outcome == 'failure'$/m); + assert.equal(input(step("Report the judgment's outcome", SUB), "known-failure-class"), "${{ steps.judge.outputs.lane-class }}"); + assert.equal(input(step("Report the judgment's outcome", SUB), "lane"), SUB_LANE); + // Only a hand-off spares the key job its red; an empty token fails the subscription job. + assert.match( + STEPS[KEY].at(-1).block, + /^ {8}if: always\(\) && steps\.agree\.outcome == 'success' && steps\.verdict\.outcome != 'success' && steps\.credential\.outputs\.present != 'false' && steps\.key-leg\.outcome != 'success'$/m, + ); + assert.match(STEPS[SUB].at(-1).block, /^ {8}if: always\(\) && steps\.agree\.outcome == 'success' && steps\.verdict\.outcome != 'success'$/m); +}); + +test("a credit stop leaves exactly one lane record, from the job that saw its end", () => { + const report = step("Report the judgment's outcome", KEY); + assert.equal(/^ {8}if: (.*)$/m.exec(report)[1], KEY_REPORT_IF); + assert.equal(input(report, "lane"), "pr-verify"); + // A key leg that is not clean hands nothing on, so the key job records the + // stop itself, as the class report-lane-outcome knows for a credit stop. + assert.equal(input(report, "known-failure-class"), KEY_CLASS); + // A clean stop skips this report; the subscription job, which runs exactly + // when key-leg-clean is true, records the run and names the stop in its lane. + assert.match(jobText(SUB), /needs\.check-intent\.outputs\.key-leg-clean == 'true'/); + assert.match(SUB_LANE, /lane-class == 'credit-stop' && '[^']*credit stop\)'/); + assert.doesNotMatch(SUB_LANE, /=[a-z]/, "no key=value token a lane reader could parse"); }); -test("the outcome is reported whenever the judge ran, and a missing verdict fails the job last", () => { - const report = step("Report the judgment's outcome"); - assert.equal(PIN.exec(/^ {8}uses: (.*)$/m.exec(report)[1])?.[1], "report-lane-outcome"); - assert.ok(index("Report the judgment's outcome") === index("Judge the pull request") + 1); - assert.match(report, /^ {8}if: steps\.judge\.outcome == 'success' \|\| steps\.judge\.outcome == 'failure'$/m); - assert.equal(input(report, "credential"), "${{ steps.judge.outputs.credential }}"); - assert.equal(input(report, "known-failure-class"), "${{ steps.judge.outputs.lane-class }}"); - assert.equal(input(report, "step-timeout-minutes"), "22"); - // The start time is taken right before the judge, so a timeout is told from other failures. - assert.equal(input(report, "step-started-at"), "${{ steps.prompt.outputs.started-at }}"); - assert.ok(index("Write the prompt and schema") === index("Judge the pull request") - 1); - assert.match(step("Write the verdict"), /^ {10}EXECUTION_FILE: \$\{\{ steps\.judge\.outputs\.execution-file \}\}$/m); - assert.match(step("Write the verdict"), /^ {10}LOG_FILE: \$\{\{ steps\.judge\.outputs\.log-file \}\}$/m); - - const last = steps.at(-1); - assert.equal(last.name, "Fail without a verdict"); - assert.match(last.block, /^ {8}if: always\(\) && steps\.agree\.outcome == 'success' && steps\.verdict\.outcome != 'success'$/m); - assert.match(last.block, /exit 1/); +test("every ci-workflows action is pinned to one release", () => { + const refs = [...text.matchAll(/uses: melodic-software\/ci-workflows\/[^@\s]+@([0-9a-f]{40}) # (v\d+\.\d+\.\d+)$/gm)].map((m) => `${m[1]} ${m[2]}`); + assert.equal(refs.length, (text.match(/melodic-software\/ci-workflows\//g) ?? []).length, "every ref is a SHA pin with its tag"); + assert.ok(refs.length >= 6); + assert.equal(new Set(refs).size, 1, `one pin: ${[...new Set(refs)].join(", ")}`); }); diff --git a/.github/scripts/pr-verify/write-verdict.mjs b/.github/scripts/pr-verify/write-verdict.mjs index 89b230dff3..9d31c7156a 100644 --- a/.github/scripts/pr-verify/write-verdict.mjs +++ b/.github/scripts/pr-verify/write-verdict.mjs @@ -124,9 +124,7 @@ export function structuredOutput(messages) { return result.structured_output; } -// The execution file is the last leg's alone, so a denial on an earlier leg -// (the key leg, before a credit stop) shows only in run-claude-cli's log, one -// object per leg. +// run-claude-cli's log, one object per leg, counts every leg's denials. export function earlierDenials(logText) { const total = logText .split("\n") @@ -137,6 +135,25 @@ export function earlierDenials(logText) { } } +// After the key leg's credit stop the subscription job judges on a new VM +// that cannot read this leg's log, so the key job records here whether the +// leg was free of denials; the subscription job runs only on clean=true. +export function checkKeyLeg(env = process.env) { + for (const key of ["LOG_FILE", "GITHUB_OUTPUT"]) { + if (!env[key]) { + throw new Error(`${key} is not set`); + } + } + let logText; + try { + logText = readFileSync(env.LOG_FILE, "utf8"); + } catch { + throw new Error("the run log is unreadable"); + } + earlierDenials(logText); + appendFileSync(env.GITHUB_OUTPUT, "clean=true\n"); +} + export const artifactName =(verdict) => `pr-verify-${verdict.pr}-${verdict.head_sha}-${verdict.verdict}`; export function main(env = process.env) { @@ -193,7 +210,11 @@ export function main(env = process.env) { if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { try { - main(); + if (process.argv[2] === "check-key-leg") { + checkKeyLeg(); + } else { + main(); + } } catch (error) { console.log(`::error::no verdict: ${error.message}`); process.exitCode = 1; diff --git a/.github/scripts/pr-verify/write-verdict.test.mjs b/.github/scripts/pr-verify/write-verdict.test.mjs index 2d2d51d44c..3563d207de 100644 --- a/.github/scripts/pr-verify/write-verdict.test.mjs +++ b/.github/scripts/pr-verify/write-verdict.test.mjs @@ -12,7 +12,7 @@ import { tmpdir } from "node:os"; import path from "node:path"; import { test } from "node:test"; -import { artifactName, buildVerdict, main, structuredOutput } from "./write-verdict.mjs"; +import { artifactName, buildVerdict, checkKeyLeg, main, structuredOutput } from "./write-verdict.mjs"; const HEAD = "a".repeat(40); const BASE = "b".repeat(40); @@ -209,7 +209,7 @@ test("main writes the verdict file and the artifact name, and nothing on bad out ["missing execution file", { EXECUTION_FILE: path.join(dir, "missing.json") }], ["unparsable execution file", { EXECUTION_FILE: unparsable }], ["unreadable spec", { EXECUTION_FILE: okFile, SPEC_FILE: path.join(dir, "missing.json") }], - // A denial on the key leg, then a credit stop and a clean subscription leg. + // A log with a denial on any leg. ["a denial on an earlier leg", { EXECUTION_FILE: okFile, LOG_FILE: log("denied-log", 1, 0) }], ["missing log file", { EXECUTION_FILE: okFile, LOG_FILE: path.join(dir, "missing.jsonl") }], ["no log file", { EXECUTION_FILE: okFile, LOG_FILE: "" }], @@ -220,3 +220,26 @@ test("main writes the verdict file and the artifact name, and nothing on bad out assert.equal(existsSync(other.GITHUB_OUTPUT), false, label); } }); + +test("checkKeyLeg clears the subscription job only for a readable log with no denial", () => { + const dir = mkdtempSync(path.join(tmpdir(), "pr-verify-key-leg-")); + const file = (name, text) => { + const target = path.join(dir, name); + writeFileSync(target, text); + return target; + }; + const clean = { LOG_FILE: file("clean.jsonl", JSON.stringify({ leg: 1, denials: { count: 0 } })), GITHUB_OUTPUT: path.join(dir, "out") }; + checkKeyLeg(clean); + assert.equal(readFileSync(clean.GITHUB_OUTPUT, "utf8"), "clean=true\n"); + + for (const [label, logFile] of [ + ["a denial", file("denied.jsonl", JSON.stringify({ leg: 1, denials: { count: 2 } }))], + ["a malformed log", file("bad.jsonl", "{not json")], + ["a missing log", path.join(dir, "missing.jsonl")], + ["no log", ""], + ]) { + const env = { LOG_FILE: logFile, GITHUB_OUTPUT: path.join(dir, `out-${label}`) }; + assert.throws(() => checkKeyLeg(env), undefined, label); + assert.equal(existsSync(env.GITHUB_OUTPUT), false, label); + } +}); diff --git a/.github/workflows/pr-verify.yml b/.github/workflows/pr-verify.yml index 59f4ab9689..7697d872a4 100644 --- a/.github/workflows/pr-verify.yml +++ b/.github/workflows/pr-verify.yml @@ -13,10 +13,19 @@ name: pr-verify # issues are data files in pr-verify-data under the working directory, and the # model reads them with the Read tool alone: its rules allow that directory # and deny .base and every other tool. The judgment runs on ci-workflows' -# run-claude-cli as the automations role, its API key first and its -# subscription token after a credit stop. The job token can only read. Every -# script and local action comes from the main checkout in .base; the Claude -# actions are ci-workflows', pinned by SHA. +# run-claude-cli as the automations role, on one credential per job. The +# check-intent job holds only the API key. The check-intent-subscription job +# holds only the subscription token and runs, on a new ubuntu-24.04 VM, only +# when check-intent's key leg ended in a credit stop with no permission +# denial, or its key is empty; then check-intent writes no verdict and does +# not fail, so one job per run judges. The second job repeats every step from +# a fresh checkout. From check-intent it reads only three outputs, each +# compared to a fixed string in its `if` (and lane-class once more to pick +# its lane record's name between two literals); none of them is written into +# a `run:` script, and it takes no artifact or cache. run-claude-cli sets +# lane-class after it restores the step files, so the model cannot set it. +# The job tokens can only read. Every script and local action comes from the +# main checkout in .base; the Claude actions are ci-workflows', pinned by SHA. # # Every local action runs from the main checkout at ./.base. zizmor's # self-repository advice (`$/...`) is ignored on those lines, as in @@ -58,6 +67,10 @@ jobs: contents: read # check out main; read the PR's commits for the diff pull-requests: read # the PR, its comments and reviews actions: read # the upstream run, and verdicts already uploaded + outputs: + lane-class: ${{ steps.judge.outputs.lane-class }} + credential-present: ${{ steps.credential.outputs.present }} + key-leg-clean: ${{ steps.key-leg.outputs.clean }} steps: - name: Check out the default branch tip uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -169,11 +182,26 @@ jobs: model-route: verify-pr output-path: ${{ runner.temp }}/pr-verify-models.json + # Only whether the secret is set leaves this step, never its value. + - name: Check the credential + id: credential + if: steps.agree.outcome == 'success' + env: + PRESENT: ${{ secrets.AUTOMATIONS_CLAUDE_API_KEY != '' }} + SECRET_NAME: AUTOMATIONS_CLAUDE_API_KEY + run: | + if [ "$PRESENT" = true ]; then + echo "present=true" >>"$GITHUB_OUTPUT" + else + echo "::warning::$SECRET_NAME is empty; this job does not judge." + echo "present=false" >>"$GITHUB_OUTPUT" + fi + # run-claude-cli refuses to run without these root-owned settings, which # need sudo, so they are written before the runner is isolated. - name: Apply the Claude managed settings if: steps.agree.outcome == 'success' - uses: melodic-software/ci-workflows/.github/actions/apply-claude-managed-settings@34c8358f2d6c2551d7d462064ea46d403df0b7ac # v0.52.0 + uses: melodic-software/ci-workflows/.github/actions/apply-claude-managed-settings@7b362428c76085a160e28656cd2c75c3147a69ff # v0.54.0 - name: Isolate the runner if: steps.agree.outcome == 'success' @@ -209,18 +237,21 @@ jobs: # continue-on-error keeps a launcher failure, which run-claude-cli # reports by failing this step, from ending the job before the outcome - # step; "Fail without a verdict" carries the red. Two legs of - # leg-timeout-minutes and the CLI install fit in timeout-minutes, which - # the outcome step repeats, and that fits in the job's 30. One job is - # allowed: the model has Read alone and the job token can only read. + # step; "Fail without a verdict" carries the red. A credit stop skips + # the verdict steps, and a clean one the outcome step too: the + # subscription job judges and records the run instead. + # One leg of leg-timeout-minutes and the CLI install fit in + # timeout-minutes, which the outcome step repeats, and that fits in the + # job's 30. The model may share its job with the upload: it has Read + # alone and the job token can only read. # Under dontAsk only the allow rule's directory is readable; the deny # rules hold if the tools or the working directory change. - name: Judge the pull request id: judge - if: steps.agree.outcome == 'success' + if: steps.credential.outputs.present == 'true' continue-on-error: true timeout-minutes: 22 - uses: melodic-software/ci-workflows/.github/actions/run-claude-cli@d0fc3b14564383022ccc0559a5f96f3698dac39b # v0.53.0 + uses: melodic-software/ci-workflows/.github/actions/run-claude-cli@7b362428c76085a160e28656cd2c75c3147a69ff # v0.54.0 env: # No work may outlive the result message the launcher reads. CLAUDE_CODE_DISABLE_BACKGROUND_TASKS: "1" @@ -232,6 +263,317 @@ jobs: You run unattended; return only the verdict the schema asks for; nothing you write is posted. api-key: ${{ secrets.AUTOMATIONS_CLAUDE_API_KEY }} + model: ${{ steps.models.outputs.model }} + effort: ${{ steps.models.outputs.effort }} + max-turns: ${{ steps.models.outputs.max-turns }} + permission-mode: dontAsk + tools: Read + allowed-tools: Read(./pr-verify-data/**) + disallowed-tools: | + Read(./.base/**) + Glob + Grep + Bash + Edit + Write + NotebookEdit + WebFetch + WebSearch + Agent + mcp__* + json-schema-file: ${{ runner.temp }}/pr-verify-schema.json + leg-timeout-minutes: "10" + + # After a credit stop: the owner's signal to rotate a key, then the + # denial check. A denial on the key leg means no verdict, so the + # subscription job runs only on this step's clean=true; continue-on-error + # lets the report record the stop, and "Fail without a verdict" carries + # the red. + - name: Check the key leg + id: key-leg + if: steps.judge.outputs.lane-class == 'credit-stop' + continue-on-error: true + env: + LOG_FILE: ${{ steps.judge.outputs.log-file }} + run: | + echo "::warning::the automations role's API key (AUTOMATIONS_CLAUDE_API_KEY) hit a credit stop; rotate a spare key in" + node .base/.github/scripts/pr-verify/write-verdict.mjs check-key-leg + echo "::notice::check-intent-subscription judges on the subscription token" + + # One lane record per run: a clean credit stop is recorded by the + # subscription job; a stop with a denial on the key leg, here, as + # rate-limit, the class report-lane-outcome knows for a credit stop. + - name: Report the judgment's outcome + if: (steps.judge.outcome == 'success' || steps.judge.outcome == 'failure') && (steps.judge.outputs.lane-class != 'credit-stop' || steps.key-leg.outcome == 'failure') + uses: melodic-software/ci-workflows/.github/actions/report-lane-outcome@7b362428c76085a160e28656cd2c75c3147a69ff # v0.54.0 + with: + outcome: ${{ steps.judge.outcome }} + execution-file: ${{ steps.judge.outputs.execution-file }} + lane: pr-verify + step-started-at: ${{ steps.prompt.outputs.started-at }} + step-timeout-minutes: 22 + credential: ${{ steps.judge.outputs.credential }} + max-turns: ${{ steps.models.outputs.max-turns }} + known-failure-class: ${{ steps.judge.outputs.lane-class == 'credit-stop' && 'rate-limit' || steps.judge.outputs.lane-class }} + + # The structured output, in the last result message of the execution + # file, is untrusted model text: read from the file and checked by the + # script, never interpolated into a command. + - name: Write the verdict + id: verdict + if: steps.judge.outcome == 'success' + env: + EXECUTION_FILE: ${{ steps.judge.outputs.execution-file }} + LOG_FILE: ${{ steps.judge.outputs.log-file }} + PR_NUMBER: ${{ steps.pr.outputs.pr-number }} + HEAD_SHA: ${{ steps.pr.outputs.head-sha }} + BASE_SHA: ${{ steps.pr.outputs.base-sha }} + HEAD_REF: ${{ steps.pr.outputs.head-ref }} + SPEC_FILE: ${{ github.workspace }}/pr-verify-data/spec.json + VERDICT_DIR: ${{ runner.temp }}/pr-verify-verdict + run: node .base/.github/scripts/pr-verify/write-verdict.mjs + + - name: Upload the verdict + if: steps.verdict.outcome == 'success' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: ${{ steps.verdict.outputs.name }} + path: ${{ runner.temp }}/pr-verify-verdict/verdict.json + if-no-files-found: error + + - name: Summarize the verdict + if: steps.verdict.outcome == 'success' + env: + VERDICT_FILE: ${{ runner.temp }}/pr-verify-verdict/verdict.json + run: | + { + echo "### Verifier verdict" + echo + echo '```json' + jq '.summary |= gsub("`"; "'"'"'") | .notes |= map(gsub("`"; "'"'"'"))' "$VERDICT_FILE" + echo '```' + } >>"$GITHUB_STEP_SUMMARY" + + # A clean credit stop or an empty key hands the judgment on; any other + # run that passed the gates and left no verdict fails here. + - name: Fail without a verdict + if: always() && steps.agree.outcome == 'success' && steps.verdict.outcome != 'success' && steps.credential.outputs.present != 'false' && steps.key-leg.outcome != 'success' + run: | + echo "::error::the judgment left no verdict for this head" + exit 1 + + # The subscription leg, as its own job on a new VM: check-intent's + # processes, files and environment do not reach it. Its steps repeat + # check-intent's from a fresh checkout but for the credential and the + # key-leg check (workflow.test.mjs holds them equal), so the gates, the + # kill switch and the hardening run again. Only this job references the + # subscription token. needs adds success(): a check-intent that failed, a + # denial on its key leg included, leaves this job skipped. + check-intent-subscription: + needs: check-intent + if: >- + github.ref == 'refs/heads/main' + && github.event.workflow_run.event == 'pull_request' + && github.event.workflow_run.conclusion == 'success' + && github.event.workflow_run.head_repository.full_name == github.repository + && ((needs.check-intent.outputs.lane-class == 'credit-stop' + && needs.check-intent.outputs.key-leg-clean == 'true') + || needs.check-intent.outputs.credential-present == 'false') + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read # check out main; read the PR's commits for the diff + pull-requests: read # the PR, its comments and reviews + actions: read # the upstream run, and verdicts already uploaded + steps: + - name: Check out the default branch tip + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: refs/heads/main + path: .base + persist-credentials: false + + - name: Set up Node + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version-file: .base/.node-version + package-manager-cache: false + + - name: Check the kill switch + id: kill + uses: ./.base/.github/actions/check-kill-switch # zizmor: ignore[self-repository] + with: + value: ${{ vars.CLAUDE_LANES_DISABLED }} + repo-value: ${{ vars.CLAUDE_LANES_REPO_DISABLED }} + + - name: Report the kill switch + if: steps.kill.outputs.proceed != 'true' + env: + REASON: ${{ steps.kill.outputs.reason }} + run: | + echo "::notice::the lanes kill switch stops this run (reason: ${REASON:-unreadable}); no verdict" + + # The upstream run's actors, the re-runner and the PR author must all be + # on the trusted-actor list. + - name: Check the trigger + id: gate + if: steps.kill.outputs.proceed == 'true' + uses: ./.base/.github/actions/check-trusted-trigger # zizmor: ignore[self-repository] + with: + trusted-actors-path: .base/.github/standards/trusted-actors/trusted-actors.json + triggering-actor: ${{ github.triggering_actor }} + + - name: Resolve the pull request + id: pr + if: steps.gate.outputs.proceed == 'true' + env: + GITHUB_TOKEN: ${{ github.token }} + UPSTREAM_RUN_ID: ${{ github.event.workflow_run.id }} + SPEC_FILE: ${{ github.workspace }}/pr-verify-data/spec.json + SPEC_TEXT_FILE: ${{ github.workspace }}/pr-verify-data/spec-text.json + TRUSTED_ACTORS_PATH: .base/.github/standards/trusted-actors/trusted-actors.json + run: | + mkdir -p "$(dirname "$SPEC_FILE")" + VERIFY_AUTHORS=$(yq '.merge."verify-authors" // "lane"' .base/docs/conventions/pr-pipeline.yaml) + LANE_IDENTITY=$(yq '.merge."agent-identity" // ""' .base/docs/conventions/pr-pipeline.yaml) + export VERIFY_AUTHORS LANE_IDENTITY + node .base/.github/scripts/pr-verify/resolve-pr.mjs + + # The gate and the resolver read the PR separately; both must name the + # same PR at the run's head, or nothing is judged. + - name: Assert the gate and the resolver agree + id: agree + if: steps.pr.outputs.proceed == 'true' + env: + GATE_PR: ${{ steps.gate.outputs.pr-number }} + GATE_HEAD: ${{ steps.gate.outputs.head-sha }} + PR_NUMBER: ${{ steps.pr.outputs.pr-number }} + HEAD_SHA: ${{ steps.pr.outputs.head-sha }} + run: | + if [ "$GATE_PR" != "$PR_NUMBER" ] || [ "$GATE_HEAD" != "$HEAD_SHA" ]; then + echo "::error::the trigger gate and the resolver name different pull requests or heads" + exit 1 + fi + + - name: Select the trusted PR text + if: steps.agree.outcome == 'success' + uses: ./.base/.github/actions/select-trusted-text # zizmor: ignore[self-repository] + with: + pr-number: ${{ steps.pr.outputs.pr-number }} + trusted-actors-path: .base/.github/standards/trusted-actors/trusted-actors.json + output-path: ${{ github.workspace }}/pr-verify-data/context.json + github-token: ${{ github.token }} + + # A blobless fetch of the two commits into its own repository outside + # the workspace. git diff reads no attributes or diff drivers from it, + # so nothing in the PR runs; the patch is a data file. + - name: Write the diff + if: steps.agree.outcome == 'success' + env: + HEAD_SHA: ${{ steps.pr.outputs.head-sha }} + BASE_SHA: ${{ steps.pr.outputs.base-sha }} + PR_NUMBER: ${{ steps.pr.outputs.pr-number }} + DATA: ${{ github.workspace }}/pr-verify-data + SRC: ${{ runner.temp }}/pr-verify-src + run: | + export GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null + mkdir -p "$DATA" + git init -q "$SRC" + git -C "$SRC" remote add origin "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" + git -C "$SRC" fetch -q --no-tags --filter=blob:none origin "$BASE_SHA" "$HEAD_SHA" + git -C "$SRC" diff --no-ext-diff --no-textconv --no-color "$BASE_SHA...$HEAD_SHA" >"$DATA/diff.patch" + jq -n --argjson pr "$PR_NUMBER" --arg head "$HEAD_SHA" --arg base "$BASE_SHA" \ + '{pr: $pr, head_sha: $head, base_sha: $base}' >"$DATA/pr.json" + echo "diff: $(wc -l <"$DATA/diff.patch") lines" + + # The judgment's model, turns and effort: the verify-pr route of the + # base config's models block. + - name: Resolve the verifier's model + id: models + if: steps.agree.outcome == 'success' + uses: ./.base/.github/actions/resolve-config # zizmor: ignore[self-repository] + with: + model-route: verify-pr + output-path: ${{ runner.temp }}/pr-verify-models.json + + # Only whether the secret is set leaves this step, never its value. + - name: Check the credential + id: credential + if: steps.agree.outcome == 'success' + env: + PRESENT: ${{ secrets.AUTOMATIONS_CLAUDE_CODE_OAUTH_TOKEN != '' }} + SECRET_NAME: AUTOMATIONS_CLAUDE_CODE_OAUTH_TOKEN + run: | + if [ "$PRESENT" = true ]; then + echo "present=true" >>"$GITHUB_OUTPUT" + else + echo "::warning::$SECRET_NAME is empty; this job does not judge." + echo "present=false" >>"$GITHUB_OUTPUT" + fi + + # run-claude-cli refuses to run without these root-owned settings, which + # need sudo, so they are written before the runner is isolated. + - name: Apply the Claude managed settings + if: steps.agree.outcome == 'success' + uses: melodic-software/ci-workflows/.github/actions/apply-claude-managed-settings@7b362428c76085a160e28656cd2c75c3147a69ff # v0.54.0 + + - name: Isolate the runner + if: steps.agree.outcome == 'success' + uses: ./.base/.github/actions/setup-runner-isolation # zizmor: ignore[self-repository] + with: + install-subprocess-isolation: true + + # Files, not step outputs: run-claude-cli reads both. A `default` model + # resolves to an empty model, which the launcher cannot pass. started-at + # lets the outcome step tell a step timeout from other failures. + - name: Write the prompt and schema + id: prompt + if: steps.agree.outcome == 'success' + env: + DATA: ${{ github.workspace }}/pr-verify-data + MODEL: ${{ steps.models.outputs.model }} + PROMPT_FILE: ${{ runner.temp }}/pr-verify-prompt.md + SCHEMA_FILE: ${{ runner.temp }}/pr-verify-schema.json + run: | + if [ -z "$MODEL" ]; then + echo "::error::the verify-pr profile's model is default, which run-claude-cli cannot pass; name an alias" + exit 1 + fi + prompt=$(<.base/.github/scripts/pr-verify/prompt.md) + prompt=${prompt//DIFF_FILE/$DATA/diff.patch} + prompt=${prompt//CONTEXT_FILE/$DATA/context.json} + prompt=${prompt//PR_FILE/$DATA/pr.json} + prompt=${prompt//SPEC_TEXT_FILE/$DATA/spec-text.json} + prompt=${prompt//SPEC_FILE/$DATA/spec.json} + printf '%s\n' "$prompt" >"$PROMPT_FILE" + printf '%s\n' '{"type":"object","properties":{"verdict":{"enum":["pass","pass-notes","fail"]},"summary":{"type":"string"},"notes":{"type":"array","items":{"type":"string"}}},"required":["verdict","summary","notes"],"additionalProperties":false}' >"$SCHEMA_FILE" + echo "started-at=$(date +%s)" >>"$GITHUB_OUTPUT" + + # continue-on-error keeps a launcher failure, which run-claude-cli + # reports by failing this step, from ending the job before the outcome + # step; "Fail without a verdict" carries the red. One leg of + # leg-timeout-minutes and the CLI install fit in timeout-minutes, which + # the outcome step repeats, and that fits in the job's 30. The model may + # share its job with the upload: it has Read alone and the job token can + # only read. + # Under dontAsk only the allow rule's directory is readable; the deny + # rules hold if the tools or the working directory change. + - name: Judge the pull request + id: judge + if: steps.credential.outputs.present == 'true' + continue-on-error: true + timeout-minutes: 22 + uses: melodic-software/ci-workflows/.github/actions/run-claude-cli@7b362428c76085a160e28656cd2c75c3147a69ff # v0.54.0 + env: + # No work may outlive the result message the launcher reads. + CLAUDE_CODE_DISABLE_BACKGROUND_TASKS: "1" + with: + lane: pr-verify + role: automations + prompt-file: ${{ runner.temp }}/pr-verify-prompt.md + preamble: >- + You run unattended; return only the verdict the schema asks for; + nothing you write is posted. subscription-token: ${{ secrets.AUTOMATIONS_CLAUDE_CODE_OAUTH_TOKEN }} model: ${{ steps.models.outputs.model }} effort: ${{ steps.models.outputs.effort }} @@ -256,11 +598,11 @@ jobs: - name: Report the judgment's outcome if: steps.judge.outcome == 'success' || steps.judge.outcome == 'failure' - uses: melodic-software/ci-workflows/.github/actions/report-lane-outcome@34c8358f2d6c2551d7d462064ea46d403df0b7ac # v0.52.0 + uses: melodic-software/ci-workflows/.github/actions/report-lane-outcome@7b362428c76085a160e28656cd2c75c3147a69ff # v0.54.0 with: outcome: ${{ steps.judge.outcome }} execution-file: ${{ steps.judge.outputs.execution-file }} - lane: pr-verify + lane: ${{ needs.check-intent.outputs.lane-class == 'credit-stop' && 'pr-verify (key AUTOMATIONS_CLAUDE_API_KEY hit a credit stop)' || 'pr-verify (key AUTOMATIONS_CLAUDE_API_KEY is empty)' }} step-started-at: ${{ steps.prompt.outputs.started-at }} step-timeout-minutes: 22 credential: ${{ steps.judge.outputs.credential }} diff --git a/docs/conventions/pr-pipeline/README.md b/docs/conventions/pr-pipeline/README.md index d9dc05f88c..c4963e12e1 100644 --- a/docs/conventions/pr-pipeline/README.md +++ b/docs/conventions/pr-pipeline/README.md @@ -1258,11 +1258,13 @@ is `split`, and Uncertainty Known, Knowable and Unknown map to `run`, `research- Each Claude lane's runs are paid by one funding role: `reviews`, `automations` or `issue-lane`. A lane on ci-workflows' `run-claude-cli` action passes its role with the role's two secrets, `_CLAUDE_API_KEY` and `_CLAUDE_CODE_OAUTH_TOKEN` (role upper-snake, so `issue-lane` -gives `ISSUE_LANE_`); the action tries the key first and the subscription token only after a -credit stop. Each lane's workflow names its role as a literal, and the runner policy admits it; no -config key records it. `intake`, `pr-unblock`, `fix-ci`, `pr-verify` and `pr-refine` take -`automations`; `claude-review` and `claude-security-review` take `reviews`; `work-resolve-issue` -takes `issue-lane`. +gives `ISSUE_LANE_`); the key runs first and the subscription token only after a credit stop. +`pr-verify` holds one credential per job: `check-intent` runs on the key, and +`check-intent-subscription` runs on the token, on a new runner, only after `check-intent`'s credit +stop with no permission denial or when the key is empty. Each lane's workflow names its role as a +literal, and the runner policy admits it; no config key records it. `intake`, `pr-unblock`, +`fix-ci`, `pr-verify` and `pr-refine` take `automations`; `claude-review` and +`claude-security-review` take `reviews`; `work-resolve-issue` takes `issue-lane`. `intake` and `pr-verify` run on `run-claude-cli` today. Every other lane runs claude-code-action with its role's OAuth token alone until it moves there, and it never gets an API key, since