Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
124 changes: 108 additions & 16 deletions .github/actions/resolve-config/callers.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -328,6 +328,7 @@ test("intake-triage.yml: each job keeps its own least-privilege permissions", ()
"apply-verdict": { contents: "read", "id-token": "write" },
"refine-select": { contents: "read", issues: "read", "pull-requests": "read" },
"refine-issue": { contents: "read" },
"refine-issue-subscription": { contents: "read" },
"apply-refinement": { contents: "read", "id-token": "write" },
"redispatch-refined": { actions: "write" },
"triage-split-children": { actions: "write" },
Expand Down Expand Up @@ -420,57 +421,148 @@ test("intake-triage.yml: apply-verdict writes as the lanes App only after a tria
assert.deepEqual(stepSecrets(job), []);
});

// The refine-issue legs as the parity check compares them: every step, in
// order, with every field, except the four places the legs may differ: the
// credential check's env and run (which secret it tests, and whether an empty
// one warns or fails), the launcher's credential input, the key leg's handoff
// report (the subscription leg never hands off), and the last step's `if`.
function refineLegShape(job) {
return job.steps
.filter((step) => step.id !== "handoff")
.map((step) => {
const shape = structuredClone(step);
if (step.id === "credential") {
delete shape.env;
delete shape.run;
}
if (step.id === "run") {
delete shape.with["api-key"];
delete shape.with["subscription-token"];
}
if (step.name === "Fail without a refinement") delete shape.if;
return shape;
});
}

test("intake-triage.yml: the refine legs' parity check fails on any other difference, step order included", () => {
const { jobs } = workflow("intake-triage.yml");
const key = jobs["refine-issue"];
const sub = jobs["refine-issue-subscription"];
assert.deepEqual(refineLegShape(key), refineLegShape(sub));
const differs = (mutate) => {
const copy = structuredClone(sub);
mutate(copy);
assert.notDeepEqual(refineLegShape(key), refineLegShape(copy));
};
const step = (job, name) => job.steps.find((s) => s.name === name);
differs((job) => delete step(job, "Refine the issue")["continue-on-error"]);
differs((job) => (step(job, "Refine the issue")["timeout-minutes"] = 30));
differs((job) => (step(job, "Refine the issue").env.CLAUDE_CODE_DISABLE_BACKGROUND_TASKS = "0"));
differs((job) => {
const settings = job.steps.findIndex((s) => s.name === "Apply the Claude managed settings");
const isolate = job.steps.findIndex((s) => s.name === "Isolate the runner");
[job.steps[settings], job.steps[isolate]] = [job.steps[isolate], job.steps[settings]];
});
differs((job) => (step(job, "Collect the refinement").run += "\ntrue"));
differs((job) => (step(job, "Upload the refinement").with["retention-days"] = 2));
differs((job) => (step(job, "Check the credential").if = "always()"));
differs((job) => (step(job, "Check the context").run = "true"));
differs((job) => job.steps.pop());
});

test("intake-triage.yml: a thin issue's refinement runs its model with no write token and writes through the lanes App only", () => {
const { jobs } = workflow("intake-triage.yml");
const select = jobs["refine-select"];
const model = jobs["refine-issue"];
const sub = jobs["refine-issue-subscription"];
const apply = jobs["apply-refinement"];
const again = jobs["redispatch-refined"];
assert.deepEqual(select.permissions, { contents: "read", issues: "read", "pull-requests": "read" });
assert.deepEqual(model.permissions, { contents: "read" }, "the model job holds no write grant");
assert.deepEqual(sub.permissions, { contents: "read" }, "the subscription leg holds no write grant");
assert.deepEqual(apply.permissions, { contents: "read", "id-token": "write" });
assert.deepEqual(again.permissions, { actions: "write" }, "the dispatch grant is not on the job with the App token");
assert.deepEqual(select.needs, ["born-triaged", "apply-verdict"]);
assert.deepEqual(model.needs, ["refine-select"]);
assert.deepEqual(apply.needs, ["born-triaged", "refine-select", "refine-issue"]);
assert.deepEqual(sub.needs, ["refine-select", "refine-issue"]);
assert.deepEqual(apply.needs, ["born-triaged", "refine-select", "refine-issue", "refine-issue-subscription"]);
assert.deepEqual(again.needs, ["apply-refinement"]);

const groups = Object.values(jobs)
.map((job) => job.concurrency?.group)
.filter(Boolean);
assert.equal(new Set(groups).size, groups.length, "each job has its own concurrency group");
for (const job of [select, model, apply]) {
for (const job of [select, model, sub, apply]) {
assert.equal(job.concurrency["cancel-in-progress"], false);
}

assert.equal(evaluate(select.if, { "needs.apply-verdict.result": "success" }), true);
assert.equal(evaluate(select.if, { "needs.apply-verdict.result": "failure" }), false);
assert.equal(evaluate(model.if, { "needs.refine-select.outputs.refine": "true" }), true);
assert.equal(evaluate(model.if, { "needs.refine-select.outputs.refine": "false" }), false);
assert.equal(evaluate(apply.if, { "needs.refine-issue.result": "success" }), true);
assert.equal(evaluate(apply.if, { "needs.refine-issue.result": "skipped" }), false);
// The subscription leg runs only after the key leg handed off or found no key.
const leg = (refine, handoff, present) => ({
"needs.refine-select.outputs.refine": refine,
"needs.refine-issue.outputs.handoff": handoff,
"needs.refine-issue.outputs.credential-present": present,
});
assert.equal(evaluate(sub.if, leg("true", "true", "true")), true);
assert.equal(evaluate(sub.if, leg("true", "", "false")), true);
assert.equal(evaluate(sub.if, leg("true", "false", "true")), false);
assert.equal(evaluate(sub.if, leg("true", "", "")), false);
assert.equal(evaluate(sub.if, leg("false", "true", "true")), false);
// apply-refinement reads whichever leg uploaded the refinement.
const refined = (key, subscription) => ({
"needs.refine-issue.outputs.refined": key,
"needs.refine-issue-subscription.outputs.refined": subscription,
});
assert.equal(evaluate(apply.if, refined("true", "")), true);
assert.equal(evaluate(apply.if, refined("false", "true")), true);
assert.equal(evaluate(apply.if, refined("false", "")), false);
assert.equal(evaluate(apply.if, refined("", "false")), false);
assert.equal(evaluate(again.if, { "needs.apply-refinement.outputs.redispatch": "true" }), true);
assert.equal(evaluate(again.if, { "needs.apply-refinement.outputs.redispatch": "" }), false);

// Every checkout is the default branch without credentials.
for (const job of [select, model, apply]) {
for (const job of [select, model, sub, apply]) {
const checkout = job.steps.find((s) => s.uses?.startsWith("actions/checkout@"));
assert.equal(checkout.with.ref, "refs/heads/main");
assert.equal(checkout.with["persist-credentials"], false);
}

// The model step: no project settings, an explicit permission mode, and
// no tool that writes or runs code.
const run = model.steps.find((s) => s.uses?.startsWith("anthropics/claude-code-action@"));
const args = run.with.claude_args;
assert.match(args, /--setting-sources user/);
assert.match(args, /--permission-mode dontAsk/);
assert.doesNotMatch(JSON.stringify(model), /bypassPermissions|dangerously/);
// The model step, in each leg: run-claude-cli (--restricted, so no user,
// project or local settings), an explicit dontAsk, Read, Grep, Glob and
// WebFetch on the hosts the prompt names, and no tool that writes or runs
// code. The legs differ only in the credential.
const prompt = model.steps.find((s) => s.id === "prompt").run;
assert.match(prompt, /denied="Bash,Edit,Write,NotebookEdit,WebSearch,Agent,Skill"/);
assert.match(prompt, /mcp__github,mcp__github_comment,mcp__github_file_ops,mcp__github_ci,mcp__github_inline_comment/);
assert.match(prompt, /WebFetch\(domain:\$domain\)/);
assert.deepEqual(stepSecrets(model), ["INTAKE_CLAUDE_CODE_OAUTH_TOKEN"]);
const domains = prompt.match(/domains="([^"]+)"/)[1].split(" ");
const launcher = (job) => job.steps.find((s) => s.id === "run");
for (const job of [model, sub]) {
const run = launcher(job);
assert.match(run.uses, /^melodic-software\/ci-workflows\/\.github\/actions\/run-claude-cli@[0-9a-f]{40}$/);
assert.equal(run.with.role, "intake");
assert.equal(run.with["permission-mode"], "dontAsk");
assert.deepEqual(run.with.tools.trim().split("\n"), ["Read", "Grep", "Glob", "WebFetch"]);
assert.deepEqual(run.with["allowed-tools"].trim().split("\n"), ["Read(./refine-data/**)", ...domains.map((d) => `WebFetch(domain:${d})`)]);
assert.deepEqual(run.with["disallowed-tools"].trim().split("\n"), ["Bash", "Edit", "Write", "NotebookEdit", "WebSearch", "Agent", "Skill", "mcp__*"]);
assert.equal(run.with["prompt-file"], "${{ runner.temp }}/refine-prompt.md");
assert.equal(run.with["json-schema-file"], "${{ runner.temp }}/refine-schema.json");
assert.doesNotMatch(JSON.stringify(job), /bypassPermissions|dangerously|--bare|setting-sources/);
assert.ok(!job.steps.some((s) => s.uses?.startsWith("anthropics/claude-code-action@")));
assert.equal(job.steps.find((s) => s.id === "prompt").run, prompt, "both legs build the same prompt");
const collect = job.steps.find((s) => s.id === "collect");
assert.equal(collect.env.MODE, "dontAsk");
assert.match(collect.run, /check-permission-mode\.sh/);
}
assert.deepEqual(refineLegShape(model), refineLegShape(sub), "the legs differ only where refineLegShape allows");
const { "api-key": keyLegKey, "subscription-token": keyLegToken } = launcher(model).with;
const { "api-key": subLegKey, "subscription-token": subLegToken } = launcher(sub).with;
assert.equal(keyLegKey, "${{ secrets.INTAKE_CLAUDE_API_KEY }}");
assert.equal(subLegToken, "${{ secrets.INTAKE_CLAUDE_CODE_OAUTH_TOKEN }}");
assert.equal(keyLegToken, undefined);
assert.equal(subLegKey, undefined);
assert.deepEqual(stepSecrets(launcher(model)), ["INTAKE_CLAUDE_API_KEY"]);
assert.deepEqual(stepSecrets(launcher(sub)), ["INTAKE_CLAUDE_CODE_OAUTH_TOKEN"]);
for (const job of [select, apply, again]) {
assert.deepEqual(stepSecrets(job), []);
}
Expand Down
3 changes: 2 additions & 1 deletion .github/lane-funding.json
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@
"category": "intake",
"lanes": [
{ "id": "intake-triage", "workflow": "intake-triage.yml", "launcher": "run-claude-cli", "triggers": ["issues", "workflow_dispatch"], "gate": "author", "currentSecrets": ["INTAKE_CLAUDE_API_KEY", "INTAKE_CLAUDE_CODE_OAUTH_TOKEN", "FALLBACK_CLAUDE_CODE_OAUTH_TOKEN"] },
{ "id": "refine-issue", "workflow": "intake-triage.yml", "launcher": "claude-code-action", "triggers": ["issues", "workflow_dispatch"], "gate": "author", "currentSecrets": ["INTAKE_CLAUDE_CODE_OAUTH_TOKEN"] }
{ "id": "refine-issue", "workflow": "intake-triage.yml", "launcher": "run-claude-cli", "triggers": ["issues", "workflow_dispatch"], "gate": "author", "currentSecrets": ["INTAKE_CLAUDE_API_KEY"] },
{ "id": "refine-issue-subscription", "workflow": "intake-triage.yml", "launcher": "run-claude-cli", "triggers": ["issues", "workflow_dispatch"], "gate": "author", "currentSecrets": ["INTAKE_CLAUDE_CODE_OAUTH_TOKEN"] }
]
},
{
Expand Down
Loading
Loading