Skip to content

ci: end a work-resolve-issue kill-switch deny neutral, keeping work-unblock's re-fire #11236

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kyle-sexton merged 4 commits into from
Oct 30, 2026
Merged

ci: end a work-resolve-issue kill-switch deny neutral, keeping work-unblock's re-fire #11236

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9a9c1dd
ci: end a work-resolve-issue kill-switch deny neutral, keeping work-u…
kyle-sexton Oct 30, 2026
06069bd
Merge remote-tracking branch 'origin/main' into fix/resolve-issue-kil…
kyle-sexton Oct 30, 2026
d2504da
ci: keep a kill-switch stop from reading as a reported or passed run
kyle-sexton Oct 30, 2026
4ff6c7d
Merge remote-tracking branch 'origin/main' into fix/resolve-issue-kil…
kyle-sexton Oct 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 35 additions & 5 deletions .github/actions/resolve-config/callers.test.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5591,6 +5591,32 @@ test("work-resolve-issue.yml: report-outcome passes apply-issue-patch's escalati
assert.equal(evaluate(job.with["escalation-reason"], context("", "skipped")), "");
});

// A kill switch that stopped apply-issue-patch green dropped this run's patch,
// so report-outcome skips and work-unblock re-fires the issue as a breaker
// trip; and a report-outcome the switch stopped green never posts its status
// as passed.
test("work-resolve-issue.yml: a kill-switch stop skips report-outcome and never reads as a passed report", () => {
const { jobs } = workflow("work-resolve-issue.yml");
const started = {
"needs.resolve-issue-fallback.result": "skipped",
"needs.resolve-issue-subscription.result": "skipped",
"needs.resolve-issue.outputs.scan-issue-run-started": "true",
};
for (const id of ["resolve-issue", "resolve-issue-subscription", "resolve-issue-fallback"]) {
started[`needs.${id}.outputs.scan-reuse-branch`] = "";
started[`needs.${id}.outputs.scan-issue-run-started`] ??= "";
}
const report = (skip) => evaluate(jobs["report-outcome"].if, { ...started, "needs.apply-issue-patch.outputs.skip-reason": skip });
assert.equal(report(""), true);
assert.equal(report("rung-up"), true, "another skip still reports");
assert.equal(report("kill-switch"), false);
const post = jobs["report-statuses"].steps.find((step) => step.env?.REPORT_RESULT !== undefined);
const result = (value, skip) => evaluate(post.env.REPORT_RESULT, { "needs.report-outcome.result": value, "needs.report-outcome.outputs.skip-reason": skip });
assert.equal(result("success", ""), "success");
assert.equal(result("failure", ""), "failure");
assert.equal(result("success", "kill-switch"), "stopped");
});

test("pr-run-activity-write.yml: a patch-* escalation reason passes the input check in issue mode only", () => {
const step = workflow("pr-run-activity-write.yml").jobs.run.steps.find((s) => s.name?.startsWith("Assert the lanes App sender id"));
const status = (reason, issueMode) =>
Expand Down Expand Up @@ -6869,6 +6895,8 @@ test("work-resolve-issue.yml: every job after the calls takes each resolve-issue
if (rest === "result") context[ref] = call === "resolve-issue-fallback" ? "skipped" : "success";
else context[ref] = call === "resolve-issue" ? first : "";
}
// Another job's value (apply-issue-patch's skip-reason) is the same in both runs.
for (const [ref] of String(expression).matchAll(/needs\.[\w-]+\.(?:result|outputs\.[\w-]+)/g)) context[ref] ??= "";
return evaluate(expression, context);
};
let reads = 0;
Expand Down Expand Up @@ -6900,6 +6928,7 @@ test("work-resolve-issue.yml: every job after the calls takes each resolve-issue
"needs.resolve-issue.outputs.scan-outcome-uploaded": first.uploaded ?? "",
"needs.resolve-issue.outputs.scan-reuse-branch": "",
"needs.resolve-issue.outputs.scan-patch-state": first.patch ?? "",
"needs.apply-issue-patch.outputs.skip-reason": "",
}));
const handedOff = { started: "true" };
const ran = { started: "true", uploaded: "true", patch: "written" };
Expand Down Expand Up @@ -7192,15 +7221,16 @@ test("every kill-switch-ok caller runs no token step and no dependent job on a k
].sort());
});

// work-unblock re-fires an issue from the failed mint step, and no
// dispatcher re-fires report-escalation, so only a pull request run of any
// other activity ends a kill-switch refusal green; those two stay red.
test("pr-run-activity-write.yml takes kill-switch-ok on a pull request run only, never for report-escalation", () => {
// work-unblock re-fires an issue from its stopped mint step (label-unblocked.mjs
// STOPPED), but no dispatcher re-fires report-escalation, so every run but
// report-escalation ends a kill-switch refusal green; that one stays red.
test("pr-run-activity-write.yml takes kill-switch-ok on every run but report-escalation", () => {
const step = workflow("pr-run-activity-write.yml").jobs.run.steps.find((s) => s.id === "lane-token");
const value = step.with["kill-switch-ok"];
const at = (issue, activity) => evaluate(value, { "inputs.issue-number": issue, "inputs.activity": activity });
assert.equal(at("", "publish-feedback"), true, "a pull request run skips");
assert.equal(at("7", "apply-issue-patch"), false, "an issue run stays red");
assert.equal(at("7", "apply-issue-patch"), true, "an issue run's patch skips");
assert.equal(at("7", "report-outcome"), true, "an issue run's report skips");
assert.equal(at("", "report-escalation"), false, "an escalation stays red");
const { jobs } = workflow("pr-unblock.yml");
assert.equal(jobs["report-escalation"].with.activity, "report-escalation", "the escalation caller passes that activity");
Expand Down
68 changes: 54 additions & 14 deletions .github/scripts/work-unblock/label-unblocked.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,12 @@ const DENIAL = {
token: /^\d{4}-\d\d-\d\dT[\d:.]+Z ##\[error\]lane-token-denied: the broker answered 403 with reason ([a-z]+(?:-[a-z]+)*)(?::|$)/,
};
const STEP_FAILED = /^\d{4}-\d\d-\d\dT[\d:.]+Z ##\[error\]Process completed with exit code [0-9]+\.$/;
// The write runner's step after MINT_STEP that runs exactly when a token was
// minted; a MINT_STEP success with it skipped is a mint that stopped green.
const STAGE_STEP = "Stage the activity from the base";
// request-lane-token's notice when kill-switch-ok ends a kill-switch refusal
// green with no token.
const STOPPED = /^\d{4}-\d\d-\d\dT[\d:.]+Z ##\[notice\]lane-token-stopped: the broker refused the token because a lanes kill switch is on(?:,|$)/;
const BREAKER = "kill-switch";
// The refused jobs' logs on 2026-10-24 were 45 KB (broker-check) and 94 KB
// (apply-issue-patch); a longer one is left unread.
Expand Down Expand Up @@ -1021,10 +1027,12 @@ async function laneRuns(actions, repo, number, addedAt) {
// How every one of `runs` (completed) did no work in its latest attempt, or
// null when one did some: no job minted a lane token, and the agent-ready run
// either had its `Request the lane token` step or broker-check's BROKER_STEP
// fail for a reason other than the kill switch, or with no reason in its log
// (`mint-failed`, which wins; the read runner resolve-issue calls has no
// fail for a reason other than the kill switch, or with no reason in its log,
// or had that mint step stop green with no token and no STOPPED notice in its
// log (`mint-failed`, which wins; the read runner resolve-issue calls has no
// MINT_STEP, so its broker ask is the one the broker can fail), had the broker
// refuse it for the kill switch at broker-check or a mint, or had broker-check
// refuse it for the kill switch at broker-check or a mint (failed, or stopped
// green with its STOPPED notice), or had broker-check
// skip its broker ask for the kill switch (`breaker`), was skipped by the WIP
// cap (`capped`: pick-slot ran, resolve-issue did not), or was skipped by the
// admit job's blocker gate (`blocked`: its BLOCKER_STEP ran and pick-slot did
Expand All @@ -1050,28 +1058,36 @@ async function didNoWork(jobsOf, refusalOf, runs) {
return null;
}
const jobs = await jobsOf(run);
const mints = jobs.flatMap((job) => (job?.steps ?? []).filter((step) => step?.name === MINT_STEP));
if (mints.some((step) => step.conclusion === "success")) {
const failedAt = (job, name) => endedAt(job, name, "failure");
if (jobs.some((job) => endedAt(job, MINT_STEP, "success") && !stoppedAt(job))) {
return null;
}
const named = (name) => namedJobs(jobs, name);
const endedAt = (job, name, conclusion) => (job?.steps ?? []).some((step) => step?.name === name && step.conclusion === conclusion);
const failedAt = (job, name) => endedAt(job, name, "failure");
const killed = named("resolve-issue").some((job) => endedAt(job, KILL_STEP, "success") && endedAt(job, BROKER_STEP, "skipped"));
const checked = named("resolve-issue").find((job) => failedAt(job, BROKER_STEP));
const minting = jobs.find((job) => failedAt(job, MINT_STEP));
const stopped = jobs.find(stoppedAt);
const capped =
named("pick-slot").some((job) => job.conclusion === "success") &&
named("resolve-issue").length > 0 &&
named("resolve-issue").every((job) => job.conclusion === "skipped");
const blocked =
named("admit").some((job) => (job?.steps ?? []).some((step) => step?.name === BLOCKER_STEP && step.conclusion === "success")) &&
named("pick-slot").every((job) => job.conclusion === "skipped");
// One log per run: broker-check's when it refused, else the first failed mint's.
const refused = killed ? null : checked ? refusalOf(checked, "check") : minting ? refusalOf(minting, "token") : null;
// One log per run: broker-check's when it refused, else the first failed
// mint's, else the first stopped mint's.
const refused = killed
? null
: checked
? refusalOf(checked, "check")
: minting
? refusalOf(minting, "token")
: stopped
? refusalOf(stopped, "stop")
: null;
if (killed || (await refused) === BREAKER) {
atLeast("breaker", run, killed || checked ? "check" : "token");
} else if (minting || checked) {
} else if (minting || checked || stopped) {
atLeast("mint-failed");
} else if (capped) {
atLeast("capped");
Expand All @@ -1084,9 +1100,10 @@ async function didNoWork(jobsOf, refusalOf, runs) {

// The reason the broker refused `job`, from its log: the failed step's last
// line before the runner's first `Process completed`, in `mode` (check or
// token). Null when that line is no such refusal or the log cannot be read
// within LOG_CAP_BYTES, or no log reader was given; the log is data, and no
// other line of it is read.
// token), or BREAKER for a `stop` mode log with a STOPPED notice line. Null
// when there is no such line or the log cannot be read within LOG_CAP_BYTES,
// or no log reader was given; the log is data, and no other line of it is
// read.
function refusalReader(jobLog, repo) {
return async (job, mode) => {
if (!jobLog || !RUN_ID.test(String(job?.id))) {
Expand All @@ -1100,6 +1117,9 @@ function refusalReader(jobLog, repo) {
return null;
}
const lines = String(log ?? "").split("\n");
if (mode === "stop") {
return lines.some((line) => STOPPED.test(line)) ? BREAKER : null;
}
const end = lines.findIndex((line) => STEP_FAILED.test(line));
return end > 0 ? (DENIAL[mode].exec(lines[end - 1])?.[1] ?? null) : null;
};
Expand All @@ -1125,6 +1145,21 @@ function jobLister(actions, repo) {
};
}

const endedAt = (job, name, conclusion) => (job?.steps ?? []).some((step) => step?.name === name && step.conclusion === conclusion);
// A mint that ended green with no token: kill-switch-ok's stop.
const stoppedAt = (job) => endedAt(job, MINT_STEP, "success") && endedAt(job, STAGE_STEP, "skipped");

// Whether a mint of one of `runs` stopped green: such a run that also minted
// did work and never reported, as a failed run did before kill-switch-ok.
async function stoppedMint(jobsOf, runs) {
for (const run of runs.filter((item) => RUN_ID.test(String(item.id)))) {
if ((await jobsOf(run)).some(stoppedAt)) {
return true;
}
}
return false;
}

const namedJobs = (jobs, name) => jobs.filter((job) => job?.name === name || String(job?.name).startsWith(`${name} /`));

// Whether one of `runs` stopped at the admit job's provenance stop.
Expand Down Expand Up @@ -1477,7 +1512,12 @@ async function refireRefusal({ github, actions, jobLog, repo, number, issue, las
const trip = stalled === "breaker" ? { run: refused.id, ended: Date.parse(refused.updated_at) || ended } : undefined;
return { reason: null, resets: due, refire: key, recorded, stall: stalled, trip };
}
if (!runs.some((run) => FAILED.has(run.conclusion)) && !(await cancelledBeforeWork(jobsOf, runs)) && !(await stoppedStale(jobsOf, runs))) {
if (
!runs.some((run) => FAILED.has(run.conclusion)) &&
!(await stoppedMint(jobsOf, runs)) &&
!(await cancelledBeforeWork(jobsOf, runs)) &&
!(await stoppedStale(jobsOf, runs))
) {
return stranded(ended, "its lane run did not fail", "no-report");
}
const key = await workflowKey();
Expand Down
95 changes: 95 additions & 0 deletions .github/scripts/work-unblock/label-unblocked.test.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2335,6 +2335,96 @@ test("breaker trips at the mint never count toward the failed-mint bound", async
assert.deepEqual(writes(calls), refired(30));
});

// With request-lane-token's kill-switch-ok, the same refusal ends the mint
// step green with no token and a `lane-token-stopped` notice: every step that
// keys on the token, `Stage the activity from the base` first, skips, and the
// run ends green. Only that shape in the job listing earns a log read.
const stoppedJob = (id, name) => ({
id,
name,
conclusion: "success",
steps: [
{ name: "Request the lane token", conclusion: "success" },
{ name: "Select the trusted PR text", conclusion: "skipped" },
{ name: "Stage the activity from the base", conclusion: "skipped" },
{ name: "Write the verdict", conclusion: "success" },
],
});
const stoppedLog = ({ stamp = "2026-10-30T01:57:51.9466327Z ", line = "lane-token-stopped: the broker refused the token because a lanes kill switch is on, a designed stop; this run takes no token and skips the rest" } = {}) =>
[
"2026-10-30T01:57:50.6158116Z ##[group]Run request-lane-token",
`${stamp}##[notice]${line}`,
"2026-10-30T01:57:52.0000000Z ##[group]Run jq -n \\",
"",
].join("\n");
const MINT_STOPPED = [
ADMITTED,
pickSlot,
BROKER_ALLOWED,
{ name: "resolve-issue / run", conclusion: "success", steps: [{ name: "Run the skill activity", conclusion: "success" }] },
stoppedJob(PATCH_MINT_ID, "apply-issue-patch / run"),
stoppedJob(REPORT_MINT_ID, "report-outcome / run"),
];
const MINT_STOPPED_LOGS = { [PATCH_MINT_ID]: stoppedLog(), [REPORT_MINT_ID]: stoppedLog() };

test("a green run whose mints the kill switch stopped is a breaker trip, read from one log", async () => {
const { github, actions, calls } = stalledRun({ jobs: MINT_STOPPED, logs: MINT_STOPPED_LOGS, conclusion: "success" });
assert.equal(await sweep({ github, actions }), 0);
tripped(calls);
assert.deepEqual(logReads(calls), [`GET /repos/${REPO}/actions/jobs/${PATCH_MINT_ID}/logs`]);
// Past the failed-mint bound on these workflow files it still goes as a trip.
const pumped = stalledRun({ jobs: MINT_STOPPED, logs: MINT_STOPPED_LOGS, conclusion: "success", ...earlierPumps(3) });
assert.equal(await sweep({ github: pumped.github, actions: pumped.actions }), 0);
tripped(pumped.calls);
// The model ran, so like a mint refusal it waits limits.reset-clamp-min after the run ended.
const early = stalledRun({ jobs: MINT_STOPPED, logs: MINT_STOPPED_LOGS, conclusion: "success", ended: "2026-10-16T11:30:00Z" });
assert.equal(await sweep({ github: early.github, actions: early.actions }), 0);
assert.deepEqual(writes(early.calls), []);
});

test("a stopped patch mint before a report the switch also stopped, or one alone, is a breaker trip", async () => {
const alone = MINT_STOPPED.filter((job) => job.id !== REPORT_MINT_ID).concat({ name: "report-outcome", conclusion: "skipped", steps: [] });
const { github, actions, calls } = stalledRun({ jobs: alone, logs: MINT_STOPPED_LOGS, conclusion: "success" });
assert.equal(await sweep({ github, actions }), 0);
tripped(calls);
});

test("a run whose stopped mint is followed by a mint that took its token did work, read with no log", async () => {
const minted = { id: REPORT_MINT_ID, name: "report-outcome / run", conclusion: "failure", steps: [{ name: "Request the lane token", conclusion: "success" }, { name: "Stage the activity from the base", conclusion: "success" }] };
const jobs = [...MINT_STOPPED.slice(0, -1), minted];
const { github, actions, calls } = stalledRun({ jobs, logs: MINT_STOPPED_LOGS, ...earlierPumps(1) });
assert.equal(await sweep({ github, actions }), 0);
assert.deepEqual(writes(calls), []);
assert.deepEqual(logReads(calls), []);
});

// The reverse order: apply-issue-patch minted and pushed, then the switch
// stopped report-outcome green. The run did work but never reported, so it
// gets a failed run's once-per-version re-fire at once, not the stranded
// wait for a green run, and it reads no log.
test("a green run whose report mint the kill switch stopped after a patch mint is re-fired like a failed run", async () => {
const minted = { id: PATCH_MINT_ID, name: "apply-issue-patch / run", conclusion: "success", steps: [{ name: "Request the lane token", conclusion: "success" }, { name: "Stage the activity from the base", conclusion: "success" }] };
const jobs = [...MINT_STOPPED.slice(0, -2), minted, stoppedJob(REPORT_MINT_ID, "report-outcome / run")];
const { github, actions, calls } = stalledRun({ jobs, logs: MINT_STOPPED_LOGS, conclusion: "success" });
assert.equal(await sweep({ github, actions }), 0);
assert.deepEqual(writes(calls), refired(30));
assert.deepEqual(logReads(calls), []);
});

test("a stopped mint whose log does not carry the kill-switch notice is a failed mint, never a breaker trip", async () => {
for (const [name, log] of [
["a lookalike notice", stoppedLog({ line: "lane-token-stopped: the broker refused the token because a lanes kill switch is onward" })],
["no timestamp", stoppedLog({ stamp: "" })],
["a warning, not a notice", stoppedLog().replace("##[notice]", "##[warning]")],
["over the size cap", `${"x".repeat(2 * 1024 * 1024)}\n${stoppedLog()}`],
["no log", undefined],
]) {
const { github, actions, calls } = stalledRun({ jobs: MINT_STOPPED, logs: log === undefined ? {} : { [PATCH_MINT_ID]: log }, conclusion: "success" });
assert.equal(await sweep({ github, actions }), 0, name);
assert.deepEqual(writes(calls), refired(30), name);
}
});

test("breaker trips past limit-stops-per-issue since a person last added agent-ready wait on the stranded backoff", async () => {
const under = stalledRun({ jobs: BROKER_REFUSED, logs: CHECK_KILLED, ...(await afterSweeps(2, BROKER_REFUSED, CHECK_KILLED)) });
assert.equal(await sweep({ github: under.github, actions: under.actions }), 0);
Expand Down Expand Up @@ -2569,6 +2659,11 @@ test("the job ids and step names the sweep matches exist in the lane workflows",
// The write runner mints; the read runner resolve-issue calls never does.
assert.ok(Object.values(jobs(workflow("pr-run-activity-write.yml"))).some((block) => steps(block).includes("Request the lane token")));
assert.ok(!Object.values(read).some((block) => steps(block).includes("Request the lane token")));
// A stopped mint is read from the stage step after it, which runs exactly when the token is set.
const write = jobs(workflow("pr-run-activity-write.yml")).run;
const named = steps(write);
assert.ok(named.indexOf("Stage the activity from the base") > named.indexOf("Request the lane token"));
assert.match(write, /^ {6}- name: Stage the activity from the base\n {8}id: stage\n {8}if: steps\.lane-token\.outputs\.token != ''\n/m);
});

// Each re-runner check matches by id. The dispatch lane cannot read its run's
Expand Down
Loading
Loading