From 5de066e45979a0c4b910191a9388590b3f75f065 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 15 Sep 2026 23:03:14 +0900 Subject: [PATCH 1/4] test(queue): capture terminal pre-execution failure --- ...ions_queue_health_terminal_preexecution.py | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 tests/test_actions_queue_health_terminal_preexecution.py diff --git a/tests/test_actions_queue_health_terminal_preexecution.py b/tests/test_actions_queue_health_terminal_preexecution.py new file mode 100644 index 0000000000..bb7ea64584 --- /dev/null +++ b/tests/test_actions_queue_health_terminal_preexecution.py @@ -0,0 +1,107 @@ +"""Regression contracts for terminal failures that never obtained a runner.""" + +from datetime import datetime, timezone +import importlib.util +import json +from pathlib import Path +from subprocess import CompletedProcess + + +ROOT = Path(__file__).resolve().parents[1] +MODULE_PATH = ROOT / "scripts/ci/actions_queue_health.py" +SPEC = importlib.util.spec_from_file_location("actions_queue_health_terminal", MODULE_PATH) +assert SPEC and SPEC.loader +queue_health = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(queue_health) + + +def _pull_request() -> dict: + """Return the exact open-PR identity used by the failed run.""" + return { + "number": 1, + "state": "open", + "base": {"ref": "main", "repo": {"full_name": "owner/repo"}}, + "head": {"sha": "head"}, + "updated_at": "2026-09-15T13:00:00Z", + } + + +def _terminal_failure_run() -> dict: + """Return a completed failure linked to the current pull-request head.""" + return { + "id": 900, + "workflow_id": 901, + "name": "required-check", + "event": "pull_request", + "status": "completed", + "conclusion": "failure", + "head_sha": "head", + "created_at": "2026-09-15T13:05:00Z", + "updated_at": "2026-09-15T13:06:00Z", + "run_attempt": 1, + "pull_requests": [{"number": 1, "head": {"sha": "head"}}], + } + + +def _terminal_failure_job() -> dict: + """Return a failed materialized job with no runner and no executed step.""" + return { + "id": 902, + "name": "required-check", + "status": "completed", + "conclusion": "failure", + "runner_id": None, + "runner_name": None, + "created_at": "2026-09-15T13:05:00Z", + "steps": [], + } + + +def test_terminal_preexecution_failure_survives_collection_and_is_not_product_failure() -> None: + """Keep failed zero-step jobs as explicit non-passing admission evidence.""" + failed_run = _terminal_failure_run() + failed_job = _terminal_failure_job() + responses: dict[str, object] = { + "repos/owner/repo": {"default_branch": "main"}, + "repos/owner/repo/pulls?state=open&per_page=100": [_pull_request()], + "repos/owner/repo/actions/runs?status=completed&head_sha=head&per_page=50": [failed_run], + "repos/owner/repo/actions/runs?status=cancelled&event=pull_request_target&per_page=50": [], + "repos/owner/repo/actions/runs/900/jobs?per_page=100": { + "total_count": 1, + "jobs": [failed_job], + }, + } + for status in ("in_progress", "pending", "queued", "requested", "waiting"): + responses[f"repos/owner/repo/actions/runs?status={status}&per_page=50"] = [] + + def runner(args: list[str], **_: object) -> CompletedProcess[str]: + """Return deterministic GitHub REST payloads for the regression specimen.""" + path = args[-1] + if path not in responses: + raise AssertionError(f"unexpected endpoint: {path}") + return CompletedProcess(args, 0, json.dumps(responses[path]), "") + + snapshot = queue_health.collect_snapshot( + ["owner/repo"], + runner=runner, + generated_at="2026-09-15T13:10:00Z", + ) + observed_runs = snapshot["repositories"][0]["runs"] + assert [run["id"] for run in observed_runs] == [900] + assert observed_runs[0]["jobs"][0]["steps_count"] == 0 + assert observed_runs[0]["jobs"][0]["runner_id"] == 0 + + report = queue_health.build_report( + snapshot, + now=datetime(2026, 9, 15, 13, 10, tzinfo=timezone.utc), + ) + row = report["runs"][0] + assert row["identity_state"] == "current_head" + assert row["run_conclusion"] == "FAILURE" + assert row["execution_state"] == "terminal_pre_execution_failure" + assert row["admission_state"] == "terminal_pre_execution_failure" + assert row["is_pending"] is False + assert row["runner_assigned"] is False + assert row["blocker"] == "terminal_pre_execution_failure_before_runner_assignment" + assert row["recommended_action"] == "inspect_actions_control_plane_without_leaf_bypass" + assert report["summary"]["terminal_pre_execution_failure_count"] == 1 From b016348d1ed35304eb2504d30093922dbf2d42ae Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 15 Sep 2026 23:04:54 +0900 Subject: [PATCH 2/4] fix(queue): classify terminal pre-execution failures --- scripts/ci/actions_queue_health.py | 36 ++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/scripts/ci/actions_queue_health.py b/scripts/ci/actions_queue_health.py index 77a0d33906..1aaea5c234 100644 --- a/scripts/ci/actions_queue_health.py +++ b/scripts/ci/actions_queue_health.py @@ -31,7 +31,7 @@ _CORE_NORMALISE_RUN = _core_module._normalise_run _CORE_BUILD_REPORT = _core_module.build_report -TERMINAL_DIAGNOSTIC_STATUSES = ("startup_failure", "cancelled") +TERMINAL_DIAGNOSTIC_STATUSES = ("startup_failure", "cancelled", "failure") TARGET_TERMINAL_DIAGNOSTIC_STATUSES = ("cancelled",) TERMINAL_DIAGNOSTIC_MAX_API_PAGES = MAX_API_PAGES @@ -458,6 +458,22 @@ def build_report( report_row["recommended_action"] = ( "inspect_actions_control_plane_without_leaf_bypass" ) + elif ( + report_row["identity_state"] == "current_head" + and report_row["run_conclusion"] == "FAILURE" + and matching_job is not None + and matching_job.get("conclusion") == "FAILURE" + and not report_row["runner_assigned"] + and matching_job.get("steps_count") == 0 + ): + report_row["execution_state"] = "terminal_pre_execution_failure" + report_row["admission_state"] = "terminal_pre_execution_failure" + report_row["blocker"] = ( + "terminal_pre_execution_failure_before_runner_assignment" + ) + report_row["recommended_action"] = ( + "inspect_actions_control_plane_without_leaf_bypass" + ) current_pending_rows = [ report_row @@ -500,6 +516,13 @@ def build_report( report["summary"]["cancelled_before_runner_assignment_count"] = ( cancelled_before_runner_assignment_count ) + terminal_pre_execution_failure_count = sum( + report_row.get("admission_state") == "terminal_pre_execution_failure" + for report_row in report["runs"] + ) + report["summary"]["terminal_pre_execution_failure_count"] = ( + terminal_pre_execution_failure_count + ) if cancelled_before_runner_assignment_count: external_action = ( "Inspect Actions runner admission, billing/usage, runner-group policy, " @@ -509,6 +532,15 @@ def build_report( if external_action not in report["summary"]["external_actions"]: report["summary"]["external_actions"].append(external_action) report["summary"]["external_actions"].sort() + if terminal_pre_execution_failure_count: + external_action = ( + "Inspect Actions control-plane admission, billing/usage, runner-group policy, " + "and scheduler state; terminal failure without runner assignment or executed " + "steps is not an executed product/security failure." + ) + if external_action not in report["summary"]["external_actions"]: + report["summary"]["external_actions"].append(external_action) + report["summary"]["external_actions"].sort() return report @@ -556,4 +588,4 @@ def main( if __name__ == "__main__": # pragma: no cover - exercised through CLI tests. - raise SystemExit(main()) + raise SystemExit(main()) \ No newline at end of file From ecbdc3507148e2ab922deba8432db87c129cf433 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 15 Sep 2026 23:10:16 +0900 Subject: [PATCH 3/4] test(queue): preserve terminal aggregate accounting --- tests/test_actions_queue_health_terminal_preexecution.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_actions_queue_health_terminal_preexecution.py b/tests/test_actions_queue_health_terminal_preexecution.py index bb7ea64584..05237ba340 100644 --- a/tests/test_actions_queue_health_terminal_preexecution.py +++ b/tests/test_actions_queue_health_terminal_preexecution.py @@ -105,3 +105,4 @@ def runner(args: list[str], **_: object) -> CompletedProcess[str]: assert row["blocker"] == "terminal_pre_execution_failure_before_runner_assignment" assert row["recommended_action"] == "inspect_actions_control_plane_without_leaf_bypass" assert report["summary"]["terminal_pre_execution_failure_count"] == 1 + assert report["summary"]["terminal_job_count"] == 1 From 06162262066710c0f550296527e1141bd8992b80 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 17 Sep 2026 17:06:11 +0900 Subject: [PATCH 4/4] test(queue): preserve queued job timing evidence --- ...ctions_queue_health_queued_job_evidence.py | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 tests/test_actions_queue_health_queued_job_evidence.py diff --git a/tests/test_actions_queue_health_queued_job_evidence.py b/tests/test_actions_queue_health_queued_job_evidence.py new file mode 100644 index 0000000000..db526b3874 --- /dev/null +++ b/tests/test_actions_queue_health_queued_job_evidence.py @@ -0,0 +1,107 @@ +"""Regression contract for queued current-head jobs that materialize after run start.""" + +from datetime import datetime, timezone +import importlib.util +import json +from pathlib import Path +from subprocess import CompletedProcess + + +ROOT = Path(__file__).resolve().parents[1] +MODULE_PATH = ROOT / "scripts/ci/actions_queue_health.py" +SPEC = importlib.util.spec_from_file_location("actions_queue_health_queued_job", MODULE_PATH) +assert SPEC and SPEC.loader +queue_health = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(queue_health) + + +def _pull_request() -> dict: + """Return the open PR whose current head owns the queued run.""" + return { + "number": 1, + "state": "open", + "base": {"ref": "main", "repo": {"full_name": "owner/repo"}}, + "head": {"sha": "head"}, + "updated_at": "2026-09-17T02:55:00Z", + } + + +def _queued_run() -> dict: + """Return an old run whose downstream job only recently became eligible.""" + return { + "id": 910, + "workflow_id": 911, + "name": "required-check", + "event": "pull_request", + "status": "queued", + "conclusion": "", + "head_sha": "head", + "created_at": "2026-09-17T00:00:00Z", + "updated_at": "2026-09-17T02:58:00Z", + "run_attempt": 1, + "pull_requests": [{"number": 1, "head": {"sha": "head"}}], + } + + +def _queued_job() -> dict: + """Return the current downstream job with its own later queue timestamp.""" + return { + "id": 912, + "name": "dispatch-current-head", + "status": "queued", + "conclusion": None, + "runner_id": None, + "runner_name": None, + "created_at": "2026-09-17T02:58:00Z", + "steps": [], + } + + +def test_queued_current_head_fetches_job_evidence_and_uses_job_queue_start() -> None: + """Time a materialized queued job from its own eligibility, not the parent run.""" + queued_run = _queued_run() + queued_job = _queued_job() + responses: dict[str, object] = { + "repos/owner/repo": {"default_branch": "main"}, + "repos/owner/repo/pulls?state=open&per_page=100": [_pull_request()], + "repos/owner/repo/actions/runs?status=queued&per_page=50": [queued_run], + "repos/owner/repo/actions/runs?status=completed&head_sha=head&per_page=50": [], + "repos/owner/repo/actions/runs?status=cancelled&event=pull_request_target&per_page=50": [], + "repos/owner/repo/actions/runs/910/jobs?per_page=100": { + "total_count": 1, + "jobs": [queued_job], + }, + } + for status in ("in_progress", "pending", "requested", "waiting"): + responses[f"repos/owner/repo/actions/runs?status={status}&per_page=50"] = [] + + requested_paths: list[str] = [] + + def runner(args: list[str], **_: object) -> CompletedProcess[str]: + """Return deterministic REST payloads and retain the exact evidence reads.""" + path = args[-1] + requested_paths.append(path) + if path not in responses: + raise AssertionError(f"unexpected endpoint: {path}") + return CompletedProcess(args, 0, json.dumps(responses[path]), "") + + snapshot = queue_health.collect_snapshot( + ["owner/repo"], + runner=runner, + generated_at="2026-09-17T03:00:00Z", + ) + assert snapshot["collection_errors"] == [] + observed_run = snapshot["repositories"][0]["runs"][0] + assert "repos/owner/repo/actions/runs/910/jobs?per_page=100" in requested_paths + assert [job["id"] for job in observed_run["jobs"]] == [912] + assert observed_run["jobs"][0]["created_at"] == "2026-09-17T02:58:00Z" + + report = queue_health.build_report( + snapshot, + now=datetime(2026, 9, 17, 3, 0, tzinfo=timezone.utc), + ) + row = report["runs"][0] + assert row["job_id"] == 912 + assert row["queue_age_source"] == "job_created_at" + assert row["queue_age_started_at"] == "2026-09-17T02:58:00Z" + assert row["queue_age_seconds"] == 120