Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 34 additions & 2 deletions scripts/ci/actions_queue_health.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@

_CORE_NORMALISE_RUN = _core_module._normalise_run
_CORE_BUILD_REPORT = _core_module.build_report
TERMINAL_DIAGNOSTIC_STATUSES = ("startup_failure", "cancelled")
TERMINAL_DIAGNOSTIC_STATUSES = ("startup_failure", "cancelled", "failure")
TARGET_TERMINAL_DIAGNOSTIC_STATUSES = ("cancelled",)
TERMINAL_DIAGNOSTIC_MAX_API_PAGES = MAX_API_PAGES

Expand Down Expand Up @@ -458,6 +458,22 @@ def build_report(
report_row["recommended_action"] = (
"inspect_actions_control_plane_without_leaf_bypass"
)
elif (
report_row["identity_state"] == "current_head"
and report_row["run_conclusion"] == "FAILURE"
and matching_job is not None
and matching_job.get("conclusion") == "FAILURE"
and not report_row["runner_assigned"]
and matching_job.get("steps_count") == 0
):
report_row["execution_state"] = "terminal_pre_execution_failure"
report_row["admission_state"] = "terminal_pre_execution_failure"
report_row["blocker"] = (
"terminal_pre_execution_failure_before_runner_assignment"
)
report_row["recommended_action"] = (
"inspect_actions_control_plane_without_leaf_bypass"
)

current_pending_rows = [
report_row
Expand Down Expand Up @@ -500,6 +516,13 @@ def build_report(
report["summary"]["cancelled_before_runner_assignment_count"] = (
cancelled_before_runner_assignment_count
)
terminal_pre_execution_failure_count = sum(
report_row.get("admission_state") == "terminal_pre_execution_failure"
for report_row in report["runs"]
)
report["summary"]["terminal_pre_execution_failure_count"] = (
terminal_pre_execution_failure_count
)
if cancelled_before_runner_assignment_count:
external_action = (
"Inspect Actions runner admission, billing/usage, runner-group policy, "
Expand All @@ -509,6 +532,15 @@ def build_report(
if external_action not in report["summary"]["external_actions"]:
report["summary"]["external_actions"].append(external_action)
report["summary"]["external_actions"].sort()
if terminal_pre_execution_failure_count:
external_action = (
"Inspect Actions control-plane admission, billing/usage, runner-group policy, "
"and scheduler state; terminal failure without runner assignment or executed "
"steps is not an executed product/security failure."
)
if external_action not in report["summary"]["external_actions"]:
report["summary"]["external_actions"].append(external_action)
report["summary"]["external_actions"].sort()
return report


Expand Down Expand Up @@ -556,4 +588,4 @@ def main(


if __name__ == "__main__": # pragma: no cover - exercised through CLI tests.
raise SystemExit(main())
raise SystemExit(main())
108 changes: 108 additions & 0 deletions tests/test_actions_queue_health_terminal_preexecution.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
"""Regression contracts for terminal failures that never obtained a runner."""

from datetime import datetime, timezone
import importlib.util
import json
from pathlib import Path
from subprocess import CompletedProcess


ROOT = Path(__file__).resolve().parents[1]
MODULE_PATH = ROOT / "scripts/ci/actions_queue_health.py"
SPEC = importlib.util.spec_from_file_location("actions_queue_health_terminal", MODULE_PATH)
assert SPEC and SPEC.loader
queue_health = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(queue_health)


def _pull_request() -> dict:
"""Return the exact open-PR identity used by the failed run."""
return {
"number": 1,
"state": "open",
"base": {"ref": "main", "repo": {"full_name": "owner/repo"}},
"head": {"sha": "head"},
"updated_at": "2026-09-15T13:00:00Z",
}


def _terminal_failure_run() -> dict:
"""Return a completed failure linked to the current pull-request head."""
return {
"id": 900,
"workflow_id": 901,
"name": "required-check",
"event": "pull_request",
"status": "completed",
"conclusion": "failure",
"head_sha": "head",
"created_at": "2026-09-15T13:05:00Z",
"updated_at": "2026-09-15T13:06:00Z",
"run_attempt": 1,
"pull_requests": [{"number": 1, "head": {"sha": "head"}}],
}


def _terminal_failure_job() -> dict:
"""Return a failed materialized job with no runner and no executed step."""
return {
"id": 902,
"name": "required-check",
"status": "completed",
"conclusion": "failure",
"runner_id": None,
"runner_name": None,
"created_at": "2026-09-15T13:05:00Z",
"steps": [],
}


def test_terminal_preexecution_failure_survives_collection_and_is_not_product_failure() -> None:
"""Keep failed zero-step jobs as explicit non-passing admission evidence."""
failed_run = _terminal_failure_run()
failed_job = _terminal_failure_job()
responses: dict[str, object] = {
"repos/owner/repo": {"default_branch": "main"},
"repos/owner/repo/pulls?state=open&per_page=100": [_pull_request()],
"repos/owner/repo/actions/runs?status=completed&head_sha=head&per_page=50": [failed_run],
"repos/owner/repo/actions/runs?status=cancelled&event=pull_request_target&per_page=50": [],
"repos/owner/repo/actions/runs/900/jobs?per_page=100": {
"total_count": 1,
"jobs": [failed_job],
},
}
for status in ("in_progress", "pending", "queued", "requested", "waiting"):
responses[f"repos/owner/repo/actions/runs?status={status}&per_page=50"] = []

def runner(args: list[str], **_: object) -> CompletedProcess[str]:
"""Return deterministic GitHub REST payloads for the regression specimen."""
path = args[-1]
if path not in responses:
raise AssertionError(f"unexpected endpoint: {path}")
return CompletedProcess(args, 0, json.dumps(responses[path]), "")

snapshot = queue_health.collect_snapshot(
["owner/repo"],
runner=runner,
generated_at="2026-09-15T13:10:00Z",
)
observed_runs = snapshot["repositories"][0]["runs"]
assert [run["id"] for run in observed_runs] == [900]
assert observed_runs[0]["jobs"][0]["steps_count"] == 0
assert observed_runs[0]["jobs"][0]["runner_id"] == 0

report = queue_health.build_report(
snapshot,
now=datetime(2026, 9, 15, 13, 10, tzinfo=timezone.utc),
)
row = report["runs"][0]
assert row["identity_state"] == "current_head"
assert row["run_conclusion"] == "FAILURE"
assert row["execution_state"] == "terminal_pre_execution_failure"
assert row["admission_state"] == "terminal_pre_execution_failure"
assert row["is_pending"] is False
assert row["runner_assigned"] is False
assert row["blocker"] == "terminal_pre_execution_failure_before_runner_assignment"
assert row["recommended_action"] == "inspect_actions_control_plane_without_leaf_bypass"
assert report["summary"]["terminal_pre_execution_failure_count"] == 1
assert report["summary"]["terminal_job_count"] == 1
Loading