From 1e972278bed3c2b42398a80302ead2b58cd286c2 Mon Sep 17 00:00:00 2001 From: David Spencer <1526975+DecisionNerd@users.noreply.github.com> Date: Sun, 30 Aug 2026 15:57:30 -0600 Subject: [PATCH 1/6] ci: establish gate registry and ESC operator --- .github/workflows/README.md | 26 ++ .github/workflows/clean-env-verify.yml | 5 +- .github/workflows/concurrency-stress-gate.yml | 2 +- .../durability-certification-gate.yml | 2 +- .github/workflows/test.yml | 5 + AGENTS.md | 6 + Makefile | 7 +- benchmarks/Makefile | 12 +- benchmarks/README.md | 17 + .../fly_tiny_qualification.py | 4 +- .../qualification_operator.py | 140 +++++++ .../tests/test_qualification_operator.py | 118 ++++++ config/gate-registry.json | 382 ++++++++++++++++++ scripts/ci/clean-env-verify.py | 19 +- scripts/ci/gate-registry.py | 208 ++++++++++ scripts/ci/test-clean-env-verify.py | 8 + scripts/ci/test-gate-registry.py | 116 ++++++ 17 files changed, 1063 insertions(+), 14 deletions(-) create mode 100644 benchmarks/harness/graphforge_bench/qualification_operator.py create mode 100644 benchmarks/tests/test_qualification_operator.py create mode 100644 config/gate-registry.json create mode 100644 scripts/ci/gate-registry.py create mode 100644 scripts/ci/test-gate-registry.py diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 59451fd9e..1e33a10fd 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -7,6 +7,20 @@ is repository ruleset **19988544** (required status check context exactly A deterministic classifier runs only the policy, language, and binding jobs relevant to the pull request. +`config/gate-registry.json` is the authoritative inventory for workflow and +operator gates. It records class, owner, canonical command, evidence contract, +freshness, and SHA binding. `scripts/ci/gate-registry.py` validates the registry +in ordinary Repository Policy CI and renders commands for Make or operators. +Every gate belongs to exactly one of four classes: required PR check, scheduled +health/stress, operator qualification, or release certification. Supporting +automation is inventoried with a non-gate role because it produces no gate +decision. Only `github-status/CI Gate` is a required PR check. + +Provider-backed qualifications are Python operator commands entered through +`pulumi env run`; Pulumi ESC owns secret projection and output filtering. +GitHub workflow files are compatibility wrappers and evidence viewers, not the +operational control plane. + **Speed is a first-class value alongside honesty.** Surfaces shed work that is not required for their objective. PR CI does **not** run full `llvm-cov`. Frequent publishing uses the **publish-track** (Binding RC → tag → @@ -161,6 +175,18 @@ plan/result documents and qualified evidence. The receipt binds the commit to a fresh 128-bit app-name nonce and never contains provider resource identifiers or credentials. +For new operator runs, use the registry-owned entry point instead of Actions +dispatch. All controller flags are passed after `ARGS=`; the Python operator +requires the exact SHA, an explicit live mode, and disposable confirmation +before it opens the named ESC environment: + +```bash +make -C benchmarks qualification-operator \ + GATE=fly-tiny \ + ESC_ENVIRONMENT=curatelabs/graphforge/qualification \ + ARGS='--expected-sha --execute --confirm-disposable ' +``` + ### `fly-tiny-recovery.yml` — Manual Fly orphan recovery This protected manual janitor downloads the one-day ownership receipt from a diff --git a/.github/workflows/clean-env-verify.yml b/.github/workflows/clean-env-verify.yml index 5042a77dd..267980af1 100644 --- a/.github/workflows/clean-env-verify.yml +++ b/.github/workflows/clean-env-verify.yml @@ -71,10 +71,7 @@ jobs: fi if [[ "$LANES" == "all" ]]; then if [[ -z "$RECORD" ]]; then - # checksums requires a release record; run every other lane when absent - for lane in pip npm cli skills reopen urls; do - args+=(--lane "$lane") - done + args+=(--default) else args+=(--all) fi diff --git a/.github/workflows/concurrency-stress-gate.yml b/.github/workflows/concurrency-stress-gate.yml index 0618482ef..606648e1e 100644 --- a/.github/workflows/concurrency-stress-gate.yml +++ b/.github/workflows/concurrency-stress-gate.yml @@ -60,7 +60,7 @@ jobs: - name: Run bounded-resource stress lane run: >- /tmp/gf-concurrency-stress/bin/python - scripts/ci/concurrency-stress-gate.py run + scripts/ci/gate-registry.py run concurrency-stress -- --seed "${GF_STRESS_SEED}" --iterations "${GF_STRESS_ITERATIONS}" --output "${{ runner.temp }}/concurrency-stress-evidence" diff --git a/.github/workflows/durability-certification-gate.yml b/.github/workflows/durability-certification-gate.yml index 752628ed1..9062a0aef 100644 --- a/.github/workflows/durability-certification-gate.yml +++ b/.github/workflows/durability-certification-gate.yml @@ -50,7 +50,7 @@ jobs: - name: Run scheduled seeded certification lane run: >- - python3 scripts/ci/durability-certification-gate.py run + python3 scripts/ci/gate-registry.py run durability-certification -- --seed "${GF_CERT_SEED}" --histories "${GF_CERT_HISTORIES}" --ops "${GF_CERT_OPS}" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 95b6c6800..b5888b946 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -74,6 +74,11 @@ jobs: - name: Test changed-path classifier run: scripts/ci/test-classify-changes.sh + - name: Validate authoritative gate registry + run: | + python3 scripts/ci/gate-registry.py validate + python3 scripts/ci/test-gate-registry.py + - name: Test CodSpeed nightly policy run: python3 scripts/ci/test-codspeed-nightly.py diff --git a/AGENTS.md b/AGENTS.md index ab67b906f..f9294543b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -52,6 +52,12 @@ Run formatting after the final edit. Review intentional snapshot changes before ## PR gate +`config/gate-registry.json` is the machine-readable gate authority. Its sole +required PR status is `github-status/CI Gate` with `sha_rule=exact_head`, matching +repository ruleset 19988544. Scheduled stress, operator qualifications, and +release certification evidence are not required PR checks. Validate changes +with `make gate-registry-check`. + Merge only when: - acceptance criteria have direct tests or deterministic evidence; diff --git a/Makefile b/Makefile index d22d3d742..70b1717b3 100644 --- a/Makefile +++ b/Makefile @@ -21,6 +21,11 @@ security: ## Run Bandit security scanner workflow-lint: ## Validate GitHub Actions workflows with pinned actionlint scripts/check-workflows.sh +.PHONY: gate-registry-check +gate-registry-check: ## Validate gate classes, owners, commands, evidence, and SHA rules + python3 scripts/ci/gate-registry.py validate + python3 scripts/ci/test-gate-registry.py + license-check: ## Verify Apache-2.0 metadata and distributed copies python3 scripts/license_check.py @@ -381,7 +386,7 @@ clean-env-verify-preflight: ## Probe public registries for VERSION (fails close clean-env-verify: ## Run clean-env lanes against public registries (post-§6 only) @test -n "$(VERSION)" || (echo "VERSION is required (e.g. VERSION=0.5.0)" && exit 2) python3 scripts/ci/clean-env-verify.py run --version "$(VERSION)" \ - $(if $(RELEASE_RECORD),--release-record "$(RELEASE_RECORD)" --all,--lane pip --lane npm --lane cli --lane skills --lane reopen --lane urls) \ + $(if $(RELEASE_RECORD),--release-record "$(RELEASE_RECORD)" --all,--default) \ $(if $(OUTPUT),--output "$(OUTPUT)",) \ $(if $(WORK),--work "$(WORK)",) diff --git a/benchmarks/Makefile b/benchmarks/Makefile index e757c5a2d..685c3e8e2 100644 --- a/benchmarks/Makefile +++ b/benchmarks/Makefile @@ -1,4 +1,4 @@ -.PHONY: install smoke smoke-python smoke-rust fly-adapter-static local-admission progressive-qualification-list progressive-qualification-plan progressive-qualification-run progressive-qualification-project-s20 progressive-qualification-binaries progressive-provider-plan +.PHONY: install smoke smoke-python smoke-rust fly-adapter-static local-admission progressive-qualification-list progressive-qualification-plan progressive-qualification-run progressive-qualification-project-s20 progressive-qualification-binaries progressive-provider-plan qualification-operator install: uv sync --locked @@ -55,7 +55,15 @@ progressive-qualification-project-s20: install # Safe control-plane admission only; this never provisions a provider. progressive-provider-plan: install test -n "$(COMMIT)" && test -n "$(MAXIMUM_SCALE)" && test -n "$(OUTPUT_DIR)" && test -n "$(PLAN_OUT)" - PYTHONPATH=$(CURDIR)/harness uv run --locked python -m graphforge_bench.progressive_provider_plan \ + PYTHONPATH=$(CURDIR)/harness uv run --locked python -m graphforge_bench.qualification_operator \ + plan-progressive -- \ --root "$(CURDIR)" --output-dir "$(OUTPUT_DIR)" --commit "$(COMMIT)" \ --maximum-scale "$(MAXIMUM_SCALE)" $(if $(PROVIDER_CAPACITY),--provider-capacity "$(PROVIDER_CAPACITY)",) \ --plan-out "$(PLAN_OUT)" + +# The registry owns the live command. Pulumi ESC supplies provider credentials; +# the Python controller still requires every gate-specific confirmation flag. +qualification-operator: install + test -n "$(GATE)" && test -n "$(ESC_ENVIRONMENT)" + PYTHONPATH=$(CURDIR)/harness uv run --locked python ../scripts/ci/gate-registry.py run "$(GATE)" -- \ + --environment "$(ESC_ENVIRONMENT)" $(ARGS) diff --git a/benchmarks/README.md b/benchmarks/README.md index 0ece79591..6ebdf1dee 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -316,6 +316,23 @@ Graph500 generator/certifier, Python/BenchExec, and a proven cgroups/I/O authority. The canonical order remains S18, S19, S20, S22, S24, S25, S26; the first failed or missing gate stops the ladder. +Provider credentials belong to Pulumi ESC rather than GitHub workflow inputs or +the caller's ambient shell. Live operator commands are rendered from +`config/gate-registry.json` and run through the Python control plane: + +```bash +make -C benchmarks qualification-operator \ + GATE=fly-tiny \ + ESC_ENVIRONMENT=curatelabs/graphforge/qualification \ + ARGS='--expected-sha --execute --confirm-disposable ' +``` + +The operator uses the shell-free form `pulumi env run -- `; +secret values are never copied into its command line or evidence. The +`progressive-ladder` command is registered now, but fails before opening ESC +until the dedicated provider image and scale BenchExec executor are present. +This is a deliberate capability boundary, not a GitHub-dispatch prerequisite. + The controller derives bulk-ingest capability from the same run's bounded ordinary `gf import-session commit --json` receipt: its construction evidence must identify a configuration of at least 65,536 rows, accepted bulk chunks, diff --git a/benchmarks/harness/graphforge_bench/fly_tiny_qualification.py b/benchmarks/harness/graphforge_bench/fly_tiny_qualification.py index 24cb48bb8..30312c197 100644 --- a/benchmarks/harness/graphforge_bench/fly_tiny_qualification.py +++ b/benchmarks/harness/graphforge_bench/fly_tiny_qualification.py @@ -1045,8 +1045,8 @@ def parser() -> argparse.ArgumentParser: return result -def main() -> int: - args = parser().parse_args() +def main(argv: list[str] | None = None) -> int: + args = parser().parse_args(argv) if (args.execute or args.cleanup_only) and not args.confirm_disposable: result = sanitized_failure("authorization_refused") _atomic_json(args.result_out, result) diff --git a/benchmarks/harness/graphforge_bench/qualification_operator.py b/benchmarks/harness/graphforge_bench/qualification_operator.py new file mode 100644 index 000000000..dc50941d2 --- /dev/null +++ b/benchmarks/harness/graphforge_bench/qualification_operator.py @@ -0,0 +1,140 @@ +"""Pulumi ESC control plane for provider-backed GraphForge qualifications. + +The public ``run`` command is the only provider execution entry point. It opens +an ESC environment and then invokes the private ``execute`` command with secret +filtering owned by Pulumi. The progressive ladder remains fail-closed until a +dedicated scale executor exists; its no-spend plan is available through +``plan-progressive``. +""" + +from __future__ import annotations + +import argparse +from collections.abc import Callable, Sequence +import os +import re +import subprocess +import sys + +from graphforge_bench import fly_tiny_qualification + +ESC_CONTEXT = "pulumi-esc-v1" +ESC_ENVIRONMENT = re.compile( + r"^[A-Za-z0-9][A-Za-z0-9_.-]*(?:/[A-Za-z0-9][A-Za-z0-9_.-]*){0,2}(?:@[A-Za-z0-9_.-]+)?$" +) +COMMIT = re.compile(r"^[0-9a-f]{40}$") +LIVE_GATES = {"fly-tiny", "fly-tiny-recovery"} +ALL_GATES = (*sorted(LIVE_GATES), "progressive-ladder") + + +class OperatorRefusalError(ValueError): + """Execution lacks a closed authority or implementation boundary.""" + + +def _forwarded(argv: Sequence[str]) -> list[str]: + values = list(argv) + return values[1:] if values[:1] == ["--"] else values + + +def _require_flag(argv: Sequence[str], flag: str) -> None: + if flag not in argv: + raise OperatorRefusalError(f"{flag} is required") + + +def _single_value(argv: Sequence[str], flag: str) -> str: + positions = [index for index, value in enumerate(argv) if value == flag] + if len(positions) != 1 or positions[0] + 1 >= len(argv): + raise OperatorRefusalError(f"{flag} must occur once with a value") + value = argv[positions[0] + 1] + if value.startswith("-"): + raise OperatorRefusalError(f"{flag} must occur once with a value") + return value + + +def validate_live_request(gate: str, argv: Sequence[str]) -> None: + if gate == "progressive-ladder": + raise OperatorRefusalError( + "progressive-ladder execution is unavailable until the dedicated provider " + "image and BenchExec scale executor are implemented" + ) + if gate not in LIVE_GATES: + raise OperatorRefusalError("qualification gate is unknown") + commit = _single_value(argv, "--expected-sha") + if COMMIT.fullmatch(commit) is None: + raise OperatorRefusalError("--expected-sha must be a lowercase full Git object ID") + _require_flag(argv, "--confirm-disposable") + _require_flag(argv, "--execute" if gate == "fly-tiny" else "--cleanup-only") + + +def esc_command(environment: str, gate: str, argv: Sequence[str]) -> tuple[str, ...]: + """Build one shell-free, secret-filtered ESC invocation.""" + if ESC_ENVIRONMENT.fullmatch(environment) is None: + raise OperatorRefusalError("Pulumi ESC environment name is invalid") + forwarded = _forwarded(argv) + validate_live_request(gate, forwarded) + return ( + "pulumi", + "env", + "run", + environment, + "--", + "env", + f"GRAPHFORGE_OPERATOR_CONTEXT={ESC_CONTEXT}", + sys.executable, + "-m", + "graphforge_bench.qualification_operator", + "execute", + "--gate", + gate, + "--", + *forwarded, + ) + + +def run_under_esc( + environment: str, + gate: str, + argv: Sequence[str], + *, + runner: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run, +) -> int: + command = esc_command(environment, gate, argv) + completed = runner(command, check=False) + return completed.returncode + + +def execute_inner(gate: str, argv: Sequence[str]) -> int: + """Dispatch only after ``pulumi env run`` established the process context.""" + if os.environ.get("GRAPHFORGE_OPERATOR_CONTEXT") != ESC_CONTEXT: + raise OperatorRefusalError("provider execution must enter through pulumi env run") + forwarded = _forwarded(argv) + validate_live_request(gate, forwarded) + return fly_tiny_qualification.main(forwarded) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="action", required=True) + run = sub.add_parser("run", help="Execute a live qualification under Pulumi ESC") + run.add_argument("--environment", required=True) + run.add_argument("--gate", choices=ALL_GATES, required=True) + execute = sub.add_parser("execute", help=argparse.SUPPRESS) + execute.add_argument("--gate", choices=ALL_GATES, required=True) + plan = sub.add_parser("plan-progressive", help="Write a no-spend next-rung plan") + plan.add_argument("arguments", nargs=argparse.REMAINDER) + args, remainder = parser.parse_known_args(argv) + try: + if args.action == "run": + return run_under_esc(args.environment, args.gate, remainder) + if args.action == "execute": + return execute_inner(args.gate, remainder) + from graphforge_bench import progressive_provider_plan + + return progressive_provider_plan.main(_forwarded(args.arguments)) + except OperatorRefusalError as error: + print(f"qualification refused: {error}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/tests/test_qualification_operator.py b/benchmarks/tests/test_qualification_operator.py new file mode 100644 index 000000000..9705148dc --- /dev/null +++ b/benchmarks/tests/test_qualification_operator.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import os +import subprocess +import sys +import unittest +from unittest.mock import patch + +from graphforge_bench.qualification_operator import ( + ESC_CONTEXT, + OperatorRefusalError, + esc_command, + execute_inner, + run_under_esc, +) + +ARGS = [ + "--expected-sha", + "a" * 40, + "--execute", + "--confirm-disposable", +] + + +class QualificationOperatorTests(unittest.TestCase): + def test_esc_command_is_shell_free_and_forwards_exact_arguments(self) -> None: + command = esc_command("curatelabs/graphforge/qualification", "fly-tiny", ARGS) + self.assertEqual( + command[:5], ("pulumi", "env", "run", "curatelabs/graphforge/qualification", "--") + ) + self.assertEqual( + command[5:11], + ( + "env", + f"GRAPHFORGE_OPERATOR_CONTEXT={ESC_CONTEXT}", + sys.executable, + "-m", + "graphforge_bench.qualification_operator", + "execute", + ), + ) + self.assertEqual(list(command[-len(ARGS) :]), ARGS) + + def test_live_execution_requires_confirmation_and_exact_sha(self) -> None: + with self.assertRaisesRegex(OperatorRefusalError, "--confirm-disposable"): + esc_command("curatelabs/graphforge/qualification", "fly-tiny", ARGS[:-1]) + with self.assertRaisesRegex(OperatorRefusalError, "--expected-sha"): + esc_command( + "curatelabs/graphforge/qualification", + "fly-tiny", + ["--execute", "--confirm-disposable"], + ) + with self.assertRaisesRegex(OperatorRefusalError, "lowercase full Git object ID"): + esc_command( + "curatelabs/graphforge/qualification", + "fly-tiny", + [ + "--expected-sha", + "main", + "--execute", + "--confirm-disposable", + ], + ) + + def test_progressive_ladder_refuses_before_opening_esc(self) -> None: + runner_called = False + + def runner(*args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]: + nonlocal runner_called + runner_called = True + return subprocess.CompletedProcess(args, 0) + + with self.assertRaisesRegex(OperatorRefusalError, "scale executor"): + run_under_esc( + "curatelabs/graphforge/qualification", + "progressive-ladder", + ARGS, + runner=runner, + ) + self.assertFalse(runner_called) + + def test_inner_execution_requires_esc_context(self) -> None: + with ( + patch.dict(os.environ, {}, clear=True), + self.assertRaisesRegex(OperatorRefusalError, "pulumi env run"), + ): + execute_inner("fly-tiny", ARGS) + + def test_inner_execution_delegates_to_existing_controller(self) -> None: + with ( + patch.dict(os.environ, {"GRAPHFORGE_OPERATOR_CONTEXT": ESC_CONTEXT}, clear=True), + patch( + "graphforge_bench.qualification_operator.fly_tiny_qualification.main", + return_value=7, + ) as inner, + ): + self.assertEqual(execute_inner("fly-tiny", ARGS), 7) + inner.assert_called_once_with(ARGS) + + def test_outer_runner_receives_argv_without_secret_values(self) -> None: + observed: tuple[str, ...] | None = None + + def runner(argv: tuple[str, ...], *, check: bool) -> subprocess.CompletedProcess[str]: + nonlocal observed + observed = argv + self.assertFalse(check) + return subprocess.CompletedProcess(argv, 0) + + self.assertEqual( + run_under_esc("curatelabs/graphforge/qualification", "fly-tiny", ARGS, runner=runner), + 0, + ) + assert observed is not None + self.assertNotIn("FLY_API_TOKEN", " ".join(observed)) + + +if __name__ == "__main__": + unittest.main() diff --git a/config/gate-registry.json b/config/gate-registry.json new file mode 100644 index 000000000..34e3a90cb --- /dev/null +++ b/config/gate-registry.json @@ -0,0 +1,382 @@ +{ + "schema": "graphforge-gate-registry/1", + "commands": { + "binding-release-candidate": [ + "python3", + "scripts/ci/validate-binding-release-candidate.py" + ], + "checkpoint-recovery": [ + "python3", + "scripts/ci/checkpoint-recovery-gate.py" + ], + "clean-environment": [ + "python3", + "scripts/ci/clean-env-verify.py", + "run" + ], + "codspeed": [ + "cargo", + "codspeed", + "build" + ], + "documentation": [ + "pnpm", + "docs:build" + ], + "epistemic-contract": [ + "python3", + "scripts/ci/epistemic-contract-gate.py", + "report" + ], + "fuzz": [ + "cargo", + "fuzz", + "run" + ], + "github-automation": [ + "python3", + "scripts/ci/gate-registry.py", + "validate" + ], + "knowledge-contract": [ + "python3", + "scripts/ci/knowledge-contract-gate.py", + "report" + ], + "matrix-gate": [ + "python3", + "scripts/ci/gate-registry.py", + "matrix" + ], + "native-admission": [ + "make", + "-C", + "benchmarks", + "local-admission" + ], + "non-cypher-surface": [ + "python3", + "scripts/ci/non-cypher-surface-gate.py" + ], + "operator": [ + "python3", + "-m", + "graphforge_bench.qualification_operator", + "run" + ], + "publication": [ + "python3", + "scripts/ci/release-publish-preflight.py" + ], + "release-certification": [ + "python3", + "scripts/ci/release-certification.py", + "aggregate" + ], + "release-credential-preflight": [ + "python3", + "scripts/ci/test-publish-cli-contract.py" + ], + "repository-policy": [ + "scripts/ci/require-gates.sh" + ], + "visualization-stress": [ + "python3", + "examples/visualization/stress/harness/run.py" + ] + }, + "workflows": [ + { + "id": "binding-release-candidate", + "path": ".github/workflows/binding-release-candidate.yml", + "class": "release_certification", + "owner": "release", + "command": "binding-release-candidate", + "args": [], + "evidence_contract": "graphforge-release-candidate-v2", + "freshness": "30d", + "sha_rule": "exact_commit" + }, + { + "id": "checkpoint-recovery", + "path": ".github/workflows/checkpoint-recovery-gate.yml", + "class": "release_certification", + "owner": "release", + "command": "checkpoint-recovery", + "args": [], + "evidence_contract": "graphforge-checkpoint-recovery/1", + "freshness": "30d", + "sha_rule": "exact_commit" + }, + { + "id": "clean-environment", + "path": ".github/workflows/clean-env-verify.yml", + "class": "release_certification", + "owner": "release", + "command": "clean-environment", + "args": [], + "evidence_contract": "graphforge-clean-env-evidence-v1", + "freshness": "release_version", + "sha_rule": "version_identity" + }, + { + "id": "codspeed", + "path": ".github/workflows/codspeed.yml", + "class": "scheduled_health_stress", + "owner": "performance", + "command": "codspeed", + "args": [], + "evidence_contract": "codspeed-measurement", + "freshness": "latest_main_change", + "sha_rule": "exact_main" + }, + { + "id": "concurrency-stress", + "path": ".github/workflows/concurrency-stress-gate.yml", + "class": "scheduled_health_stress", + "owner": "storage", + "command": "matrix-gate", + "args": [ + "--family", + "concurrency", + "--variant", + "stress" + ], + "evidence_contract": "graphforge-concurrency-stress/1", + "freshness": "14d", + "sha_rule": "exact_main" + }, + { + "id": "documentation", + "path": ".github/workflows/docs.yml", + "role": "supporting_automation", + "owner": "docs", + "command": "documentation", + "args": [], + "evidence_contract": "github-pages-build", + "freshness": "current_run", + "sha_rule": "event_sha" + }, + { + "id": "durability-certification", + "path": ".github/workflows/durability-certification-gate.yml", + "class": "scheduled_health_stress", + "owner": "storage", + "command": "matrix-gate", + "args": [ + "--family", + "durability", + "--variant", + "certification" + ], + "evidence_contract": "graphforge-durability-certification/1", + "freshness": "14d", + "sha_rule": "exact_main" + }, + { + "id": "epistemic-contract", + "path": ".github/workflows/epistemic-contract-gate.yml", + "class": "release_certification", + "owner": "release", + "command": "epistemic-contract", + "args": [], + "evidence_contract": "graphforge-epistemic-contract/1", + "freshness": "30d", + "sha_rule": "exact_commit" + }, + { + "id": "fly-tiny-qualification", + "path": ".github/workflows/fly-tiny-qualification.yml", + "class": "operator_qualification", + "owner": "benchmark-operator", + "command": "operator", + "args": [ + "--gate", + "fly-tiny" + ], + "evidence_contract": "graphforge-fly-filesystem-qualification/1", + "freshness": "24h", + "sha_rule": "exact_main", + "control_plane": "pulumi_esc" + }, + { + "id": "fly-tiny-recovery", + "path": ".github/workflows/fly-tiny-recovery.yml", + "class": "operator_qualification", + "owner": "benchmark-operator", + "command": "operator", + "args": [ + "--gate", + "fly-tiny-recovery" + ], + "evidence_contract": "graphforge-fly-adapter-result/2", + "freshness": "ownership_receipt_24h", + "sha_rule": "receipt_commit", + "control_plane": "pulumi_esc" + }, + { + "id": "fuzz", + "path": ".github/workflows/fuzz.yml", + "class": "scheduled_health_stress", + "owner": "core", + "command": "fuzz", + "args": [], + "evidence_contract": "cargo-fuzz-artifacts", + "freshness": "1d", + "sha_rule": "exact_main" + }, + { + "id": "g500-certification", + "path": ".github/workflows/g500-certification.yml", + "class": "operator_qualification", + "owner": "benchmark-operator", + "command": "operator", + "args": [ + "--gate", + "progressive-ladder" + ], + "evidence_contract": "graphforge-g500-certification/1", + "freshness": "release_candidate", + "sha_rule": "exact_commit", + "control_plane": "pulumi_esc" + }, + { + "id": "knowledge-contract", + "path": ".github/workflows/knowledge-contract-gate.yml", + "class": "release_certification", + "owner": "release", + "command": "knowledge-contract", + "args": [], + "evidence_contract": "graphforge-knowledge-contract/1", + "freshness": "30d", + "sha_rule": "exact_commit" + }, + { + "id": "native-local-admission", + "path": ".github/workflows/native-local-admission.yml", + "class": "operator_qualification", + "owner": "benchmark-operator", + "command": "native-admission", + "args": [], + "evidence_contract": "graphforge-local-admission/1", + "freshness": "runner_image", + "sha_rule": "exact_commit", + "control_plane": "local_delegated" + }, + { + "id": "non-cypher-surface", + "path": ".github/workflows/non-cypher-surface-gate.yml", + "class": "release_certification", + "owner": "release", + "command": "non-cypher-surface", + "args": [], + "evidence_contract": "graphforge-rust-non-cypher/1", + "freshness": "30d", + "sha_rule": "exact_commit" + }, + { + "id": "pr-labeler", + "path": ".github/workflows/pr-labeler.yml", + "role": "supporting_automation", + "owner": "repository", + "command": "github-automation", + "args": [], + "evidence_contract": "labels", + "freshness": "current_run", + "sha_rule": "event_sha" + }, + { + "id": "publish-track", + "path": ".github/workflows/publish-track.yml", + "class": "release_certification", + "owner": "release", + "command": "publication", + "args": [], + "evidence_contract": "graphforge-release-candidate-v2", + "freshness": "30d", + "sha_rule": "exact_main" + }, + { + "id": "publish", + "path": ".github/workflows/publish.yaml", + "class": "release_certification", + "owner": "release", + "command": "publication", + "args": [], + "evidence_contract": "graphforge-release-reconciliation/1", + "freshness": "30d", + "sha_rule": "exact_commit" + }, + { + "id": "release-certification", + "path": ".github/workflows/release-certification.yml", + "class": "release_certification", + "owner": "release", + "command": "release-certification", + "args": [], + "evidence_contract": "graphforge-release-certification/1", + "freshness": "30d", + "sha_rule": "exact_main" + }, + { + "id": "release-credential-preflight", + "path": ".github/workflows/release-credential-preflight.yml", + "class": "release_certification", + "owner": "release", + "command": "release-credential-preflight", + "args": [], + "evidence_contract": "graphforge-release-credential-preflight/1", + "freshness": "same_publication", + "sha_rule": "exact_commit" + }, + { + "id": "release-labels", + "path": ".github/workflows/release-tracking.yml", + "role": "supporting_automation", + "owner": "repository", + "command": "github-automation", + "args": [], + "evidence_contract": "labels", + "freshness": "current_run", + "sha_rule": "event_sha" + }, + { + "id": "test-suite", + "path": ".github/workflows/test.yml", + "class": "required_pr_check", + "owner": "ci", + "command": "repository-policy", + "args": [], + "evidence_contract": "github-status/CI Gate", + "freshness": "current_run", + "sha_rule": "exact_head" + }, + { + "id": "visualization-limits", + "path": ".github/workflows/visualization-limits-stress.yml", + "class": "scheduled_health_stress", + "owner": "visualization", + "command": "visualization-stress", + "args": [], + "evidence_contract": "graphforge-visualization-limits/1", + "freshness": "diagnostic", + "sha_rule": "exact_commit" + } + ], + "operator_gates": [ + { + "id": "progressive-ladder", + "class": "operator_qualification", + "owner": "benchmark-operator", + "command": "operator", + "args": [ + "--gate", + "progressive-ladder" + ], + "evidence_contract": "graphforge-progressive-provider-plan/1 + rung evidence", + "freshness": "adjacent_rung", + "sha_rule": "exact_main", + "control_plane": "pulumi_esc" + } + ] +} diff --git a/scripts/ci/clean-env-verify.py b/scripts/ci/clean-env-verify.py index 9514b8396..76c17010e 100644 --- a/scripts/ci/clean-env-verify.py +++ b/scripts/ci/clean-env-verify.py @@ -61,6 +61,7 @@ "checksums": 187, } ALL_LANES = tuple(LANE_ISSUES) +DEFAULT_LANES = tuple(name for name in ALL_LANES if name not in {"cargo", "checksums"}) class VerifyError(RuntimeError): @@ -872,7 +873,13 @@ def cmd_preflight(args: argparse.Namespace) -> int: def cmd_run(args: argparse.Namespace) -> int: version = require_version(args.version) crates = tuple(args.crate) if args.crate else DEFAULT_CRATES - lanes = list(ALL_LANES) if args.all else list(args.lane or []) + lanes = ( + list(ALL_LANES) + if args.all + else list(DEFAULT_LANES) + if args.default + else list(args.lane or []) + ) if not lanes: raise VerifyError("specify --all or one or more --lane values") unknown = [lane for lane in lanes if lane not in LANE_RUNNERS] @@ -963,8 +970,14 @@ def build_parser() -> argparse.ArgumentParser: run.add_argument("--version", default=DEFAULT_VERSION) run.add_argument("--docs-base", default=DEFAULT_DOCS_BASE) run.add_argument("--crate", action="append", default=[]) - run.add_argument("--lane", action="append", choices=list(ALL_LANES)) - run.add_argument("--all", action="store_true") + selection = run.add_mutually_exclusive_group() + selection.add_argument("--lane", action="append", choices=list(ALL_LANES)) + selection.add_argument("--all", action="store_true") + selection.add_argument( + "--default", + action="store_true", + help="Run the canonical no-release-record lane set", + ) run.add_argument("--release-record", help="Path to release record or candidate manifest JSON") run.add_argument("--work", help="Work directory (default: temp dir)") run.add_argument("--output", help="Write evidence JSON to this path") diff --git a/scripts/ci/gate-registry.py b/scripts/ci/gate-registry.py new file mode 100644 index 000000000..4ed605077 --- /dev/null +++ b/scripts/ci/gate-registry.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Validate and render GraphForge's authoritative gate registry (#1009).""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import shlex +import subprocess +import sys +from typing import Any + +ROOT = Path(__file__).resolve().parents[2] +REGISTRY = ROOT / "config" / "gate-registry.json" +CLASSES = { + "required_pr_check", + "scheduled_health_stress", + "operator_qualification", + "release_certification", +} +REQUIRED_FIELDS = {"id", "owner", "command", "args", "evidence_contract", "freshness", "sha_rule"} + + +class RegistryError(ValueError): + """The checked-in gate taxonomy is incomplete or contradictory.""" + + +def load_registry(path: Path = REGISTRY) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise RegistryError("gate registry is unavailable or malformed") from error + if not isinstance(value, dict) or value.get("schema") != "graphforge-gate-registry/1": + raise RegistryError("gate registry schema is invalid") + return value + + +def _workflow_files(root: Path) -> set[str]: + directory = root / ".github" / "workflows" + return { + path.relative_to(root).as_posix() + for pattern in ("*.yml", "*.yaml") + for path in directory.glob(pattern) + } + + +def validate_registry(value: dict[str, Any], root: Path = ROOT) -> None: + commands = value.get("commands") + workflows = value.get("workflows") + operators = value.get("operator_gates") + if ( + not isinstance(commands, dict) + or not isinstance(workflows, list) + or not isinstance(operators, list) + ): + raise RegistryError("registry collections are malformed") + for name, argv in commands.items(): + if not isinstance(name, str) or not name or not isinstance(argv, list) or not argv: + raise RegistryError("command definitions must be non-empty argv arrays") + if any(not isinstance(arg, str) or not arg for arg in argv): + raise RegistryError(f"command {name} contains an invalid argument") + if argv[0].startswith("python") and argv[1:2] != ["-m"]: + script = root / argv[1] + if not script.is_file(): + raise RegistryError(f"command {name} references a missing script") + if "/" in argv[0] and not (root / argv[0]).is_file(): + raise RegistryError(f"command {name} references a missing executable") + + ids: set[str] = set() + paths: set[str] = set() + records = workflows + operators + for record in records: + if not isinstance(record, dict) or not record.keys() >= REQUIRED_FIELDS: + raise RegistryError("gate record is missing required metadata") + gate_id = record["id"] + if not isinstance(gate_id, str) or gate_id in ids: + raise RegistryError(f"duplicate or invalid gate id: {gate_id}") + ids.add(gate_id) + command = record["command"] + if ( + command not in commands + or not isinstance(record["args"], list) + or any(not isinstance(arg, str) or not arg for arg in record["args"]) + ): + raise RegistryError(f"{gate_id}: command reference is invalid") + for field in ("owner", "evidence_contract", "freshness", "sha_rule"): + if not isinstance(record[field], str) or not record[field]: + raise RegistryError(f"{gate_id}: {field} is empty") + gate_class = record.get("class") + if gate_class is None: + if record.get("role") != "supporting_automation" or record in operators: + raise RegistryError(f"{gate_id}: non-gate role is invalid") + elif gate_class not in CLASSES: + raise RegistryError(f"{gate_id}: class is invalid") + if gate_class == "required_pr_check": + if record["sha_rule"] != "exact_head" or not record["evidence_contract"].startswith( + "github-status/" + ): + raise RegistryError(f"{gate_id}: required PR checks must bind an exact-head status") + if gate_class == "operator_qualification" and record.get("control_plane") not in { + "pulumi_esc", + "local_delegated", + }: + raise RegistryError(f"{gate_id}: operator authority is missing") + if record.get("control_plane") == "pulumi_esc" and command != "operator": + raise RegistryError(f"{gate_id}: costly qualification bypasses the Python operator") + if gate_class == "release_certification" and record["sha_rule"] not in { + "exact_commit", + "exact_main", + "version_identity", + }: + raise RegistryError(f"{gate_id}: release evidence identity is weak") + if "path" in record: + path = record["path"] + if not isinstance(path, str) or path in paths: + raise RegistryError(f"{gate_id}: workflow path is duplicate or invalid") + paths.add(path) + + actual = _workflow_files(root) + if paths != actual: + missing = sorted(actual - paths) + stale = sorted(paths - actual) + raise RegistryError(f"workflow inventory mismatch: missing={missing} stale={stale}") + + required = [item for item in workflows if item.get("class") == "required_pr_check"] + if [(item["id"], item["evidence_contract"]) for item in required] != [ + ("test-suite", "github-status/CI Gate") + ]: + raise RegistryError("required PR checks drifted from repository ruleset 19988544") + + families = { + item["id"]: (item["command"], tuple(item["args"])) + for item in workflows + if item["id"] in {"concurrency-stress", "durability-certification"} + } + if set(families) != {"concurrency-stress", "durability-certification"} or any( + command != "matrix-gate" or "--variant" not in args for command, args in families.values() + ): + raise RegistryError("durability/concurrency variants must share matrix-gate") + + release_owners = { + item["owner"] + for item in workflows + if item["id"] in {"clean-environment", "publish-track", "publish"} + } + if release_owners != {"release"}: + raise RegistryError("publication verification must have one release owner") + referenced_commands = {item["command"] for item in records} + if referenced_commands != set(commands): + raise RegistryError("command definitions must be referenced exactly by gate records") + + +def command_argv(value: dict[str, Any], gate_id: str) -> list[str]: + records = value["workflows"] + value["operator_gates"] + try: + record = next(item for item in records if item["id"] == gate_id) + except StopIteration as error: + raise RegistryError(f"unknown gate: {gate_id}") from error + return [*value["commands"][record["command"]], *record["args"]] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="action", required=True) + sub.add_parser("validate") + command = sub.add_parser("command") + command.add_argument("gate") + command.add_argument("--json", action="store_true") + run = sub.add_parser("run") + run.add_argument("gate") + matrix = sub.add_parser("matrix") + matrix.add_argument("--family", choices=("concurrency", "durability"), required=True) + matrix.add_argument("--variant", choices=("stress", "certification"), required=True) + args, passthrough = parser.parse_known_args(argv) + try: + registry = load_registry() + validate_registry(registry) + if args.action == "validate": + print(f"gate registry valid: {len(registry['workflows'])} workflows") + return 0 + if args.action == "matrix": + expected = {"concurrency": "stress", "durability": "certification"} + if expected[args.family] != args.variant: + raise RegistryError("matrix family/variant pairing is invalid") + script = { + "concurrency": "scripts/ci/concurrency-stress-gate.py", + "durability": "scripts/ci/durability-certification-gate.py", + }[args.family] + if passthrough and passthrough[0] == "--": + passthrough = passthrough[1:] + return subprocess.run( + [sys.executable, script, "run", *passthrough], cwd=ROOT, check=False + ).returncode + rendered = command_argv(registry, args.gate) + if args.action == "command": + print(json.dumps(rendered) if args.json else shlex.join(rendered)) + return 0 + if passthrough and passthrough[0] == "--": + passthrough = passthrough[1:] + return subprocess.run([*rendered, *passthrough], cwd=ROOT, check=False).returncode + except RegistryError as error: + print(f"gate registry invalid: {error}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/test-clean-env-verify.py b/scripts/ci/test-clean-env-verify.py index e5953adff..b6242c327 100644 --- a/scripts/ci/test-clean-env-verify.py +++ b/scripts/ci/test-clean-env-verify.py @@ -23,6 +23,14 @@ assert cev.DEFAULT_CRATES[-1] == "graphforge-cli" assert cev.LANE_ISSUES["cargo"] == 185 assert cev.LANE_RUNNERS["cargo"] is cev.lane_cargo +assert set(cev.LANE_RUNNERS) == set(cev.LANE_ISSUES) +assert cev.DEFAULT_LANES == ("pip", "npm", "cli", "skills", "reopen", "urls") +WORKFLOW = SCRIPT.parents[2] / ".github" / "workflows" / "clean-env-verify.yml" +MAKEFILE = SCRIPT.parents[2] / "Makefile" +for entrypoint in (WORKFLOW, MAKEFILE): + entrypoint_text = entrypoint.read_text(encoding="utf-8") + assert "--default" in entrypoint_text + assert "--lane pip" not in entrypoint_text def write_json(path: Path, value: object) -> None: diff --git a/scripts/ci/test-gate-registry.py b/scripts/ci/test-gate-registry.py new file mode 100644 index 000000000..cfcc56818 --- /dev/null +++ b/scripts/ci/test-gate-registry.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Mutation-sensitive tests for the authoritative gate registry (#1009).""" + +from __future__ import annotations + +import copy +import importlib.util +from pathlib import Path +import subprocess +import sys +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts" / "ci" / "gate-registry.py" +SPEC = importlib.util.spec_from_file_location("gate_registry", SCRIPT) +assert SPEC and SPEC.loader +GATE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(GATE) + + +class GateRegistryTests(unittest.TestCase): + def setUp(self) -> None: + self.registry = GATE.load_registry() + + def rejected(self, expected: str, registry: dict[str, object]) -> None: + with self.assertRaisesRegex(GATE.RegistryError, expected): + GATE.validate_registry(registry) + + def test_checked_in_registry_is_complete(self) -> None: + GATE.validate_registry(self.registry) + paths = {item["path"] for item in self.registry["workflows"]} + actual = { + path.relative_to(ROOT).as_posix() + for suffix in ("*.yml", "*.yaml") + for path in (ROOT / ".github" / "workflows").glob(suffix) + } + self.assertEqual(paths, actual) + + def test_new_or_stale_workflow_fails_closed(self) -> None: + mutated = copy.deepcopy(self.registry) + mutated["workflows"].pop() + self.rejected("workflow inventory mismatch", mutated) + + def test_required_check_is_exact_head_ci_gate(self) -> None: + mutated = copy.deepcopy(self.registry) + test_suite = next(item for item in mutated["workflows"] if item["id"] == "test-suite") + test_suite["sha_rule"] = "event_sha" + self.rejected("required PR checks must bind", mutated) + + def test_costly_qualification_cannot_bypass_esc_operator(self) -> None: + mutated = copy.deepcopy(self.registry) + fly = next(item for item in mutated["workflows"] if item["id"] == "fly-tiny-qualification") + fly["command"] = "native-admission" + self.rejected("bypasses the Python operator", mutated) + + def test_matrix_variants_share_one_command_definition(self) -> None: + mutated = copy.deepcopy(self.registry) + concurrency = next( + item for item in mutated["workflows"] if item["id"] == "concurrency-stress" + ) + concurrency["command"] = "repository-policy" + self.rejected("must share matrix-gate", mutated) + + def test_publication_verification_has_one_owner(self) -> None: + mutated = copy.deepcopy(self.registry) + clean = next(item for item in mutated["workflows"] if item["id"] == "clean-environment") + clean["owner"] = "ci" + self.rejected("one release owner", mutated) + + def test_command_rendering_uses_registry_argv(self) -> None: + self.assertEqual( + GATE.command_argv(self.registry, "progressive-ladder"), + [ + "python3", + "-m", + "graphforge_bench.qualification_operator", + "run", + "--gate", + "progressive-ladder", + ], + ) + + def test_matrix_dispatch_uses_current_python_and_shared_definition(self) -> None: + completed = subprocess.CompletedProcess((), 0) + with patch.object(GATE.subprocess, "run", return_value=completed) as run: + self.assertEqual( + GATE.main( + [ + "matrix", + "--family", + "concurrency", + "--variant", + "stress", + "--", + "--output", + "/tmp/evidence", + ] + ), + 0, + ) + run.assert_called_once_with( + [ + sys.executable, + "scripts/ci/concurrency-stress-gate.py", + "run", + "--output", + "/tmp/evidence", + ], + cwd=ROOT, + check=False, + ) + + +if __name__ == "__main__": + unittest.main() From bae96d2894155e29d17e396c1620b8f2b7363775 Mon Sep 17 00:00:00 2001 From: David Spencer <1526975+DecisionNerd@users.noreply.github.com> Date: Sun, 30 Aug 2026 16:17:33 -0600 Subject: [PATCH 2/6] fix: harden qualification operator invocation --- .../qualification_operator.py | 35 ++++--------------- .../tests/test_qualification_operator.py | 33 ++--------------- scripts/ci/clean-env-verify.py | 2 +- scripts/ci/gate-registry.py | 24 +++++++++++-- scripts/ci/test-gate-registry.py | 29 ++++++++++++++- 5 files changed, 59 insertions(+), 64 deletions(-) diff --git a/benchmarks/harness/graphforge_bench/qualification_operator.py b/benchmarks/harness/graphforge_bench/qualification_operator.py index dc50941d2..606d165ae 100644 --- a/benchmarks/harness/graphforge_bench/qualification_operator.py +++ b/benchmarks/harness/graphforge_bench/qualification_operator.py @@ -1,24 +1,20 @@ """Pulumi ESC control plane for provider-backed GraphForge qualifications. -The public ``run`` command is the only provider execution entry point. It opens -an ESC environment and then invokes the private ``execute`` command with secret -filtering owned by Pulumi. The progressive ladder remains fail-closed until a -dedicated scale executor exists; its no-spend plan is available through -``plan-progressive``. +The canonical ``run`` command opens an ESC environment and invokes the existing +controller inside Pulumi's secret-filtered process. It does not claim that a +caller-controlled marker can prove ESC ancestry. The progressive ladder remains +fail-closed until a dedicated scale executor exists; its no-spend plan is +available through ``plan-progressive``. """ from __future__ import annotations import argparse from collections.abc import Callable, Sequence -import os import re import subprocess import sys -from graphforge_bench import fly_tiny_qualification - -ESC_CONTEXT = "pulumi-esc-v1" ESC_ENVIRONMENT = re.compile( r"^[A-Za-z0-9][A-Za-z0-9_.-]*(?:/[A-Za-z0-9][A-Za-z0-9_.-]*){0,2}(?:@[A-Za-z0-9_.-]+)?$" ) @@ -78,15 +74,9 @@ def esc_command(environment: str, gate: str, argv: Sequence[str]) -> tuple[str, "run", environment, "--", - "env", - f"GRAPHFORGE_OPERATOR_CONTEXT={ESC_CONTEXT}", sys.executable, "-m", - "graphforge_bench.qualification_operator", - "execute", - "--gate", - gate, - "--", + "graphforge_bench.fly_tiny_qualification", *forwarded, ) @@ -103,31 +93,18 @@ def run_under_esc( return completed.returncode -def execute_inner(gate: str, argv: Sequence[str]) -> int: - """Dispatch only after ``pulumi env run`` established the process context.""" - if os.environ.get("GRAPHFORGE_OPERATOR_CONTEXT") != ESC_CONTEXT: - raise OperatorRefusalError("provider execution must enter through pulumi env run") - forwarded = _forwarded(argv) - validate_live_request(gate, forwarded) - return fly_tiny_qualification.main(forwarded) - - def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) sub = parser.add_subparsers(dest="action", required=True) run = sub.add_parser("run", help="Execute a live qualification under Pulumi ESC") run.add_argument("--environment", required=True) run.add_argument("--gate", choices=ALL_GATES, required=True) - execute = sub.add_parser("execute", help=argparse.SUPPRESS) - execute.add_argument("--gate", choices=ALL_GATES, required=True) plan = sub.add_parser("plan-progressive", help="Write a no-spend next-rung plan") plan.add_argument("arguments", nargs=argparse.REMAINDER) args, remainder = parser.parse_known_args(argv) try: if args.action == "run": return run_under_esc(args.environment, args.gate, remainder) - if args.action == "execute": - return execute_inner(args.gate, remainder) from graphforge_bench import progressive_provider_plan return progressive_provider_plan.main(_forwarded(args.arguments)) diff --git a/benchmarks/tests/test_qualification_operator.py b/benchmarks/tests/test_qualification_operator.py index 9705148dc..96a841c99 100644 --- a/benchmarks/tests/test_qualification_operator.py +++ b/benchmarks/tests/test_qualification_operator.py @@ -1,16 +1,12 @@ from __future__ import annotations -import os import subprocess import sys import unittest -from unittest.mock import patch from graphforge_bench.qualification_operator import ( - ESC_CONTEXT, OperatorRefusalError, esc_command, - execute_inner, run_under_esc, ) @@ -29,15 +25,8 @@ def test_esc_command_is_shell_free_and_forwards_exact_arguments(self) -> None: command[:5], ("pulumi", "env", "run", "curatelabs/graphforge/qualification", "--") ) self.assertEqual( - command[5:11], - ( - "env", - f"GRAPHFORGE_OPERATOR_CONTEXT={ESC_CONTEXT}", - sys.executable, - "-m", - "graphforge_bench.qualification_operator", - "execute", - ), + command[5:8], + (sys.executable, "-m", "graphforge_bench.fly_tiny_qualification"), ) self.assertEqual(list(command[-len(ARGS) :]), ARGS) @@ -79,24 +68,6 @@ def runner(*args: object, **_kwargs: object) -> subprocess.CompletedProcess[str] ) self.assertFalse(runner_called) - def test_inner_execution_requires_esc_context(self) -> None: - with ( - patch.dict(os.environ, {}, clear=True), - self.assertRaisesRegex(OperatorRefusalError, "pulumi env run"), - ): - execute_inner("fly-tiny", ARGS) - - def test_inner_execution_delegates_to_existing_controller(self) -> None: - with ( - patch.dict(os.environ, {"GRAPHFORGE_OPERATOR_CONTEXT": ESC_CONTEXT}, clear=True), - patch( - "graphforge_bench.qualification_operator.fly_tiny_qualification.main", - return_value=7, - ) as inner, - ): - self.assertEqual(execute_inner("fly-tiny", ARGS), 7) - inner.assert_called_once_with(ARGS) - def test_outer_runner_receives_argv_without_secret_values(self) -> None: observed: tuple[str, ...] | None = None diff --git a/scripts/ci/clean-env-verify.py b/scripts/ci/clean-env-verify.py index 76c17010e..38cd257d9 100644 --- a/scripts/ci/clean-env-verify.py +++ b/scripts/ci/clean-env-verify.py @@ -881,7 +881,7 @@ def cmd_run(args: argparse.Namespace) -> int: else list(args.lane or []) ) if not lanes: - raise VerifyError("specify --all or one or more --lane values") + raise VerifyError("specify --all, --default, or one or more --lane values") unknown = [lane for lane in lanes if lane not in LANE_RUNNERS] if unknown: raise VerifyError(f"unknown lanes: {', '.join(unknown)}") diff --git a/scripts/ci/gate-registry.py b/scripts/ci/gate-registry.py index 4ed605077..dc4f9f559 100644 --- a/scripts/ci/gate-registry.py +++ b/scripts/ci/gate-registry.py @@ -5,6 +5,7 @@ import argparse import json +import os from pathlib import Path import shlex import subprocess @@ -157,7 +158,21 @@ def command_argv(value: dict[str, Any], gate_id: str) -> list[str]: record = next(item for item in records if item["id"] == gate_id) except StopIteration as error: raise RegistryError(f"unknown gate: {gate_id}") from error - return [*value["commands"][record["command"]], *record["args"]] + rendered = [*value["commands"][record["command"]], *record["args"]] + if rendered[:3] == ["python3", "-m", "graphforge_bench.qualification_operator"]: + rendered[0] = sys.executable + return rendered + + +def command_environment(argv: list[str]) -> dict[str, str] | None: + """Make benchmark modules importable for registry-owned operator commands.""" + if argv[1:3] != ["-m", "graphforge_bench.qualification_operator"]: + return None + environment = os.environ.copy() + harness = str(ROOT / "benchmarks" / "harness") + inherited = environment.get("PYTHONPATH") + environment["PYTHONPATH"] = harness if not inherited else harness + os.pathsep + inherited + return environment def main(argv: list[str] | None = None) -> int: @@ -198,7 +213,12 @@ def main(argv: list[str] | None = None) -> int: return 0 if passthrough and passthrough[0] == "--": passthrough = passthrough[1:] - return subprocess.run([*rendered, *passthrough], cwd=ROOT, check=False).returncode + return subprocess.run( + [*rendered, *passthrough], + cwd=ROOT, + check=False, + env=command_environment(rendered), + ).returncode except RegistryError as error: print(f"gate registry invalid: {error}", file=sys.stderr) return 2 diff --git a/scripts/ci/test-gate-registry.py b/scripts/ci/test-gate-registry.py index cfcc56818..d3c28e69e 100644 --- a/scripts/ci/test-gate-registry.py +++ b/scripts/ci/test-gate-registry.py @@ -72,7 +72,7 @@ def test_command_rendering_uses_registry_argv(self) -> None: self.assertEqual( GATE.command_argv(self.registry, "progressive-ladder"), [ - "python3", + sys.executable, "-m", "graphforge_bench.qualification_operator", "run", @@ -111,6 +111,33 @@ def test_matrix_dispatch_uses_current_python_and_shared_definition(self) -> None check=False, ) + def test_operator_run_uses_invoking_python_and_explicit_harness_path(self) -> None: + completed = subprocess.CompletedProcess((), 0) + with patch.object(GATE.subprocess, "run", return_value=completed) as run: + self.assertEqual( + GATE.main( + [ + "run", + "fly-tiny-qualification", + "--", + "--environment", + "curatelabs/graphforge/qualification", + "--expected-sha", + "a" * 40, + "--execute", + "--confirm-disposable", + ] + ), + 0, + ) + argv = run.call_args.args[0] + environment = run.call_args.kwargs["env"] + self.assertEqual(argv[0], sys.executable) + self.assertEqual( + environment["PYTHONPATH"].split(GATE.os.pathsep)[0], + str(ROOT / "benchmarks" / "harness"), + ) + if __name__ == "__main__": unittest.main() From 947a854e34f7c886807e4a4a058d002119dc190e Mon Sep 17 00:00:00 2001 From: David Spencer <1526975+DecisionNerd@users.noreply.github.com> Date: Sun, 30 Aug 2026 16:29:49 -0600 Subject: [PATCH 3/6] fix: close gate operator bypasses --- .github/workflows/README.md | 36 +-- .github/workflows/clean-env-verify.yml | 3 - .github/workflows/concurrency-stress-gate.yml | 5 - .../durability-certification-gate.yml | 5 - .github/workflows/fly-tiny-qualification.yml | 281 +----------------- .github/workflows/fly-tiny-recovery.yml | 138 +-------- .github/workflows/g500-certification.yml | 127 +------- .../fly_tiny_qualification.py | 2 +- .../qualification_operator.py | 57 +++- .../tests/test_qualification_operator.py | 89 +++++- config/gate-registry.json | 4 + scripts/ci/gate-registry.py | 73 ++++- scripts/ci/test-gate-registry.py | 42 +++ 13 files changed, 289 insertions(+), 573 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 1e33a10fd..48e7c8bdc 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -153,27 +153,17 @@ this project. ### `g500-certification.yml` — Billion-edge certification -This protected, manual-only workflow checks out an exact commit on a -maintainer-approved ephemeral Linux scale host. It never provisions or spends. -It runs the bounded target-live SCALE-26 lifecycle, writes an atomic typed phase -journal, validates sanitized evidence, and uploads only the evidence JSON and -journal. Provider, SKU, runner registration, cost approval, and teardown remain -explicit maintainer decisions; see the +This manual compatibility wrapper validates and renders the registry-owned +operator command. It never provisions or spends. Provider, SKU, runner +registration, cost approval, and teardown remain explicit operator decisions; +the actual run must use the Python + Pulumi ESC control plane described in the [certification runbook](../../docs/development/g500-certification.md). ### `fly-tiny-qualification.yml` — Disposable Fly environment smoke -This protected manual workflow checks out the exact current `main` commit and -runs the non-ladder #958 qualification on a Blacksmith Linux runner. It builds -the commit-pinned image with hosted Docker, pushes it to the disposable app's -Fly registry path, resolves the immutable digest, runs only the 10 GiB -`performance-1x` filesystem smoke, and independently verifies teardown. A -separate recovery job has its own timeout budget after execution, and -`fly-tiny-recovery.yml` provides receipt-bound manual orphan cleanup. The -workflow uploads a one-day pre-creation ownership receipt plus sanitized -plan/result documents and qualified evidence. The receipt binds the commit to -a fresh 128-bit app-name nonce and never contains provider resource identifiers -or credentials. +This manual compatibility wrapper validates and renders the registry-owned +operator command without credentials or provider execution. Live qualification +is available only through the Python + Pulumi ESC control plane below. For new operator runs, use the registry-owned entry point instead of Actions dispatch. All controller flags are passed after `ARGS=`; the Python operator @@ -182,19 +172,17 @@ before it opens the named ESC environment: ```bash make -C benchmarks qualification-operator \ - GATE=fly-tiny \ + GATE=fly-tiny-qualification \ ESC_ENVIRONMENT=curatelabs/graphforge/qualification \ ARGS='--expected-sha --execute --confirm-disposable ' ``` ### `fly-tiny-recovery.yml` — Manual Fly orphan recovery -This protected manual janitor downloads the one-day ownership receipt from a -specified qualification run and refuses deletion unless its exact nonce-bound -app and commit match. It runs the trusted controller from current `main` and -shares the qualification concurrency lock, so it cannot race the source run. -It exists for runner loss, workflow cancellation, or a failed independent -recovery job; it cannot infer ownership from an app prefix. +This manual compatibility wrapper only validates and renders the registry-owned +recovery command. Receipt-bound orphan cleanup must be launched through the +Python + Pulumi ESC operator; Actions has no provider credential or deletion +authority. ### CodeRabbit diff --git a/.github/workflows/clean-env-verify.yml b/.github/workflows/clean-env-verify.yml index 267980af1..d901689b4 100644 --- a/.github/workflows/clean-env-verify.yml +++ b/.github/workflows/clean-env-verify.yml @@ -47,9 +47,6 @@ jobs: with: toolchain: "1.96.0" - - name: Harness unit tests - run: python3 scripts/ci/test-clean-env-verify.py - - name: Preflight public registries id: preflight run: | diff --git a/.github/workflows/concurrency-stress-gate.yml b/.github/workflows/concurrency-stress-gate.yml index 606648e1e..8f8e59381 100644 --- a/.github/workflows/concurrency-stress-gate.yml +++ b/.github/workflows/concurrency-stress-gate.yml @@ -43,11 +43,6 @@ jobs: with: python-version: "3.13" - - name: Validate stress configuration - run: | - python3 scripts/ci/concurrency-stress-gate.py validate - python3 scripts/ci/test-concurrency-stress-gate.py - - name: Build native Python wheel for mixed workload run: | uvx maturin build \ diff --git a/.github/workflows/durability-certification-gate.yml b/.github/workflows/durability-certification-gate.yml index 9062a0aef..5930a3916 100644 --- a/.github/workflows/durability-certification-gate.yml +++ b/.github/workflows/durability-certification-gate.yml @@ -43,11 +43,6 @@ jobs: with: toolchain: "1.96.0" - - name: Validate certification configuration - run: | - python3 scripts/ci/durability-certification-gate.py validate - python3 scripts/ci/test-durability-certification-gate.py - - name: Run scheduled seeded certification lane run: >- python3 scripts/ci/gate-registry.py run durability-certification -- diff --git a/.github/workflows/fly-tiny-qualification.yml b/.github/workflows/fly-tiny-qualification.yml index ae6c6b22e..6b2478409 100644 --- a/.github/workflows/fly-tiny-qualification.yml +++ b/.github/workflows/fly-tiny-qualification.yml @@ -1,283 +1,20 @@ -name: Fly tiny qualification +name: Fly Tiny Qualification (operator handoff) on: workflow_dispatch: - inputs: - commit_sha: - description: Exact current main commit to qualify - required: true - type: string - organization: - description: Fly organization slug - required: true - default: personal - type: string - region: - description: Fixed admitted Fly region - required: true - default: dfw - type: string - confirm_disposable: - description: Authorize one tiny app, 10 GiB volume, and performance-1x Machine - required: true - default: false - type: boolean permissions: contents: read -concurrency: - group: fly-tiny-qualification - cancel-in-progress: false - -env: - COMMIT_SHA: ${{ inputs.commit_sha }} - FLY_ORGANIZATION: ${{ inputs.organization }} - FLY_REGION: ${{ inputs.region }} - PREFLIGHT_ARTIFACT: fly-tiny-preflight-${{ github.run_id }}-${{ github.run_attempt }} - jobs: - prepare: - name: Admit and persist cleanup ownership - outputs: - app_name: ${{ steps.identity.outputs.app_name }} - machine_name: ${{ steps.identity.outputs.machine_name }} - volume_name: ${{ steps.identity.outputs.volume_name }} + handoff: + name: Render Python + Pulumi ESC operator command runs-on: blacksmith-4vcpu-ubuntu-2404 - environment: scale-certification - timeout-minutes: 20 + timeout-minutes: 5 steps: - - name: Require explicit authorization and credential - env: - CONFIRMED: ${{ inputs.confirm_disposable }} - FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} - run: | - test "$CONFIRMED" = true - test -n "$FLY_API_TOKEN" - case "$COMMIT_SHA" in - (????????????????????????????????????????) ;; - (*) exit 2 ;; - esac - case "$COMMIT_SHA" in (*[!0-9a-f]*) exit 2 ;; esac - - - name: Checkout exact commit - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ inputs.commit_sha }} - fetch-depth: 1 - - - name: Require exact current main + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Validate registry and render operator handoff run: | - test "$(git rev-parse HEAD)" = "$COMMIT_SHA" - git fetch --no-tags --depth=1 origin main - test "$(git rev-parse FETCH_HEAD)" = "$COMMIT_SHA" - - - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - with: - version: "0.10.9" - enable-cache: false - - - name: Install flyctl - uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1 # v1.6 - - - name: Install locked benchmark environment - working-directory: benchmarks - run: uv sync --locked - - - name: Create collision-resistant disposable identity - id: identity - run: | - python - "$GITHUB_OUTPUT" <<'PY' - from pathlib import Path - import secrets - import sys - - nonce = secrets.token_hex(16) - with Path(sys.argv[1]).open("a", encoding="utf-8") as output: - output.write(f"app_name=gf-q958-{nonce}\n") - output.write(f"volume_name=gfq958_{nonce[:20]}\n") - output.write(f"machine_name=gf-q958-machine-{nonce[:16]}\n") - PY - - - name: Run no-spend admission - env: - APP_NAME: ${{ steps.identity.outputs.app_name }} - FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} - MACHINE_NAME: ${{ steps.identity.outputs.machine_name }} - VOLUME_NAME: ${{ steps.identity.outputs.volume_name }} - run: >- - PYTHONPATH=benchmarks/harness uv run --project benchmarks python -m - graphforge_bench.fly_tiny_qualification - --expected-sha "$COMMIT_SHA" - --org "$FLY_ORGANIZATION" - --app "$APP_NAME" - --region "$FLY_REGION" - --volume-name "$VOLUME_NAME" - --machine-name "$MACHINE_NAME" - --prerequisite-955 merged - --prerequisite-956 merged - --prerequisite-957 merged - --build-authority hosted-docker - --ledger "$RUNNER_TEMP/fly-q958-ledger.json" - --evidence-out "$RUNNER_TEMP/fly-q958-evidence.json" - --result-out "$RUNNER_TEMP/fly-q958-plan.json" - - - name: Persist pre-creation ownership receipt - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: ${{ env.PREFLIGHT_ARTIFACT }} - path: | - ${{ runner.temp }}/fly-q958-ledger.json - ${{ runner.temp }}/fly-q958-plan.json - if-no-files-found: error - retention-days: 1 - - execute: - name: Hosted image and tiny Fly smoke - needs: prepare - runs-on: blacksmith-4vcpu-ubuntu-2404 - environment: scale-certification - timeout-minutes: 55 - env: - APP_NAME: ${{ needs.prepare.outputs.app_name }} - MACHINE_NAME: ${{ needs.prepare.outputs.machine_name }} - VOLUME_NAME: ${{ needs.prepare.outputs.volume_name }} - steps: - - name: Checkout exact commit - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ inputs.commit_sha }} - fetch-depth: 1 - - - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - with: - version: "0.10.9" - enable-cache: false - - - name: Install flyctl - uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1 # v1.6 - - - name: Install locked benchmark environment - working-directory: benchmarks - run: uv sync --locked - - - name: Restore pre-creation ownership receipt - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: ${{ env.PREFLIGHT_ARTIFACT }} - path: ${{ runner.temp }} - - - name: Execute one tiny qualification - timeout-minutes: 50 - env: - FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} - DOCKER_CONFIG: ${{ runner.temp }}/fly-q958-docker - run: | - git fetch --no-tags --depth=1 origin main - test "$(git rev-parse HEAD)" = "$COMMIT_SHA" - test "$(git rev-parse FETCH_HEAD)" = "$COMMIT_SHA" - mkdir -p "$DOCKER_CONFIG" - PYTHONPATH=benchmarks/harness uv run --project benchmarks python -m \ - graphforge_bench.fly_tiny_qualification \ - --expected-sha "$COMMIT_SHA" \ - --org "$FLY_ORGANIZATION" \ - --app "$APP_NAME" \ - --region "$FLY_REGION" \ - --volume-name "$VOLUME_NAME" \ - --machine-name "$MACHINE_NAME" \ - --prerequisite-955 merged \ - --prerequisite-956 merged \ - --prerequisite-957 merged \ - --build-authority hosted-docker \ - --ledger "$RUNNER_TEMP/fly-q958-ledger.json" \ - --evidence-out "$RUNNER_TEMP/fly-q958-evidence.json" \ - --result-out "$RUNNER_TEMP/fly-q958-result.json" \ - --execute \ - --confirm-disposable - - - name: Remove temporary Docker credentials - if: always() - env: - DOCKER_CONFIG: ${{ runner.temp }}/fly-q958-docker - run: rm -rf -- "$DOCKER_CONFIG" - - - name: Upload sanitized execution result - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: fly-tiny-execution-${{ inputs.commit_sha }} - path: | - ${{ runner.temp }}/fly-q958-result.json - ${{ runner.temp }}/fly-q958-evidence.json - if-no-files-found: error - retention-days: 1 - - recover: - name: Independent orphan recovery - if: ${{ always() && needs.prepare.result == 'success' }} - needs: [prepare, execute] - runs-on: blacksmith-4vcpu-ubuntu-2404 - environment: scale-certification - timeout-minutes: 30 - env: - APP_NAME: ${{ needs.prepare.outputs.app_name }} - MACHINE_NAME: ${{ needs.prepare.outputs.machine_name }} - VOLUME_NAME: ${{ needs.prepare.outputs.volume_name }} - steps: - - name: Checkout exact commit - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ inputs.commit_sha }} - fetch-depth: 1 - - - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - with: - version: "0.10.9" - enable-cache: false - - - name: Install flyctl - uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1 # v1.6 - - - name: Install locked benchmark environment - working-directory: benchmarks - run: uv sync --locked - - - name: Restore pre-creation ownership receipt - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: ${{ env.PREFLIGHT_ARTIFACT }} - path: ${{ runner.temp }} - - - name: Recover and verify teardown - env: - FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} - run: >- - PYTHONPATH=benchmarks/harness uv run --project benchmarks python -m - graphforge_bench.fly_tiny_qualification - --expected-sha "$COMMIT_SHA" - --org "$FLY_ORGANIZATION" - --app "$APP_NAME" - --region "$FLY_REGION" - --volume-name "$VOLUME_NAME" - --machine-name "$MACHINE_NAME" - --prerequisite-955 merged - --prerequisite-956 merged - --prerequisite-957 merged - --build-authority hosted-docker - --ledger "$RUNNER_TEMP/fly-q958-ledger.json" - --evidence-out "$RUNNER_TEMP/fly-q958-evidence.json" - --result-out "$RUNNER_TEMP/fly-q958-cleanup-result.json" - --cleanup-only - --confirm-disposable - - - name: Upload sanitized cleanup result - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: fly-tiny-cleanup-${{ inputs.commit_sha }} - path: ${{ runner.temp }}/fly-q958-cleanup-result.json - if-no-files-found: error - retention-days: 1 + python3 scripts/ci/gate-registry.py validate + python3 scripts/ci/gate-registry.py command fly-tiny-qualification + echo "No provider operation occurs in GitHub Actions; execute through Pulumi ESC." diff --git a/.github/workflows/fly-tiny-recovery.yml b/.github/workflows/fly-tiny-recovery.yml index 8b433591a..fe9b0ac6e 100644 --- a/.github/workflows/fly-tiny-recovery.yml +++ b/.github/workflows/fly-tiny-recovery.yml @@ -1,140 +1,20 @@ -name: Fly tiny orphan recovery +name: Fly Tiny Recovery (operator handoff) on: workflow_dispatch: - inputs: - source_run_id: - description: Qualification workflow run ID that owns the app - required: true - type: string - source_run_attempt: - description: Qualification workflow run attempt - required: true - default: "1" - type: string - organization: - description: Fly organization slug used by the qualification run - required: true - default: personal - type: string - region: - description: Fly region used by the qualification run - required: true - default: dfw - type: string - confirm_recovery: - description: Authorize cleanup of the receipt-bound disposable app - required: true - default: false - type: boolean permissions: - actions: read contents: read -concurrency: - group: fly-tiny-qualification - cancel-in-progress: false - jobs: - recover: - name: Receipt-bound manual recovery + handoff: + name: Render Python + Pulumi ESC recovery command runs-on: blacksmith-4vcpu-ubuntu-2404 - environment: scale-certification - timeout-minutes: 30 - env: - FLY_ORGANIZATION: ${{ inputs.organization }} - FLY_REGION: ${{ inputs.region }} - PREFLIGHT_ARTIFACT: fly-tiny-preflight-${{ inputs.source_run_id }}-${{ inputs.source_run_attempt }} + timeout-minutes: 5 steps: - - name: Require explicit recovery authorization - env: - CONFIRMED: ${{ inputs.confirm_recovery }} - FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} - SOURCE_RUN_ID: ${{ inputs.source_run_id }} - SOURCE_RUN_ATTEMPT: ${{ inputs.source_run_attempt }} - run: | - test "$CONFIRMED" = true - test -n "$FLY_API_TOKEN" - case "$SOURCE_RUN_ID:$SOURCE_RUN_ATTEMPT" in (*[!0-9:]*) exit 2 ;; esac - - - name: Checkout trusted recovery controller - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: main - fetch-depth: 1 - - - name: Install uv - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - with: - version: "0.10.9" - enable-cache: false - - - name: Install flyctl - uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1 # v1.6 - - - name: Install locked benchmark environment - working-directory: benchmarks - run: uv sync --locked - - - name: Restore durable ownership receipt - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: ${{ env.PREFLIGHT_ARTIFACT }} - path: ${{ runner.temp }} - github-token: ${{ github.token }} - repository: ${{ github.repository }} - run-id: ${{ inputs.source_run_id }} - - - name: Read validated receipt identity - id: identity + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Validate registry and render operator handoff run: | - PYTHONPATH=benchmarks/harness python - \ - "$RUNNER_TEMP/fly-q958-ledger.json" "$GITHUB_OUTPUT" <<'PY' - from pathlib import Path - import sys - - from graphforge_bench.fly_adapter import ResourceLedger - - ledger = ResourceLedger.load(Path(sys.argv[1])) - if ledger.owner_app is None or ledger.owner_commit is None: - raise SystemExit("ownership receipt is already consumed") - with Path(sys.argv[2]).open("a", encoding="utf-8") as output: - output.write(f"app_name={ledger.owner_app}\n") - output.write(f"commit_sha={ledger.owner_commit}\n") - PY - - - name: Recover and verify teardown - env: - APP_NAME: ${{ steps.identity.outputs.app_name }} - COMMIT_SHA: ${{ steps.identity.outputs.commit_sha }} - FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} - MACHINE_NAME: gf-q958-recovery - VOLUME_NAME: gfq958_recovery - run: >- - PYTHONPATH=benchmarks/harness uv run --project benchmarks python -m - graphforge_bench.fly_tiny_qualification - --expected-sha "$COMMIT_SHA" - --org "$FLY_ORGANIZATION" - --app "$APP_NAME" - --region "$FLY_REGION" - --volume-name "$VOLUME_NAME" - --machine-name "$MACHINE_NAME" - --prerequisite-955 merged - --prerequisite-956 merged - --prerequisite-957 merged - --build-authority hosted-docker - --ledger "$RUNNER_TEMP/fly-q958-ledger.json" - --evidence-out "$RUNNER_TEMP/fly-q958-evidence.json" - --result-out "$RUNNER_TEMP/fly-q958-manual-cleanup-result.json" - --cleanup-only - --confirm-disposable - - - name: Upload sanitized recovery result - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: fly-tiny-manual-cleanup-${{ inputs.source_run_id }} - path: ${{ runner.temp }}/fly-q958-manual-cleanup-result.json - if-no-files-found: error - retention-days: 1 + python3 scripts/ci/gate-registry.py validate + python3 scripts/ci/gate-registry.py command fly-tiny-recovery + echo "No provider operation occurs in GitHub Actions; execute through Pulumi ESC." diff --git a/.github/workflows/g500-certification.yml b/.github/workflows/g500-certification.yml index 258d95f65..453ed028c 100644 --- a/.github/workflows/g500-certification.yml +++ b/.github/workflows/g500-certification.yml @@ -1,127 +1,20 @@ -name: Billion-edge certification +name: G500 Certification (operator handoff) on: workflow_dispatch: - inputs: - commit_sha: - description: Exact 40-character commit to certify - required: true - type: string - runner_label: - description: Maintainer-approved ephemeral Linux runner label - required: true - type: string - provider: - description: Provisioned provider recorded in evidence - required: true - type: string - region: - description: Exact provisioned provider region recorded in evidence - required: true - type: string - sku: - description: Provisioned SKU recorded in evidence - required: true - type: string - os_image: - description: Immutable resolved OS image identity recorded in evidence - required: true - type: string permissions: contents: read -concurrency: - group: g500-certification - cancel-in-progress: false - jobs: - certify: - name: SCALE26 target-live certification - environment: scale-certification - runs-on: ${{ inputs.runner_label }} - # The Rust watchdog owns the 240-minute product fail-safe. Keep 30 minutes - # of job headroom so failure evidence can still be validated and uploaded. - timeout-minutes: 270 + handoff: + name: Render Python + Pulumi ESC operator command + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 5 steps: - - name: Reject branch names and malformed SHAs - env: - COMMIT_SHA: ${{ inputs.commit_sha }} - PROVIDER: ${{ inputs.provider }} - REGION: ${{ inputs.region }} - SKU: ${{ inputs.sku }} - OS_IMAGE: ${{ inputs.os_image }} - run: | - case "$COMMIT_SHA" in - (????????????????????????????????????????) ;; - (*) echo "commit_sha must contain exactly 40 characters" >&2; exit 2 ;; - esac - case "$COMMIT_SHA" in (*[!0-9a-f]*) echo "commit_sha must be lowercase hex" >&2; exit 2 ;; esac - case "$(printf '%s' "$PROVIDER" | tr '[:upper:]' '[:lower:]')" in - (""|local|localhost|developer|example|test|unknown) echo "provider must name provisioned cloud infrastructure" >&2; exit 2 ;; - esac - case "$(printf '%s' "$REGION" | tr '[:upper:]' '[:lower:]')" in - (""|local|localhost|default|example|global|test|unknown) echo "region must name the exact provisioned region" >&2; exit 2 ;; - esac - case "$(printf '%s' "$SKU" | tr '[:upper:]' '[:lower:]')" in - (""|local|default|example|generic|test|unknown) echo "sku must name the exact provisioned machine SKU" >&2; exit 2 ;; - esac - case "$(printf '%s' "$OS_IMAGE" | tr '[:upper:]' '[:lower:]')" in - (""|local|default|example|generic|latest|test|unknown|*:latest) echo "os_image must be an immutable resolved image identity" >&2; exit 2 ;; - esac - - name: Checkout exact commit - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ inputs.commit_sha }} - fetch-depth: 1 - - name: Attest exact tree and provisioned envelope - env: - COMMIT_SHA: ${{ inputs.commit_sha }} - run: | - mkdir -p "$RUNNER_TEMP/g500-sut-temp" - test "$(git rev-parse HEAD)" = "$COMMIT_SHA" - test "$(uname -s)" = Linux - test "$(awk '/MemTotal/ {print $2 * 1024}' /proc/meminfo)" -ge 137438953472 - test "$(df --output=size -B1 "$RUNNER_TEMP/g500-sut-temp" | tail -1 | tr -d ' ')" -ge 1099511627776 - case "$(stat -f -c %T "$RUNNER_TEMP/g500-sut-temp")" in ext2/ext3|xfs|btrfs) ;; (*) exit 2 ;; esac - - name: Install pinned Rust toolchain - run: rustup show active-toolchain - - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 - - name: Prepare pinned evidence validator - run: uv run --isolated --no-project --with jsonschema==4.26.0 python -c "import jsonschema" - - name: Run bounded preflight and provisioned certification - env: - CARGO_TARGET_DIR: ${{ runner.temp }}/cargo-g500-certification - # tempfile::TempDir follows TMPDIR. This binds the source project, - # spills, bundle, and clean import to the exact attested volume. - TMPDIR: ${{ runner.temp }}/g500-sut-temp - GF_G500_CERT_EVIDENCE_OUT: ${{ runner.temp }}/g500-certification-evidence.json - GF_G500_CERT_JOURNAL_OUT: ${{ runner.temp }}/g500-certification-phase-journal.json - GF_G500_CERT_PROVIDER: ${{ inputs.provider }} - GF_G500_CERT_REGION: ${{ inputs.region }} - GF_G500_CERT_SKU: ${{ inputs.sku }} - GF_G500_CERT_OS_IMAGE: ${{ inputs.os_image }} - GF_G500_CERT_EXPECTED_SHA: ${{ inputs.commit_sha }} + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Validate registry and render operator handoff run: | - export GF_G500_CERT_STARTED_EPOCH_S="$(date +%s)" - cargo test -p graphforge-api --release --test scale_g500_ladder certification_lifecycle_journals_equivalent_round_trip_and_drills -- --exact --nocapture - cargo test -p graphforge-api --release --test scale_g500_ladder certification_target_live_full_lifecycle_evidence -- --ignored --exact --nocapture --test-threads=1 - - name: Validate sanitized durable evidence - if: always() - env: - COMMIT_SHA: ${{ inputs.commit_sha }} - run: >- - uv run --isolated --no-project --with jsonschema==4.26.0 python - scripts/ci/validate-g500-certification.py - "$RUNNER_TEMP/g500-certification-evidence.json" - --expected-sha "$COMMIT_SHA" - - name: Upload evidence without project payload - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: g500-certification-${{ inputs.commit_sha }} - path: | - ${{ runner.temp }}/g500-certification-evidence.json - ${{ runner.temp }}/g500-certification-phase-journal.json - if-no-files-found: error - retention-days: 14 + python3 scripts/ci/gate-registry.py validate + python3 scripts/ci/gate-registry.py command g500-certification + echo "No provider operation occurs in GitHub Actions; execute through Pulumi ESC." diff --git a/benchmarks/harness/graphforge_bench/fly_tiny_qualification.py b/benchmarks/harness/graphforge_bench/fly_tiny_qualification.py index 30312c197..cf3cbe169 100644 --- a/benchmarks/harness/graphforge_bench/fly_tiny_qualification.py +++ b/benchmarks/harness/graphforge_bench/fly_tiny_qualification.py @@ -1018,7 +1018,7 @@ def cleanup_only( def parser() -> argparse.ArgumentParser: - result = argparse.ArgumentParser(description=__doc__) + result = argparse.ArgumentParser(description=__doc__, allow_abbrev=False) result.add_argument("--expected-sha", required=True) result.add_argument("--org", required=True) result.add_argument("--app", required=True) diff --git a/benchmarks/harness/graphforge_bench/qualification_operator.py b/benchmarks/harness/graphforge_bench/qualification_operator.py index 606d165ae..e29a8f642 100644 --- a/benchmarks/harness/graphforge_bench/qualification_operator.py +++ b/benchmarks/harness/graphforge_bench/qualification_operator.py @@ -11,6 +11,7 @@ import argparse from collections.abc import Callable, Sequence +from pathlib import Path import re import subprocess import sys @@ -21,6 +22,7 @@ COMMIT = re.compile(r"^[0-9a-f]{40}$") LIVE_GATES = {"fly-tiny", "fly-tiny-recovery"} ALL_GATES = (*sorted(LIVE_GATES), "progressive-ladder") +ROOT = Path(__file__).resolve().parents[3] class OperatorRefusalError(ValueError): @@ -81,25 +83,74 @@ def esc_command(environment: str, gate: str, argv: Sequence[str]) -> tuple[str, ) +def attest_current_main( + commit: str, + *, + root: Path = ROOT, + runner: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run, +) -> None: + """Bind provider authority to a clean checkout of current origin/main.""" + try: + head = runner( + ("git", "rev-parse", "HEAD"), + cwd=root, + check=True, + text=True, + capture_output=True, + ).stdout.strip() + status = runner( + ("git", "status", "--porcelain"), + cwd=root, + check=True, + text=True, + capture_output=True, + ).stdout.strip() + runner( + ("git", "fetch", "--no-tags", "--depth=1", "origin", "main"), + cwd=root, + check=True, + ) + fetched = runner( + ("git", "rev-parse", "FETCH_HEAD"), + cwd=root, + check=True, + text=True, + capture_output=True, + ).stdout.strip() + except (OSError, subprocess.CalledProcessError) as error: + raise OperatorRefusalError("unable to attest current origin/main") from error + if head != commit or status or fetched != commit: + raise OperatorRefusalError( + "--expected-sha must identify a clean checkout of current origin/main" + ) + + def run_under_esc( environment: str, gate: str, argv: Sequence[str], *, runner: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run, + attestor: Callable[[str], None] = attest_current_main, ) -> int: command = esc_command(environment, gate, argv) + commit = _single_value(_forwarded(argv), "--expected-sha") + attestor(commit) completed = runner(command, check=False) return completed.returncode def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) + parser = argparse.ArgumentParser(description=__doc__, allow_abbrev=False) sub = parser.add_subparsers(dest="action", required=True) - run = sub.add_parser("run", help="Execute a live qualification under Pulumi ESC") + run = sub.add_parser( + "run", help="Execute a live qualification under Pulumi ESC", allow_abbrev=False + ) run.add_argument("--environment", required=True) run.add_argument("--gate", choices=ALL_GATES, required=True) - plan = sub.add_parser("plan-progressive", help="Write a no-spend next-rung plan") + plan = sub.add_parser( + "plan-progressive", help="Write a no-spend next-rung plan", allow_abbrev=False + ) plan.add_argument("arguments", nargs=argparse.REMAINDER) args, remainder = parser.parse_known_args(argv) try: diff --git a/benchmarks/tests/test_qualification_operator.py b/benchmarks/tests/test_qualification_operator.py index 96a841c99..cd0b966f3 100644 --- a/benchmarks/tests/test_qualification_operator.py +++ b/benchmarks/tests/test_qualification_operator.py @@ -4,8 +4,10 @@ import sys import unittest +from graphforge_bench.fly_tiny_qualification import parser as fly_parser from graphforge_bench.qualification_operator import ( OperatorRefusalError, + attest_current_main, esc_command, run_under_esc, ) @@ -78,12 +80,97 @@ def runner(argv: tuple[str, ...], *, check: bool) -> subprocess.CompletedProcess return subprocess.CompletedProcess(argv, 0) self.assertEqual( - run_under_esc("curatelabs/graphforge/qualification", "fly-tiny", ARGS, runner=runner), + run_under_esc( + "curatelabs/graphforge/qualification", + "fly-tiny", + ARGS, + runner=runner, + attestor=lambda _commit: None, + ), 0, ) assert observed is not None self.assertNotIn("FLY_API_TOKEN", " ".join(observed)) + def test_stale_main_refuses_before_pulumi(self) -> None: + responses = iter(("a" * 40 + "\n", "", "b" * 40 + "\n")) + + def git_runner( + argv: tuple[str, ...], **_kwargs: object + ) -> subprocess.CompletedProcess[str]: + if argv[1] == "fetch": + return subprocess.CompletedProcess(argv, 0, "", "") + return subprocess.CompletedProcess(argv, 0, next(responses), "") + + called = False + + def pulumi_runner(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]: + nonlocal called + called = True + return subprocess.CompletedProcess((), 0) + + with self.assertRaisesRegex(OperatorRefusalError, "current origin/main"): + run_under_esc( + "curatelabs/graphforge/qualification", + "fly-tiny", + ARGS, + runner=pulumi_runner, + attestor=lambda commit: attest_current_main(commit, runner=git_runner), + ) + self.assertFalse(called) + + def test_exact_current_main_allows_pulumi(self) -> None: + responses = iter(("a" * 40 + "\n", "", "a" * 40 + "\n")) + + def git_runner( + argv: tuple[str, ...], **_kwargs: object + ) -> subprocess.CompletedProcess[str]: + if argv[1] == "fetch": + return subprocess.CompletedProcess(argv, 0, "", "") + return subprocess.CompletedProcess(argv, 0, next(responses), "") + + self.assertEqual( + run_under_esc( + "curatelabs/graphforge/qualification", + "fly-tiny", + ARGS, + runner=lambda argv, **_kwargs: subprocess.CompletedProcess(argv, 0), + attestor=lambda commit: attest_current_main(commit, runner=git_runner), + ), + 0, + ) + + def test_inner_controller_rejects_abbreviated_expected_sha(self) -> None: + with self.assertRaises(SystemExit): + fly_parser().parse_args( + [ + "--expected-s", + "b" * 40, + "--org", + "owner", + "--app", + "app", + "--region", + "iad", + "--volume-name", + "data", + "--machine-name", + "machine", + "--prerequisite-955", + "merged", + "--prerequisite-956", + "merged", + "--prerequisite-957", + "merged", + "--ledger", + "ledger.json", + "--evidence-out", + "evidence.json", + "--result-out", + "result.json", + ] + ) + if __name__ == "__main__": unittest.main() diff --git a/config/gate-registry.json b/config/gate-registry.json index 34e3a90cb..3ff3c2e7c 100644 --- a/config/gate-registry.json +++ b/config/gate-registry.json @@ -85,6 +85,10 @@ "examples/visualization/stress/harness/run.py" ] }, + "matrix_variants": { + "concurrency/stress": "scripts/ci/concurrency-stress-gate.py", + "durability/certification": "scripts/ci/durability-certification-gate.py" + }, "workflows": [ { "id": "binding-release-candidate", diff --git a/scripts/ci/gate-registry.py b/scripts/ci/gate-registry.py index dc4f9f559..d542427ef 100644 --- a/scripts/ci/gate-registry.py +++ b/scripts/ci/gate-registry.py @@ -21,6 +21,12 @@ "release_certification", } REQUIRED_FIELDS = {"id", "owner", "command", "args", "evidence_contract", "freshness", "sha_rule"} +FORBIDDEN_OPERATOR_WORKFLOW_MARKERS = ( + "FLY_API_TOKEN", + "graphforge_bench.fly_tiny_qualification", + "--execute", + "--cleanup-only", +) class RegistryError(ValueError): @@ -48,14 +54,27 @@ def _workflow_files(root: Path) -> set[str]: def validate_registry(value: dict[str, Any], root: Path = ROOT) -> None: commands = value.get("commands") + matrix_variants = value.get("matrix_variants") workflows = value.get("workflows") operators = value.get("operator_gates") if ( not isinstance(commands, dict) + or not isinstance(matrix_variants, dict) or not isinstance(workflows, list) or not isinstance(operators, list) ): raise RegistryError("registry collections are malformed") + expected_variants = { + "concurrency/stress": "scripts/ci/concurrency-stress-gate.py", + "durability/certification": "scripts/ci/durability-certification-gate.py", + } + if matrix_variants != expected_variants: + raise RegistryError( + "matrix variants must be the registry-owned concurrency/durability pair" + ) + for script in matrix_variants.values(): + if not (root / script).is_file(): + raise RegistryError(f"matrix variant references a missing script: {script}") for name, argv in commands.items(): if not isinstance(name, str) or not name or not isinstance(argv, list) or not argv: raise RegistryError("command definitions must be non-empty argv arrays") @@ -117,6 +136,19 @@ def validate_registry(value: dict[str, Any], root: Path = ROOT) -> None: if not isinstance(path, str) or path in paths: raise RegistryError(f"{gate_id}: workflow path is duplicate or invalid") paths.add(path) + workflow = (root / path).read_text(encoding="utf-8") + if record.get("control_plane") == "pulumi_esc": + marker = next( + (item for item in FORBIDDEN_OPERATOR_WORKFLOW_MARKERS if item in workflow), + None, + ) + if marker is not None: + raise RegistryError(f"{gate_id}: workflow bypasses ESC operator via {marker}") + if f"gate-registry.py command {gate_id}" not in workflow: + raise RegistryError(f"{gate_id}: workflow must render its registry command") + if gate_id in {"concurrency-stress", "durability-certification"}: + if f"gate-registry.py run {gate_id}" not in workflow: + raise RegistryError(f"{gate_id}: workflow bypasses its registry command") actual = _workflow_files(root) if paths != actual: @@ -175,16 +207,29 @@ def command_environment(argv: list[str]) -> dict[str, str] | None: return environment +def _strip_separator(argv: list[str]) -> list[str]: + return argv[1:] if argv[:1] == ["--"] else argv + + +def reject_owned_options(passthrough: list[str], owned: list[str]) -> None: + """Prevent caller arguments from overriding registry-selected options.""" + reserved = {value for value in owned if value.startswith("--")} + for value in passthrough: + option = value.split("=", 1)[0] + if option in reserved: + raise RegistryError(f"caller may not override registry-owned option {option}") + + def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) + parser = argparse.ArgumentParser(description=__doc__, allow_abbrev=False) sub = parser.add_subparsers(dest="action", required=True) - sub.add_parser("validate") - command = sub.add_parser("command") + sub.add_parser("validate", allow_abbrev=False) + command = sub.add_parser("command", allow_abbrev=False) command.add_argument("gate") command.add_argument("--json", action="store_true") - run = sub.add_parser("run") + run = sub.add_parser("run", allow_abbrev=False) run.add_argument("gate") - matrix = sub.add_parser("matrix") + matrix = sub.add_parser("matrix", allow_abbrev=False) matrix.add_argument("--family", choices=("concurrency", "durability"), required=True) matrix.add_argument("--variant", choices=("stress", "certification"), required=True) args, passthrough = parser.parse_known_args(argv) @@ -198,12 +243,9 @@ def main(argv: list[str] | None = None) -> int: expected = {"concurrency": "stress", "durability": "certification"} if expected[args.family] != args.variant: raise RegistryError("matrix family/variant pairing is invalid") - script = { - "concurrency": "scripts/ci/concurrency-stress-gate.py", - "durability": "scripts/ci/durability-certification-gate.py", - }[args.family] - if passthrough and passthrough[0] == "--": - passthrough = passthrough[1:] + script = registry["matrix_variants"][f"{args.family}/{args.variant}"] + passthrough = _strip_separator(passthrough) + reject_owned_options(passthrough, ["--family", "--variant"]) return subprocess.run( [sys.executable, script, "run", *passthrough], cwd=ROOT, check=False ).returncode @@ -211,8 +253,13 @@ def main(argv: list[str] | None = None) -> int: if args.action == "command": print(json.dumps(rendered) if args.json else shlex.join(rendered)) return 0 - if passthrough and passthrough[0] == "--": - passthrough = passthrough[1:] + passthrough = _strip_separator(passthrough) + record = next( + item + for item in registry["workflows"] + registry["operator_gates"] + if item["id"] == args.gate + ) + reject_owned_options(passthrough, record["args"]) return subprocess.run( [*rendered, *passthrough], cwd=ROOT, diff --git a/scripts/ci/test-gate-registry.py b/scripts/ci/test-gate-registry.py index d3c28e69e..70792b59e 100644 --- a/scripts/ci/test-gate-registry.py +++ b/scripts/ci/test-gate-registry.py @@ -62,6 +62,11 @@ def test_matrix_variants_share_one_command_definition(self) -> None: concurrency["command"] = "repository-policy" self.rejected("must share matrix-gate", mutated) + def test_matrix_dispatch_map_is_registry_owned(self) -> None: + mutated = copy.deepcopy(self.registry) + mutated["matrix_variants"]["concurrency/stress"] = "scripts/ci/require-gates.sh" + self.rejected("matrix variants", mutated) + def test_publication_verification_has_one_owner(self) -> None: mutated = copy.deepcopy(self.registry) clean = next(item for item in mutated["workflows"] if item["id"] == "clean-environment") @@ -138,6 +143,43 @@ def test_operator_run_uses_invoking_python_and_explicit_harness_path(self) -> No str(ROOT / "benchmarks" / "harness"), ) + def test_run_rejects_registry_owned_gate_override(self) -> None: + with patch.object(GATE.subprocess, "run") as run: + self.assertEqual( + GATE.main( + [ + "run", + "fly-tiny-qualification", + "--", + "--gate=progressive-ladder", + ] + ), + 2, + ) + run.assert_not_called() + + def test_matrix_rejects_registry_owned_variant_override(self) -> None: + with patch.object(GATE.subprocess, "run") as run: + self.assertEqual( + GATE.main( + [ + "matrix", + "--family", + "concurrency", + "--variant", + "stress", + "--", + "--variant=certification", + ] + ), + 2, + ) + run.assert_not_called() + + def test_documented_fly_gate_id_renders(self) -> None: + rendered = GATE.command_argv(self.registry, "fly-tiny-qualification") + self.assertEqual(rendered[-2:], ["--gate", "fly-tiny"]) + if __name__ == "__main__": unittest.main() From 9e8d3df2fef7cca3b5978e88370679ef9bf384a6 Mon Sep 17 00:00:00 2001 From: David Spencer <1526975+DecisionNerd@users.noreply.github.com> Date: Sun, 30 Aug 2026 16:32:44 -0600 Subject: [PATCH 4/6] fix: preserve receipt-bound recovery --- .github/workflows/non-cypher-surface-gate.yml | 1 - benchmarks/README.md | 2 +- .../qualification_operator.py | 14 ++++++------- .../tests/test_qualification_operator.py | 20 +++++++++++++++++++ 4 files changed, 28 insertions(+), 9 deletions(-) diff --git a/.github/workflows/non-cypher-surface-gate.yml b/.github/workflows/non-cypher-surface-gate.yml index 70ac41980..51a266a96 100644 --- a/.github/workflows/non-cypher-surface-gate.yml +++ b/.github/workflows/non-cypher-surface-gate.yml @@ -33,7 +33,6 @@ jobs: mkdir -p non-cypher-evidence python3 scripts/ci/non-cypher-surface-gate.py \ --report non-cypher-evidence/inventory-report.json - python3 scripts/ci/test-non-cypher-surface-gate.py - name: Execute public Rust surface matrix run: | diff --git a/benchmarks/README.md b/benchmarks/README.md index 6ebdf1dee..aac367877 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -322,7 +322,7 @@ the caller's ambient shell. Live operator commands are rendered from ```bash make -C benchmarks qualification-operator \ - GATE=fly-tiny \ + GATE=fly-tiny-qualification \ ESC_ENVIRONMENT=curatelabs/graphforge/qualification \ ARGS='--expected-sha --execute --confirm-disposable ' ``` diff --git a/benchmarks/harness/graphforge_bench/qualification_operator.py b/benchmarks/harness/graphforge_bench/qualification_operator.py index e29a8f642..2463f704c 100644 --- a/benchmarks/harness/graphforge_bench/qualification_operator.py +++ b/benchmarks/harness/graphforge_bench/qualification_operator.py @@ -84,7 +84,7 @@ def esc_command(environment: str, gate: str, argv: Sequence[str]) -> tuple[str, def attest_current_main( - commit: str, + expected_commit: str | None, *, root: Path = ROOT, runner: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run, @@ -119,10 +119,10 @@ def attest_current_main( ).stdout.strip() except (OSError, subprocess.CalledProcessError) as error: raise OperatorRefusalError("unable to attest current origin/main") from error - if head != commit or status or fetched != commit: - raise OperatorRefusalError( - "--expected-sha must identify a clean checkout of current origin/main" - ) + if status or head != fetched: + raise OperatorRefusalError("operator must use a clean checkout of current origin/main") + if expected_commit is not None and head != expected_commit: + raise OperatorRefusalError("--expected-sha must identify current origin/main") def run_under_esc( @@ -131,11 +131,11 @@ def run_under_esc( argv: Sequence[str], *, runner: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run, - attestor: Callable[[str], None] = attest_current_main, + attestor: Callable[[str | None], None] = attest_current_main, ) -> int: command = esc_command(environment, gate, argv) commit = _single_value(_forwarded(argv), "--expected-sha") - attestor(commit) + attestor(commit if gate == "fly-tiny" else None) completed = runner(command, check=False) return completed.returncode diff --git a/benchmarks/tests/test_qualification_operator.py b/benchmarks/tests/test_qualification_operator.py index cd0b966f3..ac31c61b3 100644 --- a/benchmarks/tests/test_qualification_operator.py +++ b/benchmarks/tests/test_qualification_operator.py @@ -140,6 +140,26 @@ def git_runner( 0, ) + def test_recovery_attests_current_controller_not_older_receipt_commit(self) -> None: + observed: list[str | None] = [] + recovery_args = [ + "--expected-sha", + "b" * 40, + "--cleanup-only", + "--confirm-disposable", + ] + self.assertEqual( + run_under_esc( + "curatelabs/graphforge/qualification", + "fly-tiny-recovery", + recovery_args, + runner=lambda argv, **_kwargs: subprocess.CompletedProcess(argv, 0), + attestor=observed.append, + ), + 0, + ) + self.assertEqual(observed, [None]) + def test_inner_controller_rejects_abbreviated_expected_sha(self) -> None: with self.assertRaises(SystemExit): fly_parser().parse_args( From 6a7bbef281ff310d9d8e176373d21bf676b61ca5 Mon Sep 17 00:00:00 2001 From: David Spencer <1526975+DecisionNerd@users.noreply.github.com> Date: Sun, 30 Aug 2026 16:36:54 -0600 Subject: [PATCH 5/6] test: retire provider workflow artifact fixtures --- scripts/ci/test-ci-storage-policy.py | 50 ++++++++-------------------- 1 file changed, 13 insertions(+), 37 deletions(-) diff --git a/scripts/ci/test-ci-storage-policy.py b/scripts/ci/test-ci-storage-policy.py index 09986abe7..08d34f052 100644 --- a/scripts/ci/test-ci-storage-policy.py +++ b/scripts/ci/test-ci-storage-policy.py @@ -97,12 +97,7 @@ def uses_approved(uses: str | None, action: str, *tags: str) -> bool: "native-oracle-macos-${{ github.sha }}": 1, "native-durability-aggregate-${{ github.sha }}": 1, "m6-memory-${{ github.sha }}-blacksmith-4vcpu-ubuntu-2404": 1, - "g500-certification-${{ inputs.commit_sha }}": 1, "native-local-admission-${{ matrix.authority }}-${{ github.sha }}": 1, - "${{ env.PREFLIGHT_ARTIFACT }}": 1, - "fly-tiny-execution-${{ inputs.commit_sha }}": 1, - "fly-tiny-cleanup-${{ inputs.commit_sha }}": 1, - "fly-tiny-manual-cleanup-${{ inputs.source_run_id }}": 1, } ) EXPECTED_ARTIFACT_DOWNLOADS = Counter( @@ -127,7 +122,6 @@ def uses_approved(uses: str | None, action: str, *tags: str) -> bool: "pr-node-addon-${{ github.sha }}": 1, "native-oracle-windows-${{ github.sha }}": 1, "native-oracle-macos-${{ github.sha }}": 1, - "${{ env.PREFLIGHT_ARTIFACT }}": 3, } ) EXPECTED_DEPENDENCY_KEYS = Counter( @@ -408,36 +402,18 @@ def artifact_contracts(text: str) -> tuple[list[str], list[str]]: return uploaded, downloaded -def validate_g500_artifact_negative_fixtures() -> None: - text = (WORKFLOWS / "g500-certification.yml").read_text() - artifact_contracts(text) - g500_marker = " name: g500-certification-${{ inputs.commit_sha }}" - g500_start = text.index(g500_marker) - retention = " retention-days: 14" - retention_at = text.index(retention, g500_start) - retention_mutation = ( - text[:retention_at] + " retention-days: 30" + text[retention_at + len(retention) :] - ) - assert retention_mutation[g500_start:].count("retention-days: 30") == 1 - mutations = ( - text.replace( - "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1", - "actions/upload-artifact@65f0bc87b66c8c4f1891f20b5c7a8028c9d7c796 # v4.6.2", - 1, - ), - retention_mutation, - text.replace( - " ${{ runner.temp }}/g500-certification-phase-journal.json", - " ${{ runner.temp }}/project.gfpb", - 1, - ), - ) - for mutation in mutations: - try: - artifact_contracts(mutation) - except AssertionError: - continue - raise AssertionError("G500 artifact policy accepted an unsafe mutation") +def validate_operator_handoffs_have_no_artifacts() -> None: + for workflow, gate in ( + ("g500-certification.yml", "g500-certification"), + ("fly-tiny-qualification.yml", "fly-tiny-qualification"), + ("fly-tiny-recovery.yml", "fly-tiny-recovery"), + ): + text = (WORKFLOWS / workflow).read_text() + uploaded, downloaded = artifact_contracts(text) + assert not uploaded and not downloaded, f"operator handoff transfers artifacts: {workflow}" + assert f"gate-registry.py command {gate}" in text, ( + f"operator handoff does not render its registry command: {workflow}" + ) def cache_contracts(text: str) -> tuple[list[str], list[str]]: @@ -667,7 +643,7 @@ def main() -> None: test_suite = texts[WORKFLOWS / "test.yml"] validate_test_suite_trigger(test_suite) validate_required_run_negative_fixtures() - validate_g500_artifact_negative_fixtures() + validate_operator_handoffs_have_no_artifacts() validate_ci_gate_cutover(test_suite) jobs = workflow_jobs(test_suite) for job_id, runner in ( From 03ab9164dde4303cf14e7e690812632fb0d4086c Mon Sep 17 00:00:00 2001 From: David Spencer <1526975+DecisionNerd@users.noreply.github.com> Date: Sun, 30 Aug 2026 16:56:11 -0600 Subject: [PATCH 6/6] fix: enforce fail-closed workflow handoffs --- .github/workflows/fly-tiny-qualification.yml | 5 +++-- .github/workflows/fly-tiny-recovery.yml | 5 +++-- .github/workflows/g500-certification.yml | 5 +++-- scripts/ci/gate-registry.py | 5 ++++- scripts/ci/test-ci-storage-policy.py | 17 ++++++++++++++--- scripts/ci/test-gate-registry.py | 5 +++++ 6 files changed, 32 insertions(+), 10 deletions(-) diff --git a/.github/workflows/fly-tiny-qualification.yml b/.github/workflows/fly-tiny-qualification.yml index 6b2478409..3d2cdc189 100644 --- a/.github/workflows/fly-tiny-qualification.yml +++ b/.github/workflows/fly-tiny-qualification.yml @@ -13,8 +13,9 @@ jobs: timeout-minutes: 5 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Validate registry and render operator handoff + - name: Validate registry + run: python3 scripts/ci/gate-registry.py validate + - name: Render operator handoff run: | - python3 scripts/ci/gate-registry.py validate python3 scripts/ci/gate-registry.py command fly-tiny-qualification echo "No provider operation occurs in GitHub Actions; execute through Pulumi ESC." diff --git a/.github/workflows/fly-tiny-recovery.yml b/.github/workflows/fly-tiny-recovery.yml index fe9b0ac6e..9d42363f4 100644 --- a/.github/workflows/fly-tiny-recovery.yml +++ b/.github/workflows/fly-tiny-recovery.yml @@ -13,8 +13,9 @@ jobs: timeout-minutes: 5 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Validate registry and render operator handoff + - name: Validate registry + run: python3 scripts/ci/gate-registry.py validate + - name: Render operator handoff run: | - python3 scripts/ci/gate-registry.py validate python3 scripts/ci/gate-registry.py command fly-tiny-recovery echo "No provider operation occurs in GitHub Actions; execute through Pulumi ESC." diff --git a/.github/workflows/g500-certification.yml b/.github/workflows/g500-certification.yml index 453ed028c..8cb0e7bc1 100644 --- a/.github/workflows/g500-certification.yml +++ b/.github/workflows/g500-certification.yml @@ -13,8 +13,9 @@ jobs: timeout-minutes: 5 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Validate registry and render operator handoff + - name: Validate registry + run: python3 scripts/ci/gate-registry.py validate + - name: Render operator handoff run: | - python3 scripts/ci/gate-registry.py validate python3 scripts/ci/gate-registry.py command g500-certification echo "No provider operation occurs in GitHub Actions; execute through Pulumi ESC." diff --git a/scripts/ci/gate-registry.py b/scripts/ci/gate-registry.py index d542427ef..c9a891082 100644 --- a/scripts/ci/gate-registry.py +++ b/scripts/ci/gate-registry.py @@ -136,7 +136,10 @@ def validate_registry(value: dict[str, Any], root: Path = ROOT) -> None: if not isinstance(path, str) or path in paths: raise RegistryError(f"{gate_id}: workflow path is duplicate or invalid") paths.add(path) - workflow = (root / path).read_text(encoding="utf-8") + try: + workflow = (root / path).read_text(encoding="utf-8") + except OSError as error: + raise RegistryError(f"{gate_id}: workflow file is unavailable: {path}") from error if record.get("control_plane") == "pulumi_esc": marker = next( (item for item in FORBIDDEN_OPERATOR_WORKFLOW_MARKERS if item in workflow), diff --git a/scripts/ci/test-ci-storage-policy.py b/scripts/ci/test-ci-storage-policy.py index 08d34f052..f9876b4ed 100644 --- a/scripts/ci/test-ci-storage-policy.py +++ b/scripts/ci/test-ci-storage-policy.py @@ -411,9 +411,20 @@ def validate_operator_handoffs_have_no_artifacts() -> None: text = (WORKFLOWS / workflow).read_text() uploaded, downloaded = artifact_contracts(text) assert not uploaded and not downloaded, f"operator handoff transfers artifacts: {workflow}" - assert f"gate-registry.py command {gate}" in text, ( - f"operator handoff does not render its registry command: {workflow}" - ) + command = f"python3 scripts/ci/gate-registry.py command {gate}" + matches = [ + scalar + for body in workflow_jobs(text).values() + for scalar in job_required_run_scalars(body, command) + ] + assert matches, f"operator handoff does not execute its registry command: {workflow}" + inactive = text.replace(command, f'echo "{command}"', 1) + inactive_matches = [ + scalar + for body in workflow_jobs(inactive).values() + for scalar in job_required_run_scalars(body, command) + ] + assert not inactive_matches, f"inactive operator handoff passed policy: {workflow}" def cache_contracts(text: str) -> tuple[list[str], list[str]]: diff --git a/scripts/ci/test-gate-registry.py b/scripts/ci/test-gate-registry.py index 70792b59e..c6a6917fc 100644 --- a/scripts/ci/test-gate-registry.py +++ b/scripts/ci/test-gate-registry.py @@ -42,6 +42,11 @@ def test_new_or_stale_workflow_fails_closed(self) -> None: mutated["workflows"].pop() self.rejected("workflow inventory mismatch", mutated) + def test_missing_registered_workflow_is_a_registry_error(self) -> None: + mutated = copy.deepcopy(self.registry) + mutated["workflows"][0]["path"] = ".github/workflows/does-not-exist.yml" + self.rejected("workflow file is unavailable", mutated) + def test_required_check_is_exact_head_ci_gate(self) -> None: mutated = copy.deepcopy(self.registry) test_suite = next(item for item in mutated["workflows"] if item["id"] == "test-suite")