diff --git a/changelog.d/8214-ratchet-selftest-full-repin.md b/changelog.d/8214-ratchet-selftest-full-repin.md new file mode 100644 index 0000000000..8a4f07513b --- /dev/null +++ b/changelog.d/8214-ratchet-selftest-full-repin.md @@ -0,0 +1,26 @@ +Fixed the GC ratchet's own test suite demanding a selective re-pin receipt from +every pinned baseline, which made `windows-build` red on every open PR. + +`accepted_deterministic_deltas` is the receipt for a *selective* re-pin — the +dangerous kind, which can turn one red row green while leaving no +machine-readable answer to which rows moved or why. A *full* re-pin carries +artifact-wide provenance instead, and the validator says so explicitly +(`if receipt is None: return`). #8204 moved 130 of 168 cells, so it correctly +shipped no receipt; three tests that hard-subscripted the key on the live +pinned baseline errored with `KeyError`. The gate punished the correct action. + +Those tests had frozen one historical selective re-pin — #8069's exact 21 cells +and causes — into assertions against whatever baseline happens to be current, +which could only stay green by the world never changing. + +The two tamper tests remain, but build their fixture synthetically from the pin +rather than assuming the pinned artifact carries a receipt: a fixture taken +from the artifact under test cannot independently test it. The structural +invariant survives — a receipt, if present, must name real probes/metrics, +agree with the pinned medians, and reference declared causes. And the contract +#8204 exercised is now a test rather than a docstring: a full re-pin with no +receipt is valid, so the next full re-pin will not red the gate again. + +Sabotage-tested: removing the pinned-median comparison, accepting any +timestamp, or making a missing receipt a defect each fails the corresponding +test. Test-only; no runtime, codegen or baseline changes. diff --git a/tests/test_gc_ratchet.py b/tests/test_gc_ratchet.py index 67a6959952..76bcce51e9 100644 --- a/tests/test_gc_ratchet.py +++ b/tests/test_gc_ratchet.py @@ -55,6 +55,73 @@ TOLERANCES_PATH = REPO_ROOT / "benchmarks" / "gc_ratchet" / "tolerances.json" +def _artifact_with_synthetic_receipt(): + """The pinned artifact plus a well-formed selective-re-pin receipt. + + Derived FROM the pin rather than hard-coded. The tests that tamper with a + receipt are testing the *validator*, not whatever happens to be pinned + today -- sourcing their fixture from the live baseline is what tied them to + one historical selective re-pin and broke them at the next full one. + + Two cells, because a single-cell receipt cannot catch an inspector that + validates only `cells[0]`. + """ + artifact = json.loads(DEFAULT_ARTIFACT.read_text(encoding="utf-8")) + artifact.pop("accepted_deterministic_deltas", None) + + chosen = [] + for probe_name, probe in sorted(artifact["probes"].items()): + for metric in sorted(probe.get("metrics", {})): + if metric not in DETERMINISTIC_METRICS: + continue + pinned = probe["metrics"][metric].get("median") + if isinstance(pinned, bool) or not isinstance(pinned, (int, float)): + continue + chosen.append((probe_name, metric, pinned)) + break + if len(chosen) == 2: + break + assert len(chosen) == 2, "pinned artifact has too few deterministic cells to build a receipt" + + cause = "b" * 40 + artifact["accepted_deterministic_deltas"] = { + "commit": "a" * 40, + "code_tree": "c" * 40, + "generated_at": "2026-01-01T00:00:00+00:00", + "notes": "Synthetic receipt built by the test suite from the pinned artifact.", + "measurement": { + "platform": "test-harness", + "repeats": 3, + "traced_runs": 2, + "binaries": { + name: {"size": 1, "sha256": "d" * 64} + for name in ("perry", "libperry_runtime.a", "libperry_stdlib.a") + }, + }, + "causes": { + cause: { + "pull_request": 1, + "category": "synthetic", + "evidence": "constructed by the test suite", + } + }, + "cells": [ + { + "probe": probe, + "metric": metric, + # previous must differ from accepted, or the inspector reports + # "records no change" -- a receipt row for a cell that did not + # move is itself a defect. + "previous_median": pinned + 1, + "accepted_median": pinned, + "causes": [cause], + } + for probe, metric, pinned in chosen + ], + } + return artifact + + def _shipped_tolerances(): return json.loads(TOLERANCES_PATH.read_text(encoding="utf-8")) @@ -490,48 +557,67 @@ def test_pinned_artifact_records_provenance(self): for key in ("perry", "libperry_runtime.a", "libperry_stdlib.a"): self.assertRegex(binaries[key]["sha256"], r"^[0-9a-f]{64}$") - def test_selective_refresh_names_every_accepted_cell_and_cause(self): + def test_a_full_re_pin_without_a_receipt_is_valid(self): + """The contract #8204 exercised, which nothing covered. + + `accepted_deterministic_deltas` is the receipt for a SELECTIVE re-pin -- + the dangerous kind, which can turn one red row green while leaving no + machine-readable answer to which rows moved or why. A FULL re-pin has + artifact-wide provenance instead, and the validator says so explicitly: + `if receipt is None: return`. + + This existed only as a docstring. #8204 did a full re-pin (130 of 168 + cells moved), correctly carried no receipt, and three tests here that + hard-subscripted the key errored with `KeyError`, reddening + `windows-build` on every open PR. The gate punished the correct action, + so pin the permission as a test rather than a comment. + """ artifact = json.loads(DEFAULT_ARTIFACT.read_text(encoding="utf-8")) - receipt = artifact["accepted_deterministic_deltas"] - expected = { - ("02_survivor_promotion", "copied_objects"), - ("03_cross_gen_writes", "copied_objects"), - ("03_cross_gen_writes", "copied_bytes"), - ("03_cross_gen_writes", "freed_bytes"), - ("04_dead_after_deep_stack", "copied_objects"), - ("04_dead_after_deep_stack", "freed_bytes"), - ("05_closure_capture", "copied_objects"), - ("05_closure_capture", "freed_bytes"), - ("06_string_retention", "freed_bytes"), - ("08_map_set_sidetables", "copied_objects"), - ("08_map_set_sidetables", "copied_bytes"), - ("08_map_set_sidetables", "freed_bytes"), - ("12_large_live_set", "copied_objects"), - ("12_large_live_set", "promoted_bytes"), - ("12_large_live_set", "freed_bytes"), - ("13_large_eden_survivors", "heap_used_bytes"), - ("13_large_eden_survivors", "freed_bytes"), - ("14_grow_then_churn", "copied_objects"), - ("14_grow_then_churn", "copied_bytes"), - ("14_grow_then_churn", "promoted_bytes"), - ("14_grow_then_churn", "freed_bytes"), - } - actual = {(cell["probe"], cell["metric"]) for cell in receipt["cells"]} - self.assertEqual(actual, expected) - self.assertEqual( - {cause["pull_request"] for cause in receipt["causes"].values()}, - {7928, 7960, 7961}, + self.assertNotIn( + "accepted_deterministic_deltas", + artifact, + "the pinned baseline is a full re-pin; update this test if that changes", ) + validate_artifact(artifact) - def test_selective_refresh_receipt_cannot_disagree_with_the_pin(self): + def test_a_receipt_on_the_pin_must_name_real_cells_and_real_causes(self): + """The durable half of the old cell-by-cell assertion. + + What that test actually pinned was one historical selective re-pin: + #8069's exact 21 cells and causes {7928, 7960, 7961}. That is a snapshot, + not an invariant -- any later re-pin breaks it by construction, which is + precisely what happened. The invariant worth keeping is structural: a + receipt, IF present, must describe the artifact it ships with. + """ artifact = json.loads(DEFAULT_ARTIFACT.read_text(encoding="utf-8")) + receipt = artifact.get("accepted_deterministic_deltas") + if receipt is None: + self.skipTest("pinned baseline is a full re-pin (no selective receipt)") + probes = artifact["probes"] + for cell in receipt["cells"]: + self.assertIn(cell["probe"], probes) + self.assertIn(cell["metric"], probes[cell["probe"]]["metrics"]) + self.assertEqual( + cell["accepted_median"], + probes[cell["probe"]]["metrics"][cell["metric"]]["median"], + f"{cell['probe']}.{cell['metric']} receipt disagrees with the pin", + ) + for commit in cell["causes"]: + self.assertIn(commit, receipt["causes"]) + for cause in receipt["causes"].values(): + self.assertIsInstance(cause["pull_request"], int) + self.assertGreater(cause["pull_request"], 0) + + def test_selective_refresh_receipt_cannot_disagree_with_the_pin(self): + artifact = _artifact_with_synthetic_receipt() + validate_artifact(artifact) # control: the fixture itself is valid tampered = copy.deepcopy(artifact) tampered["accepted_deterministic_deltas"]["cells"][0]["accepted_median"] += 1 with self.assertRaisesRegex(RatchetError, "does not match pinned median"): validate_artifact(tampered) def test_selective_refresh_receipt_rejects_a_malformed_timestamp(self): - artifact = json.loads(DEFAULT_ARTIFACT.read_text(encoding="utf-8")) + artifact = _artifact_with_synthetic_receipt() tampered = copy.deepcopy(artifact) tampered["accepted_deterministic_deltas"]["generated_at"] = "unknown" with self.assertRaisesRegex(RatchetError, "ISO-8601 UTC timestamp"):