From 9e151dcb5240c34662fb8e2815e746e9bf6f6d25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 16 Aug 2026 17:48:37 +0200 Subject: [PATCH 1/2] test(gc-ratchet): stop the self-tests demanding a selective re-pin receipt `windows-build` is red on every open PR. It fails at "GC structural audits (Windows)" with three errors in the ratchet's own test suite: KeyError: 'accepted_deterministic_deltas' FAILED (errors=3, skipped=11) The artifact is not at fault, and neither is #8204. `accepted_deterministic_ deltas` is the receipt for a SELECTIVE re-pin -- the dangerous kind, which can turn one red row green while leaving no machine-readable answer to which rows moved or why. A FULL re-pin carries artifact-wide provenance instead. The inspector's own docstring says so ("Older and synthetic artifacts may omit the receipt") and the validator implements it: `if receipt is None: return`. #8204 moved 130 of 168 cells -- a full re-pin -- so it correctly shipped no receipt. Three tests here hard-subscripted the key on the *live pinned baseline* and errored. The gate punished the correct action. What those tests actually pinned was one historical selective re-pin: #8069's exact 21 cells and causes {7928, 7960, 7961}, frozen into assertions against whatever baseline happens to be current. That is a snapshot, not an invariant. It could only stay green by the world never changing, and any later full re-pin breaks it by construction. So: - The two tamper tests (a receipt disagreeing with the pin; a malformed timestamp) are genuinely valuable -- they test the VALIDATOR. They now build their fixture synthetically from the pin rather than assuming the pinned artifact carries a receipt. A fixture taken from the artifact under test cannot independently test it. Two cells, not one, so an inspector that validated only `cells[0]` would not pass. - #8069's specific 21 cells are gone. The durable invariant they reached for stays: a receipt, IF present, must name real probes/metrics, agree with the pinned medians, and reference declared causes. - Added the case #8204 exercised and nothing covered: a full re-pin with no receipt is VALID. That contract existed only as a docstring, which is why the trap was armed. Without this test, the next full re-pin reds the gate again. Sabotage-tested, because three assertions that cannot fail would be worse than the errors they replace. Baseline: all three pass. Remove the pinned-median comparison and the disagreement test fails; accept any timestamp and the timestamp test fails; make a missing receipt a defect and the full-re-pin test fails. 98 tests, OK (1 skipped -- the receipt-present invariant, correctly skipped while the pin is a full re-pin). --- tests/test_gc_ratchet.py | 150 ++++++++++++++++++++++++++++++--------- 1 file changed, 118 insertions(+), 32 deletions(-) diff --git a/tests/test_gc_ratchet.py b/tests/test_gc_ratchet.py index 67a6959952..76bcce51e9 100644 --- a/tests/test_gc_ratchet.py +++ b/tests/test_gc_ratchet.py @@ -55,6 +55,73 @@ TOLERANCES_PATH = REPO_ROOT / "benchmarks" / "gc_ratchet" / "tolerances.json" +def _artifact_with_synthetic_receipt(): + """The pinned artifact plus a well-formed selective-re-pin receipt. + + Derived FROM the pin rather than hard-coded. The tests that tamper with a + receipt are testing the *validator*, not whatever happens to be pinned + today -- sourcing their fixture from the live baseline is what tied them to + one historical selective re-pin and broke them at the next full one. + + Two cells, because a single-cell receipt cannot catch an inspector that + validates only `cells[0]`. + """ + artifact = json.loads(DEFAULT_ARTIFACT.read_text(encoding="utf-8")) + artifact.pop("accepted_deterministic_deltas", None) + + chosen = [] + for probe_name, probe in sorted(artifact["probes"].items()): + for metric in sorted(probe.get("metrics", {})): + if metric not in DETERMINISTIC_METRICS: + continue + pinned = probe["metrics"][metric].get("median") + if isinstance(pinned, bool) or not isinstance(pinned, (int, float)): + continue + chosen.append((probe_name, metric, pinned)) + break + if len(chosen) == 2: + break + assert len(chosen) == 2, "pinned artifact has too few deterministic cells to build a receipt" + + cause = "b" * 40 + artifact["accepted_deterministic_deltas"] = { + "commit": "a" * 40, + "code_tree": "c" * 40, + "generated_at": "2026-01-01T00:00:00+00:00", + "notes": "Synthetic receipt built by the test suite from the pinned artifact.", + "measurement": { + "platform": "test-harness", + "repeats": 3, + "traced_runs": 2, + "binaries": { + name: {"size": 1, "sha256": "d" * 64} + for name in ("perry", "libperry_runtime.a", "libperry_stdlib.a") + }, + }, + "causes": { + cause: { + "pull_request": 1, + "category": "synthetic", + "evidence": "constructed by the test suite", + } + }, + "cells": [ + { + "probe": probe, + "metric": metric, + # previous must differ from accepted, or the inspector reports + # "records no change" -- a receipt row for a cell that did not + # move is itself a defect. + "previous_median": pinned + 1, + "accepted_median": pinned, + "causes": [cause], + } + for probe, metric, pinned in chosen + ], + } + return artifact + + def _shipped_tolerances(): return json.loads(TOLERANCES_PATH.read_text(encoding="utf-8")) @@ -490,48 +557,67 @@ def test_pinned_artifact_records_provenance(self): for key in ("perry", "libperry_runtime.a", "libperry_stdlib.a"): self.assertRegex(binaries[key]["sha256"], r"^[0-9a-f]{64}$") - def test_selective_refresh_names_every_accepted_cell_and_cause(self): + def test_a_full_re_pin_without_a_receipt_is_valid(self): + """The contract #8204 exercised, which nothing covered. + + `accepted_deterministic_deltas` is the receipt for a SELECTIVE re-pin -- + the dangerous kind, which can turn one red row green while leaving no + machine-readable answer to which rows moved or why. A FULL re-pin has + artifact-wide provenance instead, and the validator says so explicitly: + `if receipt is None: return`. + + This existed only as a docstring. #8204 did a full re-pin (130 of 168 + cells moved), correctly carried no receipt, and three tests here that + hard-subscripted the key errored with `KeyError`, reddening + `windows-build` on every open PR. The gate punished the correct action, + so pin the permission as a test rather than a comment. + """ artifact = json.loads(DEFAULT_ARTIFACT.read_text(encoding="utf-8")) - receipt = artifact["accepted_deterministic_deltas"] - expected = { - ("02_survivor_promotion", "copied_objects"), - ("03_cross_gen_writes", "copied_objects"), - ("03_cross_gen_writes", "copied_bytes"), - ("03_cross_gen_writes", "freed_bytes"), - ("04_dead_after_deep_stack", "copied_objects"), - ("04_dead_after_deep_stack", "freed_bytes"), - ("05_closure_capture", "copied_objects"), - ("05_closure_capture", "freed_bytes"), - ("06_string_retention", "freed_bytes"), - ("08_map_set_sidetables", "copied_objects"), - ("08_map_set_sidetables", "copied_bytes"), - ("08_map_set_sidetables", "freed_bytes"), - ("12_large_live_set", "copied_objects"), - ("12_large_live_set", "promoted_bytes"), - ("12_large_live_set", "freed_bytes"), - ("13_large_eden_survivors", "heap_used_bytes"), - ("13_large_eden_survivors", "freed_bytes"), - ("14_grow_then_churn", "copied_objects"), - ("14_grow_then_churn", "copied_bytes"), - ("14_grow_then_churn", "promoted_bytes"), - ("14_grow_then_churn", "freed_bytes"), - } - actual = {(cell["probe"], cell["metric"]) for cell in receipt["cells"]} - self.assertEqual(actual, expected) - self.assertEqual( - {cause["pull_request"] for cause in receipt["causes"].values()}, - {7928, 7960, 7961}, + self.assertNotIn( + "accepted_deterministic_deltas", + artifact, + "the pinned baseline is a full re-pin; update this test if that changes", ) + validate_artifact(artifact) - def test_selective_refresh_receipt_cannot_disagree_with_the_pin(self): + def test_a_receipt_on_the_pin_must_name_real_cells_and_real_causes(self): + """The durable half of the old cell-by-cell assertion. + + What that test actually pinned was one historical selective re-pin: + #8069's exact 21 cells and causes {7928, 7960, 7961}. That is a snapshot, + not an invariant -- any later re-pin breaks it by construction, which is + precisely what happened. The invariant worth keeping is structural: a + receipt, IF present, must describe the artifact it ships with. + """ artifact = json.loads(DEFAULT_ARTIFACT.read_text(encoding="utf-8")) + receipt = artifact.get("accepted_deterministic_deltas") + if receipt is None: + self.skipTest("pinned baseline is a full re-pin (no selective receipt)") + probes = artifact["probes"] + for cell in receipt["cells"]: + self.assertIn(cell["probe"], probes) + self.assertIn(cell["metric"], probes[cell["probe"]]["metrics"]) + self.assertEqual( + cell["accepted_median"], + probes[cell["probe"]]["metrics"][cell["metric"]]["median"], + f"{cell['probe']}.{cell['metric']} receipt disagrees with the pin", + ) + for commit in cell["causes"]: + self.assertIn(commit, receipt["causes"]) + for cause in receipt["causes"].values(): + self.assertIsInstance(cause["pull_request"], int) + self.assertGreater(cause["pull_request"], 0) + + def test_selective_refresh_receipt_cannot_disagree_with_the_pin(self): + artifact = _artifact_with_synthetic_receipt() + validate_artifact(artifact) # control: the fixture itself is valid tampered = copy.deepcopy(artifact) tampered["accepted_deterministic_deltas"]["cells"][0]["accepted_median"] += 1 with self.assertRaisesRegex(RatchetError, "does not match pinned median"): validate_artifact(tampered) def test_selective_refresh_receipt_rejects_a_malformed_timestamp(self): - artifact = json.loads(DEFAULT_ARTIFACT.read_text(encoding="utf-8")) + artifact = _artifact_with_synthetic_receipt() tampered = copy.deepcopy(artifact) tampered["accepted_deterministic_deltas"]["generated_at"] = "unknown" with self.assertRaisesRegex(RatchetError, "ISO-8601 UTC timestamp"): From d4a797e9c4b4517ff7a15a0e09ed3099a93df4eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sun, 16 Aug 2026 17:49:14 +0200 Subject: [PATCH 2/2] docs(changelog): add the 8214 fragment --- .../8214-ratchet-selftest-full-repin.md | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 changelog.d/8214-ratchet-selftest-full-repin.md diff --git a/changelog.d/8214-ratchet-selftest-full-repin.md b/changelog.d/8214-ratchet-selftest-full-repin.md new file mode 100644 index 0000000000..8a4f07513b --- /dev/null +++ b/changelog.d/8214-ratchet-selftest-full-repin.md @@ -0,0 +1,26 @@ +Fixed the GC ratchet's own test suite demanding a selective re-pin receipt from +every pinned baseline, which made `windows-build` red on every open PR. + +`accepted_deterministic_deltas` is the receipt for a *selective* re-pin — the +dangerous kind, which can turn one red row green while leaving no +machine-readable answer to which rows moved or why. A *full* re-pin carries +artifact-wide provenance instead, and the validator says so explicitly +(`if receipt is None: return`). #8204 moved 130 of 168 cells, so it correctly +shipped no receipt; three tests that hard-subscripted the key on the live +pinned baseline errored with `KeyError`. The gate punished the correct action. + +Those tests had frozen one historical selective re-pin — #8069's exact 21 cells +and causes — into assertions against whatever baseline happens to be current, +which could only stay green by the world never changing. + +The two tamper tests remain, but build their fixture synthetically from the pin +rather than assuming the pinned artifact carries a receipt: a fixture taken +from the artifact under test cannot independently test it. The structural +invariant survives — a receipt, if present, must name real probes/metrics, +agree with the pinned medians, and reference declared causes. And the contract +#8204 exercised is now a test rather than a docstring: a full re-pin with no +receipt is valid, so the next full re-pin will not red the gate again. + +Sabotage-tested: removing the pinned-median comparison, accepting any +timestamp, or making a missing receipt a defect each fails the corresponding +test. Test-only; no runtime, codegen or baseline changes.