From 5cbff76ebe570a1c015188060a90a36640c866ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 31 Jul 2026 21:47:18 +0200 Subject: [PATCH 1/8] test(repsel): make emission determinism a hard gate on every host (#7131) The knob-isolation gate detected a nondeterministic host and SKIPPED its emission half rather than reporting phantom diffs. That workaround existed because objects were nondeterministic on ELF (#7131); #7135 fixed the cause, so the skip now hides a regression instead of a known defect. - nondeterminism is a hard failure, on every host, before any knob is judged - --require-emission removed: it selected between a real check and a skip, and only one of those is still a decision worth having - new census-determinism subcommand + self-test: the standalone instrument the issue asks for, so any agent can establish object-hash validity on the host it is measuring on --- scripts/compiler_output_harness/cli.py | 30 ++- .../repsel_determinism.py | 222 ++++++++++++++++++ .../repsel_knob_isolation.py | 114 +++++---- 3 files changed, 301 insertions(+), 65 deletions(-) create mode 100644 scripts/compiler_output_harness/repsel_determinism.py diff --git a/scripts/compiler_output_harness/cli.py b/scripts/compiler_output_harness/cli.py index 3c410eb01f..c8148207e3 100644 --- a/scripts/compiler_output_harness/cli.py +++ b/scripts/compiler_output_harness/cli.py @@ -7,6 +7,8 @@ from .common import DEFAULT_BENCHMARK_RUNS, HarnessError from .repsel_census import census from .repsel_census import self_test as census_self_test +from .repsel_determinism import DEFAULT_REPEAT, check_determinism +from .repsel_determinism import self_test as determinism_self_test from .repsel_knob_isolation import check_isolation from .repsel_knob_isolation import self_test as isolation_self_test from .spec import WORKLOADS @@ -145,11 +147,6 @@ def build_parser() -> argparse.ArgumentParser: iso_p.add_argument("--knob", action="append", help="restrict to named knob(s)") iso_p.add_argument("--compile-timeout", type=int, default=300) iso_p.add_argument("--jobs", type=int, default=4, help="parallel compiles") - iso_p.add_argument( - "--require-emission", - action="store_true", - help="fail instead of skipping when the host cannot emit objects deterministically", - ) iso_p.add_argument("--keep-objects", action="store_true") iso_p.set_defaults(func=check_isolation) @@ -159,6 +156,29 @@ def build_parser() -> argparse.ArgumentParser: ) iso_self_p.set_defaults(func=isolation_self_test) + # Emission determinism (#7131). The precondition every object-hash A/B in + # this repo assumes and that nothing checked until it was false for months + # on Linux only. + det_p = sub.add_parser( + "census-determinism", + help="assert the compiler emits byte-identical objects for identical inputs", + ) + det_p.add_argument("--perry") + det_p.add_argument("--baseline") + det_p.add_argument("--workload", action="append", help="restrict to named workload(s)") + det_p.add_argument( + "--repeat", type=int, default=DEFAULT_REPEAT, help="compiles per workload (min 2)" + ) + det_p.add_argument("--compile-timeout", type=int, default=300) + det_p.add_argument("--jobs", type=int, default=4, help="parallel compiles") + det_p.set_defaults(func=check_determinism) + + det_self_p = sub.add_parser( + "census-determinism-self-test", + help="check the determinism verdict logic without compiling", + ) + det_self_p.set_defaults(func=determinism_self_test) + return parser diff --git a/scripts/compiler_output_harness/repsel_determinism.py b/scripts/compiler_output_harness/repsel_determinism.py new file mode 100644 index 0000000000..68ccb0a9d5 --- /dev/null +++ b/scripts/compiler_output_harness/repsel_determinism.py @@ -0,0 +1,222 @@ +"""Object-emission determinism check (#7131). + +Object-hash A/B is the cheapest and least deniable instrument this project has +for "did this change what ships": compile twice, hash the objects, and a +difference is not an opinion. Every representation-selection finding of the +#7113/#7119/#7121/#7128 series rests on it. + +The instrument is only sound if the compiler is a function of its inputs. It +was not, on ELF: + + clang -c perry_llvm___.ll -o out.o + +records the **source basename** of the translation unit into the object as an +`STT_FILE` symbol, so two identical compiles differed by exactly the digits of +the pid and the clock (#7131 — 26/26 census workloads nondeterministic on a +Raspberry Pi 5, 10 bytes apart on `suite_01_startup`). Mach-O keeps that name +in the debug map rather than the `.o`, which is why macOS looked clean while +carrying the same defect. `#7135` fixed it by content-addressing the `.ll` +basename; this module is the check that says so, and keeps saying so. + +Measured properties of the ELF path, so a future reader does not have to +re-derive which names matter (aarch64 Debian clang 19.1.7, no `-g`): + +* the `.ll` **source basename** IS recorded — `STT_FILE`, `.strtab`; +* its **directory** and the process **CWD** are NOT (that needs DWARF, i.e. + `-g`, which Perry only passes under `PERRY_DEBUG_SYMBOLS`); +* the `-o` **output** path is NOT recorded anywhere; +* `ld -r` (the multi-codegen-unit merge, #5391) records neither its input nor + its output paths. + +So the `.o` and partial-link staging names may keep their uniquifying counter — +only the `.ll` name had to become a function of content. + +Why this is a check and not a comment: the property is invisible on the host +most of this project's compiler work happens on. A macOS-only reviewer cannot +tell a fixed compiler from a broken one, which is exactly how the defect +survived from #509 to #7131. Run it on Linux. +""" + +from __future__ import annotations + +import argparse +import platform +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Any, Callable + +from .capture import resolve_perry +from .common import HarnessError, REPO_ROOT +from .repsel_census import DEFAULT_BASELINE, compile_and_census, load_baseline + + +#: How many times each workload is compiled. Two is the minimum that can +#: observe a difference; more is a stronger sample for a flaky (rather than +#: systematically clock-keyed) source of variation. +DEFAULT_REPEAT = 2 + + +def nondeterminism_report(varied: list[str], total: int, *, repeat: int = 2) -> str: + """The message shown when the same compiler twice disagreed with itself. + + Kept in one place because two callers need to say the same thing: this + check, and the knob-isolation gate's determinism control (#7128), whose + every object comparison is void if this property does not hold. + """ + shown = ", ".join(varied[:4]) + ("…" if len(varied) > 4 else "") + return ( + f"OBJECT EMISSION IS NONDETERMINISTIC: {len(varied)}/{total} workload(s) " + f"compiled {repeat}x with identical flags and environment produced " + f"different bytes ({shown}).\n" + "\n" + " This is a REGRESSION, not a host property. It was one until #7131:\n" + " the temp `.ll` name carried pid + wall-clock nanos, and clang records\n" + " a translation unit's source basename into the ELF object. #7135\n" + " content-addressed that name. If this fires again, object-hash A/B is\n" + " invalid on this host and every measurement taken through it is void.\n" + "\n" + " To localise: keep two objects for one workload and compare\n" + " readelf -sW a.o | grep FILE # the #7131 shape: names differ here\n" + " cmp -l a.o b.o | head # anywhere else is a NEW cause\n" + " A differing `STT_FILE` symbol is the old defect returning. Differences\n" + " elsewhere in `.text` are a nondeterministic codegen ordering instead —\n" + " the macOS analogue was closure-source iteration order permuting\n" + " `@.str.N` numbering per process (#7038/#7039)." + ) + + +def _digest_objects(paths: list[str]) -> str: + import hashlib + + h = hashlib.sha256() + for path in sorted(paths): + h.update(Path(path).read_bytes()) + return h.hexdigest() + + +def verdict( + digests: dict[str, list[str]], *, repeat: int, printer: Callable[[str], None] = print +) -> int: + """Turn per-workload digest lists into an exit code. + + Split from the compile loop so the verdict can be exercised without a + compiler — a gate whose decision logic is only reachable through a 52-compile + run is a gate nobody re-checks. + """ + names = sorted(digests) + if not names: + raise HarnessError("no workloads were compiled; nothing was checked") + for name in names: + seen = digests[name] + if len(seen) < 2: + raise HarnessError( + f"{name} was compiled {len(seen)}x; determinism needs at least 2 " + "observations, so this run proves nothing" + ) + varied = [n for n in names if len(set(digests[n])) > 1] + + printer("Per-workload emission") + printer("---------------------") + for name in names: + mark = "DIFFERS" if name in varied else "same" + printer(f" {name:<34} {mark:>8} {digests[name][0][:16]}") + printer("") + + if varied: + printer(nondeterminism_report(varied, len(names), repeat=repeat)) + return 1 + printer( + f"Emission is deterministic: {len(names)}/{len(names)} workload(s) " + f"compiled {repeat}x produced byte-identical objects. Object-hash A/B " + "is valid on this host." + ) + return 0 + + +def check_determinism(args: argparse.Namespace) -> int: + """Compile every census workload `--repeat` times and compare the bytes.""" + perry = resolve_perry(getattr(args, "perry", None)) + baseline = load_baseline(Path(args.baseline) if args.baseline else DEFAULT_BASELINE) + workloads: list[dict[str, Any]] = baseline["workloads"] + if getattr(args, "workload", None): + wanted = set(args.workload) + workloads = [w for w in workloads if w["name"] in wanted] + missing = wanted - {w["name"] for w in workloads} + if missing: + raise HarnessError(f"unknown workload(s): {', '.join(sorted(missing))}") + if not workloads: + raise HarnessError("no workloads selected") + + repeat = max(2, int(args.repeat)) + + print("Object-emission determinism (#7131)") + print("===================================\n") + print(f"compiler: {' '.join(perry)}") + print(f"host: {platform.system()} {platform.machine()}") + print(f"corpus: {len(workloads)} workload(s) x {repeat} compile(s)\n") + + import shutil + import tempfile + + tmp = Path(tempfile.mkdtemp(prefix="repsel-determinism-")) + try: + # Repeats run through the same pool as the corpus on purpose: two + # workers holding IDENTICAL IR now share one content-addressed `.ll`, + # so racing them is part of the subject, not a confound (#7135 CR). + jobs = [(w, i) for w in workloads for i in range(repeat)] + + def run(job: tuple[dict[str, Any], int]) -> tuple[str, str]: + workload, index = job + source = REPO_ROOT / workload["source"] + census = compile_and_census( + perry, + source, + timeout=args.compile_timeout, + object_out=tmp / workload["name"] / str(index) / "out.o", + ) + return workload["name"], _digest_objects(census["objects"]) + + with ThreadPoolExecutor(max_workers=max(1, args.jobs)) as pool: + observed = list(pool.map(run, jobs)) + finally: + shutil.rmtree(tmp, ignore_errors=True) + + digests: dict[str, list[str]] = {} + for name, digest in observed: + digests.setdefault(name, []).append(digest) + return verdict(digests, repeat=repeat) + + +def self_test(_args: argparse.Namespace) -> int: + """Prove the verdict can go red, and that it refuses a vacuous run. + + CLAUDE.md failure mode 4: a gate must assert its subject was live. A + determinism check handed one observation per workload has compared nothing, + and must say so rather than printing a green line. + """ + quiet: Callable[[str], None] = lambda _line: None + + assert verdict({"w": ["a", "a"], "v": ["b", "b"]}, repeat=2, printer=quiet) == 0 + + assert verdict({"w": ["a", "a"], "v": ["b", "c"]}, repeat=2, printer=quiet) == 1 + assert verdict({"w": ["a", "b"]}, repeat=2, printer=quiet) == 1 + + # Nondeterminism on a LATER repeat must count too — a check that only + # compared the first two observations would miss a 1-in-3 flake. + assert verdict({"w": ["a", "a", "z"]}, repeat=3, printer=quiet) == 1 + + for vacuous in ({}, {"w": ["a"]}): + try: + verdict(vacuous, repeat=2, printer=quiet) # type: ignore[arg-type] + except HarnessError: + pass + else: # pragma: no cover - the assertion below is the failure report + raise AssertionError( + f"verdict({vacuous!r}) returned a verdict having compared nothing" + ) + + report = nondeterminism_report(["w", "v"], 26, repeat=2) + assert "2/26" in report and "#7131" in report and "STT_FILE" in report, report + + print("repsel determinism self-test OK") + return 0 diff --git a/scripts/compiler_output_harness/repsel_knob_isolation.py b/scripts/compiler_output_harness/repsel_knob_isolation.py index 8123bdece7..9e82303e78 100644 --- a/scripts/compiler_output_harness/repsel_knob_isolation.py +++ b/scripts/compiler_output_harness/repsel_knob_isolation.py @@ -38,9 +38,11 @@ "noisy" proves nothing: * **determinism** — the same compiler, same flags, twice, must produce the same - bytes. On aarch64 Linux it does not (the LLVM module name embeds pid + - nanotime), so this check refuses to run the emission half there rather than - reporting 26 phantom diffs; + bytes. This used to be false on aarch64 Linux (the temp `.ll` name carried pid + + nanotime and clang records a unit's source basename into the ELF object), so + the gate detected the host and skipped the emission half there. #7131/#7135 + content-addressed that name; the skip is gone and a disagreement is now a + hard failure on every host — see [`repsel_determinism`]; * **inert-variable** — `K=1` and an unrelated `PERRY_TOTALLY_UNRELATED=0` must both reproduce the default object bit-for-bit. If they do not, the diff signal is not attributable to the knob at all. @@ -77,6 +79,7 @@ from .capture import resolve_perry from .common import HarnessError, REPO_ROOT +from .repsel_determinism import nondeterminism_report from .repsel_census import ( CENSUS_KEYS, DEFAULT_BASELINE, @@ -315,7 +318,7 @@ def run(job: tuple[str, str, Path, dict[str, str], Path]) -> tuple[tuple[str, st with ThreadPoolExecutor(max_workers=max(1, args.jobs)) as pool: results: dict[tuple[str, str], Arm] = dict(pool.map(run, jobs)) - return _verdict(workloads, knobs, results, args) + return _verdict(workloads, knobs, results) finally: if not args.keep_objects: shutil.rmtree(tmp, ignore_errors=True) @@ -327,53 +330,48 @@ def _verdict( workloads: list[dict[str, Any]], knobs: tuple[Knob, ...], results: dict[tuple[str, str], Arm], - args: argparse.Namespace, ) -> int: names = [w["name"] for w in workloads] # ── control 1: determinism ──────────────────────────────────────────── + # Every object comparison below is void unless the compiler is a function + # of its inputs. Until #7131/#7135 it was not on ELF, and this control + # SKIPPED the emission half there — which meant the half of the gate that + # caught the `PERRY_CANONICAL_STR_LOCALS` defect could not run on Linux at + # all. The skip is gone: a disagreement here is a regression to fix, not a + # host to route around (CLAUDE.md — a mode that still exists is a decision + # that hasn't been made). nondeterministic = [ n for n in names if results[(n, "default")].digest != results[(n, "default#2")].digest ] - emission_checkable = not nondeterministic if nondeterministic: + print(nondeterminism_report(nondeterministic, len(names))) + print() print( - "OBJECT EMISSION IS NONDETERMINISTIC on this host: " - f"{len(nondeterministic)}/{len(names)} workload(s) compiled twice with " - "identical flags produced different bytes " - f"({', '.join(nondeterministic[:4])}{'…' if len(nondeterministic) > 4 else ''}).\n" - " Known cause on aarch64 Linux: the LLVM module name embeds pid +\n" - " nanotime and lands in the object. The emission half of this gate is\n" - " SKIPPED — it would report a diff for every arm. Run it on a host\n" - " where the compiler is deterministic (macOS today).\n" + "Knob isolation FAILED at its determinism control: nothing below it can " + "be trusted, so no knob was judged." ) - if args.require_emission: - print( - "--require-emission was passed, so a host that cannot compare objects " - "is a failure rather than a partial run." - ) - return 1 + return 1 failures: list[str] = [] notes: list[str] = [] # ── control 2: an inert variable must not move the object ───────────── - if emission_checkable: + for n in names: + if results[(n, f"inert:{INERT_VAR}=0")].digest != results[(n, "default")].digest: + failures.append( + f"CONTROL: {n} compiled differently with {INERT_VAR}=0 set, an env var " + "the compiler does not read. The object diff below is not attributable " + "to any knob." + ) + for knob in knobs: for n in names: - if results[(n, f"inert:{INERT_VAR}=0")].digest != results[(n, "default")].digest: + if results[(n, f"{knob.env}=1")].digest != results[(n, "default")].digest: failures.append( - f"CONTROL: {n} compiled differently with {INERT_VAR}=0 set, an env var " - "the compiler does not read. The object diff below is not attributable " - "to any knob." + f"CONTROL: {n} compiled differently with {knob.env}=1, which is the " + "default. The knob is keyed into codegen beyond its documented " + "off-state." ) - for knob in knobs: - for n in names: - if results[(n, f"{knob.env}=1")].digest != results[(n, "default")].digest: - failures.append( - f"CONTROL: {n} compiled differently with {knob.env}=1, which is the " - "default. The knob is keyed into codegen beyond its documented " - "off-state." - ) # ── rule 1 / rule 2, per knob ───────────────────────────────────────── rows: list[str] = [] @@ -420,8 +418,6 @@ def _verdict( ) moved_counts += int(lost) - if not emission_checkable: - continue differs = off.digest != base.digest moved_objects += int(differs) promotes = ( @@ -451,7 +447,7 @@ def _verdict( "anywhere in the corpus. Either the representation stopped firing or the " "knob no longer reaches it; both make every A/B through it vacuous." ) - if emission_checkable and moved_objects == 0: + if moved_objects == 0: failures.append( f"DEAD KNOB: {knob.env}=0 left every object in the corpus byte-identical. " "An arm that emits the same bytes as the default cannot be evidence about " @@ -489,12 +485,6 @@ def _verdict( ) return 1 - if not emission_checkable: - print( - "Count isolation OK on every knob. EMISSION isolation was not checked " - "(nondeterministic host)." - ) - return 0 print("Knob isolation OK: every knob moves its own representation and nothing else.") return 0 @@ -512,7 +502,6 @@ def self_test(_args: argparse.Namespace) -> int: two branches that catch them are exercised on every run rather than only on a host with a compiler. """ - ns = argparse.Namespace(require_emission=False) workloads = [{"name": "w", "source": "x.ts"}, {"name": "v", "source": "y.ts"}] def arm(counts: dict[str, int], digest: str, **signals: int) -> Arm: @@ -549,7 +538,7 @@ def table(default: dict[str, Arm], off: dict[str, Arm], knob: Knob) -> dict[tupl }, i32, ) - verdict = _capture(_verdict, workloads, (i32,), leak, ns) + verdict = _capture(_verdict, workloads, (i32,), leak) assert verdict.code == 1, verdict.out assert "COUNT LEAK" in verdict.out and "ptr-shape-consumed" in verdict.out, verdict.out @@ -560,7 +549,7 @@ def table(default: dict[str, Arm], off: dict[str, Arm], knob: Knob) -> dict[tupl {"w": arm({"canonical-str": 0}, "cc"), "v": arm({}, "zz")}, strk, ) - verdict = _capture(_verdict, workloads, (strk,), emission, ns) + verdict = _capture(_verdict, workloads, (strk,), emission) assert verdict.code == 1, verdict.out assert "EMISSION LEAK" in verdict.out, verdict.out @@ -571,7 +560,7 @@ def table(default: dict[str, Arm], off: dict[str, Arm], knob: Knob) -> dict[tupl {"w": arm({"canonical-str": 0}, "cc"), "v": arm({}, "bb")}, strk, ) - verdict = _capture(_verdict, workloads, (strk,), clean, ns) + verdict = _capture(_verdict, workloads, (strk,), clean) assert verdict.code == 0, verdict.out # A knob that moves nothing is dead, not clean. @@ -580,7 +569,7 @@ def table(default: dict[str, Arm], off: dict[str, Arm], knob: Knob) -> dict[tupl {"w": arm({"canonical-str": 1}, "aa"), "v": arm({}, "bb")}, strk, ) - verdict = _capture(_verdict, workloads, (strk,), dead, ns) + verdict = _capture(_verdict, workloads, (strk,), dead) assert verdict.code == 1 and "DEAD KNOB" in verdict.out, verdict.out # An inert variable that moves the object means the diff is not the knob's. @@ -590,22 +579,27 @@ def table(default: dict[str, Arm], off: dict[str, Arm], knob: Knob) -> dict[tupl strk, ) contaminated[("v", f"inert:{INERT_VAR}=0")] = arm({}, "qq") - verdict = _capture(_verdict, workloads, (strk,), contaminated, ns) + verdict = _capture(_verdict, workloads, (strk,), contaminated) assert verdict.code == 1 and "CONTROL" in verdict.out, verdict.out - # A nondeterministic host must skip the emission half, not fail it — and - # must still run the count half. + # A nondeterministic compiler fails the gate outright (#7131). It used to + # skip the emission half, which is how the half that caught defect B above + # became unrunnable on Linux — the host where it mattered most. Note the + # arms here are otherwise CLEAN: the determinism control must reject them on + # its own, before any knob is judged. flaky = table( {"w": arm({"canonical-str": 1}, "aa"), "v": arm({}, "bb")}, - {"w": arm({"canonical-str": 0}, "cc"), "v": arm({}, "zz")}, + {"w": arm({"canonical-str": 0}, "cc"), "v": arm({}, "bb")}, strk, ) flaky[("w", "default#2")] = arm({"canonical-str": 1}, "AA") - verdict = _capture(_verdict, workloads, (strk,), flaky, ns) - assert verdict.code == 0 and "NONDETERMINISTIC" in verdict.out.upper(), verdict.out - strict = argparse.Namespace(require_emission=True) - verdict = _capture(_verdict, workloads, (strk,), flaky, strict) + verdict = _capture(_verdict, workloads, (strk,), flaky) assert verdict.code == 1, verdict.out + assert "NONDETERMINISTIC" in verdict.out.upper(), verdict.out + assert "#7131" in verdict.out, verdict.out + # …and it must stop there rather than reporting per-knob verdicts drawn + # from bytes it just declared untrustworthy. + assert "EMISSION LEAK" not in verdict.out and "DEAD KNOB" not in verdict.out, verdict.out # `PERRY_STATIC_STRING_LOWERING` owns no census key: it must move no count, # and rule 2 must NOT demand a byte-identical object of it. @@ -616,14 +610,14 @@ def table(default: dict[str, Arm], off: dict[str, Arm], knob: Knob) -> dict[tupl {"w": arm({"canonical-str": 1}, "cc"), "v": arm({}, "dd")}, static, ) - verdict = _capture(_verdict, workloads, (static,), ok, ns) + verdict = _capture(_verdict, workloads, (static,), ok) assert verdict.code == 0, verdict.out moved = table( {"w": arm({"canonical-str": 1}, "aa"), "v": arm({}, "bb")}, {"w": arm({"canonical-str": 0}, "cc"), "v": arm({}, "dd")}, static, ) - verdict = _capture(_verdict, workloads, (static,), moved, ns) + verdict = _capture(_verdict, workloads, (static,), moved) assert verdict.code == 1 and "COUNT LEAK" in verdict.out, verdict.out # A documented proof dependency may lower the downstream key and only that. @@ -634,7 +628,7 @@ def table(default: dict[str, Arm], off: dict[str, Arm], knob: Knob) -> dict[tupl {"w": arm({"int-valued-ta": 0, "canonical-i32": 2}, "cc"), "v": arm({}, "bb")}, intk, ) - verdict = _capture(_verdict, workloads, (intk,), down_ok, ns) + verdict = _capture(_verdict, workloads, (intk,), down_ok) assert verdict.code == 0, verdict.out # …but never raise it. A knob that ADDS another representation's promotions # is a leak no withdrawn proof can explain. @@ -643,7 +637,7 @@ def table(default: dict[str, Arm], off: dict[str, Arm], knob: Knob) -> dict[tupl {"w": arm({"int-valued-ta": 0, "canonical-i32": 4}, "cc"), "v": arm({}, "bb")}, intk, ) - verdict = _capture(_verdict, workloads, (intk,), down_bad, ns) + verdict = _capture(_verdict, workloads, (intk,), down_bad) assert verdict.code == 1 and "RAISED" in verdict.out, verdict.out # An UNDOCUMENTED cross-representation move is still a leak: the Str knob # has no dependency on canonical-i32, so the identical shape must go red. @@ -652,7 +646,7 @@ def table(default: dict[str, Arm], off: dict[str, Arm], knob: Knob) -> dict[tupl {"w": arm({"canonical-str": 0, "canonical-i32": 2}, "cc"), "v": arm({}, "bb")}, strk, ) - verdict = _capture(_verdict, workloads, (strk,), undocumented, ns) + verdict = _capture(_verdict, workloads, (strk,), undocumented) assert verdict.code == 1 and "COUNT LEAK" in verdict.out, verdict.out print("repsel knob-isolation self-test OK") From 2a9262f4b0ca4c84703c2850be3fdf2f05140731 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 31 Jul 2026 21:50:26 +0200 Subject: [PATCH 2/8] docs(repsel): record which temp names reach the object, drop the Linux caveat (#7131) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - linker.rs: a measured table of which paths clang/ld actually record in an ELF object (only the .ll BASENAME does), so a future temp-path change can be reviewed without re-deriving it, plus the PERRY_DEBUG_SYMBOLS caveat - census README: replace 'it is not deterministic on aarch64 Linux' with the check that establishes whether it is - CI: run the determinism check on the ubuntu-latest repsel-census job — the ELF host that can actually observe a relapse — and both verdict self-tests - correct the two unreleased changelog fragments that still describe the skip and the removed --require-emission flag --- .github/workflows/test.yml | 14 ++++++++ benchmarks/repsel_census/README.md | 34 ++++++++++++++++--- changelog.d/7133-repsel-knob-isolation.md | 11 +++--- .../7135-deterministic-llvm-temp-names.md | 4 +-- crates/perry-codegen/src/linker.rs | 17 ++++++++++ 5 files changed, 68 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8a8a7f7985..2ada5dc60b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1216,11 +1216,25 @@ jobs: - name: Census verdict self-test run: | python3 scripts/compiler_output_regression.py census-self-test + python3 scripts/compiler_output_regression.py census-knob-isolation-self-test + python3 scripts/compiler_output_regression.py census-determinism-self-test python3 -m unittest tests.test_repsel_census - name: Build compiler run: cargo build -p perry + # #7131. Every object comparison in this job (and in every + # representation-selection A/B this repo has taken) assumes the compiler + # is a function of its inputs. On ELF it was not — for months, and only + # on ELF, which is why macOS review never saw it. This runner is x86_64 + # Linux, so it is the host that can actually observe a relapse. + - name: Emission determinism + run: | + python3 scripts/compiler_output_regression.py census-determinism \ + --perry target/debug/perry \ + --repeat 2 \ + --jobs 4 + - name: Promotion census run: | python3 scripts/compiler_output_regression.py census \ diff --git a/benchmarks/repsel_census/README.md b/benchmarks/repsel_census/README.md index e9b2bbde7b..5cc1bafd03 100644 --- a/benchmarks/repsel_census/README.md +++ b/benchmarks/repsel_census/README.md @@ -236,11 +236,35 @@ Per knob, with that knob at `0` and every other at its default: some object. Rule 1 catches the first defect, rule 2 the second (it leaves every count -untouched). Two controls guard the diff: the compiler must be deterministic -(**it is not on aarch64 Linux** — the LLVM module name embeds pid + nanotime, so -the emission half is skipped there rather than reporting 26 phantoms), and both -`X=1` and an env var the compiler does not read must reproduce the default -object bit-for-bit. +untouched). Two controls guard the diff: the compiler must be deterministic, and +both `X=1` and an env var the compiler does not read must reproduce the default +object bit-for-bit. A determinism failure aborts the run — it is not a host +property to route around (see below). + +## Emission determinism (#7131) + +Every object comparison on this page assumes the compiler is a function of its +inputs. On ELF it was not: the temp `.ll` name carried pid + wall-clock nanos, +and clang records a translation unit's **source basename** into the object as an +`STT_FILE` symbol, so two identical compiles differed by exactly those digits — +26/26 census workloads on a Raspberry Pi 5, 10 bytes apart on +`suite_01_startup`. Mach-O keeps that name in the debug map instead of the `.o`, +which is why macOS looked clean while carrying the same defect. #7135 +content-addressed the `.ll` basename. + +Check it before trusting any object-level result on a host you have not measured +on: + +```bash +python3 scripts/compiler_output_regression.py census-determinism \ + --perry --repeat 2 --jobs 4 +``` + +The knob-isolation gate runs the same control inline and **fails** on a +disagreement. It used to skip its emission half instead, which meant the half +that caught the `PERRY_CANONICAL_STR_LOCALS` defect could not run on Linux at +all — the host where object-hash A/B is most useful, because it is the one with +an unprivileged instruction-retired counter. One documented exception, downward only: `PERRY_INT_VALUED_LOCALS=0` lowers `canonical-i32` on `fixture_int_valued_ta` (3 → 2), because diff --git a/changelog.d/7133-repsel-knob-isolation.md b/changelog.d/7133-repsel-knob-isolation.md index 0ea878e791..b27c38b29c 100644 --- a/changelog.d/7133-repsel-knob-isolation.md +++ b/changelog.d/7133-repsel-knob-isolation.md @@ -54,8 +54,9 @@ cannot be selected. A knob that *raises* another representation's count is still a leak. - Object emission is **nondeterministic on aarch64 Linux** (the LLVM temp module - name embeds pid + nanotime and lands in the ELF object — filed as #7131), so - the emission half detects the host and skips rather than reporting 26 phantom - diffs. `--require-emission` turns that into a failure where determinism is - expected. + Object emission was **nondeterministic on aarch64 Linux** when this gate + landed (the temp `.ll` name embedded pid + nanotime and clang records a unit's + source basename into the ELF object — filed as #7131), so the emission half + detected the host and skipped rather than reporting 26 phantom diffs. #7135 + fixed the cause and the skip was removed: a determinism disagreement is now a + hard failure on every host. diff --git a/changelog.d/7135-deterministic-llvm-temp-names.md b/changelog.d/7135-deterministic-llvm-temp-names.md index b1b3ae6607..19b8ab4aff 100644 --- a/changelog.d/7135-deterministic-llvm-temp-names.md +++ b/changelog.d/7135-deterministic-llvm-temp-names.md @@ -12,8 +12,8 @@ rayon workers never race the clang output file (#509). Writes go through a unique `.tmp` + rename so concurrent same-content producers never leave a partial file. -This restores object-hash A/B on Linux (`repsel_census`, `census-knob-isolation ---require-emission`, and every "did emission change" claim). +This restores object-hash A/B on Linux (`repsel_census`, `census-knob-isolation`, +and every "did emission change" claim). Content-addressed `.ll` files are no longer unlinked after a successful compile: concurrent identical-IR workers share that path, so a per-call delete could race diff --git a/crates/perry-codegen/src/linker.rs b/crates/perry-codegen/src/linker.rs index 4dad530820..0b79fc551f 100644 --- a/crates/perry-codegen/src/linker.rs +++ b/crates/perry-codegen/src/linker.rs @@ -30,6 +30,23 @@ static CLANG_PROBE: OnceLock> = OnceLock::new(); /// in the `.ll` name made two identical compiles produce different objects on /// Linux (#7131). The `.ll` is content-addressed instead; uniqueness of /// concurrent same-content writes is handled by an atomic rename. +/// +/// Which names actually reach the object, measured rather than assumed +/// (aarch64 Debian clang 19.1.7, ELF, no `-g`) — so nobody has to re-derive +/// this when reviewing a temp-path change: +/// +/// | name | recorded in the `.o`? | +/// |---------------------------------|--------------------------------| +/// | `.ll` **basename** | YES — `STT_FILE` in `.symtab` | +/// | `.ll` **directory**, process CWD| no (needs DWARF, i.e. `-g`) | +/// | `-o` output path | no | +/// | `ld -r` input / output paths | no (`compile_units_to_object`) | +/// +/// That is the whole reason only the `.ll` basename had to change: the counter +/// may stay in every *output* name, where it costs nothing and still closes +/// #509. The one caveat is `PERRY_DEBUG_SYMBOLS`, which adds `-g` and pulls the +/// absolute `.ll` path plus `DW_AT_comp_dir` into DWARF — objects built with it +/// are reproducible only for a fixed `TMPDIR` and working directory. static TEMP_NONCE_COUNTER: AtomicU64 = AtomicU64::new(0); /// FNV-1a 64-bit over `ll_text`. Stable across platforms and rustc versions From 7e5e981ae28f82dd9d55f5f111729c299c2a3b87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 31 Jul 2026 21:58:46 +0200 Subject: [PATCH 3/8] fix(codegen): restore per-process uniqueness of the LLVM temp OBJECT path (#7131) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #7135 content-addressed both temp names. The .ll had to become a pure function of the IR — clang records a unit's source basename into the ELF object, which was #7131. The .o did not, and lost the pid it used to carry. TEMP_NONCE_COUNTER is per-process state and every process starts it at 0, so two 'perry' processes compiling identical IR both chose perry_llvm__0.o. compile_ll_to_object deletes the object once it has read it, so they deleted it out from under each other. Measured on macOS at a3b31c0d8: 8 of 12 concurrent same-source compiles failed with Failed to read clang output at .../perry_llvm_eee31bbdd9dc24a5_0.o This is #509 again, one scope out — the pid that used to prevent it was removed as collateral. The output path is recorded nowhere (measured: clang records only the .ll BASENAME, as an STT_FILE symbol; not the directory, not the CWD, not -o, and ld -r records neither its inputs nor its output), so uniquifiers are free on the .o and mandatory. Same for the atomic-write staging .tmp, which two processes also reached with the same hash and the same counter. cargo fmt: linker.rs was left unformatted by #7135, so lint is red on main. --- crates/perry-codegen/src/linker.rs | 132 ++++++++++++++++++++++++----- 1 file changed, 111 insertions(+), 21 deletions(-) diff --git a/crates/perry-codegen/src/linker.rs b/crates/perry-codegen/src/linker.rs index 0b79fc551f..b460827dc8 100644 --- a/crates/perry-codegen/src/linker.rs +++ b/crates/perry-codegen/src/linker.rs @@ -62,27 +62,61 @@ fn ll_content_hash(ll_text: &str) -> u64 { h } -/// Content-addressed `.ll` path + unique `.o` path under `tmp_dir`. +/// Content-addressed `.ll` path + per-process-unique `.o` path under `tmp_dir`. /// -/// The `.ll` basename is a function of the IR bytes alone so two compiles of -/// the same module record the same source name in the object (Linux ELF -/// determinism, #7131). The `.o` basename still carries a per-call counter so -/// concurrent workers never race the clang output file (#509). -/// Returns `(ll_path, obj_path, counter)` — `counter` is also used for the -/// atomic-write staging filename. -fn llvm_temp_paths(tmp_dir: &Path, ll_text: &str) -> (PathBuf, PathBuf, u64) { +/// The two names are asymmetric **on purpose**, and each half has already been +/// got wrong once: +/// +/// * The `.ll` basename is a function of the IR bytes ALONE. clang records a +/// translation unit's source basename into the object, so anything else in +/// this name (pid, clock) lands in the shipped bytes — that was #7131. +/// * The `.o` basename must be unique per *process as well as* per call. The +/// output path is not recorded anywhere, so uniquifiers are free here, and +/// they are mandatory: `compile_ll_to_object` deletes the object once it has +/// read it, so two concurrent `perry` processes compiling identical IR that +/// agree on the name will delete it out from under each other. The counter +/// alone does not achieve this — it is per-process state, and every process +/// starts it at 0, so two processes with the same IR both pick +/// `..._0.o`. Measured before this was fixed: **8 of 12** concurrent +/// same-source compiles failed with "Failed to read clang output … No such +/// file or directory". This is #509 again, one scope out. +/// +/// `pid` and `counter` are parameters rather than read in here so the property +/// above is testable without spawning processes. +fn llvm_temp_paths_for( + tmp_dir: &Path, + ll_text: &str, + pid: u32, + counter: u64, +) -> (PathBuf, PathBuf) { let hash = ll_content_hash(ll_text); - let counter = TEMP_NONCE_COUNTER.fetch_add(1, Ordering::Relaxed); let ll_path = tmp_dir.join(format!("perry_llvm_{hash:016x}.ll")); - let obj_path = tmp_dir.join(format!("perry_llvm_{hash:016x}_{counter:x}.o")); - (ll_path, obj_path, counter) + let obj_path = tmp_dir.join(format!("perry_llvm_{hash:016x}_{pid:x}_{counter:x}.o")); + (ll_path, obj_path) +} + +/// `llvm_temp_paths_for` with this process's pid and the next counter value. +/// Returns `(ll_path, obj_path, pid, counter)` — the last two also name the +/// atomic-write staging file. +fn llvm_temp_paths(tmp_dir: &Path, ll_text: &str) -> (PathBuf, PathBuf, u32, u64) { + let pid = std::process::id(); + let counter = TEMP_NONCE_COUNTER.fetch_add(1, Ordering::Relaxed); + let (ll_path, obj_path) = llvm_temp_paths_for(tmp_dir, ll_text, pid, counter); + (ll_path, obj_path, pid, counter) +} + +/// Staging name for the atomic `.ll` write. Must be unique per process for the +/// same reason the `.o` is: two processes holding identical IR reach this with +/// the same content hash and the same counter value. +fn ll_staging_path(ll_path: &Path, pid: u32, counter: u64) -> PathBuf { + ll_path.with_extension(format!("ll.tmp.{pid:x}.{counter:x}")) } /// Write `ll_text` to a content-addressed path. Concurrent workers with the /// same IR may race; we write via a unique `.tmp` then `rename` into place so /// readers never see a partial file. A lost race (dest already exists) is fine /// — the winner already wrote the same content. -fn write_ll_atomically(ll_path: &Path, ll_text: &str, counter: u64) -> Result<()> { +fn write_ll_atomically(ll_path: &Path, ll_text: &str, pid: u32, counter: u64) -> Result<()> { // Fast path: already present (common under parallel multi-module compile // when two units share nothing but we re-hit the same hash only on true // content match — overwrite is still safe because the content is identical). @@ -96,7 +130,7 @@ fn write_ll_atomically(ll_path: &Path, ll_text: &str, counter: u64) -> Result<() } } } - let tmp = ll_path.with_extension(format!("ll.tmp.{counter}")); + let tmp = ll_staging_path(ll_path, pid, counter); { let mut f = fs::File::create(&tmp) .with_context(|| format!("Failed to create temp .ll file at {}", tmp.display()))?; @@ -113,9 +147,8 @@ fn write_ll_atomically(ll_path: &Path, ll_text: &str, counter: u64) -> Result<() return Ok(()); } } - fs::write(ll_path, ll_text.as_bytes()).with_context(|| { - format!("Failed to write temp .ll file at {}", ll_path.display()) - }) + fs::write(ll_path, ll_text.as_bytes()) + .with_context(|| format!("Failed to write temp .ll file at {}", ll_path.display())) } } } @@ -384,8 +417,8 @@ pub fn compile_ll_to_object(ll_text: &str, target_triple: Option<&str>) -> Resul let tmp_dir = env::temp_dir(); // #7131: content-address the `.ll` basename (clang embeds it into the // object on ELF). #509: keep the `.o` unique via the per-call counter. - let (ll_path, obj_path, write_nonce) = llvm_temp_paths(&tmp_dir, ll_text); - write_ll_atomically(&ll_path, ll_text, write_nonce)?; + let (ll_path, obj_path, write_pid, write_nonce) = llvm_temp_paths(&tmp_dir, ll_text); + write_ll_atomically(&ll_path, ll_text, write_pid, write_nonce)?; let plan = build_clang_compile_plan( clang.clone(), @@ -1537,8 +1570,8 @@ mod tests { // basename still differs via the counter. let tmp = env::temp_dir(); let ir = "define void @f() {\n ret void\n}\n"; - let (ll_a, obj_a, _) = llvm_temp_paths(&tmp, ir); - let (ll_b, obj_b, _) = llvm_temp_paths(&tmp, ir); + let (ll_a, obj_a, _, _) = llvm_temp_paths(&tmp, ir); + let (ll_b, obj_b, _, _) = llvm_temp_paths(&tmp, ir); assert_eq!( ll_a.file_name(), ll_b.file_name(), @@ -1550,7 +1583,7 @@ mod tests { ".o basenames must stay unique across calls (#509)" ); // Different IR → different .ll basename. - let (ll_c, _, _) = llvm_temp_paths(&tmp, "define void @g() {\n ret void\n}\n"); + let (ll_c, _, _, _) = llvm_temp_paths(&tmp, "define void @g() {\n ret void\n}\n"); assert_ne!(ll_a.file_name(), ll_c.file_name()); // No pid / wall-clock digits of variable width — only hex hash. let name = ll_a.file_name().unwrap().to_string_lossy(); @@ -1568,6 +1601,63 @@ mod tests { ); } + #[test] + fn object_temp_name_is_unique_across_processes_but_ll_is_not() { + // The regression this test exists for: #7135 content-addressed BOTH + // temp names, so the `.o` lost the pid it used to carry. Two `perry` + // processes compiling identical IR then agreed on the object path — + // and `compile_ll_to_object` deletes the object after reading it, so + // they deleted each other's. Measured on macOS before the fix: 8 of 12 + // concurrent same-source compiles failed with + // Failed to read clang output at …/perry_llvm__0.o + // Both processes start TEMP_NONCE_COUNTER at 0, so the counter cannot + // separate them; only the pid can. + let tmp = env::temp_dir(); + let ir = "define void @f() {\n ret void\n}\n"; + + // Same IR, same counter, DIFFERENT process. + let (ll_p1, obj_p1) = llvm_temp_paths_for(&tmp, ir, 1111, 0); + let (ll_p2, obj_p2) = llvm_temp_paths_for(&tmp, ir, 2222, 0); + assert_eq!( + ll_p1.file_name(), + ll_p2.file_name(), + "the .ll is what clang records into the object; it must stay a pure \ + function of the IR across processes (#7131)" + ); + assert_ne!( + obj_p1.file_name(), + obj_p2.file_name(), + "two processes with identical IR must NOT share an object path — \ + they delete it out from under each other (#509 across processes)" + ); + + // Same process, different call: the counter still has to separate + // in-process rayon workers. + let (_, obj_c0) = llvm_temp_paths_for(&tmp, ir, 1111, 0); + let (_, obj_c1) = llvm_temp_paths_for(&tmp, ir, 1111, 1); + assert_ne!(obj_c0.file_name(), obj_c1.file_name()); + + // The atomic-write staging name needs the same separation: both + // processes reach it with the same hash and the same counter, and + // `File::create` truncates. + assert_ne!( + ll_staging_path(&ll_p1, 1111, 0).file_name(), + ll_staging_path(&ll_p1, 2222, 0).file_name(), + "staging .tmp name must be per-process" + ); + assert_ne!( + ll_staging_path(&ll_p1, 1111, 0).file_name(), + ll_staging_path(&ll_p1, 1111, 1).file_name(), + "staging .tmp name must be per-call" + ); + + // …and the staging file must never be mistaken for the real `.ll`. + assert_ne!( + ll_staging_path(&ll_p1, 1111, 0).file_name(), + ll_p1.file_name() + ); + } + #[test] fn ll_content_hash_is_stable_for_fixed_input() { // Pin the FNV-1a value so a future hash swap is intentional. From 8c94081c5b3f228e56da6f52c4b3a6600cefd45b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 31 Jul 2026 22:03:48 +0200 Subject: [PATCH 4/8] changelog: #7131 Linux emission determinism (PR #7140) --- .../7140-linux-emission-determinism.md | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 changelog.d/7140-linux-emission-determinism.md diff --git a/changelog.d/7140-linux-emission-determinism.md b/changelog.d/7140-linux-emission-determinism.md new file mode 100644 index 0000000000..0e0ffcfd87 --- /dev/null +++ b/changelog.d/7140-linux-emission-determinism.md @@ -0,0 +1,87 @@ +**codegen:** the LLVM temp *object* path is per-process unique again, and +emission determinism is now a gate on every host instead of a Linux caveat +(#7131). + +### The regression #7135 left behind + +#7135 fixed #7131 by content-addressing the temp `.ll` name. It content-addressed +the temp `.o` name too, and that half was wrong: the `.o` used to carry the pid, +and lost it. + +`TEMP_NONCE_COUNTER` is per-process state and every process starts it at `0`, so +two `perry` processes compiling identical IR both chose +`perry_llvm__0.o`. `compile_ll_to_object` deletes the object once it has +read it — so they deleted it out from under each other. Measured at `a3b31c0d8`, +four concurrent compiles of one census fixture, three rounds: **8 of 12 failed** + +``` +Failed to read clang output at /tmp/perry_llvm_eee31bbdd9dc24a5_0.o: +No such file or directory (os error 2) +``` + +This is #509 again, one scope out: the pid that used to prevent it was removed +as collateral. It bites any parallel build of the same input — including every +A/B harness in `scripts/compiler_output_harness/`. After the fix, 0 of 12. + +The atomic-write staging file had the same defect (two processes reach +`…​.ll.tmp.0` with the same hash and the same counter, and `File::create` +truncates), and is fixed the same way. + +### Which names actually reach the object + +The two names are asymmetric on purpose, and both halves have now been got wrong +once. Measured rather than assumed — aarch64 Debian clang 19.1.7, ELF, no `-g`: + +| name | recorded in the `.o`? | +|---|---| +| `.ll` **basename** | **yes** — `STT_FILE` in `.symtab` | +| `.ll` directory, process CWD | no (needs DWARF, i.e. `-g`) | +| `-o` output path | no | +| `ld -r` input / output paths | no | + +So the `.ll` must be a pure function of the IR, and uniquifiers are both free +and mandatory on every *output* name. That table is now a comment on +`TEMP_NONCE_COUNTER` so the next temp-path change can be reviewed without +re-deriving it. + +Residual, stated plainly: under `PERRY_DEBUG_SYMBOLS` clang emits DWARF, which +pulls the absolute `.ll` path and `DW_AT_comp_dir` into the object. Those builds +are reproducible only for a fixed `TMPDIR` and working directory. Nothing else +in the emission path is known to vary. + +### The gate + +`census-knob-isolation` detected a nondeterministic host and **skipped** its +emission half. That workaround existed because of #7131 — so the half of the gate +that caught the `PERRY_CANONICAL_STR_LOCALS` leak could not run on Linux at all, +the host where object-hash A/B is most useful (it is the one with an unprivileged +instruction-retired counter). The skip and its `--require-emission` escape hatch +are gone: a determinism disagreement is a hard failure, on every host, before any +knob is judged. + +- **`census-determinism`** (new) — compile the census corpus N times and compare + the bytes; the standalone instrument for establishing that object-hash evidence + is valid on the host you are measuring on. Repeats run concurrently on purpose: + identical IR now shares one content-addressed `.ll`, so racing it is part of the + subject — and that is what caught the collision above. +- Runs on the `ubuntu-latest` `repsel-census` CI job, the ELF host that can + actually observe a relapse, alongside both verdict self-tests. + +### Verified + +Red-then-green on a Raspberry Pi 5 (aarch64 Linux) across all 26 census +workloads, three arms built sequentially from one target dir with distinct +binary hashes: + +| arm | serial | concurrent | +|---|---|---| +| `22367565f` (before #7135) | **26/26 nondeterministic** | — | +| `a3b31c0d8` (#7135 merged) | 26/26 identical | **fails: object-path collision** | +| this branch | 26/26 identical | 26/26 identical | + +macOS (`Darwin arm64`): 26/26 workloads × 3 compiles → one hash each, so the +#7039 closure-iteration-order fix has not regressed. Mach-O does not record the +`.ll` basename at all, which is why the original defect was invisible there. + +`cargo fmt`: `linker.rs` was left unformatted by #7135, so `lint` is red on +`main` independently of this change; formatting it is included here. From 2c83dada9c3fcd31f979abc39f4551c20a7e1862 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 31 Jul 2026 22:18:44 +0200 Subject: [PATCH 5/8] changelog: fold in the measured Linux + macOS numbers (#7131) --- .../7140-linux-emission-determinism.md | 45 +++++++++++++++---- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/changelog.d/7140-linux-emission-determinism.md b/changelog.d/7140-linux-emission-determinism.md index 0e0ffcfd87..ef46712f14 100644 --- a/changelog.d/7140-linux-emission-determinism.md +++ b/changelog.d/7140-linux-emission-determinism.md @@ -21,7 +21,18 @@ No such file or directory (os error 2) This is #509 again, one scope out: the pid that used to prevent it was removed as collateral. It bites any parallel build of the same input — including every -A/B harness in `scripts/compiler_output_harness/`. After the fix, 0 of 12. +A/B harness in `scripts/compiler_output_harness/`. + +Both platforms, four concurrent compiles of one census fixture, before → after: + +| host | `a3b31c0d8` | this branch | +|---|---|---| +| macOS `arm64` | **8 / 12 failed** | 0 / 12 | +| Raspberry Pi 5, aarch64 Linux | **3 / 16 failed** | 0 / 16 | + +It is a race, so the rate is timing-dependent — the slower host loses fewer. It +is also why this was found by running the determinism repeats *concurrently* +rather than in sequence: a serial check never opens the window. The atomic-write staging file had the same defect (two processes reach `…​.ll.tmp.0` with the same hash and the same counter, and `File::create` @@ -70,18 +81,34 @@ knob is judged. ### Verified Red-then-green on a Raspberry Pi 5 (aarch64 Linux) across all 26 census -workloads, three arms built sequentially from one target dir with distinct -binary hashes: +workloads. Three arms built sequentially from one target dir — same toolchain by +construction — with three distinct binary hashes: -| arm | serial | concurrent | -|---|---|---| -| `22367565f` (before #7135) | **26/26 nondeterministic** | — | -| `a3b31c0d8` (#7135 merged) | 26/26 identical | **fails: object-path collision** | -| this branch | 26/26 identical | 26/26 identical | +| arm | `census-determinism` | +|---|---| +| `22367565f` (before #7135) | exit 1 — **26/26 nondeterministic** | +| `a3b31c0d8` (#7135 as merged) | exit 0 — 26/26 identical | +| this branch | exit 0 — 26/26 identical (also at `--repeat 3 --jobs 3`) | + +`suite_01_startup`, compiled twice serially, with the mechanism visible: + +``` +pre : objects DIFFER in 12 bytes + STT_FILE perry_llvm_217502_1785528949373123236_0.ll + STT_FILE perry_llvm_217533_1785528951945773193_0.ll +post: objects IDENTICAL + STT_FILE perry_llvm_2791e842224ea99c.ll (both runs) +``` macOS (`Darwin arm64`): 26/26 workloads × 3 compiles → one hash each, so the #7039 closure-iteration-order fix has not regressed. Mach-O does not record the -`.ll` basename at all, which is why the original defect was invisible there. +`.ll` basename at all (verified directly), which is why the original defect was +invisible there. + +**No behavioural change**, by the strongest available measure: across all 26 +workloads the objects emitted by `a3b31c0d8` and by this branch are +**byte-identical, 0/26 differences**. This change renames temp files and nothing +else, and the emitted bytes say so. `cargo fmt`: `linker.rs` was left unformatted by #7135, so `lint` is red on `main` independently of this change; formatting it is included here. From 7db975407c22093afbf8eac90780bec8e22271a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 31 Jul 2026 22:20:29 +0200 Subject: [PATCH 6/8] refactor(harness): one digest_objects, and correct two claims the measurements disproved MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - knob-isolation and the determinism check had identical private _digest implementations; share one so 'the objects are the same' cannot drift - the module docstring said output names 'may keep their uniquifying counter'. The counter alone is not enough — it is per-process state every process starts at 0 — which is the collision this check then found - Mach-O does not keep the .ll basename in the debug map, it does not record it at all (verified directly); the byte delta is ~10, measured 12 --- .../repsel_determinism.py | 32 ++++++++++++------- .../repsel_knob_isolation.py | 11 ++----- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/scripts/compiler_output_harness/repsel_determinism.py b/scripts/compiler_output_harness/repsel_determinism.py index 68ccb0a9d5..541e639821 100644 --- a/scripts/compiler_output_harness/repsel_determinism.py +++ b/scripts/compiler_output_harness/repsel_determinism.py @@ -13,10 +13,10 @@ records the **source basename** of the translation unit into the object as an `STT_FILE` symbol, so two identical compiles differed by exactly the digits of the pid and the clock (#7131 — 26/26 census workloads nondeterministic on a -Raspberry Pi 5, 10 bytes apart on `suite_01_startup`). Mach-O keeps that name -in the debug map rather than the `.o`, which is why macOS looked clean while -carrying the same defect. `#7135` fixed it by content-addressing the `.ll` -basename; this module is the check that says so, and keeps saying so. +Raspberry Pi 5, ~10 bytes apart on `suite_01_startup`). Mach-O does not record +that name in the `.o` at all, which is why macOS looked clean while carrying the +same defect. #7135 fixed it by content-addressing the `.ll` basename; this +module is the check that says so, and keeps saying so. Measured properties of the ELF path, so a future reader does not have to re-derive which names matter (aarch64 Debian clang 19.1.7, no `-g`): @@ -28,8 +28,13 @@ * `ld -r` (the multi-codegen-unit merge, #5391) records neither its input nor its output paths. -So the `.o` and partial-link staging names may keep their uniquifying counter — -only the `.ll` name had to become a function of content. +So only the `.ll` name had to become a function of content — and, symmetrically, +every *output* name must stay unique per **process**, not merely per call. #7135 +content-addressed both and dropped the pid from the `.o`; two `perry` processes +compiling identical IR then agreed on the object path and deleted it out from +under each other, because the counter that was left is per-process state that +every process starts at 0. This check found that, because it runs its repeats +concurrently — a serial check never opens the window. Why this is a check and not a comment: the property is invisible on the host most of this project's compiler work happens on. A macOS-only reviewer cannot @@ -40,7 +45,10 @@ from __future__ import annotations import argparse +import hashlib import platform +import shutil +import tempfile from concurrent.futures import ThreadPoolExecutor from pathlib import Path from typing import Any, Callable @@ -85,9 +93,12 @@ def nondeterminism_report(varied: list[str], total: int, *, repeat: int = 2) -> ) -def _digest_objects(paths: list[str]) -> str: - import hashlib +def digest_objects(paths: list[str]) -> str: + """SHA-256 over every object a compile emitted, in a stable order. + Shared with the knob-isolation gate so "the objects are the same" means one + thing in this package rather than two implementations that could drift. + """ h = hashlib.sha256() for path in sorted(paths): h.update(Path(path).read_bytes()) @@ -155,9 +166,6 @@ def check_determinism(args: argparse.Namespace) -> int: print(f"host: {platform.system()} {platform.machine()}") print(f"corpus: {len(workloads)} workload(s) x {repeat} compile(s)\n") - import shutil - import tempfile - tmp = Path(tempfile.mkdtemp(prefix="repsel-determinism-")) try: # Repeats run through the same pool as the corpus on purpose: two @@ -174,7 +182,7 @@ def run(job: tuple[dict[str, Any], int]) -> tuple[str, str]: timeout=args.compile_timeout, object_out=tmp / workload["name"] / str(index) / "out.o", ) - return workload["name"], _digest_objects(census["objects"]) + return workload["name"], digest_objects(census["objects"]) with ThreadPoolExecutor(max_workers=max(1, args.jobs)) as pool: observed = list(pool.map(run, jobs)) diff --git a/scripts/compiler_output_harness/repsel_knob_isolation.py b/scripts/compiler_output_harness/repsel_knob_isolation.py index 9e82303e78..483ce89652 100644 --- a/scripts/compiler_output_harness/repsel_knob_isolation.py +++ b/scripts/compiler_output_harness/repsel_knob_isolation.py @@ -79,13 +79,13 @@ from .capture import resolve_perry from .common import HarnessError, REPO_ROOT -from .repsel_determinism import nondeterminism_report from .repsel_census import ( CENSUS_KEYS, DEFAULT_BASELINE, compile_and_census, load_baseline, ) +from .repsel_determinism import digest_objects, nondeterminism_report #: `SpecParamRep::label()` spelling for a canonical-i32 parameter slot. @@ -219,13 +219,6 @@ def _spec_abi_i32_slots(report: dict[str, Any]) -> int: return total -def _digest(paths: list[str]) -> str: - h = hashlib.sha256() - for path in sorted(paths): - h.update(Path(path).read_bytes()) - return h.hexdigest() - - def _compile_arm( perry: list[str], source: Path, @@ -249,7 +242,7 @@ def _compile_arm( "spec-abi-i32-slot": _spec_abi_i32_slots(report), "consumed-receiver": int(census.get("consumed_receiver", 0)), }, - digest=_digest(census["objects"]), + digest=digest_objects(census["objects"]), objects=list(census["objects"]), ) From e6dc4428e4d5922265b885c6bc5f312292165c25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 31 Jul 2026 22:27:40 +0200 Subject: [PATCH 7/8] docs(census): correct the Mach-O claim to what was measured (#7131) The 'debug map' explanation came from the issue and is not what I observed: Mach-O does not record the .ll basename in the .o at all (no -g, no debug map). Also the real byte delta and the concurrency note. --- benchmarks/repsel_census/README.md | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/benchmarks/repsel_census/README.md b/benchmarks/repsel_census/README.md index 5cc1bafd03..0a54f586b0 100644 --- a/benchmarks/repsel_census/README.md +++ b/benchmarks/repsel_census/README.md @@ -247,19 +247,30 @@ Every object comparison on this page assumes the compiler is a function of its inputs. On ELF it was not: the temp `.ll` name carried pid + wall-clock nanos, and clang records a translation unit's **source basename** into the object as an `STT_FILE` symbol, so two identical compiles differed by exactly those digits — -26/26 census workloads on a Raspberry Pi 5, 10 bytes apart on -`suite_01_startup`. Mach-O keeps that name in the debug map instead of the `.o`, -which is why macOS looked clean while carrying the same defect. #7135 -content-addressed the `.ll` basename. +26/26 census workloads on a Raspberry Pi 5, 12 bytes apart on `suite_01_startup`: + +``` +run1 STT_FILE perry_llvm_217502_1785528949373123236_0.ll +run2 STT_FILE perry_llvm_217533_1785528951945773193_0.ll +``` + +Mach-O does not record that name in the `.o` at all, which is why macOS looked +clean while carrying the same defect — and why this cannot be reviewed on a Mac. +#7135 content-addressed the `.ll` basename. Check it before trusting any object-level result on a host you have not measured -on: +on (the whole corpus twice is ~7 s): ```bash python3 scripts/compiler_output_regression.py census-determinism \ --perry --repeat 2 --jobs 4 ``` +Repeats run **concurrently** on purpose. Identical IR now shares one +content-addressed `.ll`, so racing it is part of the subject — and that is what +caught #7135's other half, where the `.o` name lost its pid and two `perry` +processes compiling the same file deleted each other's object. + The knob-isolation gate runs the same control inline and **fails** on a disagreement. It used to skip its emission half instead, which meant the half that caught the `PERRY_CANONICAL_STR_LOCALS` defect could not run on Linux at From 169296368bc45978cb587f9ab75a4fc94e9e3b37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 31 Jul 2026 22:46:28 +0200 Subject: [PATCH 8/8] changelog: matrix verdict, FAIL=0 over 546 rows (#7131) --- changelog.d/7140-linux-emission-determinism.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/changelog.d/7140-linux-emission-determinism.md b/changelog.d/7140-linux-emission-determinism.md index ef46712f14..5138d394a2 100644 --- a/changelog.d/7140-linux-emission-determinism.md +++ b/changelog.d/7140-linux-emission-determinism.md @@ -108,7 +108,10 @@ invisible there. **No behavioural change**, by the strongest available measure: across all 26 workloads the objects emitted by `a3b31c0d8` and by this branch are **byte-identical, 0/26 differences**. This change renames temp files and nothing -else, and the emitted bytes say so. +else, and the emitted bytes say so. The GC x representation-selection matrix +against the pinned Node 26.5.1 oracle agrees — +`gc_repsel_matrix.sh --arms all --pressure 8` on the Pi: +`PASS=426 UNVER=119 XFAIL=1 FAIL=0` over 546 rows. `cargo fmt`: `linker.rs` was left unformatted by #7135, so `lint` is red on `main` independently of this change; formatting it is included here.