From 8091863873d71fa2ecfcfef842adcbe6120e1653 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 12 Aug 2026 19:30:47 +0200 Subject: [PATCH 1/4] fix(ci): read Rust sources as UTF-8 in the Windows GC audits (#7977) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `scripts/check_thread_locals.py` walked `crates/perry-runtime/src` with a bare `Path.read_text()`, which decodes with `locale.getencoding()` — cp1252 on a GitHub Windows runner. Fifteen runtime sources carry a byte cp1252 cannot map (0x81/0x8d/0x8f/0x90/0x9d); `i18n.rs` is reached first, at offset 31552 with 923 newlines before it, so `core.autocrlf` puts the failure at position 32475 — matching the reported traceback to the byte. That is the FIRST step of `windows-build`, so the seven steps behind it were `skipped` on every PR: no Windows build of the runtime, stdlib or either Windows UI crate, no Windows run of the `perry-runtime` unit tests, no Windows parity smoke, no VERSIONINFO check, no COFF-trimming test. #7882 fixed the path-separator half of this class in this same file and the encoding in `gc_runtime_root_holders.py`; these four readers were missed. - route all reads/writes through `read_source()` / `write_source()` helpers that pass `encoding="utf-8"` (and `newline=""`, so `--update` is byte-stable across hosts). The verified `files` map is unchanged. - fix a latent instance in `gc_runtime_root_holders.py`'s fixture writer and a bare `read_text()` in `tests/test_gc_ratchet.py`. - add `scripts/check_locale_independent_io.py`: an AST scan of exactly the six Python files the Windows audit step runs, failing on locale-defaulted text I/O. Static rather than `PYTHONWARNDEFAULTENCODING` because that only fires on executed calls, and because `test_gc_ratchet.py` embeds `open(...)` inside probe source *literals* that an AST correctly ignores and a grep would not. - run it in `lint` (Linux, per-PR, already required) so the class is caught before it reaches Windows, and set `PYTHONUTF8=1` on the Windows step as belt-and-braces. Validated: main's version exits 1 with `UnicodeDecodeError` under a non-UTF-8 locale and the fixed version exits 0; the full eight-command Windows audit sequence passes under that locale; and replanting the exact defect makes the new gate exit 1 naming the call site, so a green run means the detector works. --- .github/workflows/test.yml | 26 +++ gc-handoff/CIFIX-NOTES.md | 114 ++++++++++++ scripts/check_locale_independent_io.py | 244 +++++++++++++++++++++++++ scripts/check_thread_locals.py | 64 +++++-- scripts/gc_runtime_root_holders.py | 6 +- tests/test_gc_ratchet.py | 2 +- 6 files changed, 434 insertions(+), 22 deletions(-) create mode 100644 gc-handoff/CIFIX-NOTES.md create mode 100755 scripts/check_locale_independent_io.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5c08ebe694..462b0eff0a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -263,6 +263,23 @@ jobs: python3 scripts/check_gc_doc_claims.py --self-test python3 scripts/check_gc_doc_claims.py + # #7977. The audits above ALSO run as `windows-build`'s first step, where + # a bare `read_text()` decodes cp1252 and dies on the 15 runtime sources + # carrying 0x81/0x8d/0x8f/0x90/0x9d. That took the whole Windows job down + # — including the only Windows run of the perry-runtime unit tests — + # before any of it executed. #7882 fixed three of the four readers; this + # is what stops the fourth miss from being found on Windows again. + # + # It runs HERE, on Linux, in a REQUIRED context, precisely because the + # defect is invisible on Linux at runtime: the scan is static, so the + # class is caught per-PR rather than only when a Windows runner gets to + # it. `--self-test` plants each shape, so the checker can say no. + - name: Windows-portable text I/O in the Windows-CI audits + if: ${{ !cancelled() }} + run: | + python3 scripts/check_locale_independent_io.py --self-test + python3 scripts/check_locale_independent_io.py + # Node is a correctness input (see CLAUDE.md "TypeScript Parity Status"): # an oracle that cannot run a gap test drops it from the gate instead of # failing it. #6367 made `.node-version` the single pin, and the pin has @@ -1111,6 +1128,15 @@ jobs: - name: GC structural audits (Windows) shell: bash + env: + # #7977 belt-and-braces. These scripts read Rust sources that contain + # bytes cp1252 has no mapping for; the ACTUAL fix is an explicit + # `encoding="utf-8"` at every call site, enforced on Linux by + # `scripts/check_locale_independent_io.py` in `lint`. This makes a + # future miss in a script that gate does not yet cover degrade to + # "works anyway" instead of taking the whole job — and with it the + # only Windows run of the perry-runtime unit tests — down at step one. + PYTHONUTF8: "1" run: | set -euo pipefail python scripts/gc_runtime_root_holders.py --self-test diff --git a/gc-handoff/CIFIX-NOTES.md b/gc-handoff/CIFIX-NOTES.md new file mode 100644 index 0000000000..ba70fa39ea --- /dev/null +++ b/gc-handoff/CIFIX-NOTES.md @@ -0,0 +1,114 @@ +# CI gate repair: #7977, #7971, #7970 + +Working notes. Written incrementally; the PR body is the summary. + +--- + +## #7977 — `windows-build` red before it runs anything [FIXED] + +### What this gate covers (and what was therefore unprotected) + +`windows-build` is the **only Windows execution of anything** in per-PR CI. Its +step list, in order: + +1. GC structural audits (Windows) <- died here, 27 s in +2. Install Rust toolchain / LLVM 22 / Node +3. Build compiler + runtime + `perry-ui-windows` + `perry-ui-windows-winui` +4. **`perry-runtime` unit tests** (`RUST_TEST_THREADS=1`, #7356) <- the big one +5. Windows parity harness smoke +6. `perry.exe` VERSIONINFO resource check +7. COFF duplicate-symbol archive trimming test + +Steps 2-7 were `skipped` on **every** PR whose Windows run executed the current +`test.yml`. So for the duration: **no Windows compile of the runtime, the stdlib +or either Windows UI crate; no Windows run of the `perry-runtime` unit tests; no +Windows parity smoke; no VERSIONINFO check; no COFF-trimming test.** A Windows-only +miscompile, a Windows-only test failure, or a broken `perry-ui-windows` build would +all have merged green-by-skipping. + +### Root cause (confirmed to the byte) + +`scripts/check_thread_locals.py` reads Rust sources with a bare +`Path.read_text()`, which decodes with `locale.getencoding()` — **cp1252** on a +GitHub Windows runner. 15 files under `crates/perry-runtime/src` carry a byte +cp1252 has no mapping for (0x81/0x8d/0x8f/0x90/0x9d). Reproduced locally: + +``` +15 files undecodable as cp1252 + crates/perry-runtime/src/i18n.rs offset 31552 0x8d + crates/perry-runtime/src/intl/duration_format.rs offset 17349 0x81 + ... +i18n.rs: offset=31552 newlines_before=923 crlf_position=32475 (issue says 32475) +``` + +The issue's arithmetic is exact: `core.autocrlf` widens 923 `\n` to `\r\n`, so +31552 + 923 = **32475**, the position in the traceback. + +`#7882` ("make GC structural audits portable") fixed the *path-separator* half of +this class in this same file and the *encoding* in `gc_runtime_root_holders.py`, +but missed these four readers. Same shape, one file over. + +### Changed + +- `scripts/check_thread_locals.py` — all 5 reads and 15 writes now go through + `read_source()` / `write_source()` helpers that pass `encoding="utf-8"` + (and `newline=""` on write, so `--update` is byte-stable across hosts). + The gated content is **unchanged**: the `files` map the checker verifies is + identical before and after (asserted, see validation). +- `scripts/gc_runtime_root_holders.py` — one **latent** instance of the same + class in the self-test's fixture writer. ASCII today, so it has not bitten; + fixed with the rest. +- `tests/test_gc_ratchet.py:1369` — bare `read_text()` on the shipped baseline. +- `scripts/check_locale_independent_io.py` — **new gate** (below). +- `.github/workflows/test.yml` — new `lint` step running that gate; + `PYTHONUTF8: "1"` on the Windows audit step as belt-and-braces. + +### The new gate, and why it is static + +`scripts/check_locale_independent_io.py` AST-scans exactly the six Python files +`windows-build`'s audit step executes, and fails on `open()` / `read_text()` / +`write_text()` / `Path.open()` without an explicit `encoding=` (binary modes +exempt). + +Static, not `PYTHONWARNDEFAULTENCODING`, for two reasons: + +- `EncodingWarning` only fires on a call that **executes**. A bare `read_text()` + on an error path or in a subcommand CI does not invoke warns nobody and ships. +- `tests/test_gc_ratchet.py` writes Python probes as *string literals* containing + `open(...)`. Those are not calls; the AST correctly ignores them, a grep would not. + +It runs in **`lint`** — Linux, per-PR, and a **required** context — because the +defect is invisible on Linux at runtime. That is the point: the class is now +caught before it can reach Windows, rather than by taking the Windows job down. + +### Validation + +| check | result | +|---|---| +| reproduce main's failure under a non-UTF-8 locale | `UnicodeDecodeError: 'ascii' codec can't decode byte 0xe2`, exit **1** | +| same command, fixed | exit **0**, `thread-local policy OK: 173 hot declarations, 129 raw blocks in 91 recorded cold files` | +| full 8-command Windows audit sequence, non-UTF-8 locale | all **PASS** | +| `--update` output vs shipped allowlist | `files` map **identical** | +| **sabotage**: replant the exact #7977 defect | new gate exits **1** and names `check_thread_locals.py:115` | +| remove the replant | new gate exits **0** | +| new gate `--self-test` | OK — 6 flagged shapes, accepted shapes clean, scope asserted | + +The sabotage arm is the one that matters: a green run of this gate means the +detector works, not that nothing was tried. + +### Not changed (deliberate) + +- `_hot_declarations` in `thread_local_cold_allowlist.json` reads 163 against an + actual 173. **Pre-existing drift on `main`, not caused by this change**, and + `verify()` does not gate on that field — only on `files`. Left alone to keep + the diff reviewable; worth a one-line `--update` in a separate change. +- `gc_runtime_root_holders.py` raises `UnicodeEncodeError` when *printing* an + em-dash under an ASCII locale. Measured: its output **is** cp1252-encodable, so + this is an artifact of the stricter ASCII simulation and **not** a Windows + defect. `PYTHONUTF8=1` on the step covers it regardless. + +### Promotion + +Nothing promoted. `lint` is **already** a required context; this adds a step to it. +That is deliberate — CLAUDE.md hazard 2 is the step people forget, so the gate is +placed where that step does not exist. It is green locally on this tree. diff --git a/scripts/check_locale_independent_io.py b/scripts/check_locale_independent_io.py new file mode 100755 index 0000000000..31deb64cdb --- /dev/null +++ b/scripts/check_locale_independent_io.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +"""Keep host-locale text I/O out of the scripts CI runs on Windows (#7977). + +WHY THIS EXISTS +=============== + +`open()`, `Path.read_text()` and `Path.write_text()` decode and encode with +`locale.getencoding()` when no `encoding=` is given. On Linux and macOS that is +UTF-8, so a bare call is indistinguishable from a correct one; on a GitHub +Windows runner it is **cp1252**, which has no mapping for 0x81/0x8d/0x8f/0x90/ +0x9d. Fifteen files under `crates/perry-runtime/src` contain one of those bytes. + +`scripts/check_thread_locals.py::cfg_test_module_files` walked the crate with a +bare `read_text()` and died on `i18n.rs` with `UnicodeDecodeError: 'charmap' +codec can't decode byte 0x8d in position 32475` — offset 31552 plus the 923 +newlines `core.autocrlf` had widened, to the byte. That was the FIRST step of +`windows-build`, so the eleven steps behind it — including the only Windows run +of the `perry-runtime` unit tests — were `skipped` on every PR. + +#7882 ("make GC structural audits portable") fixed the path-separator half of +this class and the encoding in `gc_runtime_root_holders.py`, but missed this one +call site. This checker is what stops the next miss: a locale-dependent call is +now a **Linux** failure in the per-PR `lint` job, so it can no longer reach +Windows and take the job down before the audits it gates have run. + +WHY STATIC AND NOT `PYTHONWARNDEFAULTENCODING` +============================================== + +Python's own `EncodingWarning` (`-X warn_default_encoding`) reports the same +defect, and is used here as the self-test's oracle. But it only fires on a call +that actually *executes*: a bare `read_text()` on an error path, or in a +subcommand CI does not invoke, warns nobody and ships. The scan below reads the +AST, so an unexecuted branch is caught exactly like a hot one. + +Strings are not code: an `open(...)` inside a probe's source-code *literal* (as +in `tests/test_gc_ratchet.py`, which writes Python probes as text) is invisible +to the AST and correctly not flagged. That is the second reason not to grep. + +THIS CHECK IS DESIGNED TO BE ABLE TO FAIL +========================================= + +`--self-test` plants each flagged shape and each accepted shape in a synthetic +module and asserts the scan separates them, so the checker cannot quietly stop +being able to say no. It also asserts the scanned file list is non-empty and +that every named file exists — a scope that silently shrinks to nothing is +CLAUDE.md's fourth way a gate cannot fail. +""" + +from __future__ import annotations + +import argparse +import ast +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent + +# Exactly the Python that `windows-build`'s "GC structural audits (Windows)" +# step executes, plus this checker. Keep in lockstep with that step: a script +# added there and not here is unguarded, and `--self-test` fails if a name here +# stops existing. +SCANNED = ( + "scripts/check_locale_independent_io.py", + "scripts/check_thread_locals.py", + "scripts/check_gc_doc_claims.py", + "scripts/gc_runtime_root_holders.py", + "benchmarks/gc_ratchet/gc_ratchet.py", + "tests/test_gc_ratchet.py", +) + +# `open(...)` and the `Path` text helpers. `Path.open()` is included: it is the +# same defaulting. Binary modes are exempt (there is no encoding to get wrong), +# which the mode check below establishes. +TEXT_METHODS = {"read_text", "write_text", "open"} + + +def _keyword(call: ast.Call, name: str) -> ast.expr | None: + for kw in call.keywords: + if kw.arg == name: + return kw.value + if kw.arg is None: # `**kwargs` — cannot prove absence, treat as given. + return kw.value + return None + + +def _is_binary(call: ast.Call, func_name: str) -> bool: + """True when the call cannot have an encoding (binary mode).""" + if func_name == "read_text" or func_name == "write_text": + return False + mode: ast.expr | None = _keyword(call, "mode") + if mode is None: + # Positional mode: `open(p, "rb")` / `p.open("rb")`. + idx = 1 if func_name == "open" and not _is_method(call) else 0 + if func_name == "open" and _is_method(call): + idx = 0 + elif func_name == "open": + idx = 1 + if len(call.args) > idx: + mode = call.args[idx] + return isinstance(mode, ast.Constant) and isinstance(mode.value, str) and "b" in mode.value + + +def _is_method(call: ast.Call) -> bool: + return isinstance(call.func, ast.Attribute) + + +def _func_name(call: ast.Call) -> str | None: + if isinstance(call.func, ast.Attribute): + return call.func.attr + if isinstance(call.func, ast.Name): + return call.func.id + return None + + +def scan_source(src: str, rel: str) -> list[str]: + """Flagged call sites in one module, as human-readable problems.""" + problems: list[str] = [] + tree = ast.parse(src) + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + name = _func_name(node) + if name not in TEXT_METHODS: + continue + # A bare `open(...)` that is not `builtins.open` (e.g. `zipfile.open`) + # still defaults the same way when it is a text stream; flagging it is + # the safe direction and the fix is identical. + if _is_binary(node, name): + continue + if _keyword(node, "encoding") is not None: + continue + problems.append( + f"{rel}:{node.lineno}: `{name}(...)` without `encoding=`. " + f"It decodes with the host locale — cp1252 on a Windows runner, " + f"which cannot read the 15 runtime sources carrying 0x81/0x8d/" + f"0x8f/0x90/0x9d (#7977). Pass `encoding=\"utf-8\"`." + ) + return sorted(problems) + + +def verify(root: Path, scanned: tuple[str, ...]) -> list[str]: + problems: list[str] = [] + if not scanned: + return ["nothing was scanned — the gate's scope is empty (see hazard 4)"] + for rel in scanned: + path = root / rel + if not path.exists(): + problems.append( + f"{rel}: listed in SCANNED but missing. Either it moved (update " + f"the list) or the Windows audit step no longer runs it — a " + f"stale entry is scope nobody has to justify." + ) + continue + problems.extend(scan_source(path.read_text(encoding="utf-8"), rel)) + return problems + + +GOOD_SOURCE = ''' +import io +from pathlib import Path + +def ok(p: Path, q): + a = p.read_text(encoding="utf-8") + p.write_text(a, encoding="utf-8", newline="") + with open(q, "r", encoding="utf-8") as fh: + fh.read() + with open(q, "rb") as fh: # binary: no encoding to get wrong + fh.read() + with p.open("rb") as fh: + fh.read() + p.write_bytes(b"x") + embedded = """ + body = open(source).read() + with open(out, "w") as handle: + handle.write(body) + """ # a STRING, not a call — must not flag + return a, embedded +''' + +BAD_SHAPES = ( + ("read_text", "def f(p):\n return p.read_text()\n"), + ("write_text", "def f(p):\n p.write_text('x')\n"), + ("open-builtin", "def f(q):\n return open(q).read()\n"), + ("open-mode-w", "def f(q):\n return open(q, 'w')\n"), + ("path-open", "def f(p):\n return p.open()\n"), + ("unreached", "def f(p):\n if False:\n return p.read_text()\n return ''\n"), +) + + +def self_test() -> int: + failures: list[str] = [] + + if scan_source(GOOD_SOURCE, "good.py"): + failures.append( + "the accepted shapes were flagged: " + "; ".join(scan_source(GOOD_SOURCE, "good.py")) + ) + for label, src in BAD_SHAPES: + if not scan_source(src, "bad.py"): + failures.append(f"a bare `{label}` call passed the scan") + + # Scope liveness: an empty or stale list is the failure mode this gate is + # least able to notice about itself. + if verify(REPO, ()) == []: + failures.append("an empty scan scope was reported clean") + missing = verify(REPO, ("scripts/definitely-not-here.py",)) + if not missing: + failures.append("a missing scanned file was reported clean") + + if failures: + for f in failures: + print(f"self-test FAILED: {f}", file=sys.stderr) + return 1 + print( + f"check_locale_independent_io self-test: OK " + f"({len(BAD_SHAPES)} flagged shapes, accepted shapes clean, scope asserted)" + ) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--self-test", action="store_true", help="prove the checker can say no") + args = parser.parse_args() + if args.self_test: + return self_test() + + problems = verify(REPO, SCANNED) + if problems: + print("locale-dependent text I/O in a Windows-CI script:\n", file=sys.stderr) + for p in problems: + print(f" {p}", file=sys.stderr) + print( + "\nThese run in `windows-build`'s GC structural audits, which is the " + "FIRST step of the job — a UnicodeDecodeError there skips the only " + "Windows run of the perry-runtime unit tests (#7977).", + file=sys.stderr, + ) + return 1 + print(f"locale-independent I/O OK: {len(SCANNED)} Windows-CI scripts scanned") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check_thread_locals.py b/scripts/check_thread_locals.py index da07860aa2..b2db1a1481 100755 --- a/scripts/check_thread_locals.py +++ b/scripts/check_thread_locals.py @@ -100,6 +100,30 @@ def repo_relative(path: PurePath, root: PurePath) -> str: """Return a stable repository-relative key on every host platform.""" return path.relative_to(root).as_posix() + +def read_source(path: Path) -> str: + """Read a repository file as UTF-8 on every host (#7977). + + A bare `Path.read_text()` decodes with `locale.getencoding()`, which is + cp1252 on a GitHub Windows runner. Fifteen files under + `crates/perry-runtime/src` carry a byte that cp1252 has no mapping for + (0x81/0x8d/0x8f/0x90/0x9d), so the walk died on `i18n.rs` with a + `UnicodeDecodeError` before the checker examined a single declaration — + taking the whole `windows-build` job, and the eleven steps behind it, + down with it. The sources are UTF-8; say so. + """ + return path.read_text(encoding="utf-8") + + +def write_source(path: Path, text: str) -> None: + """Write a file as UTF-8 with LF endings on every host (#7977). + + `newline=""` suppresses Windows' `\\n` -> `\\r\\n` translation so + `--update` produces the same allowlist bytes everywhere. + """ + path.write_text(text, encoding="utf-8", newline="") + + # `#[cfg(test)] mod ;` — the whole file is a test module. CFG_TEST_MOD_RE = re.compile( r"(?m)^[ \t]*#\[cfg\(test\)\]\s*\n[ \t]*(?:pub(?:\([^)]*\))?\s+)?mod\s+([A-Za-z_0-9]+)\s*;" @@ -165,7 +189,7 @@ def cfg_test_module_files(root: Path, crates: list[str]) -> set[str]: continue path = Path(dirpath) / name rel = repo_relative(path, root) - src = path.read_text() + src = read_source(path) parent = path.parent if name in ("lib.rs", "mod.rs") else path.with_suffix("") gated = set(CFG_TEST_MOD_RE.findall(src)) edges = [] @@ -219,7 +243,7 @@ def scan(root: Path, crates: list[str]) -> tuple[dict[str, int], int]: continue path = Path(dirpath) / name rel = repo_relative(path, root) - src = path.read_text() + src = read_source(path) hot_declarations += sum( len(DECL_RE.findall(body)) for body in block_bodies(src, HOT_RE) ) @@ -232,7 +256,7 @@ def scan(root: Path, crates: list[str]) -> tuple[dict[str, int], int]: def hot_slot_capacity(root: Path) -> int: - src = (root / "crates/perry-runtime/src/tls_hot.rs").read_text() + src = read_source(root / "crates/perry-runtime/src/tls_hot.rs") m = re.search(r"pub const HOT_SLOT_CAPACITY: usize = (\d+);", src) if not m: raise SystemExit("could not read HOT_SLOT_CAPACITY from tls_hot.rs") @@ -241,7 +265,7 @@ def hot_slot_capacity(root: Path) -> int: def verify(root: Path, crates: list[str], allowlist_path: Path) -> list[str]: raw, hot_declarations = scan(root, crates) - recorded = json.loads(allowlist_path.read_text())["files"] + recorded = json.loads(read_source(allowlist_path))["files"] problems = [] for rel, count in sorted(raw.items()): @@ -283,7 +307,7 @@ def verify(root: Path, crates: list[str], allowlist_path: Path) -> list[str]: def write_allowlist(root: Path, crates: list[str], allowlist_path: Path) -> None: raw, hot_declarations = scan(root, crates) - allowlist_path.write_text( + write_source(allowlist_path, json.dumps( { "_comment": ( @@ -313,18 +337,18 @@ def self_test() -> int: root = Path(tmp) src_dir = root / "crates/perry-runtime/src" src_dir.mkdir(parents=True) - (src_dir / "tls_hot.rs").write_text( + write_source(src_dir / "tls_hot.rs", "pub const HOT_SLOT_CAPACITY: usize = 768;\n" "thread_local! { static HOT: u8 = const { 0 }; }\n" ) - (src_dir / "cold.rs").write_text("thread_local! { static A: u8 = const { 0 }; }\n") - (src_dir / "hot.rs").write_text( + write_source(src_dir / "cold.rs", "thread_local! { static A: u8 = const { 0 }; }\n") + write_source(src_dir / "hot.rs", "crate::perry_thread_local! { static B: u8 = const { 0 }; }\n" ) allowlist = root / "allow.json" write_allowlist(root, CRATES, allowlist) - recorded = json.loads(allowlist.read_text()) + recorded = json.loads(read_source(allowlist)) if "crates/perry-runtime/src/tls_hot.rs" in recorded["files"]: failures.append("tls_hot.rs must be excluded from the allowlist") if recorded["_hot_declarations"] != 1: @@ -335,13 +359,13 @@ def self_test() -> int: failures.append("a freshly written allowlist must verify clean") # 1. A new raw declaration in an unlisted file must fail. - (src_dir / "new.rs").write_text("thread_local! { static C: u8 = const { 0 }; }\n") + write_source(src_dir / "new.rs", "thread_local! { static C: u8 = const { 0 }; }\n") if not verify(root, CRATES, allowlist): failures.append("a new raw `thread_local!` in an unlisted file passed") (src_dir / "new.rs").unlink() # 2. A second raw declaration in an already-listed file must fail. - (src_dir / "cold.rs").write_text( + write_source(src_dir / "cold.rs", "thread_local! { static A: u8 = const { 0 }; }\n" "thread_local! { static D: u8 = const { 0 }; }\n" ) @@ -349,7 +373,7 @@ def self_test() -> int: failures.append("a raw `thread_local!` added to a listed file passed") # 3. A stale entry must fail. - (src_dir / "cold.rs").write_text( + write_source(src_dir / "cold.rs", "crate::perry_thread_local! { static A: u8 = const { 0 }; }\n" ) if not verify(root, CRATES, allowlist): @@ -357,20 +381,20 @@ def self_test() -> int: # 4. Blowing the slot ceiling must fail. write_allowlist(root, CRATES, allowlist) - (src_dir / "hot.rs").write_text( + write_source(src_dir / "hot.rs", "crate::perry_thread_local! {\n" + "".join(f" static H{i}: u8 = const {{ 0 }};\n" for i in range(800)) + "}\n" ) if not any("HOT_SLOT_CAPACITY" in p for p in verify(root, CRATES, allowlist)): failures.append("exceeding HOT_SLOT_CAPACITY passed") - (src_dir / "hot.rs").write_text( + write_source(src_dir / "hot.rs", "crate::perry_thread_local! { static B: u8 = const { 0 }; }\n" ) # 5. The three `#[cfg(test)]` shapes are out of scope — and, the half # that can go wrong quietly, REMOVING the gate puts them back in it. - (src_dir / "cold.rs").write_text("thread_local! { static A: u8 = const { 0 }; }\n") + write_source(src_dir / "cold.rs", "thread_local! { static A: u8 = const { 0 }; }\n") write_allowlist(root, CRATES, allowlist) gated = { "attribute": "#[cfg(test)]\nthread_local! { static G: u8 = const { 0 }; }\n", @@ -381,23 +405,23 @@ def self_test() -> int: ), } for shape, body in gated.items(): - (src_dir / "gated.rs").write_text(body) + write_source(src_dir / "gated.rs", body) if verify(root, CRATES, allowlist): failures.append(f"a `#[cfg(test)]` {shape} declaration was counted") - (src_dir / "gated.rs").write_text(body.replace("#[cfg(test)]\n", "")) + write_source(src_dir / "gated.rs", body.replace("#[cfg(test)]\n", "")) if not verify(root, CRATES, allowlist): failures.append(f"an UNGATED {shape} declaration passed") (src_dir / "gated.rs").unlink() # 6. A whole file declared `#[cfg(test)] mod ;` is out of scope, # and drops back in when the parent stops gating it. - (src_dir / "probes.rs").write_text( + write_source(src_dir / "probes.rs", "thread_local! { static G: u8 = const { 0 }; }\n" ) - (src_dir / "lib.rs").write_text("#[cfg(test)]\nmod probes;\n") + write_source(src_dir / "lib.rs", "#[cfg(test)]\nmod probes;\n") if verify(root, CRATES, allowlist): failures.append("a `#[cfg(test)] mod ;` file was counted") - (src_dir / "lib.rs").write_text("mod probes;\n") + write_source(src_dir / "lib.rs", "mod probes;\n") if not verify(root, CRATES, allowlist): failures.append("an ungated `mod ;` file passed") diff --git a/scripts/gc_runtime_root_holders.py b/scripts/gc_runtime_root_holders.py index c5cb8ec076..cd514ad04d 100755 --- a/scripts/gc_runtime_root_holders.py +++ b/scripts/gc_runtime_root_holders.py @@ -709,7 +709,11 @@ def _scan_tree(extra: dict[str, str] | None = None) -> list[dict]: for rel, body in tree.items(): path = root / rel path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(body) + # Explicit UTF-8: a bare write_text() encodes with the host locale + # (cp1252 on a Windows runner), which is the #7977 class. The + # fixtures are ASCII today, so this is the latent half — the read + # side of the same shape is what took `windows-build` down. + path.write_text(body, encoding="utf-8", newline="") holders, _ = scan(root) return holders diff --git a/tests/test_gc_ratchet.py b/tests/test_gc_ratchet.py index e606b9609c..b39fb59581 100644 --- a/tests/test_gc_ratchet.py +++ b/tests/test_gc_ratchet.py @@ -1366,7 +1366,7 @@ class RebaseStableProvenanceTests(unittest.TestCase): def test_the_shipped_baseline_records_a_code_tree(self): baseline = Path(__file__).resolve().parents[1] / "benchmarks" / "gc_ratchet" / "baseline" / "gc-ratchet-v1.json" - artifact = json.loads(baseline.read_text()) + artifact = json.loads(baseline.read_text(encoding="utf-8")) self.assertIn( "code_tree", artifact, From 8243ea89cbce2095224b5c56f6c7b57a37f99ff5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 12 Aug 2026 19:39:29 +0200 Subject: [PATCH 2/4] fix(ci): make llvm-inprocess say what it did, and stop it greening on a skip (#7971) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three problems, all of which let the gate report success while asserting nothing. 1. VACUOUS GREEN. On a PR that touched no IR-affecting path the workflow ran `changes=success, native-backend=skipped` and concluded `success`. Sampled runs 31505530279, 31499833415 and 31476724152 are all that shape. "Green" meant "not relevant", but read as "the in-process backend passed". Add a `llvm-inprocess-complete` fan-in that states the verdict in the log and the step summary — EXERCISED vs NOT EXERCISED — and that can fail in two directions: `native-backend` failing or being cancelled, and `native-backend` being SKIPPED on a non-PR event. Every non-PR event sets `relevant=true` unconditionally, so a skip there means the post-merge anchor has stopped anchoring, which is how #7856 starved this gate for eight days unnoticed. 2. NO DIAGNOSTIC. `Native-mode smoke` ran bare `grep -q` / `cmp` under `set -euo pipefail`, with each compiler's stderr redirected to a file nothing ever printed. The three 2026-08-11 `main` failures therefore ended at "Generating code..." with a naked `exit 1`. Rewrite it around `run` / `assert_grep` / `assert_same` helpers: the failing command is named with its real exit status, captured stdout and stderr are dumped, a parity failure prints the diff, and every liveness assert says what it is protecting. (The status is captured after the command rather than inside an `if !` branch, where `$?` is the negated status and always 0.) 3. STALE CORPORA, INVISIBLE. The unit gate asserts `corpus_spike ... ok` to prove the corpus tests ran, but not that the corpora are current. All three were frozen on 2026-08-03, 151 codegen commits ago, and contain zero `addrspace(1)` — so they stayed green while the end-to-end arm could not build a single RS4GC root slot. Print corpus age and the IR-affecting commit count so that gap is visible instead of rediscovered. The underlying backend defect this arm was correctly reporting is filed as #7982: `PERRY_LLVM_INPROCESS=native` cannot construct `ptr addrspace(1)` roots. It is reproduced locally and is not a one-line fix, so it is left to that issue rather than folded into a CI change. --- .github/workflows/llvm-inprocess.yml | 219 ++++++++++++++++++++++++--- 1 file changed, 198 insertions(+), 21 deletions(-) diff --git a/.github/workflows/llvm-inprocess.yml b/.github/workflows/llvm-inprocess.yml index bb0ecb3789..3e03203b17 100644 --- a/.github/workflows/llvm-inprocess.yml +++ b/.github/workflows/llvm-inprocess.yml @@ -120,28 +120,126 @@ jobs: echo "$out" | grep -q "dialect::tests::corpus_exception_handling ... ok" echo "$out" | grep -q "inprocess::tests::rs4gc_schedules_in_process ... ok" + + # The tracked `.ll` corpora above are a SNAPSHOT of what the compiler + # emitted when they were last refreshed (#7302/#7307/#7310, 2026-08-03). + # `corpus_spike ... ok` therefore proves the dialect reader can build + # THAT IR — not the IR this commit emits. When the end-to-end arm below + # goes red while these stay green, that gap is the first thing to check, + # so print it rather than leaving the next reader to rediscover it. + - name: Corpus currency (diagnostic, not a gate) + if: ${{ !cancelled() }} + run: | + set -uo pipefail + newest=$(git log -1 --format=%ct -- experiments/llvm-inprocess-spike/*.ll) + behind=$(git log --oneline --since="@${newest}" -- crates/perry-codegen/src \ + crates/perry-hir/src crates/perry-transform/src | wc -l | tr -d ' ') + echo "tracked .ll corpora last refreshed: $(git log -1 --format='%h %ad' \ + --date=short -- experiments/llvm-inprocess-spike/*.ll)" + echo "IR-affecting commits since then (codegen+hir+transform): ${behind}" + { + echo "### llvm-inprocess corpus currency" + echo "" + echo "- corpora refreshed: \`$(git log -1 --format='%h %ad' --date=short \ + -- experiments/llvm-inprocess-spike/*.ll)\`" + echo "- IR-affecting commits since: **${behind}**" + echo "" + echo "The unit corpus gates assert the reader handles that snapshot." + echo "Only the end-to-end smoke below exercises the IR this commit emits." + } >> "$GITHUB_STEP_SUMMARY" + - name: Native-mode smoke — liveness, behavior parity, object-byte verdicts run: | - set -euo pipefail + # NOTE: deliberately NOT `set -e`. Every check below reports what it + # was doing and dumps the captured output before exiting. The previous + # version used bare `grep -q` / `cmp` under `set -euo pipefail`, so the + # 2026-08-11 `main` failures ended at "Generating code..." with a naked + # `exit 1` and no diagnostic at all — three runs, untriageable (#7971). + # The compiler's own message was fine and named the offending IR line; + # it went to a captured file that nothing ever printed. That is #7982. + # The compilers' stderr carries the liveness banner, so it is captured + # to a file; that file is what went unread. It is now always dumped on + # failure, and the failing COMMAND is named. + set -uo pipefail export PERRY_RUNTIME_DIR="$PWD/target/perry-dev" export PERRY_NO_AUTO_OPTIMIZE=1 BIN=target/perry-dev/perry SRC=experiments/llvm-inprocess-spike/spike.ts + EH=test-files/test_gap_7302_invoke_eh_paths.ts + W=/tmp/inproc; mkdir -p "$W" - "$BIN" "$SRC" -o /tmp/spike_text - /tmp/spike_text > /tmp/text.out + dump() { + for f in "$@"; do + [ -s "$f" ] || continue + echo "--- $f (last 80 lines) ---" + tail -80 "$f" + done + } - PERRY_LLVM_INPROCESS=native "$BIN" "$SRC" -o /tmp/spike_native 2> /tmp/native.err - grep -q "in-process LLVM backend active" /tmp/native.err - /tmp/spike_native > /tmp/native.out - cmp /tmp/text.out /tmp/native.out + # run