diff --git a/benchmarks/public_baseline.py b/benchmarks/public_baseline.py index 7e3f646c18..9e54cf93ca 100755 --- a/benchmarks/public_baseline.py +++ b/benchmarks/public_baseline.py @@ -130,10 +130,46 @@ def _is_resolved_path(value: Any) -> bool: return bool(text) and (os.path.isabs(text) or os.sep in text) +def _cargo_profile_tables(data: bytes) -> bytes: + """The `[profile.*]` tables of a Cargo manifest, and nothing else. + + #7282: `Cargo.toml` was fingerprinted whole-file (modulo the version line, + which #7264's normalization already neutralized). Everything else in it — + a new `perry-ext-*` workspace member, a dependency bump, a `[workspace]` + restructure — invalidated the published baseline without being able to + change a measured number. #6758/#6761's restructuring tripped it with the + `.ts` kernels untouched, and the artifact then sat 40+ commits stale on a + REQUIRED check, so every later `lint` step never ran at all and every merge + needed an `--admin` bypass. + + What genuinely can move a number is the build profile: `opt-level`, `lto`, + `codegen-units`, `panic`. So only those tables participate. + + Extraction is deliberately textual and conservative — a TOML parser is not + guaranteed available in the CI Python, and this must agree byte-for-byte + between the generator and the checker. A section header ends the capture + unless it is itself a `[profile...` header, so `[profile.release.package.x]` + subtables are kept. + """ + kept: list[bytes] = [] + capturing = False + for line in data.splitlines(keepends=True): + stripped = line.strip() + if stripped.startswith(b"[") and stripped.endswith(b"]"): + capturing = stripped.startswith(b"[profile") + if capturing: + kept.append(line) + return b"".join(kept) + + def _fingerprint_bytes(name: str) -> bytes: data = (ROOT / name).read_bytes() if name == "Cargo.toml": + # The version normalization stays: `[workspace.package] version` is not + # in a profile table, but keeping the substitution makes the intent + # explicit if the extraction below is ever widened. data = _CARGO_VERSION_RE.sub(b'version = "0.0.0"', data) + data = _cargo_profile_tables(data) return data diff --git a/benchmarks/results/public-node-bun-v1.json b/benchmarks/results/public-node-bun-v1.json index c5a9841042..a597108217 100644 --- a/benchmarks/results/public-node-bun-v1.json +++ b/benchmarks/results/public-node-bun-v1.json @@ -5,8 +5,8 @@ "perry_version": "perry 0.5.1355", "generated_at": "2026-08-08T11:42:16Z", "freshness": { - "source_fingerprint": "57fe15fcce9a80de22d56d4075cb4cc196a8246ff2920f61ff8a0a11299d9ddf", - "harness_fingerprint": "45ac06e91e37d0873f543d02d6f1d1af9470d014c0dd7fa01a1e22b57707adb9" + "source_fingerprint": "65a1218e02c95b4aa8ed22065ca33e4653addcd81a77a4fb1a810e5153e07887", + "harness_fingerprint": "28117b86b2bcca9cc4e6f418f156bfcce0b4bf0851698c3e88525737221df49b" }, "host": { "os_version": "26.5.1", diff --git a/changelog.d/7821-narrow-baseline-fingerprint.md b/changelog.d/7821-narrow-baseline-fingerprint.md new file mode 100644 index 0000000000..0d1642ba29 --- /dev/null +++ b/changelog.d/7821-narrow-baseline-fingerprint.md @@ -0,0 +1,22 @@ +**The public-baseline freshness gate no longer fires on `Cargo.toml` changes that cannot move a measured number** (#7282, proposal 1 — the issue's "single biggest win"). + +`Cargo.toml` was fingerprinted whole-file. A new `perry-ext-*` workspace member, a dependency bump or a `[workspace]` restructure invalidated the published artifact without touching a benchmarked kernel — #6758/#6761's restructuring did exactly that, and the artifact then sat **40+ commits stale on a REQUIRED check**, so every later `lint` step (file-size, GC store-site inventory, addr-class audit, #7253's gate-wiring check) never executed in CI at all, and every merge needed an `--admin` bypass. + +Only the `[profile.*]` tables now participate — `opt-level`, `lto`, `codegen-units`, `panic`, i.e. the things that genuinely change comparability. Extraction is textual and conservative (no TOML parser is guaranteed in the CI Python, and generator and checker must agree byte-for-byte); a section header ends the capture unless it is itself `[profile…`, so `[profile.release.package.x]` subtables are kept. Measured: 19,466 bytes → 8,681, all 45 profile tables retained, `[workspace.package]` and every dependency line gone. + +**Sabotage-verified in both directions**, by exit code rather than by message: + +| planted change | `ci_public_baseline_check.py` | +|---|--:| +| `[profile.release] opt-level = 3 → 2` | **exit 2** — "benchmark inputs changed; regenerate it" | +| new `crates/perry-ext-sabotage` workspace member | **exit 0** | + +So the gate keeps blocking absolutely on what matters and stops firing on what does not — the issue's explicit requirement that it stay required and hard-failing. + +**On the artifact's stored digests.** Narrowing the fingerprint changes both keys, so they are recomputed in the same commit. That is sound rather than an attestation: the artifact **matches under the broad fingerprint today** (verified before the change), and the narrow fingerprint covers a strict subset of those inputs — an artifact valid under the broad one is necessarily valid under the narrower one. No measurement was re-run and none needed to be: everything in the file except the two `freshness` digests is byte-identical, asserted programmatically. Independently, the `[profile.*]` extract is unchanged from the artifact's own commit (`38ff7eccc`) through HEAD, and across `HEAD~40` and `HEAD~120`. + +Also worth recording: the issue's headline complaint — "`Cargo.toml` changes on every version bump" — was **already fixed** before this change. `_fingerprint_bytes` normalizes the `version = "…"` line to `0.0.0`, which is why the gate was green today despite `0.5.1355 → 0.5.1461`. This change addresses what was left. + +Scope is proposal 1 only. `HARNESS_PATHS` still fingerprints scripts whose error handling cannot change a number (#7265's `run.sh` fix is the cited example); proposal 2's measurement-affecting/plumbing split is untouched. The deliberate circularity is preserved — `ci_public_baseline_check.py` stays out of `public_baseline.py` so the checker's own file is not in `HARNESS_PATHS`. + +Verified: `tests/test_public_baseline.py` 7 passed, `benchmarks/ci_public_baseline_check.py` exit 0, file-size gate clean.