From 07236dccd7dfca0e5e501c77e24f645b7ee523f6 Mon Sep 17 00:00:00 2001 From: Nick Petalas Date: Thu, 29 Oct 2026 13:17:48 +0000 Subject: [PATCH] perf(research): reuse replay planner ancestor checks --- .../BENCHMARK.json | 86 ++++++++ .../replay-planner-ancestor-cache/README.md | 151 ++++++++++++++ .../SOURCES.json | 24 +++ .../replay-planner-ancestor-cache/TASK.json | 61 ++++++ .../replay-planner-ancestor-cache/VERIFY.json | 36 ++++ .../benchmark.py | 76 +++++++ .../replay-planner-ancestor-cache/planner.py | 95 +++++++++ .../replay-planner-ancestor-cache/verify.py | 196 ++++++++++++++++++ 8 files changed, 725 insertions(+) create mode 100644 research/replay-planner-ancestor-cache/BENCHMARK.json create mode 100644 research/replay-planner-ancestor-cache/README.md create mode 100644 research/replay-planner-ancestor-cache/SOURCES.json create mode 100644 research/replay-planner-ancestor-cache/TASK.json create mode 100644 research/replay-planner-ancestor-cache/VERIFY.json create mode 100644 research/replay-planner-ancestor-cache/benchmark.py create mode 100644 research/replay-planner-ancestor-cache/planner.py create mode 100644 research/replay-planner-ancestor-cache/verify.py diff --git a/research/replay-planner-ancestor-cache/BENCHMARK.json b/research/replay-planner-ancestor-cache/BENCHMARK.json new file mode 100644 index 0000000000..d0bd52dc8f --- /dev/null +++ b/research/replay-planner-ancestor-cache/BENCHMARK.json @@ -0,0 +1,86 @@ +{ + "schema_version": 1, + "claim_type": "empirical tooling improvement", + "timestamp_utc": "2026-10-29T13:11:10.625011+00:00", + "head": "3f0152a32eeebac0df11946dbd50cabe5a7f7839", + "python": "3.12.15 (main, Oct 14 2026, 00:55:37) [Clang 23.1.3 ]", + "platform": "macOS-27.0.1-arm64-arm-64bit", + "tracked_paths_sha256": "e2d42dd7de100ce4155860425a3b9d0f1ef943c1e70bd26b5ae9fc4ccd445022", + "sparse_checkout_sha256": "5230ac40920ff55e2fb4be70ddb3075488002bf79dddccae95d54ea7b2def5c6", + "source_sha256": { + "scripts/research_package.py": "e1256016e7f51827d04fa2179255989f639ae1d97afaa9c16265f85a0c6abd70", + "research/replay-planner-ancestor-cache/planner.py": "54ed1ecd59ae2aef90755e96d304f445407763907c55e526294b01c7cf47cd49", + "research/replay-planner-ancestor-cache/benchmark.py": "1cca5ee36d6a17603bc2ebb9b4a6074bf6ab271eb3ffbec7678800856be4e2e6", + "research/replay-planner-ancestor-cache/VERIFY.json": "91f0aa22c030ef0ee00b65bf91d4c3d65531e4795b3110d46948e00a42185bde" + }, + "measurement": { + "commands": [ + [ + "/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12", + "-I", + "-W", + "error", + "-B", + "/Users/nick/git/imb-research-11/scripts/research_package.py", + "--changed-since", + "0000000000000000000000000000000000000000" + ], + [ + "/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12", + "-I", + "-W", + "error", + "-B", + "/Users/nick/git/imb-research-11/research/replay-planner-ancestor-cache/planner.py", + "--changed-since", + "0000000000000000000000000000000000000000" + ] + ], + "pairs": 9, + "samples_seconds": [ + [ + 0.89816625, + 0.884204958, + 0.970985542, + 0.888555875, + 0.890791583, + 0.902384959, + 0.928754333, + 0.8907095, + 0.905556583 + ], + [ + 0.817896708, + 0.801496583, + 0.852495667, + 0.823127125, + 0.826971417, + 0.819598375, + 0.83802175, + 0.823856, + 0.867351208 + ] + ], + "median_seconds": [ + 0.89816625, + 0.823856 + ], + "range_seconds": [ + [ + 0.884204958, + 0.970985542 + ], + [ + 0.801496583, + 0.867351208 + ] + ], + "fraction_less_time": 0.0827355180624968, + "errors": 0, + "complete_outputs_equal": true, + "selected_packages": 3788, + "stdout_sha256": "e0a4c3bcf5545c3334dc8a36b2509c7b57df4c25cbdb9395898db385d56e82d0", + "stderr_sha256": "0ccfcc32adec026de1f50bb05cf4d5ac9e555a7589d844786ace39f5ed9c5e93", + "stderr": "Planning includes 3759 unchanged sparse manifests from HEAD; materialize packages and inputs before replay.\n" + } +} diff --git a/research/replay-planner-ancestor-cache/README.md b/research/replay-planner-ancestor-cache/README.md new file mode 100644 index 0000000000..9f070c96c2 --- /dev/null +++ b/research/replay-planner-ancestor-cache/README.md @@ -0,0 +1,151 @@ +# Reuse checked ancestors within one complete replay plan + +The opt-in planner reduces complete local planning time by **8.27%**, from +median **0.898166250 s** to **0.823856000 s** over nine alternating pairs. +All eighteen commands succeed and produce identical selection JSON and sparse +warnings for **3,788 packages**, including **3,759** manifests authenticated from +sparse HEAD. Before times range from 0.884204958 to 0.970985542 seconds; after +times range from 0.801496583 to 0.867351208 seconds. + +This is a scoped equivalence proof with finite performance evidence, not a +mathematical bound or supplier-admission result. This contribution was +produced by Nick Petalas. The selected frontier is unchanged. + +## The change + +[JoyboyBrian's PR #6087](https://github.com/CrocSwap/integer-mult-bounds/pull/6087) +already avoids checking an identical full input path repeatedly within a plan. +Different inputs and manifests still share ancestors. The new [planner.py](planner.py) +checks each distinct existing ancestor once and reuses that result only within +the same invocation of `plan_details`. + +```text +begin plan with an empty prefix table +for each canonical path: + reuse already checked prefixes + inspect new components for symlinks and junctions + stop at a missing component or a regular-file ancestor +run the unchanged selection and manifest-hash logic +discard the table on success or failure +``` + +The command authenticates the exact shared runner against [SOURCES.json](SOURCES.json), +loads those bytes into a private module without consulting bytecode, and temporarily +replaces only that module's path guard during a synchronous plan. Its parser, +sparse-manifest authentication, Git environment, exact HEAD/diff spelling, +complete-input cache, selection, hashes and output logic remain in use. It does +not modify the shared runner or accelerate the default CI command. A changed +runner fails authentication until this package is deliberately updated. + +## Scoped equivalence proof + +The inherited contract requires the checkout to remain stable throughout one +plan. Each `lstat` therefore returns the same metadata or exception for a given +path during that plan. Every call first runs the original canonical-path validator. +The first uncached lookup performs the original symlink and junction tests. +Only a checked directory permits descent. A checked regular file permits the +same return as the original guard: a descendant would raise the suppressed +`NotADirectoryError`. A missing prefix similarly suppresses deeper lookups. +Thus the cached result reproduces every later decision under that prefix. +Permission and other unexpected errors propagate with their original type and +message and are not cached. Rejected symlinks and junctions are never cached. + +Induct on the finite components of each path and then on the paths visited by +the unchanged planner. The replacement guard returns the same path or raises +the same error at each visit. Every manifest read, input-validation result, +selection cause and manifest hash is consequently unchanged. A table is created +inside each `plan_details` call and the guard is restored in `finally`; the next +plan must inspect the filesystem again. Concurrent filesystem mutation is +outside the inherited contract. The private module has one synchronous caller. + +## Evidence and reproduction + +[BENCHMARK.json](BENCHMARK.json) is generated by [benchmark.py](benchmark.py). +It binds implementation, harness, manifest and shared-source hashes, Git path +inventory and sparse configuration, interpreter, platform, all eighteen samples, +error count and stdout/stderr digests. Each timed command includes process +startup, candidate source authentication, all planning and captured output. +Every pair reverses which side runs first. No precomputed result is timed. + +An untimed profile put repeated component traversal and `lstat` work in the +original guard at about one quarter of complete baseline planning time. The +measured end-to-end reduction is smaller; JSON, Git blob reads and timeout +supervision still run. The machine had 18 logical CPUs and load averages 2.92, +3.19 and 2.79 at the initial profile. Both sides use identical interpreter flags +and the existing caches. These are warm-filesystem macOS measurements on a +shared machine, not cold-cache or Linux/Windows speedup claims. + +From the repository root, using the local interpreter: + +```sh +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -I -W error -B research/replay-planner-ancestor-cache/planner.py --changed-since origin/main +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -I -W error -B research/replay-planner-ancestor-cache/verify.py +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -I -O -W error -B research/replay-planner-ancestor-cache/verify.py +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -B scripts/research_package.py --package research/replay-planner-ancestor-cache +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -B scripts/check_research_commit.py --package research/replay-planner-ancestor-cache --check-source-history --check-modes +``` + +Another standard-library CPython 3.12 executable may replace the absolute path. +The direct marker is `PASS_REPLAY_PLANNER_ANCESTOR_CACHE`; the registered marker +is `PASS_REGISTERED_RESEARCH_PACKAGE`. Each direct run generates a receipt bound +to every package file and writes no package files. Optional timing reproduction, +which CI does not run: + +```sh +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -I -W error -B research/replay-planner-ancestor-cache/benchmark.py --pairs 9 --output /absolute/scratch/benchmark.json +``` + +Future checkouts and hosts need not reproduce exact times or package counts. +Registered replay checks the committed measurement's bindings and arithmetic. +The direct verifier covers 18 path cases, four symlink/junction cases, two lookup +failure cases, eight complete fixture-plan cases, changed-source rejection and +measurement-receipt validation. It checks every fixture consumer, duplicate +inputs, component ordering, package changes, shared controls and manifest hashes. +Later calls reject a new alias and recover after its removal. Both successful +and failed calls restore the guard. Normal and optimized modes pass. Fixture +Git inventory is explicit; the real timed commands perform original Git and +sparse authentication. + +## Resource and diagnostic limits + +Added loops visit finite path components, manifest inputs or fixed test cases. +The cache stores at most one entry per distinct prefix encountered in one plan; +memory is proportional to the total stored prefix characters and is released +when that plan ends. Registered commands have 120-second timeouts. Each optional +benchmark subprocess has a 60-second timeout and the harness permits at most +twenty pairs. + +Python's managed allocation is a Power-of-Ten rule 3 deviation limited to this +finite table. Its compensating controls are the finite prefix bound and release +at the end of the call; tests check reuse and next-plan invalidation. Heap peak +was not measured. Shared integration requiring a hard memory limit must first +measure and bound this table for its repository-size limit. + +Executing the authenticated existing runner is a rule 8 deviation: the input is +one SHA256-pinned source, not generated code. A changed-source control rejects +before execution, and the namespace is private. This preserves inspectable +shared checks without duplicating them. Reassess this loader if the runner's +dependencies or execution model change. + +Warning-strict normal/optimized and in-memory syntax checks are used. No Python +linter was installed locally. Native Windows junction execution, a fully +materialized checkout benchmark, CI timing, selected supplier replay, Lean, +Docker and complete multiplication obligations are unrun. Independent human +and formal proof-kernel review are unclaimed. + +## Open next questions + +Should this validator be incorporated into the shared planner? First exercise +real Windows junctions and measure complete plans in a fully materialized +checkout. Those platform and workload checks remain the blockers. The present +opt-in command provides a reviewable implementation and reproduction harness. + +## Credit + +JoyboyBrian retains PR #6087's complete-input reuse and tests. Douglas Colkitt +retains the shared replay runner. Ali Romman retains the earlier complete-plan +benchmark context; the exact-path, symlink/junction, Git-source, sparse-manifest +and deadline contributors retain their original source notices. The shared +source is consumed unchanged. The new prefix validator, private CLI, scoped +equivalence proof, behavioral controls and timing harness were produced by +Nick Petalas. Apache-2.0. diff --git a/research/replay-planner-ancestor-cache/SOURCES.json b/research/replay-planner-ancestor-cache/SOURCES.json new file mode 100644 index 0000000000..e6ad2a8a83 --- /dev/null +++ b/research/replay-planner-ancestor-cache/SOURCES.json @@ -0,0 +1,24 @@ +{ + "schema_version": 1, + "base_commit": "3f0152a32eeebac0df11946dbd50cabe5a7f7839", + "files": [ + { + "path": "certificates/selected-result.json", + "usage": "provenance", + "git_blob": "d59b5035e7b4cbaa9d688a1d32dd0d6eb79c466a", + "sha256": "9b8d0eea95b3bf0dfcb7938a732a6439b87a2685dec2a5ab9e525b3b34debbfa" + }, + { + "path": "docs/research/agent-task-catalog.json", + "usage": "provenance", + "git_blob": "3404fbdd5e7529a0674eef26114b5071e4586cec", + "sha256": "ead6cdc9d0af6d58e8a49744d8ec8c7c15e13d2ba93c276c434bcf9b51b93a81" + }, + { + "path": "scripts/research_package.py", + "usage": "input", + "git_blob": "091b4c8f58bac9a9580d76025b887dc5c873d05f", + "sha256": "e1256016e7f51827d04fa2179255989f639ae1d97afaa9c16265f85a0c6abd70" + } + ] +} diff --git a/research/replay-planner-ancestor-cache/TASK.json b/research/replay-planner-ancestor-cache/TASK.json new file mode 100644 index 0000000000..1c3feb2650 --- /dev/null +++ b/research/replay-planner-ancestor-cache/TASK.json @@ -0,0 +1,61 @@ +{ + "schema_version": 1, + "task_id": "replay-runtime", + "priority_revision": 8, + "base_commit": "3f0152a32eeebac0df11946dbd50cabe5a7f7839", + "original_card": { + "id": "replay-runtime", + "question": "Can one measured replay bottleneck be sped up without dropping coverage?", + "starting_points": [ + "scripts/run_isolated_tests.py", + "scripts/verify_replay_coverage.py", + "certificates/selected-result.json" + ], + "methods": [ + "Profile one isolated replay and optimize the measured bottleneck.", + "Design source-bound reuse with explicit dependency invalidation and negative controls." + ], + "acceptance_gate": "Report before/after wall time on the same fixture, retained checks and invalidation behavior.", + "first_step": { + "command": [ + "python3", + "-B", + "scripts/research_selected_baseline.py" + ], + "expected": "PASS_RESEARCH_BASELINE_BINDINGS_ONLY", + "inspect": "Resolve the current selected review, separate complex/ordinary BIT/admitted BIT/outer budgets, and compare the card against live handoffs before starting. Historical ceilings require exact source/model equivalence.", + "scope": "Source-binding orientation only; not a scientific replay or all-size proof. Materialize selected inputs if this checkout is sparse." + }, + "first_experiment": "Measure one unchanged fixture, preserve check coverage, and test one optimization with invalidation controls.", + "bottleneck_link": "Faster trustworthy replays increase review throughput and let mechanism candidates be checked sooner.", + "allowed_variations": [ + "Profile one stage or design one source-bound cache boundary." + ], + "scope_change_triggers": [ + "Skipping coverage or comparing different fixtures to claim speedup." + ], + "overlap_terms": [ + "replay runtime", + "cache invalidation", + "isolated tests" + ] + }, + "scope_status": "changed", + "actual_scope": "Reuse successful filesystem ancestor checks within one complete replay-planning invocation; deliver a source-pinned opt-in CLI with exact output and failure controls.", + "scope_change_reason": "The profiled coordination planner is a bounded prerequisite for replay rather than a scientific verifier. This adds prefix reuse beyond PR6087 exact-full-input reuse.", + "bottleneck_link": "Less planning overhead lets unchanged complete replay coverage reach its isolated jobs sooner; no mathematical or supplier saving is inferred.", + "budget_minutes": 60, + "stop_rule": "Ship the first reliable complete-command improvement after exact equivalence and rejection controls; do not extend into shared-runner changes.", + "outcome": "completed", + "evidence": "verify.py behavioral differentials; BENCHMARK.json generated complete-command timing receipt; README.md scoped equivalence argument.", + "author": "Nick Petalas", + "ai_assistance": "Independent human and formal proof-kernel review are unclaimed.", + "open_questions": [ + { + "question": "Should this invocation-local ancestor validator be incorporated into the shared planner?", + "why_it_matters": "The present opt-in command does not accelerate the default CI path.", + "first_step": "Run the same differential controls on native Windows junctions and benchmark a fully materialized checkout before proposing shared integration.", + "blocked_by": "Native Windows and full-checkout performance have not been measured." + } + ] +} diff --git a/research/replay-planner-ancestor-cache/VERIFY.json b/research/replay-planner-ancestor-cache/VERIFY.json new file mode 100644 index 0000000000..a71456d89e --- /dev/null +++ b/research/replay-planner-ancestor-cache/VERIFY.json @@ -0,0 +1,36 @@ +{ + "schema_version": 1, + "status": "ready", + "python": "3.12", + "requirements": [], + "inputs": [ + "scripts/research_package.py" + ], + "commands": [ + { + "argv": [ + "{python}", + "-I", + "-W", + "error", + "-B", + "{package}/verify.py" + ], + "timeout_seconds": 120, + "checks": "Exact path-guard and complete-plan differential controls, next-call invalidation, errors and source authentication." + }, + { + "argv": [ + "{python}", + "-I", + "-O", + "-W", + "error", + "-B", + "{package}/verify.py" + ], + "timeout_seconds": 120, + "checks": "The same behavioral controls with assertions disabled." + } + ] +} diff --git a/research/replay-planner-ancestor-cache/benchmark.py b/research/replay-planner-ancestor-cache/benchmark.py new file mode 100644 index 0000000000..7656dff371 --- /dev/null +++ b/research/replay-planner-ancestor-cache/benchmark.py @@ -0,0 +1,76 @@ +"""Generate complete-command timing evidence; never run by the generic replay.""" +import argparse +import datetime +import hashlib +import json +from pathlib import Path +import platform +import statistics +import subprocess +import sys +import time + +PACKAGE = Path(__file__).resolve().parent +ROOT = PACKAGE.parents[1] + + +def sha256(raw): + return hashlib.sha256(raw).hexdigest() + + +def run(argv, cwd): + started = time.perf_counter_ns() + result = subprocess.run(argv, cwd=cwd, capture_output=True, timeout=60, check=True) + return result, (time.perf_counter_ns() - started) / 1_000_000_000 + + +def benchmark(pairs): + common = [sys.executable, "-I", "-W", "error", "-B"] + suffix = ["--changed-since", "0" * 40] + commands = [common + [str(ROOT / "scripts/research_package.py")] + suffix, + common + [str(PACKAGE / "planner.py")] + suffix] + samples, reference = [[], []], None + for pair in range(pairs): + for side in ((0, 1) if pair % 2 == 0 else (1, 0)): + result, seconds = run(commands[side], ROOT) + observed = (result.stdout, result.stderr) + if reference is None: + reference = observed + if observed != reference: + raise ValueError("complete planner output or warning changed") + samples[side].append(seconds) + assert reference is not None + include = json.loads(reference[0])["include"] + medians = [statistics.median(side) for side in samples] + return {"commands": commands, "pairs": pairs, "samples_seconds": samples, + "median_seconds": medians, + "range_seconds": [[min(side), max(side)] for side in samples], + "fraction_less_time": 1 - medians[1] / medians[0], "errors": 0, + "complete_outputs_equal": True, "selected_packages": len(include), + "stdout_sha256": sha256(reference[0]), "stderr_sha256": sha256(reference[1]), + "stderr": reference[1].decode("utf-8")} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--pairs", type=int, default=9, choices=range(5, 21)) + args = parser.parse_args() + head, _ = run(["git", "rev-parse", "HEAD"], ROOT) + tracked, _ = run(["git", "ls-tree", "-r", "--name-only", "-z", "HEAD"], ROOT) + sparse, _ = run(["git", "sparse-checkout", "list"], ROOT) + result = {"schema_version": 1, "claim_type": "empirical tooling improvement", + "timestamp_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "head": head.stdout.decode().strip(), "python": sys.version, + "platform": platform.platform(), "tracked_paths_sha256": sha256(tracked.stdout), + "sparse_checkout_sha256": sha256(sparse.stdout), + "source_sha256": {str(path.relative_to(ROOT)): sha256(path.read_bytes()) + for path in [ROOT / "scripts/research_package.py", PACKAGE / "planner.py", + PACKAGE / "benchmark.py", PACKAGE / "VERIFY.json"]}, + "measurement": benchmark(args.pairs)} + args.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + print(json.dumps(result["measurement"], sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/research/replay-planner-ancestor-cache/planner.py b/research/replay-planner-ancestor-cache/planner.py new file mode 100644 index 0000000000..3d440e7141 --- /dev/null +++ b/research/replay-planner-ancestor-cache/planner.py @@ -0,0 +1,95 @@ +"""Opt-in replay planner with invocation-local filesystem ancestor reuse. + +Produced by Nick Petalas. Apache-2.0. +""" +import hashlib +import json +from pathlib import Path +import stat +import types + +PACKAGE = Path(__file__).resolve().parent +ROOT = PACKAGE.parents[1] + + +def load_runner(root=ROOT): + """Execute exactly the authenticated source, without consulting bytecode.""" + source = root / "scripts/research_package.py" + pins = json.loads((PACKAGE / "SOURCES.json").read_text(encoding="utf-8")) + rows = [row for row in pins["files"] if row["path"] == "scripts/research_package.py"] + if len(rows) != 1 or rows[0]["usage"] != "input": + raise ValueError("exactly one live runner pin is required") + raw = source.read_bytes() + if hashlib.sha256(raw).hexdigest() != rows[0]["sha256"]: + raise ValueError("runner source SHA256 mismatch") + module = types.ModuleType("ancestor_cache_runner") + module.__file__ = str(source) + exec(compile(raw, str(source), "exec"), module.__dict__) + return module + + +class PathChecks: + """Reuse successful component checks only while one plan owns this instance.""" + + def __init__(self, root, canonical, require): + self.root = root + self.canonical = canonical + self.require = require + self.checked = {} + self.lookups = 0 + self.hits = 0 + + def __call__(self, root, name): + self.require(root == self.root, "path cache cannot cross repository roots") + parts = self.canonical(name).parts + prefix = "" + for part in parts: + prefix = part if not prefix else prefix + "/" + part + if prefix in self.checked: + self.hits += 1 + if not self.checked[prefix]: + break + continue + self.lookups += 1 + try: + info = (root / prefix).lstat() + except (FileNotFoundError, NotADirectoryError): + self.checked[prefix] = False + break + self.require(not stat.S_ISLNK(info.st_mode), + "symlink in repository-relative path: " + name) + self.require(getattr(info, "st_reparse_tag", 0) + != getattr(stat, "IO_REPARSE_TAG_MOUNT_POINT", None), + "junction in repository-relative path: " + name) + self.checked[prefix] = stat.S_ISDIR(info.st_mode) + if not self.checked[prefix]: + break + return root / name + + +def install(runner): + """Each plan gets fresh state; other runner entry points retain their guards.""" + original_plan = runner.plan_details + original_guard = runner.symlink_free_relative_path + + def plan_details(root, changed): + checks = PathChecks(root, runner.canonical_relative_path, runner.require) + runner.symlink_free_relative_path = checks + try: + return original_plan(root, changed) + finally: + runner.symlink_free_relative_path = original_guard + + runner.plan_details = plan_details + return runner + + +def main(): + # This private module is used by one synchronous CLI invocation. Delegating + # its original main preserves all argument, output and failure contracts. + runner = install(load_runner()) + runner.main() + + +if __name__ == "__main__": + main() diff --git a/research/replay-planner-ancestor-cache/verify.py b/research/replay-planner-ancestor-cache/verify.py new file mode 100644 index 0000000000..8c8eb4864a --- /dev/null +++ b/research/replay-planner-ancestor-cache/verify.py @@ -0,0 +1,196 @@ +"""Behavioral controls for one-plan ancestor reuse. Standard library only.""" +import hashlib +import json +from pathlib import Path +import stat +import statistics +import tempfile +import types +from unittest.mock import patch + +PACKAGE = Path(__file__).resolve().parent +namespace = {"__file__": str(PACKAGE / "planner.py"), "__name__": "planner_checks"} +exec(compile((PACKAGE / "planner.py").read_bytes(), "planner.py", "exec"), namespace) +load_runner = namespace["load_runner"] +PathChecks = namespace["PathChecks"] +install = namespace["install"] + + +def require(ok, message): + if not ok: + raise ValueError(message) + + +def outcome(function, root, name): + try: + return ("return", str(function(root, name))) + except (OSError, ValueError, TypeError) as error: + return (type(error).__name__, str(error)) + + +def write(root, name, value): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(value, encoding="utf-8") + + +def check_guard(root, runner): + write(root, "shared/branch/a.txt", "a") + write(root, "shared/branch/b.txt", "b") + write(root, "plain", "a regular file") + names = ["shared/branch/a.txt", "shared/branch/b.txt", "absent/child/a", + "absent/child/b", "plain/child/a", "plain/child/b", "shared", + "./shared", "shared//branch", "../outside", "/absolute", "", + "shared/../plain", "shared\\branch", None, 3, [], {}] + cache = PathChecks(root, runner.canonical_relative_path, runner.require) + for name in names: + expected = outcome(runner.symlink_free_relative_path, root, name) + require(outcome(cache, root, name) == expected, "guard behavior changed") + require(cache.hits > 0 and cache.lookups == len(cache.checked), + "successful prefixes were not reused") + require(cache.checked["absent"] is False and cache.checked["plain"] is False, + "terminal component status changed") + return len(names) + + +def check_aliases(root, runner): + aliases = [("alias", "shared"), ("dangling", "missing"), + ("leaf", "shared/branch/a.txt")] + for name, target in aliases: + (root / name).symlink_to(target) + cache = PathChecks(root, runner.canonical_relative_path, runner.require) + for name in ["alias/branch/a.txt", "dangling/child", "leaf"]: + expected = outcome(runner.symlink_free_relative_path, root, name) + require(expected[0] == "ValueError" and outcome(cache, root, name) == expected, + "symlink was not rejected identically") + original = Path.lstat + + def junction(path, *args, **kwargs): + if path == root / "junction": + return types.SimpleNamespace(st_mode=stat.S_IFDIR, st_reparse_tag=0xA0000003) + return original(path, *args, **kwargs) + + with patch.object(stat, "IO_REPARSE_TAG_MOUNT_POINT", 0xA0000003, create=True), \ + patch.object(Path, "lstat", junction): + expected = outcome(runner.symlink_free_relative_path, root, "junction/child") + require(expected[0] == "ValueError" + and outcome(cache, root, "junction/child") == expected, + "junction was not rejected identically") + return 4 + + +def check_plan(root, baseline, candidate): + names = ["shared/branch/a.txt", "shared/branch/b.txt"] + for package in ("alpha", "alpha-extra", "beta"): + write(root, "research/" + package + "/VERIFY.json", + json.dumps({"inputs": names + names[:1]})) + tracked = frozenset(path.relative_to(root).as_posix() + for path in root.rglob("*") if path.is_file() and not path.is_symlink()) + changes = [[], names[:1], names + ["research/alpha/check.py"], + ["scripts/research_package.py"], ["shared/branch/removed.txt"]] + original = candidate.symlink_free_relative_path + with patch.object(baseline, "tracked_input_paths", return_value=tracked), \ + patch.object(candidate, "tracked_input_paths", return_value=tracked): + for changed in changes: + expected = baseline.plan_details(root, changed) + require(candidate.plan_details(root, changed) == expected, + "complete plan causes or manifest hashes changed") + require(candidate.symlink_free_relative_path is original, + "successful plan leaked its cache") + write(root, "research/beta/VERIFY.json", json.dumps({"inputs": ["shared/BRANCH/a.txt"]})) + expected = outcome(baseline.plan_details, root, names) + require(expected[0] == "ValueError" + and outcome(candidate.plan_details, root, names) == expected, + "exact HEAD spelling was not retained") + require(candidate.symlink_free_relative_path is original, + "failing plan leaked its cache") + write(root, "research/beta/VERIFY.json", json.dumps({"inputs": names})) + # Same checked input becomes an alias between plans. A new plan must + # inspect it again rather than retaining the previous plan's success. + leaf = root / names[0] + leaf.unlink() + leaf.symlink_to("b.txt") + expected = outcome(baseline.plan_details, root, names) + require(expected[0] == "ValueError" + and outcome(candidate.plan_details, root, names) == expected, + "next plan reused stale filesystem validation") + leaf.unlink() + leaf.write_text("restored", encoding="utf-8") + require(candidate.plan_details(root, names) == baseline.plan_details(root, names), + "next plan failed to recover after a rejected alias") + return len(changes) + 3 + + +def check_failure(root, runner): + original = Path.lstat + + def denied(path, *args, **kwargs): + if path == root / "shared/branch": + raise PermissionError("fixture ancestor lookup denied") + return original(path, *args, **kwargs) + + cache = PathChecks(root, runner.canonical_relative_path, runner.require) + with patch.object(Path, "lstat", denied): + expected = outcome(runner.symlink_free_relative_path, root, "shared/branch/a.txt") + require(expected[0] == "PermissionError" + and outcome(cache, root, "shared/branch/a.txt") == expected, + "ancestor lookup failure was swallowed") + require("shared/branch" not in cache.checked, "a failed lookup was cached") + require(outcome(cache, root, "shared/branch/a.txt")[0] == "return", + "failed lookup poisoned a later retry") + return 2 + + +def check_source(root): + write(root, "scripts/research_package.py", "raise RuntimeError('unauthenticated source ran')") + try: + load_runner(root) + except ValueError as error: + require(str(error) == "runner source SHA256 mismatch", "wrong source rejection") + return 1 + raise ValueError("changed runner source was executed") + + +def check_benchmark(): + receipt = json.loads((PACKAGE / "BENCHMARK.json").read_text(encoding="utf-8")) + expected = {"scripts/research_package.py"} | { + "research/replay-planner-ancestor-cache/" + name + for name in ("planner.py", "benchmark.py", "VERIFY.json")} + require(set(receipt["source_sha256"]) == expected, "benchmark source coverage changed") + for name, digest in receipt["source_sha256"].items(): + require(hashlib.sha256((PACKAGE.parents[1] / name).read_bytes()).hexdigest() == digest, + "benchmark source hash mismatch: " + name) + measured = receipt["measurement"] + samples = measured["samples_seconds"] + require(len(samples) == 2 and measured["pairs"] == 9 + and all(len(side) == 9 and all(type(x) is float and x > 0 for x in side) + for side in samples), "benchmark samples are incomplete") + medians = [statistics.median(side) for side in samples] + require(medians == measured["median_seconds"] + and [[min(side), max(side)] for side in samples] == measured["range_seconds"] + and 1 - medians[1] / medians[0] == measured["fraction_less_time"], + "benchmark summary does not match samples") + require(measured["errors"] == 0 and measured["complete_outputs_equal"] is True + and measured["selected_packages"] == 3788 and min(samples[0]) > max(samples[1]), + "benchmark acceptance predicate failed") + return 1 + + +def main(): + baseline, candidate = load_runner(), install(load_runner()) + with tempfile.TemporaryDirectory(prefix="planner-ancestor-controls-") as directory: + root = Path(directory) + counts = {"guard_cases": check_guard(root, baseline), + "alias_cases": check_aliases(root, baseline), + "lookup_failure_cases": check_failure(root, baseline), + "plan_cases": check_plan(root, baseline, candidate), + "source_corruption_cases": check_source(root), + "benchmark_receipt_cases": check_benchmark()} + hashes = {path.name: hashlib.sha256(path.read_bytes()).hexdigest() + for path in sorted(PACKAGE.iterdir()) if path.is_file()} + print(json.dumps({"marker": "PASS_REPLAY_PLANNER_ANCESTOR_CACHE", "cases": counts, + "package_sha256": hashes}, sort_keys=True)) + + +if __name__ == "__main__": + main()