diff --git a/docs/research/replay-planner-input-reuse-benchmark.json b/docs/research/replay-planner-input-reuse-benchmark.json new file mode 100644 index 0000000000..1e56bad2b2 --- /dev/null +++ b/docs/research/replay-planner-input-reuse-benchmark.json @@ -0,0 +1,101 @@ +{ + "baseline": "721c03822c421b7f7fcbb2598993b947c4d88419", + "head": "721c03822c421b7f7fcbb2598993b947c4d88419", + "python": "3.14.6 (main, Jun 10 2026, 10:03:53) [Clang 21.0.0 (clang-2100.0.123.102)]", + "platform": "macOS-27.0-arm64-arm-64bit-Mach-O", + "source_sha256": { + "before": "84a45190a6a96c20993a0d4a1045fbf24de660d2e6fef8f8a447bb3d1aa78db6", + "after": "694d04550ab3cb46c03ad3cdf8be0429898fbee60ccabf1ea26fbade8d7d9895" + }, + "scope": "Complete plan equality on one stable checkout; no verifier runs.", + "cases": { + "all-tracked": { + "changed_paths": 128068, + "selected_packages": 3678, + "complete_plan_sha256": "0db78110c2e6a071d9d6486daf4d6fc3553821afa70df477bb255aed698dcfd2", + "seconds": { + "before": [ + 2.9264107919998423, + 1.287484750000658, + 1.277621124999314, + 1.452889375000268, + 1.2720515000000887 + ], + "after": [ + 0.7315727920004065, + 0.7162379999999757, + 0.7512017909994029, + 0.9831258750000416, + 0.732671124999797 + ] + }, + "median_ratio": 1.7572478374946396 + }, + "shared-control": { + "changed_paths": 1, + "selected_packages": 3678, + "complete_plan_sha256": "80f05bbd5f845875092de5955af3786720a75ab4974b136414e36c45e9d3f979", + "seconds": { + "before": [ + 1.1906395000005432, + 1.1812420420001217, + 1.2033583749998797, + 1.2103980410001896, + 1.2037691249997806 + ], + "after": [ + 0.6428887920001216, + 0.6389516670005833, + 0.6400463330001003, + 0.6461026669994681, + 0.6397243329993216 + ] + }, + "median_ratio": 1.880111349688322 + }, + "package-file": { + "changed_paths": 1, + "selected_packages": 22, + "complete_plan_sha256": "f434daa2502211dd78f214845bc6b9219b73b5a15cb641db3aad7962cb68c7ab", + "seconds": { + "before": [ + 1.1752807499997289, + 1.6841356249997261, + 1.1927050410004085, + 1.1669380829998772, + 1.176353208999899 + ], + "after": [ + 0.6379960409994965, + 0.6744783750000352, + 0.6313787500002945, + 0.6304629589994875, + 0.642034832999343 + ] + }, + "median_ratio": 1.8438252487538984 + }, + "unrelated": { + "changed_paths": 1, + "selected_packages": 1, + "complete_plan_sha256": "c2b6a0cc41489ccadea83a398f5fc5e701294ea96271825c3e727fbb850606bc", + "seconds": { + "before": [ + 1.181231083000057, + 1.2318855419998727, + 1.5036698750000141, + 1.2036627910001698, + 1.20319683300022 + ], + "after": [ + 0.7455231249996359, + 0.9656177920005575, + 1.3634524589997454, + 0.6298098329998538, + 0.6532685000001948 + ] + }, + "median_ratio": 1.61452106666813 + } + } +} diff --git a/docs/research/replay-planner-input-reuse.md b/docs/research/replay-planner-input-reuse.md new file mode 100644 index 0000000000..b3563181a5 --- /dev/null +++ b/docs/research/replay-planner-input-reuse.md @@ -0,0 +1,112 @@ +# Reuse input validation within one replay plan + +At base `721c03822c421b7f7fcbb2598993b947c4d88419`, the planner validates +each declared input occurrence independently. Shared files therefore repeat +the same exact-path check and component `lstat` calls across many packages. +A read-only profile of the complete planner counted 36,576 path-guard calls +for 14,638 distinct paths, including manifest discovery. The guard accounted +for 2.319 seconds of a 4.069-second profiled run. This identifies redundant +work; those instrumented timings are not the benchmark below. + +The change retains a set of successfully validated **declared input names** +inside `plan_details`. A name enters it only after both the existing filesystem +guard and exact HEAD-or-changed-path check succeed. Non-string values still +reach the original canonical-path error. No validation state survives a call. +Manifest discovery, reading, hashing, HEAD acquisition and all selection causes +are unchanged. In particular, skipping a repeated validation does not skip a +consumer or remove a repeated input from the manifest's authenticated bytes. + +The existing contract requires a stable checkout during planning. For a fixed +root, tracked-path set and filesystem, successful repeated checks have the same +result, so replacing them with membership checks preserves the complete plan. +This is not an atomic filesystem snapshot. A later invocation must check again, +including after a file becomes a symlink or junction, HEAD changes, a deleted +input leaves the current diff, or a manifest changes its spelling. +The set retains O(U) distinct input names for that call; peak memory was not +benchmarked. + +## Measurement + +The existing benchmark by Ali Romman compares complete `plan_details` results, +not only package counts. It ran five alternating pairs per case on the same +3,678-manifest checkout. All 40 complete results matched their corresponding +baseline, including ordering, causes and manifest SHA256 values. + +| Changed paths | Selected packages | Baseline median | Candidate median | Ratio | +| --- | ---: | ---: | ---: | ---: | +| All tracked paths | 3,678 | 1.287 s | 0.733 s | 1.757x | +| Shared runner | 3,678 | 1.203 s | 0.640 s | 1.880x | +| One package file | 22 | 1.176 s | 0.638 s | 1.844x | +| README.md | 1 | 1.204 s | 0.746 s | 1.615x | + +[Raw samples, environment, source hashes and full-result digests](replay-planner-input-reuse-benchmark.json) +bind this evidence. The benchmark was run before committing the implementation: +its `head` identifies the checkout baseline; `source_sha256.after` identifies +the exact candidate bytes. The latter are committed in +`585e1404f` and remain unchanged in the documentation commit. + +These are local macOS 27 arm64 / CPython 3.14.6 measurements with warm filesystem +caches, repeated calls in one process, and other work on the host. The raw samples +include cold/noisy outliers. No fresh-process, cold-cache, Linux, Windows or +whole-CI speedup is claimed. Reproduce with: + +```sh +python3 -B scripts/benchmark_research_plan.py \ + --baseline 721c03822c421b7f7fcbb2598993b947c4d88419 --samples 5 +``` + +## Validation + +Six new tests use temporary real Git repositories and independently constructed +expected plans. They check shared-input validation counts without losing either +consumer, every cause or either hash; fresh permission and symlink rejection; +HEAD/deletion/diff freshness; manifest reload and exact-case/canonical spelling; +and non-string diagnostics, including unhashable values. The unmodified baseline +fails the redundant-validation test. Separate injected persistent-cache and +skipped-tracking-guard controls are rejected by the freshness/spelling tests. + +The following nine modules pass normally and with `-O`, under `-B -W error`: +92 tests per mode on both CPython 3.14.6 and 3.12.2. Existing platform/tool +skips total eight per mode on 3.14 and four per mode on 3.12. + +```sh +for test in plan_input_reuse input_paths symlink_inputs sparse_planning \ + plan_explanations replay_selection matrix_batches git_root_binding onboarding +do + python3 -B -W error "tests/test_research_${test}.py" + python3 -O -B -W error "tests/test_research_${test}.py" +done +``` + +The committed shared runner and new test were also exported with `git archive` +into a fresh Git-free tree, made read-only, and run normally and optimized. +All six tests pass in each mode and exported source hashes remain unchanged. +The tests create their own temporary Git fixtures outside that read-only tree. +This shared-code change has no registered research package: package-specific +source-history and package-export hooks are not applicable. + +Native Windows junction execution, hosted CI, complete historical replay and +scientific admission were not performed. Existing junction and sparse guards +are retained; no selected certificate or frozen source is modified. + +## Scope, attribution and open next questions + +This refines the `replay-runtime` card, seed +`48bb2ae90f98183ae8b15a7200992341`, on priority revision 8. +The [canonical investigation](https://github.com/CrocSwap/integer-mult-bounds/discussions/6077) +records the claim. The selected PR6030 baseline passed source-binding checks; +this work establishes no new mathematical bound or supplier admission. + +Credit Douglas Colkitt for the registered runner, Ali Romman for the retained +complete-plan benchmark, and the earlier exact-path, junction and sparse-input +guards. Per-invocation reuse is standard; the contribution is its bounded +application and measured coverage-preserving effect in this planner. +Prepared by JoyboyBrian with substantial OpenAI Codex assistance. Apache-2.0. +Independent human review is not claimed. + +The next useful check is to run the same exact-output benchmark and native +junction regressions on a Windows checkout, reporting source hashes and the +complete receipts. That can establish host-specific behavior; a Windows host +is required. No broader cache or additional optimization is justified by this +result alone. Concurrent filesystem mutation remains outside the existing +stable-checkout contract. diff --git a/scripts/research_package.py b/scripts/research_package.py index 48fe9feb46..24326db165 100644 --- a/scripts/research_package.py +++ b/scripts/research_package.py @@ -907,6 +907,8 @@ def plan_details(root, changed): changed_packages.setdefault("/".join(parts[:2]), []).append(name) packages = [] sparse_count = 0 + # Planning requires a stable checkout; reuse only within this invocation. + validated_inputs = set() for path, raw, sparse in planning_manifests(root, head_tracked): sparse_count += sparse package = path.parent.relative_to(root).as_posix() @@ -915,9 +917,12 @@ def plan_details(root, changed): inputs = manifest.get("inputs", []) require(isinstance(inputs, list), "declare input paths for CI invalidation") for name in inputs: + if isinstance(name, str) and name in validated_inputs: + continue # Do not require existence: a deleted input must still trigger replay. symlink_free_relative_path(root, name) require_tracked_input(name, tracked) + validated_inputs.add(name) causes = { "declared_inputs": sorted(changed.intersection(inputs)), "package_files": changed_packages.get(package, []), diff --git a/tests/test_research_plan_input_reuse.py b/tests/test_research_plan_input_reuse.py new file mode 100644 index 0000000000..e78aecbf8b --- /dev/null +++ b/tests/test_research_plan_input_reuse.py @@ -0,0 +1,118 @@ +"""Shared inputs retain every consumer and are revalidated on each plan. + +JoyboyBrian, with substantial OpenAI Codex assistance. Apache-2.0. +""" +import hashlib +import importlib.util +import json +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location('input_reuse_runner', ROOT / 'scripts/research_package.py') +runner = importlib.util.module_from_spec(spec) +spec.loader.exec_module(runner) + + +class PlanInputReuseTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.write('inputs/shared.txt', 'shared') + self.write('inputs/other.txt', 'other') + self.manifests = {} + for name in ('alpha', 'beta'): + raw = json.dumps({'inputs': ['inputs/shared.txt', 'inputs/shared.txt', + 'inputs/other.txt']}) + self.write('research/' + name + '/VERIFY.json', raw) + self.manifests[name] = raw.encode() + self.git('init', '-q') + self.commit() + + def write(self, name, text): + path = self.root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding='utf-8') + + def git(self, *args): + return subprocess.run(['git', *args], cwd=self.root, check=True, + capture_output=True, timeout=15).stdout + + def commit(self): + self.git('add', '--all') + self.git('-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', + '-c', 'core.autocrlf=false', 'commit', '-qm', 'fixture') + + def test_each_shared_input_is_checked_once_without_losing_consumers_or_causes(self): + changed = ['inputs/shared.txt', 'research/alpha/check.py', 'scripts/research_package.py'] + with patch.object(runner, 'symlink_free_relative_path', + wraps=runner.symlink_free_relative_path) as filesystem, \ + patch.object(runner, 'require_tracked_input', wraps=runner.require_tracked_input) as tracked: + result = runner.plan_details(self.root, changed) + for name in ('inputs/shared.txt', 'inputs/other.txt'): + self.assertEqual(sum(call.args[1] == name for call in filesystem.call_args_list), 1) + self.assertEqual(sum(call.args[0] == name for call in tracked.call_args_list), 1) + self.assertEqual(result, [ + {'package': 'research/' + name, + 'causes': {'declared_inputs': ['inputs/shared.txt'], + 'package_files': ['research/alpha/check.py'] if name == 'alpha' else [], + 'shared_controls': ['scripts/research_package.py']}, + 'manifest_sha256': hashlib.sha256(raw).hexdigest()} + for name, raw in self.manifests.items()]) + + def test_next_plan_rechecks_filesystem_and_propagates_lookup_failures(self): + runner.plan_details(self.root, []) + original = Path.lstat + shared = self.root / 'inputs/shared.txt' + + def denied(path, *args, **kwargs): + if path == shared: + raise PermissionError('shared input lookup denied') + return original(path, *args, **kwargs) + + with patch.object(Path, 'lstat', denied): + with self.assertRaisesRegex(PermissionError, 'shared input lookup denied'): + runner.plan_details(self.root, []) + + def test_next_plan_rejects_a_new_symlink_alias(self): + runner.plan_details(self.root, []) + shared = self.root / 'inputs/shared.txt' + shared.unlink() + try: + shared.symlink_to('other.txt') + except OSError as error: + self.skipTest('symlink creation unavailable: ' + str(error)) + with self.assertRaisesRegex(ValueError, 'symlink in repository-relative path'): + runner.plan_details(self.root, ['inputs/other.txt']) + + def test_next_plan_reloads_head_and_does_not_remember_deleted_diff_paths(self): + runner.plan_details(self.root, []) + (self.root / 'inputs/shared.txt').unlink() + self.commit() + rows = runner.plan_details(self.root, ['inputs/shared.txt']) + self.assertEqual([row['package'] for row in rows], ['research/alpha', 'research/beta']) + for row in rows: + self.assertEqual(row['causes']['declared_inputs'], ['inputs/shared.txt']) + with self.assertRaisesRegex(ValueError, 'exact Git-tracked input path'): + runner.plan_details(self.root, []) + + def test_next_plan_reloads_manifest_and_preserves_exact_spelling(self): + runner.plan_details(self.root, ['inputs/shared.txt']) + for name in ('inputs/SHARED.txt', './inputs/shared.txt'): + self.write('research/beta/VERIFY.json', json.dumps({'inputs': [name]})) + with self.subTest(name=name), self.assertRaises(ValueError): + runner.plan_details(self.root, ['inputs/shared.txt']) + + def test_nonstring_input_keeps_the_canonical_path_error(self): + for name in ([], {}, None, 1): + self.write('research/beta/VERIFY.json', json.dumps({'inputs': ['inputs/shared.txt', name]})) + with self.subTest(name=name), self.assertRaisesRegex(ValueError, 'canonical relative path must be a string'): + runner.plan_details(self.root, ['inputs/shared.txt']) + + +if __name__ == '__main__': + unittest.main()