From 68186ffb68582d8f81495de7655d9950e310ce56 Mon Sep 17 00:00:00 2001 From: Nick Petalas Date: Thu, 29 Oct 2026 16:31:51 +0000 Subject: [PATCH 1/2] perf(research): reuse the checked archive for input coverage --- docs/research/export-input-coverage.md | 4 +- .../export-single-scan/BENCHMARK-small.json | 160 +++ .../export-single-scan/BENCHMARK-toolkit.json | 160 +++ research/export-single-scan/PROFILE.json | 112 ++ research/export-single-scan/README.md | 171 +++ research/export-single-scan/RESULT.json | 49 + research/export-single-scan/SOURCES.json | 46 + research/export-single-scan/TASK.json | 63 + research/export-single-scan/VALIDATION.json | 85 ++ research/export-single-scan/VERIFY.json | 79 ++ .../baseline/export_research_package.py | 252 ++++ .../baseline/research_package.py | 1053 +++++++++++++++++ research/export-single-scan/benchmark.py | 84 ++ research/export-single-scan/verify.py | 175 +++ scripts/export_research_package.py | 67 +- tests/test_research_export_input_coverage.py | 62 + 16 files changed, 2595 insertions(+), 27 deletions(-) create mode 100644 research/export-single-scan/BENCHMARK-small.json create mode 100644 research/export-single-scan/BENCHMARK-toolkit.json create mode 100644 research/export-single-scan/PROFILE.json create mode 100644 research/export-single-scan/README.md create mode 100644 research/export-single-scan/RESULT.json create mode 100644 research/export-single-scan/SOURCES.json create mode 100644 research/export-single-scan/TASK.json create mode 100644 research/export-single-scan/VALIDATION.json create mode 100644 research/export-single-scan/VERIFY.json create mode 100644 research/export-single-scan/baseline/export_research_package.py create mode 100644 research/export-single-scan/baseline/research_package.py create mode 100644 research/export-single-scan/benchmark.py create mode 100644 research/export-single-scan/verify.py diff --git a/docs/research/export-input-coverage.md b/docs/research/export-input-coverage.md index aafad3e770..1ba001b160 100644 --- a/docs/research/export-input-coverage.md +++ b/docs/research/export-input-coverage.md @@ -18,7 +18,9 @@ The option also reads the bundled `VERIFY.json` and checks these declarations: A successful console receipt includes `declared_inputs_checked`, with the number of declarations checked. Zero is valid for a self-contained package. The command uses no Git process and does not extract files or run the verifier. The manifest's -second read must match its checked payload hash. Manifest bytes are limited to +second read reuses the open archive and its checked member index, and must match +its checked payload hash. The command closes the archive on success or failure. +Manifest bytes are limited to 16 MiB and input declarations to 100,000 entries. This is a declaration-coverage check, not full `VERIFY.json` schema validation or diff --git a/research/export-single-scan/BENCHMARK-small.json b/research/export-single-scan/BENCHMARK-small.json new file mode 100644 index 0000000000..7056c92b4a --- /dev/null +++ b/research/export-single-scan/BENCHMARK-small.json @@ -0,0 +1,160 @@ +{ + "archive_bytes": 2324480, + "archive_sha256": "568604c6c3cf6ecbda5ff86102e7a675dfa047fd64fdd3aa813da180bbbe09f9", + "commands_per_side_per_pair": 5, + "complete_stdout": "PASS_RESEARCH_EXPORT {\"commit\": \"a5bff3bf133e482541c09eba4e773884bdb2b1b7\", \"package\": \"research/final4610-independent-admission\", \"files\": 82, \"declared_inputs_checked\": 51}\n", + "elapsed_ns": [ + [ + [ + 40535375, + 40328917, + 40592333, + 40010709, + 40946792 + ], + [ + 40141833, + 40378875, + 40242208, + 40036208, + 40503125 + ], + [ + 41392666, + 40478625, + 40398791, + 40137958, + 40199917 + ], + [ + 41817458, + 40782750, + 40814334, + 39955875, + 40146375 + ], + [ + 40438209, + 39944042, + 39986834, + 39949292, + 40283500 + ], + [ + 40890709, + 40913750, + 40495792, + 40276125, + 40507750 + ], + [ + 41103583, + 40287167, + 40731541, + 41759167, + 40823041 + ], + [ + 40777833, + 41380125, + 40550125, + 40687292, + 40666083 + ], + [ + 41398208, + 40275209, + 40780375, + 40356291, + 40405625 + ] + ], + [ + [ + 40152458, + 42102500, + 40285083, + 40468875, + 40626750 + ], + [ + 41413625, + 40388084, + 39879041, + 39751708, + 40112917 + ], + [ + 39316500, + 39956250, + 39559750, + 39818375, + 40188792 + ], + [ + 39744875, + 40215791, + 40301583, + 39632709, + 40619083 + ], + [ + 39605709, + 41376542, + 41108041, + 40421083, + 39949917 + ], + [ + 39514375, + 40073833, + 42060334, + 40252333, + 40177291 + ], + [ + 39641041, + 39923500, + 40035834, + 39509125, + 39606709 + ], + [ + 40018292, + 39808167, + 40290083, + 39877834, + 39804250 + ], + [ + 40525333, + 40392875, + 40348458, + 39584000, + 39745833 + ] + ] + ], + "environment": { + "load": [ + 3.47412109375, + 3.14794921875, + 2.67626953125 + ], + "logical_cpus": 18, + "platform": "macOS-27.0.1-arm64-arm-64bit", + "python": "3.12.15 (main, Oct 14 2026, 00:55:37) [Clang 23.1.3 ]" + }, + "errors": 0, + "median_ns": [ + 40495792, + 40073833 + ], + "pairs": 9, + "source_sha256": [ + "7126afaebff2328b8d64ba1ce992c497103820284f52d4590a558f97f13965fe", + "68d1a58056c8ab83690b11833d7e362d5e3b2b2632274415b240c676293547e4" + ], + "successful_commands": 90, + "wall_time_reduction": 0.0104198233732532 +} diff --git a/research/export-single-scan/BENCHMARK-toolkit.json b/research/export-single-scan/BENCHMARK-toolkit.json new file mode 100644 index 0000000000..6485e4a3f4 --- /dev/null +++ b/research/export-single-scan/BENCHMARK-toolkit.json @@ -0,0 +1,160 @@ +{ + "archive_bytes": 13701120, + "archive_sha256": "97d88fe02c1538ecd3570ff028bf1cfea268ced91d9ae811c738701c41e25767", + "commands_per_side_per_pair": 5, + "complete_stdout": "PASS_RESEARCH_EXPORT {\"commit\": \"a5bff3bf133e482541c09eba4e773884bdb2b1b7\", \"package\": \"research/p104-designer-toolkit-v2\", \"files\": 529, \"declared_inputs_checked\": 0}\n", + "elapsed_ns": [ + [ + [ + 59001875, + 59626916, + 59468792, + 60037708, + 58884792 + ], + [ + 58664833, + 58957125, + 59150250, + 60545125, + 59882417 + ], + [ + 58791792, + 58914334, + 58512791, + 58851208, + 61036167 + ], + [ + 59640125, + 59497333, + 59072542, + 58337125, + 59460375 + ], + [ + 60365083, + 60283375, + 60331375, + 59081792, + 60716667 + ], + [ + 60784541, + 59800375, + 61234791, + 61354625, + 59097292 + ], + [ + 59707667, + 58984041, + 58772750, + 59100667, + 59210750 + ], + [ + 59131000, + 59396417, + 59763458, + 58973167, + 59013250 + ], + [ + 59632625, + 59838917, + 59693708, + 59371334, + 59013000 + ] + ], + [ + [ + 54202334, + 53876583, + 54875917, + 54344583, + 55683958 + ], + [ + 56009166, + 55127458, + 54762708, + 54889916, + 54171000 + ], + [ + 55549916, + 55924625, + 53355958, + 55792750, + 54451417 + ], + [ + 54862083, + 54226333, + 54685167, + 54870416, + 54812417 + ], + [ + 55419791, + 54900875, + 55402542, + 56023292, + 56586834 + ], + [ + 56488375, + 55935542, + 55517125, + 55330375, + 55094584 + ], + [ + 54964250, + 55091916, + 56029959, + 56343125, + 54413584 + ], + [ + 54905875, + 54732583, + 54202375, + 55323875, + 54573708 + ], + [ + 56250250, + 55052417, + 55164000, + 54847541, + 54148292 + ] + ] + ], + "environment": { + "load": [ + 3.4287109375, + 3.1337890625, + 2.6689453125 + ], + "logical_cpus": 18, + "platform": "macOS-27.0.1-arm64-arm-64bit", + "python": "3.12.15 (main, Oct 14 2026, 00:55:37) [Clang 23.1.3 ]" + }, + "errors": 0, + "median_ns": [ + 59396417, + 54964250 + ], + "pairs": 9, + "source_sha256": [ + "7126afaebff2328b8d64ba1ce992c497103820284f52d4590a558f97f13965fe", + "68d1a58056c8ab83690b11833d7e362d5e3b2b2632274415b240c676293547e4" + ], + "successful_commands": 90, + "wall_time_reduction": 0.07462010713541867 +} diff --git a/research/export-single-scan/PROFILE.json b/research/export-single-scan/PROFILE.json new file mode 100644 index 0000000000..f3a11b7892 --- /dev/null +++ b/research/export-single-scan/PROFILE.json @@ -0,0 +1,112 @@ +{ + "baseline": { + "rows": [ + { + "calls": 1, + "cumulative_seconds": 0.009423167000000001, + "file": "tarfile.py", + "function": "_load", + "line": 2869, + "primitive_calls": 1, + "self_seconds": 4.0748000000000004e-05 + }, + { + "calls": 1, + "cumulative_seconds": 0.009424042, + "file": "tarfile.py", + "function": "getmembers", + "line": 2077, + "primitive_calls": 1, + "self_seconds": 8.33e-07 + }, + { + "calls": 1, + "cumulative_seconds": 0.00964225, + "file": "export_research_package.py", + "function": "check_declared_inputs", + "line": 142, + "primitive_calls": 1, + "self_seconds": 4.9118000000000006e-05 + }, + { + "calls": 1064, + "cumulative_seconds": 0.019141883000000002, + "file": "tarfile.py", + "function": "next", + "line": 2764, + "primitive_calls": 1064, + "self_seconds": 0.000746635 + }, + { + "calls": 1, + "cumulative_seconds": 0.052266417, + "file": "export_research_package.py", + "function": "check_bundle", + "line": 178, + "primitive_calls": 1, + "self_seconds": 0.0015643220000000002 + } + ], + "total_seconds": 0.07324024799999984 + }, + "candidate": { + "rows": [ + { + "calls": 1, + "cumulative_seconds": 0.048918, + "file": "export_research_package.py", + "function": "check_bundle", + "line": 230, + "primitive_calls": 1, + "self_seconds": 1.5625e-05 + }, + { + "calls": 1, + "cumulative_seconds": 6.660000000000001e-07, + "file": "tarfile.py", + "function": "getmembers", + "line": 2077, + "primitive_calls": 1, + "self_seconds": 6.240000000000001e-07 + }, + { + "calls": 1, + "cumulative_seconds": 0.029078834, + "file": "export_research_package.py", + "function": "read_bundle", + "line": 181, + "primitive_calls": 1, + "self_seconds": 0.001071925 + }, + { + "calls": 1, + "cumulative_seconds": 0.000138917, + "file": "export_research_package.py", + "function": "check_declared_inputs", + "line": 142, + "primitive_calls": 1, + "self_seconds": 5.1273e-05 + }, + { + "calls": 532, + "cumulative_seconds": 0.012503646, + "file": "tarfile.py", + "function": "next", + "line": 2764, + "primitive_calls": 532, + "self_seconds": 0.000408279 + }, + { + "calls": 1, + "cumulative_seconds": 0.0169815, + "file": "export_research_package.py", + "function": "check_bundle_inventory", + "line": 202, + "primitive_calls": 1, + "self_seconds": 0.000503751 + } + ], + "total_seconds": 0.06984487599999982 + }, + "scope": "Untimed cProfile diagnostics only; not the reported wall-time comparison." +} diff --git a/research/export-single-scan/README.md b/research/export-single-scan/README.md new file mode 100644 index 0000000000..a69d631f63 --- /dev/null +++ b/research/export-single-scan/README.md @@ -0,0 +1,171 @@ +# Reuse the archive index during offline input checking + +This contribution was produced by Nick Petalas. + +The complete offline `check --check-inputs` command takes **7.46% less median +wall time** on the recorded 529-file export, from 59.396417 ms to 54.964250 ms. +All 90 complete commands succeed and emit identical output. Each of the nine +paired batches improves. This is a measured tooling improvement with a scoped +equivalence argument and finite conformance evidence. It supplies no new +mathematical theorem, native supplier or kappa. + +The smaller 82-file export gives medians of 40.495792 and 40.073833 ms with +overlapping ranges and one losing paired batch. That comparison is inconclusive. +The reported gain applies to the larger workload on this host. + +## Change and equivalence argument + +The old checker opens the tar, hashes each member and checks the receipt, then +closes it. Declaration checking opens the same path again; `getmember` loads +every header into a second tar index before rereading `VERIFY.json`. + +The changed checker keeps the first handle open through declaration checking. +Its completed iteration has already populated the index. `getmember` therefore +uses that index and rereads only the manifest payload. The reread still has to +match the size and SHA256 recorded during integrity checking. All receipt, +path, tree, mode, hash, JSON, declaration and inventory predicates are retained. +The direct `check_declared_inputs(path, receipt, seen)` call still opens and +closes its own handle. A caller-supplied handle is borrowed and remains open. +The outer command owns its handle and closes it on success and exceptions. + +For a stable finite archive, the first scan produces the same member sequence, +payload hashes and receipt. Receipt validation is unchanged. The old second +scan and the retained first index describe the same members, so they locate +the same manifest. The exact same payload and declaration predicates then +return the same full receipt or rejection. The changed program removes one +header scan, without caching payload hashes across invocations. This is a +source-level argument, supported by the complete-outcome differential fixtures; +finite tests alone do not prove equivalence for every archive. + +Archive mutation during checking is outside this stable-input equivalence +claim. The retained same-handle reread detects an in-place manifest change after +hashing. Reusing one handle also avoids switching to a different file merely +because its pathname is replaced. It does not provide a filesystem snapshot. +There is no new extraction or Git-provenance claim. + +## Measurements + +The primary fixture is the official committed export of +`research/p104-designer-toolkit-v2` at +`a5bff3bf133e482541c09eba4e773884bdb2b1b7`: 529 payload files, 13,701,120 archive +bytes, and zero external declarations. The command still performs manifest +schema and declaration-coverage validation. The secondary fixture is the +82-file `research/final4610-independent-admission` export, with 51 declarations. +These packages are workload bytes only; their scientific verifiers were not +run or audited for this contribution. + +[BENCHMARK-toolkit.json](BENCHMARK-toolkit.json) and +[BENCHMARK-small.json](BENCHMARK-small.json) are generated raw observations. +Each has nine alternating pairs of batches, five fresh complete CLI processes +per side per batch, with the starting side alternating by pair. Timing includes +startup, imports, archive I/O, validation and output capture. Every process has +Git absent from PATH. Output equality, zero errors and unchanged archive hashes +are checked. There are 45 observations per side per fixture, 180 commands total. +No timing assertion is imposed on CI. + +The primary before range is 58.337125 to 61.354625 ms; the after range is +53.355958 to 56.586834 ms. Measurements use CPython 3.12.15 on macOS 27.0.1 +arm64, with 18 logical CPUs and load averages recorded in each observation. +Concurrent work was present and could not be stopped; alternating pairs expose +both sides to that load. The gap exceeds the observed primary ranges. + +An untimed `cProfile` run attributes about 10 ms of its 73 ms total to the old +declaration check, chiefly the second header scan. Profiling overhead is excluded +from the benchmark. Imports, path/tree checks and payload hashing remain costs +of the complete command. Removing that measured stage is consistent with the +4.432167 ms complete-command median reduction; no microbenchmark ratio is used. + +## Reproduce + +From the repository root, using the standard library and CPython 3.12: + +```sh +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -W error -B research/export-single-scan/verify.py +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -O -W error -B research/export-single-scan/verify.py +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -B scripts/research_package.py --package research/export-single-scan +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -B scripts/check_research_commit.py --package research/export-single-scan --check-source-history +``` + +Replace the interpreter path with another CPython 3.12 installation if needed. +The direct verifier authenticates the retained original source, compares full +accepted receipts and rejection type/message, validates recorded timing +arithmetic and checks the generated source-bound [RESULT.json](RESULT.json). +Its 44 finite archives cover receipt position, executable modes, payload counts, +malformed declarations, missing and extra inputs, and corrupted content, with +default and declaration-enabled checking. The registered commands additionally +run the complete exporter regression suite in normal and optimized Python. +The direct comparison has 67 accepted and 21 rejected outcome pairs. Each +exporter suite runs 46 tests, with 45 passes and the existing native-Windows-only +publication test skipped on macOS. Four new tests cover one-open completion, +failure cleanup, borrowed-handle ownership and real post-hash manifest mutation. + +To repeat the performance measurement, choose fresh external paths: + +```sh +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -B scripts/export_research_package.py export --package research/p104-designer-toolkit-v2 --ref a5bff3bf133e482541c09eba4e773884bdb2b1b7 --output /tmp/toolkit-benchmark.tar +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -W error -B research/export-single-scan/benchmark.py --archive /tmp/toolkit-benchmark.tar --output /tmp/toolkit-benchmark.json +``` + +The baseline directory contains the two exact original scripts needed to run +the old CLI, retaining their notices. [SOURCES.json](SOURCES.json) binds them +to the base Git blobs and SHA256 values. The benchmark never replaces outputs. +Receipt regeneration uses `verify.py --emit-to NEW_EXTERNAL_FILE`; ordinary +verification is read-only apart from private temporary fixtures. + +## Scope and credit + +Card `replay-runtime`, priority revision 8, is refined to one standard transport +check before replay. The selected binding-only baseline passed. Scientific base +and selected source bindings are pinned in [TASK.json](TASK.json) and +[SOURCES.json](SOURCES.json). The selected frontier and its five assumptions are +unchanged. Lean, Docker, selected scientific replay, mathematical predecessor +audits, native Windows execution and cross-platform timing were not run. +Independent human and formal proof-kernel review are unclaimed. + +Credit Tekalign / PR3138 for the exporter; Ali Romman / PR5744 and PR5728 for +declared coverage and typed metadata; roger-sato / PR3629 and PR3641 for path +and tree checks; Mihir Srivastava for portable fixtures; Jose Carlos Gonzalez +Roque / PR6189 for Windows publication; and Douglas Colkitt and the retained +Git-reader contributors. Original notices and source bytes are unchanged. +The new contribution is archive-index reuse, its lifecycle controls and measured +complete-command effect. New work is Apache-2.0. + +The [advance claim](https://github.com/CrocSwap/integer-mult-bounds/discussions/1166#discussioncomment-19272236) +records the target. Live open and merged PRs, recent account work, handoffs, +local reservations and distinctive main README searches were checked. The +overlap cache had complete remote pagination for 310 open PRs and 5,907 +Discussion records, but omitted 8,534 main-tree local notes. Additional targeted +main and merged-PR searches address this object. Unpublished work and arbitrary +historical diffs remain outside coverage. No novelty guarantee is inferred. + +## Open next questions + +Does the same change improve complete-command time on Linux and Windows, or on +larger real exports? Start with the pinned official export and the same +alternating benchmark. Native host measurements are the missing evidence; the +macOS percentage is not a portable performance guarantee. No further optimization +or follow-on target is reserved. + +## Implementation constraints + +The changed checker has structured, nonrecursive control flow and functions +under 60 physical lines. It uses one owned archive and one scoped manifest +stream; borrowed handles use `nullcontext`. Returned counts and CLI failures +are checked. Resource ownership, missing inputs, direct-call fallback and +changed-manifest rejection have explicit regressions. Warning-strict normal and +optimized tests and in-memory compilation supply diagnostics; no external +Python linter was available. The package harness is bounded to 44 archives, +at most 31 external files per archive and 20 benchmark pairs of ten processes +per side. Each process has a 60-second timeout. + +Power of Ten rules 2 and 3 have a retained finite-file API deviation in the +shared checker. Tar traversal ends at the finite input's EOF, and payload reads +end at each member's declared extent; no new fixed archive-size cap is imposed. +Imposing one would narrow the existing accepted domain. CPython, `tarfile`, JSON +and hashing allocate dynamically. The member inventory is bounded by the finite +archive, while manifest and receipt reads retain their 16 MiB bounds and input +declarations retain their 100,000-entry bound. The changed path retains one tar +index instead of building two sequential indexes. Fixed-capacity operation and +concurrent filesystem mutation require a separately specified interface before +any service with those requirements is built. The finite conformance and full +exporter suites are the compensating checks for this change. diff --git a/research/export-single-scan/RESULT.json b/research/export-single-scan/RESULT.json new file mode 100644 index 0000000000..bd78c38ed9 --- /dev/null +++ b/research/export-single-scan/RESULT.json @@ -0,0 +1,49 @@ +{ + "conformance": { + "accept": 67, + "complete_outcomes_sha256": "564c51a8f4251593f2e18f54d926aa6a84697e90f656014df10897905d628dc5", + "fixtures": 44, + "reject": 21 + }, + "input_sha256": { + "scripts/export_research_package.py": "68d1a58056c8ab83690b11833d7e362d5e3b2b2632274415b240c676293547e4", + "scripts/research_package.py": "e1256016e7f51827d04fa2179255989f639ae1d97afaa9c16265f85a0c6abd70", + "tests/test_research_export.py": "d161d4641f7cdb71ee1c26b0f3dedb16919ca0ff874a42f150d60c54c159b3ba", + "tests/test_research_export_drive_paths.py": "65ccee24438634afcc521b1504d1332f0c889f756486ea70ff9b911332d635ec", + "tests/test_research_export_input_coverage.py": "0f12c1b3d1b5a223c89b600ab34de91a30d4b1decd22d220be6b7d44fc819554", + "tests/test_research_export_input_ref.py": "dd68f2aba8807bd2d85d8fa84de322f06efc987f5bc9dd6b9e14d43129c10064", + "tests/test_research_export_metadata.py": "2692df0a5a519f562b3fba0e01d6f0694ce26c79751659dec01af0034632fb87", + "tests/test_research_export_publication.py": "f9b3f0287051cf91c1faf49f12eb126c0fb444b058daf2b607d3910a55a5676e", + "tests/test_research_export_topology.py": "3b2e95a8fbc5a817384286f4a44315e9497238add8bcdb3be8559f9f62397208" + }, + "package_sha256": { + "BENCHMARK-small.json": "4f261a6709cb5cc33f97268b205357fc81804251b0268037b7dd6508f011eb5c", + "BENCHMARK-toolkit.json": "78146a0d0905055d3629237404d1fa7efe8dd5aade69695b121927b3b9241d15", + "PROFILE.json": "27a4b48bf480652ba8d27bfacfb4d2b7e7f7cff570b83683dd28adba8d1671d9", + "README.md": "8fd66e8a83c149bee89335f3281d061fedac49d2c4a0a6f104a41bee6bcdf28c", + "SOURCES.json": "f699c23d9fd1ee46f4ded7a21b0f5ad65be292c1867b895e36a70cd312b90419", + "TASK.json": "06638e8d16d96892ea8ac8a2eb3a3c32f97e2d725271f207b1e62efb082e6742", + "VALIDATION.json": "24e959de5f350fc3e58cbe4195d7c5ea98a40142cf4b438830427539d4672f93", + "VERIFY.json": "8d37fe36f0513ea3a86d0b7ed781a3023412cd720fe717b437b5e2dde8a869ec", + "baseline/export_research_package.py": "7126afaebff2328b8d64ba1ce992c497103820284f52d4590a558f97f13965fe", + "baseline/research_package.py": "e1256016e7f51827d04fa2179255989f639ae1d97afaa9c16265f85a0c6abd70", + "benchmark.py": "331ea6d2c29dbff752fbb036490f8a60ff70e011f18a719a9426ff1473b1cedb", + "verify.py": "60e44e62b245545b77733750d476e279df2b14493292c6175a3df76018392f9e" + }, + "recorded_measurements": { + "small": { + "improving_batches": 8, + "median_ns": [ + 40495792, + 40073833 + ] + }, + "toolkit": { + "improving_batches": 9, + "median_ns": [ + 59396417, + 54964250 + ] + } + } +} diff --git a/research/export-single-scan/SOURCES.json b/research/export-single-scan/SOURCES.json new file mode 100644 index 0000000000..bd52284ce7 --- /dev/null +++ b/research/export-single-scan/SOURCES.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "base_commit": "a5bff3bf133e482541c09eba4e773884bdb2b1b7", + "files": [ + { + "path": "certificates/selected-result.json", + "usage": "provenance", + "git_blob": "d59b5035e7b4cbaa9d688a1d32dd0d6eb79c466a", + "sha256": "9b8d0eea95b3bf0dfcb7938a732a6439b87a2685dec2a5ab9e525b3b34debbfa" + }, + { + "path": "docs/research/agent-task-catalog.json", + "usage": "provenance", + "git_blob": "3404fbdd5e7529a0674eef26114b5071e4586cec", + "sha256": "ead6cdc9d0af6d58e8a49744d8ec8c7c15e13d2ba93c276c434bcf9b51b93a81" + }, + { + "path": "scripts/export_research_package.py", + "usage": "provenance", + "git_blob": "25b0a12a8073f589867ca7f2e564ea5ae011bdf5", + "sha256": "7126afaebff2328b8d64ba1ce992c497103820284f52d4590a558f97f13965fe" + }, + { + "path": "scripts/research_package.py", + "usage": "provenance", + "git_blob": "091b4c8f58bac9a9580d76025b887dc5c873d05f", + "sha256": "e1256016e7f51827d04fa2179255989f639ae1d97afaa9c16265f85a0c6abd70" + } + ], + "retained_baseline": [ + { + "path": "scripts/export_research_package.py", + "usage": "provenance", + "git_blob": "25b0a12a8073f589867ca7f2e564ea5ae011bdf5", + "sha256": "7126afaebff2328b8d64ba1ce992c497103820284f52d4590a558f97f13965fe", + "package_path": "baseline/export_research_package.py" + }, + { + "path": "scripts/research_package.py", + "usage": "provenance", + "git_blob": "091b4c8f58bac9a9580d76025b887dc5c873d05f", + "sha256": "e1256016e7f51827d04fa2179255989f639ae1d97afaa9c16265f85a0c6abd70", + "package_path": "baseline/research_package.py" + } + ] +} diff --git a/research/export-single-scan/TASK.json b/research/export-single-scan/TASK.json new file mode 100644 index 0000000000..38a564f5e0 --- /dev/null +++ b/research/export-single-scan/TASK.json @@ -0,0 +1,63 @@ +{ + "schema_version": 1, + "task_id": "replay-runtime", + "priority_revision": 8, + "base_commit": "a5bff3bf133e482541c09eba4e773884bdb2b1b7", + "original_card": { + "id": "replay-runtime", + "question": "Can one measured replay bottleneck be sped up without dropping coverage?", + "starting_points": [ + "scripts/run_isolated_tests.py", + "scripts/verify_replay_coverage.py", + "certificates/selected-result.json" + ], + "methods": [ + "Profile one isolated replay and optimize the measured bottleneck.", + "Design source-bound reuse with explicit dependency invalidation and negative controls." + ], + "acceptance_gate": "Report before/after wall time on the same fixture, retained checks and invalidation behavior.", + "first_step": { + "command": [ + "python3", + "-B", + "scripts/research_selected_baseline.py" + ], + "expected": "PASS_RESEARCH_BASELINE_BINDINGS_ONLY", + "inspect": "Resolve the current selected review, separate complex/ordinary BIT/admitted BIT/outer budgets, and compare the card against live handoffs before starting. Historical ceilings require exact source/model equivalence.", + "scope": "Source-binding orientation only; not a scientific replay or all-size proof. Materialize selected inputs if this checkout is sparse." + }, + "first_experiment": "Measure one unchanged fixture, preserve check coverage, and test one optimization with invalidation controls.", + "bottleneck_link": "Faster trustworthy replays increase review throughput and let mechanism candidates be checked sooner.", + "allowed_variations": [ + "Profile one stage or design one source-bound cache boundary." + ], + "scope_change_triggers": [ + "Skipping coverage or comparing different fixtures to claim speedup." + ], + "overlap_terms": [ + "replay runtime", + "cache invalidation", + "isolated tests" + ] + }, + "scope_status": "refined", + "actual_scope": "Reuse the checked tar handle and its populated index during complete offline declared-input checking.", + "scope_change_reason": "Refine the replay-runtime card to one repeated tar-index scan in the standard offline pre-replay export checker.", + "bottleneck_link": "Remove duplicated archive parsing from a standard pre-replay integrity command, preserving its checks.", + "budget_minutes": 60, + "stop_rule": "Ship the first measured complete-command gain with unchanged outcomes and effective resource and corruption controls.", + "outcome": "completed", + "evidence": "BENCHMARK-toolkit.json and BENCHMARK-small.json; generated RESULT.json; registered normal/optimized conformance and exporter suites.", + "author": "Nick Petalas", + "ai_assistance": "Independent human and formal proof-kernel review are unclaimed.", + "open_questions": [ + { + "question": "Does archive-index reuse help larger archives on Linux and Windows?", + "why_it_matters": "The measured percentage is specific to the recorded macOS workload.", + "first_step": "Export the same pinned package and run benchmark.py in alternating batches on the other host.", + "blocked_by": "No cross-platform timing data is supplied." + } + ], + "claim_url": "https://github.com/CrocSwap/integer-mult-bounds/discussions/1166#discussioncomment-19272236", + "playbook": "performance-issue" +} diff --git a/research/export-single-scan/VALIDATION.json b/research/export-single-scan/VALIDATION.json new file mode 100644 index 0000000000..f68411b6d2 --- /dev/null +++ b/research/export-single-scan/VALIDATION.json @@ -0,0 +1,85 @@ +{ + "python": "CPython 3.12.15", + "registered_commands": [ + { + "argv": [ + "/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12", + "-W", + "error", + "-B", + "research/export-single-scan/verify.py" + ], + "elapsed_seconds": 0.20851995889097452, + "exit": "success", + "timeout_seconds": 60 + }, + { + "argv": [ + "/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12", + "-O", + "-W", + "error", + "-B", + "research/export-single-scan/verify.py" + ], + "elapsed_seconds": 0.2110392909962684, + "exit": "success", + "timeout_seconds": 60 + }, + { + "argv": [ + "/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12", + "-W", + "error", + "-B", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-p", + "test_research_export*.py", + "-v" + ], + "elapsed_seconds": 5.7126572909764946, + "exit": "success", + "timeout_seconds": 120 + }, + { + "argv": [ + "/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12", + "-O", + "-W", + "error", + "-B", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-p", + "test_research_export*.py", + "-v" + ], + "elapsed_seconds": 6.616088875103742, + "exit": "success", + "timeout_seconds": 120 + } + ], + "normal_and_optimized_exporter_tests": { + "run": 46, + "passed": 45, + "skipped": 1, + "skip_reason": "native Windows rename contract" + }, + "claim_type": "measured tooling improvement with scoped equivalence and finite conformance evidence", + "unrun": [ + "hosted CI pending", + "native Windows tests and timing", + "Linux timing", + "scientific predecessor replay", + "selected scientific cold replay", + "Lean", + "Docker" + ] +} diff --git a/research/export-single-scan/VERIFY.json b/research/export-single-scan/VERIFY.json new file mode 100644 index 0000000000..c3b068240e --- /dev/null +++ b/research/export-single-scan/VERIFY.json @@ -0,0 +1,79 @@ +{ + "schema_version": 1, + "status": "ready", + "python": "3.12", + "requirements": [], + "inputs": [ + "scripts/export_research_package.py", + "scripts/research_package.py", + "tests/test_research_export.py", + "tests/test_research_export_drive_paths.py", + "tests/test_research_export_input_coverage.py", + "tests/test_research_export_input_ref.py", + "tests/test_research_export_metadata.py", + "tests/test_research_export_publication.py", + "tests/test_research_export_topology.py" + ], + "commands": [ + { + "argv": [ + "{python}", + "-W", + "error", + "-B", + "{package}/verify.py" + ], + "timeout_seconds": 60, + "checks": "Authenticate retained baseline, compare complete acceptance/rejection outputs on finite archives, validate recorded benchmark arithmetic and source-bound receipt." + }, + { + "argv": [ + "{python}", + "-O", + "-W", + "error", + "-B", + "{package}/verify.py" + ], + "timeout_seconds": 60, + "checks": "Authenticate retained baseline, compare complete acceptance/rejection outputs on finite archives, validate recorded benchmark arithmetic and source-bound receipt." + }, + { + "argv": [ + "{python}", + "-W", + "error", + "-B", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-p", + "test_research_export*.py", + "-v" + ], + "timeout_seconds": 120, + "checks": "Complete exporter regression suite, including path/type/topology/input coverage, publication, exactly one archive open, ownership and changed-manifest rejection." + }, + { + "argv": [ + "{python}", + "-O", + "-W", + "error", + "-B", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-p", + "test_research_export*.py", + "-v" + ], + "timeout_seconds": 120, + "checks": "Complete exporter regression suite, including path/type/topology/input coverage, publication, exactly one archive open, ownership and changed-manifest rejection." + } + ] +} diff --git a/research/export-single-scan/baseline/export_research_package.py b/research/export-single-scan/baseline/export_research_package.py new file mode 100644 index 0000000000..25b0a12a80 --- /dev/null +++ b/research/export-single-scan/baseline/export_research_package.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 +"""Export committed package bytes and explicit inputs; check bundles without Git. + +Tekalign, with substantial OpenAI Codex assistance. Apache-2.0. +""" +import argparse +from contextlib import closing +import hashlib +import io +import json +import ntpath +import os +from pathlib import Path +import re +import subprocess +import tarfile +import tempfile + +from research_package import canonical_relative_path, git, git_blobs, require + +ROOT = Path(__file__).resolve().parents[1] +RECEIPT = '.research-export.json' +FORMAT = 'research-package-export-v1' + + +def unique_json_keys(pairs): + result = {} + for key, value in pairs: + require(key not in result, 'duplicate export JSON key: ' + key) + result[key] = value + return result + + +def bundle_relative_path(name): + canonical_relative_path(name) + # A POSIX-relative name such as C:/file or C:file can discard the extraction + # directory on Windows. Apply the bundle policy on every producer/check host. + require(not ntpath.splitdrive(name)[0], 'drive-qualified export path: ' + name) + + +def inventory(root, package, ref, input_ref=None): + require(re.fullmatch(r'research/[a-z0-9][a-z0-9-]*', package), + 'package must be research/') + commit = git(root, 'rev-parse', '--verify', '--end-of-options', + ref + '^{commit}').decode('ascii').strip() + # Read declarations from the publication, never the working manifest. + with closing(git_blobs(root, [commit + ':' + package + '/VERIFY.json'])) as rows: + values = list(rows) + manifest = json.loads(values[0][1], object_pairs_hook=unique_json_keys) + require(isinstance(manifest, dict), 'VERIFY.json must be an object') + inputs = manifest.get('inputs') + require(isinstance(inputs, list), 'VERIFY.json inputs must be a list') + for name in inputs: + bundle_relative_path(name) + require(name != RECEIPT, 'reserved export path: ' + name) + require(len(inputs) == len(set(inputs)), 'duplicate declared input') + contexts = [(commit, [package, *inputs])] + if input_ref is not None: + require(all(name != package and not name.startswith(package + '/') for name in inputs), + 'separate input commit requires external declared inputs') + input_commit = git(root, 'rev-parse', '--verify', '--end-of-options', + input_ref + '^{commit}').decode('ascii').strip() + contexts = [(commit, [package]), (input_commit, inputs)] + records = [] + for tree, paths in contexts: + if paths: + records.extend(git(root, '--literal-pathspecs', 'ls-tree', '-r', '-z', '--full-tree', + tree, '--', *paths).split(b'\0')) + entries = {} + for record in filter(None, records): + header, raw_name = record.split(b'\t', 1) + mode, kind, obj = header.split() + name = raw_name.decode('utf-8') + bundle_relative_path(name) + require(mode in (b'100644', b'100755') and kind == b'blob', + 'non-regular committed file: ' + name) + require(name.startswith(package + '/') or name in inputs, + 'input must name one file, not a directory: ' + name) + entries[name] = (int(mode, 8) & 0o777, obj.decode('ascii')) + require(set(inputs) <= entries.keys(), 'missing declared input: ' + + ', '.join(sorted(set(inputs) - entries.keys()))) + require(package + '/VERIFY.json' in entries, 'missing committed manifest') + require_file_tree([*entries, RECEIPT]) + return commit, entries + + +def require_file_tree(names): + """Regular-file members cannot also be implicit directories for other files.""" + files = set(names) + for name in sorted(files): + for parent in canonical_relative_path(name).parents: + require(parent.as_posix() not in files, + 'file/directory collision: ' + parent.as_posix() + ' and ' + name) + + +def member(name, data, mode=0o644): + info = tarfile.TarInfo(name) + info.size = len(data) + info.mode = mode + # TarInfo defaults give zero timestamps/owners and empty owner names. + return info, io.BytesIO(data) + + +def export(root, package, ref, output, input_ref=None): + # Resolve a moving input ref once, before collecting either snapshot. + input_commit = (git(root, 'rev-parse', '--verify', '--end-of-options', + input_ref + '^{commit}').decode('ascii').strip() + if input_ref is not None else None) + commit, entries = inventory(root, package, ref, input_commit) + receipt = {'format': FORMAT, 'commit': commit, 'package': package, 'files': []} + if input_commit is not None: + receipt['input_commit'] = input_commit + output = Path(output) + require(not output.exists() and not output.is_symlink(), 'output already exists') + # Publish only after Git's final status and the archive close both succeed. + fd, name = tempfile.mkstemp(prefix='.research-export-', dir=output.parent) + try: + with os.fdopen(fd, 'wb') as stream, tarfile.open(fileobj=stream, mode='w', format=tarfile.PAX_FORMAT) as archive: + names = sorted(entries) + with closing(git_blobs(root, [entries[n][1] for n in names])) as rows: + for (obj, data), path in zip(rows, names): + mode, expected = entries[path] + require(obj == expected, 'Git blob mismatch: ' + path) + archive.addfile(*member(path, data, mode)) + receipt['files'].append({'path': path, 'mode': mode, 'size': len(data), + 'sha256': hashlib.sha256(data).hexdigest()}) + data = (json.dumps(receipt, sort_keys=True, indent=2) + '\n').encode('utf-8') + archive.addfile(*member(RECEIPT, data)) + # Both operations refuse an existing destination, including one created + # after the check above. Windows rename needs no hard-link permission; + # POSIX rename would overwrite, so retain link there. The staging file + # is in the destination directory and all writer handles are closed. + if os.name == 'nt': + os.rename(name, output) + else: + os.link(name, output) + finally: + Path(name).unlink(missing_ok=True) + return receipt + + +def check_declared_inputs(path, receipt, seen): + """Check the bundled manifest's file declarations without extraction or Git.""" + name = receipt['package'] + '/VERIFY.json' + with tarfile.open(path, mode='r:') as archive: + item = archive.getmember(name) + require(item.isfile() and item.size <= 16 * 1024 * 1024, + 'invalid or oversized bundled manifest') + with archive.extractfile(item) as stream: + raw = stream.read(16 * 1024 * 1024 + 1) + require(len(raw) == seen[name]['size'] and hashlib.sha256(raw).hexdigest() == seen[name]['sha256'], + 'bundled manifest changed during declaration check') + + def unique(pairs): + result = {} + for key, value in pairs: + require(key not in result, 'duplicate bundled manifest key: ' + key) + result[key] = value + return result + + manifest = json.loads(raw.decode('utf-8'), object_pairs_hook=unique) + require(isinstance(manifest, dict) and isinstance(manifest.get('inputs'), list), + 'bundled VERIFY.json needs an inputs list') + inputs = manifest['inputs'] + require(len(inputs) <= 100000, 'bundled input count exceeds 100000') + for entry in inputs: + bundle_relative_path(entry) + require(entry != RECEIPT, 'reserved declared input: ' + entry) + declared = set(inputs) + require(len(inputs) == len(declared), 'duplicate bundled input') + missing = declared - seen.keys() + require(not missing, 'missing bundled declared input: ' + ', '.join(sorted(missing))) + extra = {entry for entry in seen if not entry.startswith(receipt['package'] + '/') and entry not in declared} + require(not extra, 'undeclared external bundle file: ' + ', '.join(sorted(extra))) + return len(inputs) + + +def check_bundle(path, check_inputs=False): + """Check declared inventory, modes and payload hashes; never extract or execute.""" + seen, receipt = {}, None + with tarfile.open(path, mode='r:') as archive: + for item in archive: + bundle_relative_path(item.name) + require(item.isfile(), 'non-regular archive member: ' + item.name) + require(item.name not in seen, 'duplicate archive member: ' + item.name) + require(item.mode in (0o644, 0o755), 'unsupported mode: ' + item.name) + digest = hashlib.sha256() + with archive.extractfile(item) as data: + if item.name == RECEIPT: + require(item.size <= 16 * 1024 * 1024, 'oversized export receipt') + receipt = json.load(data, object_pairs_hook=unique_json_keys) + else: + for block in iter(lambda: data.read(1024 * 1024), b''): + digest.update(block) + seen[item.name] = {'path': item.name, 'mode': item.mode, 'size': item.size, + 'sha256': digest.hexdigest()} + require_file_tree(seen) + require(isinstance(receipt, dict) and receipt.get('format') == FORMAT, 'missing or invalid export receipt') + require(isinstance(receipt.get('commit'), str) and + re.fullmatch(r'(?:[0-9a-f]{40}|[0-9a-f]{64})', receipt['commit']), 'invalid commit label') + if 'input_commit' in receipt: + require(isinstance(receipt['input_commit'], str) and + re.fullmatch(r'(?:[0-9a-f]{40}|[0-9a-f]{64})', receipt['input_commit']), 'invalid input commit label') + require(isinstance(receipt.get('package'), str) and + re.fullmatch(r'research/[a-z0-9][a-z0-9-]*', receipt['package']), 'invalid package label') + files = receipt.get('files') + require(isinstance(files, list), 'invalid receipt files') + expected = {} + for row in files: + require(isinstance(row, dict), 'invalid receipt row') + require(type(row.get('size')) is int and row['size'] >= 0, 'invalid receipt size') + require(type(row.get('mode')) is int and row['mode'] in (0o644, 0o755), + 'invalid receipt mode') + name = row.get('path') + bundle_relative_path(name) + require(name != RECEIPT and name not in expected, 'duplicate or reserved receipt path') + expected[name] = row + del seen[RECEIPT] + require(seen == expected, 'bundle inventory, mode, size or SHA256 mismatch') + require(receipt['package'] + '/VERIFY.json' in seen, 'missing bundled manifest') + if check_inputs: + receipt = {**receipt, 'declared_inputs_checked': check_declared_inputs(path, receipt, seen)} + return receipt + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest='command', required=True) + make = commands.add_parser('export') + make.add_argument('--package', required=True) + make.add_argument('--ref', default='HEAD') + make.add_argument('--input-ref', help='explicit commit for all external declared inputs; default: --ref') + make.add_argument('--output', type=Path, required=True) + check = commands.add_parser('check') + check.add_argument('archive', type=Path) + check.add_argument('--check-inputs', action='store_true', + help='also check bundled VERIFY.json input coverage; see docs/research/export-input-coverage.md') + args = parser.parse_args() + try: + result = (export(ROOT, args.package, args.ref, args.output, args.input_ref) if args.command == 'export' + else check_bundle(args.archive, args.check_inputs)) + except (OSError, ValueError, TypeError, KeyError, RecursionError, tarfile.TarError, subprocess.SubprocessError) as exc: + parser.exit(1, str(exc) + '\n') + print('PASS_RESEARCH_EXPORT ' + json.dumps({k: result[k] for k in ('commit', 'package')} | + {'files': len(result['files'])} | + ({'input_commit': result['input_commit']} if 'input_commit' in result else {}) | + ({'declared_inputs_checked': result['declared_inputs_checked']} + if 'declared_inputs_checked' in result else {}))) + + +if __name__ == '__main__': + main() diff --git a/research/export-single-scan/baseline/research_package.py b/research/export-single-scan/baseline/research_package.py new file mode 100644 index 0000000000..091b4c8f58 --- /dev/null +++ b/research/export-single-scan/baseline/research_package.py @@ -0,0 +1,1053 @@ +#!/usr/bin/env python3 +"""Validate and replay explicitly registered research packages, never every verify.py. + +Douglas Colkitt, with OpenAI Codex assistance. Apache-2.0. +""" +import argparse +import errno +from contextlib import closing, contextmanager +from itertools import chain +import hashlib +import importlib.metadata +import json +import os +from pathlib import Path, PurePosixPath +import re +import signal +import stat +import subprocess +import sys +import tempfile +import threading + +ROOT = Path(__file__).resolve().parents[1] +GIT_BLOB_RESPONSE_TIMEOUT = 90 + + +def require(ok, message): + if not ok: + raise ValueError(message) + + +def canonical_relative_path(name): + # Git's changed paths use one repository-relative POSIX spelling. Accepting + # aliases here would let a declared input evade the planner's exact match. + require(isinstance(name, str), "canonical relative path must be a string") + path = PurePosixPath(name) + require(bool(path.parts) and not path.is_absolute() and ".." not in path.parts + and "\\" not in name and path.as_posix() == name, + "use a canonical repository-relative POSIX path: " + name) + return path + + +def symlink_free_relative_path(root, name): + """Reject symlink/junction components without requiring a deleted input to exist. + + Git reports changes under the target's path, not a symlink alias declared + by its consumer. Inspect existing components below the repository root; + the checkout must remain stable during planning, validation and replay. + """ + parts = canonical_relative_path(name).parts + current = root + for part in parts: + current = current / part + try: + info = current.lstat() + except (FileNotFoundError, NotADirectoryError): + break + require(not stat.S_ISLNK(info.st_mode), "symlink in repository-relative path: " + name) + # Windows junctions have directory mode, not S_IFLNK. Inspect the same + # lstat result so even a dangling junction is rejected before traversal. + require(getattr(info, "st_reparse_tag", 0) != getattr(stat, "IO_REPARSE_TAG_MOUNT_POINT", None), + "junction in repository-relative path: " + name) + return root / name + + +def relative_file(root, name): + result = symlink_free_relative_path(root, name) + require(result.resolve().is_relative_to(root.resolve()) and result.is_file(), + "missing or external file: " + name) + return result + + +# Adapted from the package-local policy in +# research/t74-2530-source-partner-audit/audit_common.py (Apache-2.0). +# Keep repository configuration and linked-worktree .git discovery intact; +# inherited shell overrides must not choose a different repository or index. +_GIT_REDIRECT_VARS = frozenset(( + "GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_OBJECT_DIRECTORY", + "GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_NAMESPACE", "GIT_CEILING_DIRECTORIES", + "GIT_COMMON_DIR", "GIT_CONFIG_PARAMETERS", "GIT_CONFIG_COUNT", +)) +_GIT_CONFIG_PREFIXES = ("GIT_CONFIG_KEY_", "GIT_CONFIG_VALUE_") + + +def git_source_env(): + """Use cwd's repository and stored objects, without changing the parent. + + Git can ignore cwd when inherited selection variables point elsewhere. + Config-injection variables can also redirect worktree/index operations. + This policy is only for Git metadata/source subprocesses, not verifiers. + """ + env = {name: value for name, value in os.environ.items() + if name not in _GIT_REDIRECT_VARS and not name.startswith(_GIT_CONFIG_PREFIXES)} + # Replacement refs can report an original ID while returning other bytes. + env["GIT_NO_REPLACE_OBJECTS"] = "1" + return env + + +def git(root, *args): + return subprocess.run(["git", *args], cwd=root, env=git_source_env(), + capture_output=True, check=True).stdout + + +def tracked_input_paths(root): + """Use HEAD spelling, never case folding, filesystem resolution or the index. + + Staged/untracked files are not replay inputs until committed. NUL framing + preserves Git filenames containing whitespace, including CR and LF. + """ + raw = subprocess.run( + ["git", "ls-tree", "-r", "--name-only", "-z", "HEAD"], cwd=root, env=git_source_env(), + capture_output=True, check=True, timeout=30).stdout + return frozenset(name.decode("utf-8") for name in raw.split(b"\0") if name) + + +def require_tracked_input(name, paths): + canonical_relative_path(name) + require(name in paths, "use an exact Git-tracked input path from HEAD: " + name) + + +def git_source_rows(root, base, names): + """Read one pinned object at a time through one Git process; no persistent cache. + + Each request is consumed before the next is sent. This avoids pipe deadlocks + and retains the old one-blob memory bound, even for large source archives. + Newline paths and non-blob objects retain the legacy Git-show semantics. + The caller must close this generator if source validation fails mid-stream. + """ + with subprocess.Popen(["git", "cat-file", "--batch"], cwd=root, env=git_source_env(), + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as process: + try: + for name in names: + expression = base + ":" + name + if "\n" in expression or "\r" in expression: + yield git(root, "rev-parse", expression).decode().strip(), git(root, "show", expression) + continue + process.stdin.write(expression.encode("utf-8") + b"\n") + process.stdin.flush() + header = process.stdout.readline().rstrip(b"\n").split(b" ") + require(len(header) == 3 and re.fullmatch(rb"[0-9a-f]{40,64}", header[0]) + and header[2].isdigit(), "missing or invalid Git source: " + name) + size = int(header[2]) + payload = process.stdout.read(size) + require(len(payload) == size and process.stdout.read(1) == b"\n", + "truncated Git source: " + name) + if header[1] != b"blob": + payload = git(root, "show", expression) + yield header[0].decode("ascii"), payload + process.stdin.close() + code = process.wait(timeout=90) + require(code == 0, "Git source reader failed: " + process.stderr.read().decode("utf-8", errors="replace")) + finally: + if not process.stdin.closed: + process.stdin.close() + if process.poll() is None: + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +class _UnsupportedNulBatch(Exception): + """Git rejected the initial NUL-mode request before yielding any source.""" + + +def _closed_stdin_error(error): + # Windows can report EINVAL instead of EPIPE when a child closes stdin. + # This classifies the pipe error only; the retry still requires exit 129. + return (isinstance(error, BrokenPipeError) + or (os.name == "nt" and error.errno == errno.EINVAL)) + + +def _stop_git_blob_process(process): + """Stop this reader's owned POSIX group even after its leader exits.""" + try: + if os.name == "posix": + # Reap an exited leader first: signaling a group containing only + # its unreaped zombie can report EPERM on macOS. A surviving + # descendant keeps the group alive after the leader is reaped. + process.poll() + os.killpg(process.pid, signal.SIGKILL) + else: + process.kill() + except ProcessLookupError: + pass + + +@contextmanager +def _git_blob_response_deadline(stop): + """Bound one response, excluding time spent by its consumer. + + On POSIX the reader owns a session: ordinary descendants retaining a pipe + are stopped with the leader. Escaped sessions and separate git() calls are + outside this contract; non-POSIX retains the direct-child deadline. + """ + expired = threading.Event() + + def terminate(): + expired.set() + stop() + + timer = threading.Timer(GIT_BLOB_RESPONSE_TIMEOUT, terminate) + timer.start() + try: + yield + finally: + timer.cancel() + timer.join() + if expired.is_set(): + raise subprocess.TimeoutExpired("git cat-file blob response", GIT_BLOB_RESPONSE_TIMEOUT) + + +def _git_blob_batch(root, specs, nul): + argv = ["git", "cat-file", "--batch"] + (["-z"] if nul else []) + # Usage diagnostics from the unsupported-option probe are expected. Avoid + # a second pipe that could block while we consume the framed stdout reply. + with subprocess.Popen(argv, cwd=root, env=git_source_env(), stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + start_new_session=(os.name == "posix")) as process: + stopped = False + + def stop(): + nonlocal stopped + if not stopped: + _stop_git_blob_process(process) + stopped = True + + try: + for index, spec in enumerate(specs): + require("\0" not in spec, "invalid Git object request") + request = spec + if not nul and ("\n" in spec or "\r" in spec): + # Resolve one literal argument, never split a path into requests. + request = git(root, "rev-parse", "--verify", "--end-of-options", spec).decode("ascii").strip() + require(re.fullmatch(r"[0-9a-f]{40,64}", request), "invalid Git object ID") + with _git_blob_response_deadline(stop): + try: + process.stdin.write(request.encode("utf-8") + (b"\0" if nul else b"\n")) + process.stdin.flush() + except OSError as exc: + if not _closed_stdin_error(exc): + raise + header = b"" + else: + header = process.stdout.readline() + if nul and index == 0 and not header and process.wait(timeout=90) == 129: + raise _UnsupportedNulBatch(spec) + fields = header.rstrip(b"\n").split(b" ") + require(len(fields) == 3 and re.fullmatch(rb"[0-9a-f]{40,64}", fields[0]) + and fields[1] == b"blob" and fields[2].isdigit(), + "missing or non-blob Git source: " + spec) + size = int(fields[2]) + pinned = process.stdout.read(size) + require(len(pinned) == size and process.stdout.read(1) == b"\n", + "incomplete Git blob response: " + spec) + yield fields[0].decode("ascii"), pinned + process.stdin.close() + require(process.wait(timeout=90) == 0, "Git source reader failed") + finally: + # Terminate ordinary descendants before closing buffered pipes. This + # also covers successful leader exit and an early generator close. + if os.name == "posix": + stop() + # Option rejection can close stdin before our first write/flush. + try: + process.stdin.close() + except OSError as exc: + if not _closed_stdin_error(exc): + raise + process.stdout.close() + if process.poll() is None: + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +def git_blobs(root, specs): + """Stream exact blobs, retaining the one-process NUL protocol on newer Git. + + Git without -z rejects it with usage status 129. Retry only before any + response, replaying the consumed first request through the line protocol. + The fallback resolves CR/LF paths to object IDs before requesting content. + The caller must close this iterator if validation stops before exhaustion. + """ + specs = iter(specs) + sentinel = object() + first = next(specs, sentinel) + if first is sentinel: + return + with closing(_git_blob_batch(root, chain((first,), specs), nul=True)) as rows: + try: + yield from rows + except _UnsupportedNulBatch as exc: + with closing(_git_blob_batch(root, chain((exc.args[0],), specs), nul=False)) as legacy: + yield from legacy + + +def meaningful(value): + return isinstance(value, str) and bool(value.strip()) and not re.search(r"\bTODO\b", value, re.I) + + +def brief(value): + text = json.dumps(value, ensure_ascii=True, sort_keys=True) + return text if len(text) <= 120 else text[:117] + "..." + + +def require_key(mapping, key, where, shape=None): + """Name the missing key and its expected shape instead of a bare KeyError.""" + require(isinstance(mapping, dict), + where + " must be a JSON object; found " + type(mapping).__name__ + ": " + brief(mapping)) + require(key in mapping, where + "." + key + " is missing" + + ("; expected " + shape if shape else "")) + return mapping[key] + + +def require_text(mapping, key, where, shape="a non-empty string without TODO"): + """Name the key and distinguish missing, wrong-shaped and weak values.""" + require(isinstance(mapping, dict), + where + " must be a JSON object; found " + type(mapping).__name__ + ": " + brief(mapping)) + require(key in mapping, where + "." + key + " is missing; expected " + shape) + value = mapping[key] + require(isinstance(value, str) and meaningful(value), + where + "." + key + " must be " + shape + "; found " + type(value).__name__ + ": " + brief(value)) + + +def card_differences(own, pinned): + """Name every key by which a TASK.json original_card differs from its pin.""" + if not isinstance(own, dict): + return "TASK.json original_card is " + type(own).__name__ + ", expected a JSON object" + details = [] + missing = [key for key in pinned if key not in own] + extra = [key for key in own if key not in pinned] + if missing: + details.append("missing keys: " + ", ".join(missing)) + if extra: + details.append("unexpected keys: " + ", ".join(extra)) + for key in (key for key in pinned if key in own and own[key] != pinned[key]): + details.append(key + " differs (TASK.json: " + brief(own[key]) + "; pinned: " + brief(pinned[key]) + ")") + return "; ".join(details) or "cards compare unequal without a key difference" + + +def source_pin(row, base): + """Resolve one SOURCES.json row to its exact Git object expression. + + Rows without an explicit pin keep the historical base_commit: form. + A file vendored from a pull-request head is pinned exactly with + head:<40-hex sha>:, so its bytes resolve at that commit, not at the + package base. An explicit but unparseable pin fails closed. + """ + name = row["path"] + pin = row.get("pin") + if pin is None: + return base + ":" + name + if pin == "base_commit:" + name: + return base + ":" + name + # Git filenames can contain LF; it is part of the exact path, not a delimiter. + match = re.fullmatch(r"head:([0-9a-f]{40}):(.+)", pin, re.DOTALL) if isinstance(pin, str) else None + if match is not None and match.group(2) == name: + return match.group(1) + ":" + name + raise ValueError("SOURCES.json pin for " + name + " is not a supported pin form: " + brief(pin) + + "; accepted pin forms are base_commit: and head:<40-hex sha>:") + + +def file_equals_bytes(path, expected): + """Compare a stable regular file exactly, with at most a 1 MiB buffer. + + Read the whole initial extent even after a mismatch, propagating I/O errors. + The caller retains responsibility for path validation and checkout stability. + """ + with path.open("rb") as stream: + size = os.fstat(stream.fileno()).st_size + capacity = min(1024 * 1024, max(1, size)) + buffer = bytearray(capacity) + actual = memoryview(buffer) + wanted = memoryview(expected) + matches = size == len(expected) + for offset in range(0, size, capacity): + count = stream.readinto(buffer) + assert 0 <= count <= capacity + if count != min(capacity, size - offset) or actual[:count] != wanted[offset:offset + count]: + matches = False + tail = stream.read(1) + return matches and not tail + + +def validate(root, package, check_pins=True): + parts = PurePosixPath(package).parts + require(len(parts) == 2 and parts[0] == "research" and re.fullmatch(r"[a-z0-9][a-z0-9-]*", parts[1]), + "package must be research/") + data = {} + for name in ("TASK.json", "SOURCES.json", "VERIFY.json"): + data[name] = json.loads(relative_file(root, package + "/" + name).read_text(encoding="utf-8")) + version = require_key(data[name], "schema_version", name) + require(version == 1, name + ".schema_version must be 1; found " + brief(version)) + task, sources, manifest = (data[n] for n in ("TASK.json", "SOURCES.json", "VERIFY.json")) + scope_status = require_key(task, "scope_status", "TASK.json", "unchanged, refined or changed") + require(scope_status in ("unchanged", "refined", "changed"), + "TASK.json.scope_status must be unchanged, refined or changed; found " + brief(scope_status)) + for field in ("actual_scope", "bottleneck_link", "stop_rule", "evidence", "author", "ai_assistance"): + require_text(task, field, "TASK.json") + if scope_status == "unchanged": + question = require_key(require_key(task, "original_card", "TASK.json", + "the pinned catalogue card object"), + "question", "TASK.json.original_card") + require(task["actual_scope"] == question, + "different wording/scope requires refined or changed plus a reason" + + " (TASK.json.actual_scope " + brief(task["actual_scope"]) + + " differs from TASK.json.original_card.question " + brief(question) + ")") + else: + require_text(task, "scope_change_reason", "TASK.json", + "a non-empty string without TODO explaining the scope refinement/change") + outcome = require_key(task, "outcome", "TASK.json", + "partial, completed, blocked or negative-result") + require(outcome in ("partial", "completed", "blocked", "negative-result"), + "TASK.json.outcome must be partial, completed, blocked or negative-result; found " + brief(outcome)) + questions = task.get("open_questions") + require(isinstance(questions, list), + "TASK.json.open_questions must be a list; found " + type(questions).__name__ + ": " + brief(questions)) + if not questions: + require_text(task, "no_open_questions_reason", "TASK.json", + "a non-empty string without TODO explaining an empty next-question list") + for position, question in enumerate(questions): + for field in ("question", "why_it_matters", "first_step", "blocked_by"): + require_text(question, field, "TASK.json.open_questions[" + str(position) + "]") + readme = relative_file(root, package + "/README.md").read_text(encoding="utf-8") + require("## Open next questions" in readme, "README.md needs an \"## Open next questions\" section") + status = require_key(manifest, "status", "VERIFY.json") + require(status == "ready", + "VERIFY.json.status must be \"ready\"; complete the draft before requesting replay; found " + brief(status)) + python = require_key(manifest, "python", "VERIFY.json") + require(python == "3.12", + "VERIFY.json.python must be \"3.12\" (generic replay currently uses Python 3.12); found " + brief(python)) + requirements = manifest.get("requirements", []) + require(isinstance(requirements, list), + "VERIFY.json.requirements must be a list of exact \"name==version\" pins; found " + + type(requirements).__name__ + ": " + brief(requirements)) + require(len(requirements) == len(set(requirements)), "VERIFY.json.requirements must not repeat a pin") + for item in requirements: + require(isinstance(item, str) + and re.fullmatch(r"(?:numpy|sympy|scipy)==[0-9]+(?:\.[0-9]+){1,2}", item), + "generic CI supports exact numpy/sympy/scipy pins; use a dedicated workflow for other environments" + + "; offending entry: " + brief(item)) + names = [item.split("==")[0] for item in requirements] + require(len(names) == len(set(names)), "declare only one version per dependency") + # Validation already treats an omitted list as empty; execution must receive + # that same default without rewriting the contributor's manifest on disk. + manifest["requirements"] = requirements + commands = manifest.get("commands") + require(isinstance(commands, list) and 1 <= len(commands) <= 8, + "VERIFY.json.commands must be a list of 1..8 explicit verification commands; found " + + type(commands).__name__ + ": " + brief(commands)) + total = 0 + for position, command in enumerate(commands): + path = "VERIFY.json.commands[" + str(position) + "]" + require(isinstance(command, dict), + path + " must be a JSON object; found " + type(command).__name__ + ": " + brief(command)) + argv = command.get("argv") + require(isinstance(argv, list) and len(argv) >= 2 and all(isinstance(a, str) for a in argv), + path + ".argv must be a list of at least 2 strings; found " + brief(argv)) + require(argv[0] == "{python}", path + ".argv must explicitly use {python}; no implicit shell") + require_text(command, "checks", path, "a non-empty string without TODO describing command coverage") + seconds = command.get("timeout_seconds") + require(type(seconds) is int, + path + ".timeout_seconds must be an integer number of seconds; found " + + type(seconds).__name__ + ": " + brief(seconds)) + require(1 <= seconds <= 600, + path + ".timeout_seconds " + str(seconds) + + (" exceeds the 600 s per-command cap" if seconds > 600 + else " is below the 1 s per-command minimum")) + total += seconds + require(total <= 1200, "declared replay budget " + str(total) + + " s (sum of timeout_seconds) exceeds the 1200 s total cap" + + "; use a dedicated job for larger checks") + inputs = manifest.get("inputs") + require(isinstance(inputs, list), + "VERIFY.json.inputs must be a list of canonical repository-relative paths; found " + + type(inputs).__name__ + ": " + brief(inputs)) + tracked = tracked_input_paths(root) + for name in inputs: + symlink_free_relative_path(root, name) + require_tracked_input(name, tracked) + relative_file(root, name) + files = require_key(sources, "files", "SOURCES.json") + require(files, "SOURCES.json.files is empty; declare the source inventory") + base = require_key(sources, "base_commit", "SOURCES.json", "a 40-64 hex string pinning an exact base commit") + require(isinstance(base, str) and re.fullmatch(r"[0-9a-f]{40,64}", base), + "SOURCES.json.base_commit must be a 40-64 hex string pinning an exact base commit; found " + brief(base)) + task_base = require_key(task, "base_commit", "TASK.json", + "a 40-64 hex string equal to SOURCES.json.base_commit") + require(task_base == base, "TASK.json.base_commit " + brief(task_base) + + " must equal SOURCES.json.base_commit " + brief(base)) + for row in files: + require(isinstance(row, dict) and isinstance(row.get("path"), str), + "SOURCES.json.files rows must be JSON objects with a string path; offending row: " + brief(row)) + require(row.get("usage") in ("input", "provenance"), + "SOURCES.json.files row " + row["path"] + " must declare usage \"input\" or \"provenance\"; found " + + brief(row.get("usage"))) + undeclared = sorted({r["path"] for r in sources["files"] if r["usage"] == "input"} - set(inputs)) + require(not undeclared, + "every SOURCES.json files row with usage \"input\" must also be declared in VERIFY.json inputs" + + " (they drive CI invalidation); undeclared: " + ", ".join(undeclared)) + if check_pins: + catalog_name = "docs/research/agent-task-catalog.json" + require(catalog_name in {r["path"] for r in sources["files"]}, + "SOURCES.json.files must pin the originating task catalogue " + catalog_name + + " (a provenance row pinned at base_commit)") + names = [r["path"] for r in sources["files"]] + for name in names: + p = PurePosixPath(name) + require(not p.is_absolute() and ".." not in p.parts, "invalid source path") + specs = [source_pin(row, sources["base_commit"]) for row in sources["files"]] + require(all(spec == sources["base_commit"] + ":" + catalog_name + for name, spec in zip(names, specs) if name == catalog_name), + "originating task catalogue must be pinned at base_commit") + with closing(git_blobs(root, specs)) as objects: + for index, (obj, pinned) in enumerate(objects): + row = sources["files"][index] + name = row["path"] + require(obj == row["git_blob"], "source blob provenance mismatch: " + name) + require(hashlib.sha256(pinned).hexdigest() == row["sha256"], "source SHA256 provenance mismatch: " + name) + if row["usage"] == "input": + require(file_equals_bytes(relative_file(root, name), pinned), "working source drift: " + name) + if name == catalog_name: + catalog = json.loads(pinned) + cards = [t for tr in catalog["tracks"] for t in tr["tasks"] if t["id"] == task["task_id"]] + require(len(cards) == 1, + "TASK.json task_id " + brief(task["task_id"]) + " matches " + str(len(cards)) + + " cards in pinned " + catalog_name + " at base_commit " + sources["base_commit"] + + "; need exactly one") + original_card = require_key(task, "original_card", "TASK.json", + "the pinned catalogue card object (copy --print-card output verbatim)") + if original_card != cards[0]: + raise ValueError("original card in TASK.json must equal the pinned catalogue card object exactly; " + "copy it verbatim from " + catalog_name + " at base_commit " + sources["base_commit"] + + ": " + card_differences(original_card, cards[0])) + return task, manifest + + +def catalogue_card(root, package, task_id): + """The exact original_card object a hand-authored TASK.json must copy. + + Reads the catalogue blob pinned at the package base commit, never the + working tree, because validate() compares original_card against that pin. + """ + sources = json.loads(relative_file(root, package + "/SOURCES.json").read_text(encoding="utf-8")) + name = "docs/research/agent-task-catalog.json" + rows = require_key(sources, "files", "SOURCES.json") + row = next((r for r in rows if isinstance(r, dict) and r.get("path") == name), None) + require(row is not None, "SOURCES.json.files must pin the originating task catalogue " + name) + base = require_key(sources, "base_commit", "SOURCES.json", "a 40-64 hex string pinning an exact base commit") + spec = source_pin(row, base) + require(spec == base + ":" + name, "originating task catalogue must be pinned at base_commit") + catalog = json.loads(git(root, "show", spec)) + cards = [t for tr in catalog["tracks"] for t in tr["tasks"] if t["id"] == task_id] + require(len(cards) == 1, + "task_id " + brief(task_id) + " matches " + str(len(cards)) + + " cards in pinned " + name + " at base_commit " + base + "; need exactly one") + return cards[0] + + +_WINDOWS_GATE = r""" +import ctypes +from ctypes import wintypes +import json +import subprocess +import sys +api = ctypes.WinDLL('kernel32', use_last_error=True) +api.OpenEventW.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.LPCWSTR] +api.OpenEventW.restype = wintypes.HANDLE +api.WaitForSingleObject.argtypes = [wintypes.HANDLE, wintypes.DWORD] +api.WaitForSingleObject.restype = wintypes.DWORD +api.CloseHandle.argtypes = [wintypes.HANDLE] +api.CloseHandle.restype = wintypes.BOOL +event = api.OpenEventW(0x00100000, False, sys.argv[1]) +if not event: + raise ctypes.WinError(ctypes.get_last_error()) +try: + if api.WaitForSingleObject(event, 30000) != 0: + raise RuntimeError('replay ownership gate was not released') +finally: + api.CloseHandle(event) +# No verification command or descendant starts until the parent owns this job. +sys.exit(subprocess.call(json.loads(sys.argv[2]))) +""" + +def run_windows_command(argv, cwd, timeout, env): + """Own ordinary verification descendants even after their leader exits. + + The gated launcher joins a kill-on-close Job Object before it may spawn + the requested command. A late taskkill cannot recover an exited leader's + tree. Job handles are not inherited and breakaway is not enabled. + """ + import ctypes + from ctypes import wintypes + import time + import uuid + + class BasicLimits(ctypes.Structure): + _fields_ = [("ProcessTime", ctypes.c_int64), ("JobTime", ctypes.c_int64), + ("Flags", wintypes.DWORD), ("MinWorkingSet", ctypes.c_size_t), + ("MaxWorkingSet", ctypes.c_size_t), ("ActiveProcessLimit", wintypes.DWORD), + ("Affinity", ctypes.c_size_t), ("Priority", wintypes.DWORD), + ("Scheduling", wintypes.DWORD)] + + class IOCounts(ctypes.Structure): + _fields_ = [(name, ctypes.c_uint64) for name in + ("ReadOps", "WriteOps", "OtherOps", "ReadBytes", "WriteBytes", "OtherBytes")] + + class ExtendedLimits(ctypes.Structure): + _fields_ = [("Basic", BasicLimits), ("IO", IOCounts), + ("ProcessMemory", ctypes.c_size_t), ("JobMemory", ctypes.c_size_t), + ("PeakProcessMemory", ctypes.c_size_t), ("PeakJobMemory", ctypes.c_size_t)] + + class Accounting(ctypes.Structure): + _fields_ = [(name, ctypes.c_int64) for name in + ("UserTime", "KernelTime", "PeriodUserTime", "PeriodKernelTime")] + [ + (name, wintypes.DWORD) for name in + ("PageFaults", "TotalProcesses", "ActiveProcesses", "TerminatedProcesses")] + + api = ctypes.WinDLL("kernel32", use_last_error=True) + + def native(name, result, *arguments): + fn = getattr(api, name) + fn.restype, fn.argtypes = result, arguments + return fn + + handle, dword, boolean = wintypes.HANDLE, wintypes.DWORD, wintypes.BOOL + create_job = native("CreateJobObjectW", handle, ctypes.c_void_p, wintypes.LPCWSTR) + set_info = native("SetInformationJobObject", boolean, handle, ctypes.c_int, ctypes.c_void_p, dword) + assign = native("AssignProcessToJobObject", boolean, handle, handle) + terminate = native("TerminateJobObject", boolean, handle, wintypes.UINT) + query = native("QueryInformationJobObject", boolean, handle, ctypes.c_int, + ctypes.c_void_p, dword, ctypes.c_void_p) + create_event = native("CreateEventW", handle, ctypes.c_void_p, boolean, boolean, wintypes.LPCWSTR) + set_event = native("SetEvent", boolean, handle) + close = native("CloseHandle", boolean, handle) + + def checked(ok): + if not ok: + raise ctypes.WinError(ctypes.get_last_error()) + return ok + + job = checked(create_job(None, None)) + event, process, assigned = None, None, False + try: + limits = ExtendedLimits() + limits.Basic.Flags = 0x2000 # JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE + checked(set_info(job, 9, ctypes.byref(limits), ctypes.sizeof(limits))) + name = "Local\\research-replay-" + uuid.uuid4().hex + event = checked(create_event(None, True, False, name)) + # A name collision must not attach to an already-signaled event. + if ctypes.get_last_error() == 183: + raise ValueError("replay ownership gate name already exists") + process = subprocess.Popen( + # The gate itself must not run sitecustomize/PYTHONPATH startup + # code before ownership. The requested command retains env/argv. + # Windows accepts Unicode argv directly. ASCII JSON escaping can + # expand an otherwise valid command past its UTF-16 launch limit. + [sys.executable, "-I", "-S", "-c", _WINDOWS_GATE, name, + json.dumps(argv, ensure_ascii=False)], + cwd=cwd, env=env, start_new_session=True) + checked(assign(job, int(process._handle))) + assigned = True + checked(set_event(event)) + try: + return process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + raise ValueError("verification timed out") from None + finally: + try: + if process is not None: + if assigned: + checked(terminate(job, 1)) + # Termination is asynchronous. Require the job to report + # no active processes before releasing its ownership handle. + deadline = time.monotonic() + 10 + while True: + state = Accounting() + checked(query(job, 1, ctypes.byref(state), ctypes.sizeof(state), None)) + if not state.ActiveProcesses: + break + if time.monotonic() >= deadline: + raise ValueError("verification process cleanup timed out") + time.sleep(0.01) + elif process.poll() is None: + # Assignment failed before the gate opened: no command child + # exists, so only this known gated launcher needs stopping. + process.kill() + process.wait(timeout=10) + finally: + try: + if event: + checked(close(event)) + finally: + checked(close(job)) + +def run_command(argv, cwd, timeout): + # Assertion checks must not disappear because of the caller's environment. + # Keep explicit manifest options (including deliberate -O controls) intact. + # PYTHONPATH precedes the standard library on a child's sys.path, so a stray + # same-named module could end a verifier with exit code 0 before any check. + env = os.environ.copy() + env.pop("PYTHONOPTIMIZE", None) + # Ambient inspection can turn a verifier exception into a successful REPL + # exit at terminal EOF. Keep explicit -i and the caller's stdin unchanged. + env.pop("PYTHONINSPECT", None) + env.pop("PYTHONPATH", None) + if os.name == "nt": + code = run_windows_command(argv, cwd, timeout, env) + require(code == 0, f"verification command exited {code}") + return + previous_handler = None + terminating = False + waiting = False + process = None + + def terminate(signum, frame): + nonlocal terminating + if not terminating: + terminating = True + # Only interrupt the protected wait. Launch/assignment and cleanup + # defer delivery; repeated signals cannot interrupt group cleanup. + if waiting: + raise SystemExit(128 + signum) + + # Own a group for POSIX cleanup of this command and its descendants. + try: + if (threading.current_thread() is threading.main_thread() + and signal.getsignal(signal.SIGTERM) == signal.SIG_DFL): + previous_handler = signal.SIG_DFL + signal.signal(signal.SIGTERM, terminate) + if terminating: + raise SystemExit(143) + process = subprocess.Popen(argv, cwd=cwd, env=env, start_new_session=True) + try: + try: + waiting = True + if terminating: + raise SystemExit(143) + code = process.wait(timeout=timeout) + finally: + waiting = False + except subprocess.TimeoutExpired: + raise ValueError("verification timed out") from None + require(code == 0, f"verification command exited {code}") + finally: + try: + if process is not None: + if hasattr(os, "killpg"): + # The leader may have exited while descendants hold its group. + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + elif process.poll() is None: + subprocess.run(["taskkill", "/F", "/T", "/PID", str(process.pid)], capture_output=True) + process.wait() + finally: + if previous_handler is not None: + signal.signal(signal.SIGTERM, previous_handler) + if terminating: + raise SystemExit(143) + + +def isolated_python_argv(argv, cache_directory): + """Place our cache option after interpreter flags, before the entry point. + + -B prevents writes, not reads of existing .pyc files. A fresh namespace + keeps preexisting source-adjacent and externally configured caches out of + this registered interpreter. It is not inherited by child interpreters. + """ + require(argv and argv[0] == sys.executable, "cache isolation requires the registered Python interpreter") + index = 1 + options = [argv[0]] + while index < len(argv): + option = argv[index] + if option in ("-", "--") or not option.startswith("-"): + break + if option == "--check-hash-based-pycs": + require(index + 1 < len(argv), "missing Python hash-cache option argument") + options.extend(argv[index:index + 2]) + index += 2 + continue + if option.startswith("--"): + options.append(option) + index += 1 + continue + # Python permits combined short flags, attached -c/-m payloads, and + # attached or separate -W/-X arguments. Stop scanning an option once + # its argument begins: letters inside that argument are not flags. + entry = False + consumed = 1 + replace_cache = False + for offset, flag in enumerate(option[1:], start=1): + if flag in "cm": + entry = True + break + if flag in "WX": + if offset == len(option) - 1: + require(index + 1 < len(argv), "missing Python interpreter option argument") + consumed = 2 + value = argv[index + 1] if consumed == 2 else option[offset + 1:] + if flag == "X" and value.split("=", 1)[0] == "pycache_prefix": + replace_cache = True + if offset > 1: + options.append(option[:offset]) + break + if entry: + break + if not replace_cache: + options.extend(argv[index:index + consumed]) + index += consumed + # CPython's effective prefix can use the first duplicate -X value even + # when sys._xoptions shows the last. Remove only earlier cache-prefix + # settings; script arguments remain untouched, even when spelled -X. + return [*options, "-X", "pycache_prefix=" + str(cache_directory), *argv[index:]] + + +def run_python_replay(argv, cwd, timeout): + with tempfile.TemporaryDirectory(prefix="research-replay-pycache-") as cache: + run_command(isolated_python_argv(argv, cache), cwd, timeout) + + +def run_package(root, package, install=False): + task, manifest = validate(root, package) + require(sys.version_info[:2] == (3, 12), "run replay with Python 3.12 (metadata check is version-independent)") + if install and manifest["requirements"]: + run_command([sys.executable, "-m", "pip", "install", "--disable-pip-version-check", *manifest["requirements"]], root, 300) + for requirement in manifest["requirements"]: + name, version = requirement.split("==") + require(importlib.metadata.version(name) == version, "dependency version mismatch: " + requirement) + # Coverage labels are UTF-8 metadata but a redirected Windows stdout may + # use cp1252. Escape unrepresentable display characters instead of aborting + # before replay. This changes only our stream, not child encoding or env. + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(errors="backslashreplace") + for command in manifest["commands"]: + argv = [sys.executable if item == "{python}" else item.replace("{package}", package) for item in command["argv"]] + print("CHECK " + command["checks"], flush=True) + run_python_replay(argv, root, command["timeout_seconds"]) + print("PASS_REGISTERED_RESEARCH_PACKAGE " + package + " outcome=" + task["outcome"]) + + +def planning_manifests(root, tracked): + """Include deliberate sparse omissions without disguising deletions or drift. + + The fallback follows the chooser's PR3102 contract: one stage-zero regular + skip-worktree index entry must match the exact HEAD mode and blob. This is + declaration discovery only, not permission to replay absent package code. + """ + committed = {name for name in tracked + if name.startswith("research/") and name.count("/") == 2 + and name.endswith("/VERIFY.json")} + # Explicit scandir retains error detection; only an absent directory is OK. + try: + with os.scandir(root / "research") as entries: + local = {Path(entry.path) / "VERIFY.json" for entry in entries} + except FileNotFoundError: + local = set() + present, missing = {}, set() + for path in sorted(local | {root / name for name in committed}): + name = path.relative_to(root).as_posix() + symlink_free_relative_path(root, name) + try: + mode = path.lstat().st_mode + except (FileNotFoundError, NotADirectoryError): + if name in committed: + missing.add(name) + continue + require(stat.S_ISREG(mode), "non-regular replay manifest: " + name) + present[name] = path + fallback = {} + if missing: + indexed, objects = {}, {} + for record in git(root, "ls-files", "-t", "--stage", "-z", "--", "research").split(b"\0"): + if not record: + continue + header, raw_name = record.split(b"\t", 1) + name = raw_name.decode("utf-8") + if name in missing: + indexed.setdefault(name, []).append(header.split()) + for record in git(root, "ls-tree", "-r", "-z", "HEAD", "--", "research").split(b"\0"): + if not record: + continue + header, raw_name = record.split(b"\t", 1) + name = raw_name.decode("utf-8") + if name in missing: + objects[name] = header.split() + for name in sorted(missing): + metadata = objects.get(name, []) + require(len(metadata) == 3 and metadata[0] in (b"100644", b"100755") + and metadata[1] == b"blob" + and indexed.get(name) == [[b"S", metadata[0], metadata[2], b"0"]], + "missing replay manifest is not an unchanged sparse HEAD input: " + name + + "; materialize it or commit its intended deletion") + # A regular file occupying the package directory is not an omission. + require(not (root / name).parent.exists() or (root / name).parent.is_dir(), + "non-directory replay package: " + name) + names = sorted(missing) + with closing(git_blobs(root, [objects[n][2].decode("ascii") for n in names])) as rows: + for (oid, raw), name in zip(rows, names): + require(oid.encode("ascii") == objects[name][2], "sparse manifest blob mismatch: " + name) + fallback[name] = raw + # Preserve Path's component ordering: package "a" precedes "a-extra". + for name in sorted(present.keys() | fallback.keys(), key=lambda name: PurePosixPath(name).parts): + yield root / name, fallback[name] if name in fallback else present[name].read_bytes(), name in fallback + + +def plan_details(root, changed): + """One selection pass with exact causes; no replay or acceptance decision.""" + changed = set(changed) + # A removed input is absent from HEAD, including an old case-only rename + # spelling on case-insensitive filesystems. Git's exact changed paths remain + # eligible so its consumer is selected, rather than rejected or skipped. + head_tracked = tracked_input_paths(root) + tracked = head_tracked | changed + replay_changed = sorted(changed.intersection({ + "scripts/research_package.py", ".github/workflows/research-packages.yml", + ".github/workflows/research-package-batch.yml"})) + # Group literal Git paths once. Keep removed paths and exact spellings. + changed_packages = {} + for name in sorted(changed): + parts = name.split("/", 2) + if len(parts) == 3 and parts[0] == "research": + changed_packages.setdefault("/".join(parts[:2]), []).append(name) + packages = [] + sparse_count = 0 + # Planning requires a stable checkout; reuse only within this invocation. + validated_inputs = set() + for path, raw, sparse in planning_manifests(root, head_tracked): + sparse_count += sparse + package = path.parent.relative_to(root).as_posix() + manifest = json.loads(raw.decode("utf-8")) + # Parse only for selection; validate fully in the isolated replay job. + inputs = manifest.get("inputs", []) + require(isinstance(inputs, list), "declare input paths for CI invalidation") + for name in inputs: + if isinstance(name, str) and name in validated_inputs: + continue + # Do not require existence: a deleted input must still trigger replay. + symlink_free_relative_path(root, name) + require_tracked_input(name, tracked) + validated_inputs.add(name) + causes = { + "declared_inputs": sorted(changed.intersection(inputs)), + "package_files": changed_packages.get(package, []), + "shared_controls": replay_changed, + } + if any(causes.values()): + packages.append({"package": package, "causes": causes, + "manifest_sha256": hashlib.sha256(raw).hexdigest()}) + if sparse_count: + print(f"Planning includes {sparse_count} unchanged sparse manifests from HEAD; " + "materialize packages and inputs before replay.", file=sys.stderr) + return packages + + +def plan(root, changed): + return [row["package"] for row in plan_details(root, changed)] + + +def write_plan_explanation(path, base, head, changed, details): + """Publish a complete sidecar before stdout/GitHub matrix publication.""" + matched = {name for row in details for names in row["causes"].values() for name in names} + report = { + "schema_version": 1, "base": base, "head": head, + "mode": "all-tracked" if base is None else "commit-diff", + "scope": "Selection from working or authenticated sparse HEAD manifests, not replay or mathematical acceptance. " + "Manifest hashes bind the declarations read; checkout must remain stable.", + "changed_paths": sorted(set(changed) - {""}), "selected": details, + "unmatched_paths": sorted(set(changed) - matched - {""}), + } + fd, temporary = tempfile.mkstemp(prefix=".replay-plan-", dir=path.parent) + try: + with os.fdopen(fd, "w", encoding="utf-8") as stream: + json.dump(report, stream, indent=2) + stream.write("\n") + os.replace(temporary, path) + finally: + Path(temporary).unlink(missing_ok=True) + + +def replay_matrix(packages, batch_size=None): + """Retain the flat CLI format, or group complete coverage into bounded matrices.""" + if batch_size is None: + return {"include": [{"package": p} for p in packages]} + require(type(batch_size) is int and 1 <= batch_size <= 256, + "matrix batch size must be 1..256") + batches = [packages[i:i + batch_size] for i in range(0, len(packages), batch_size)] + require(len(batches) <= 256, + "more than 256 matrix batches required; no packages were truncated") + return {"include": [{"packages": batch} for batch in batches]} + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--package") + ap.add_argument("--check-only", action="store_true") + ap.add_argument("--print-card", metavar="TASK_ID", + help="print the exact pinned catalogue card object to copy into TASK.json original_card") + ap.add_argument("--install-deps", action="store_true", help="explicit opt-in to install manifest's allowed pinned dependencies") + ap.add_argument("--changed-since", help="plan affected opt-in packages relative to this Git commit") + ap.add_argument("--github-output", type=Path) + ap.add_argument("--explain-plan", type=Path, + help="write exact replay-selection causes to JSON; matrix output stays unchanged") + ap.add_argument("--matrix-batch-size", type=int, + help="group planned packages for bounded reusable-workflow matrices (1..256)") + args = ap.parse_args() + if args.explain_plan and (not args.changed_since or args.package or args.print_card): + ap.error("--explain-plan requires --changed-since without --package or --print-card") + if args.explain_plan and args.github_output and args.explain_plan.resolve() == args.github_output.resolve(): + ap.error("--explain-plan and --github-output must use different files") + try: + if args.print_card: + require(args.package, + "--print-card needs --package research/ to resolve its pinned catalogue") + sys.stdout.write(json.dumps(catalogue_card(ROOT, args.package, args.print_card), indent=2) + "\n") + elif args.changed_since: + if set(args.changed_since) == {"0"}: + base = None + changed = git(ROOT, "ls-files", "-z").decode().split("\0") + else: + base = git(ROOT, "rev-parse", "--verify", args.changed_since + "^{commit}").decode().strip() + # A rename must invalidate both the removed input and its new path. + changed = git(ROOT, "diff", "--no-renames", "--name-only", "-z", base, "HEAD").decode().split("\0") + details = plan_details(ROOT, changed) + packages = [row["package"] for row in details] + matrix = json.dumps(replay_matrix(packages, args.matrix_batch_size)) + if args.explain_plan: + head = git(ROOT, "rev-parse", "--verify", "HEAD^{commit}").decode().strip() + write_plan_explanation(args.explain_plan, base, head, changed, details) + print(matrix) + if args.github_output: + with args.github_output.open("a", encoding="utf-8") as out: + out.write(f"matrix={matrix}\ncount={len(packages)}\n") + elif args.package: + if args.check_only: + validate(ROOT, args.package) + print("PASS_PACKAGE_METADATA_AND_PINS (not a mathematical acceptance check)") + else: + run_package(ROOT, args.package, args.install_deps) + else: + ap.error("use --package, --print-card or --changed-since") + except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError, importlib.metadata.PackageNotFoundError) as exc: + ap.exit(1, str(exc) + "\n") + + +if __name__ == "__main__": + main() diff --git a/research/export-single-scan/benchmark.py b/research/export-single-scan/benchmark.py new file mode 100644 index 0000000000..39b74e7e93 --- /dev/null +++ b/research/export-single-scan/benchmark.py @@ -0,0 +1,84 @@ +"""Time complete offline commands on one immutable export; writes externally only.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import platform +import statistics +import subprocess +import sys +import time + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[1] + + +def measure(script, archive, repetitions, cwd): + env = dict(os.environ, PATH='', PYTHONDONTWRITEBYTECODE='1') + env.pop('PYTHONPATH', None) + env.pop('PYTHONHOME', None) + argv = [sys.executable, '-W', 'error', '-B', str(script), + 'check', str(archive), '--check-inputs'] + values, outputs = [], [] + for _ in range(repetitions): + start = time.perf_counter_ns() + run = subprocess.run(argv, cwd=cwd, env=env, capture_output=True, timeout=60) + elapsed = time.perf_counter_ns() - start + if run.returncode or run.stderr: + raise ValueError(f'command failed: {run.returncode}, {run.stderr!r}') + prefix, raw = run.stdout.decode().strip().split(' ', 1) + result = json.loads(raw) + if prefix != 'PASS_RESEARCH_EXPORT' or 'declared_inputs_checked' not in result: + raise ValueError('command did not complete declaration checking') + values.append(elapsed) + outputs.append(run.stdout) + return values, outputs + + +def benchmark(archive, pairs, batch, cwd): + scripts = [HERE / 'baseline/export_research_package.py', + ROOT / 'scripts/export_research_package.py'] + before = hashlib.sha256(archive.read_bytes()).hexdigest() + runs, outputs = [[], []], [] + for index in range(pairs): + for side in (index % 2, 1 - index % 2): + times, texts = measure(scripts[side], archive, batch, cwd) + runs[side].append(times) + outputs.extend(texts) + if len(set(outputs)) != 1: + raise ValueError('complete outputs differ') + if hashlib.sha256(archive.read_bytes()).hexdigest() != before: + raise ValueError('benchmark input changed') + medians = [statistics.median([n for row in side for n in row]) for side in runs] + return {'archive_sha256': before, 'archive_bytes': archive.stat().st_size, + 'pairs': pairs, 'commands_per_side_per_pair': batch, + 'elapsed_ns': runs, 'median_ns': medians, + 'wall_time_reduction': 1 - medians[1] / medians[0], + 'successful_commands': 2 * pairs * batch, 'errors': 0, + 'complete_stdout': outputs[0].decode(), + 'source_sha256': [hashlib.sha256(p.read_bytes()).hexdigest() for p in scripts]} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--archive', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--pairs', type=int, default=9, choices=range(5, 21)) + parser.add_argument('--batch', type=int, default=5, choices=range(1, 11)) + args = parser.parse_args() + archive, output = args.archive.resolve(), args.output.resolve() + if output.is_relative_to(ROOT) or output.exists(): + raise ValueError('output must be a new external file') + result = benchmark(archive, args.pairs, args.batch, output.parent) + result['environment'] = {'python': sys.version, 'platform': platform.platform(), + 'logical_cpus': os.cpu_count(), 'load': os.getloadavg()} + with output.open('x') as stream: + json.dump(result, stream, indent=2, sort_keys=True) + stream.write('\n') + print(json.dumps({k: result[k] for k in ('median_ns', 'wall_time_reduction', + 'successful_commands', 'errors')})) + + +if __name__ == '__main__': + main() diff --git a/research/export-single-scan/verify.py b/research/export-single-scan/verify.py new file mode 100644 index 0000000000..76e2168728 --- /dev/null +++ b/research/export-single-scan/verify.py @@ -0,0 +1,175 @@ +"""Exact export-check conformance and recorded benchmark arithmetic.""" +import argparse +import hashlib +import importlib.util +import io +import json +from pathlib import Path +import statistics +import sys +import tarfile +import tempfile + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[1] +PACKAGE = 'research/fixture' +RECEIPT = '.research-export.json' + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=True) + + +def load(path, name): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def bindings(): + sources = json.loads((HERE / 'SOURCES.json').read_text()) + rows = sources['retained_baseline'] + require({row['package_path'] for row in rows} == + {'baseline/export_research_package.py', 'baseline/research_package.py'}, + 'baseline inventory changed') + for row in rows: + data = (HERE / row['package_path']).read_bytes() + require(hashlib.sha256(data).hexdigest() == row['sha256'], 'baseline source changed') + blob = hashlib.sha1(b'blob ' + str(len(data)).encode() + b'\0' + data).hexdigest() + require(blob == row['git_blob'], 'baseline Git blob changed') + + +def bundle(path, count, order, mode, raw=None, extra=False, corrupt=False): + names = [f'inputs/item-{index}.bin' for index in range(count)] + files = {PACKAGE + '/README.md': b'scope', + PACKAGE + '/VERIFY.json': json.dumps({'inputs': names}).encode() if raw is None else raw} + files.update({name: bytes([index, 0, 255]) for index, name in enumerate(names)}) + if extra: + files['undeclared.bin'] = b'extra' + receipt = {'format': 'research-package-export-v1', 'commit': 'a' * 40, + 'package': PACKAGE, 'files': [ + {'path': name, 'mode': mode, 'size': len(data), + 'sha256': hashlib.sha256(data).hexdigest()} for name, data in files.items()]} + rows = list(files.items()) + rows.insert(min(order, len(rows)), (RECEIPT, json.dumps(receipt).encode())) + with tarfile.open(path, 'w') as archive: + for name, data in rows: + info = tarfile.TarInfo(name) + info.size, info.mode = len(data), mode + archive.addfile(info, io.BytesIO(data)) + if corrupt: + with tarfile.open(path, 'r:') as archive: + offset = archive.getmember(PACKAGE + '/README.md').offset_data + with path.open('r+b') as stream: + stream.seek(offset) + require(stream.write(b'X') == 1, 'incomplete corruption fixture') + + +def outcome(module, path, check_inputs): + try: + return ['accept', canonical(module.check_bundle(path, check_inputs=check_inputs))] + except (OSError, ValueError, TypeError, KeyError, RecursionError, tarfile.TarError) as exc: + return ['reject', type(exc).__name__, str(exc)] + + +def conformance(before, after, directory): + path = Path(directory) / 'fixture.tar' + counts = {'accept': 0, 'reject': 0} + witnesses = [] + valid = [(count, order, mode, None, False, False) + for count in (0, 1, 7, 31) for order in (0, 1, 100) for mode in (0o644, 0o755)] + malformed = [b'[]', b'{"inputs":1}', b'{"inputs":[],"inputs":[]}', + b'{"inputs":["x","x"]}', b'{"inputs":["../x"]}', + b'{"inputs":["C:/x"]}', b'{"inputs":["missing.bin"]}', + b'{"inputs":[".research-export.json"]}', b'{"inputs":[true]}'] + invalid = [(0, order, 0o644, raw, False, False) + for order in (0, 100) for raw in malformed] + invalid += [(0, 100, 0o644, None, True, False), (0, 0, 0o644, None, False, True)] + for index, fixture in enumerate(valid + invalid): + bundle(path, *fixture) + digest = hashlib.sha256(path.read_bytes()).hexdigest() + for check_inputs in (False, True): + old = outcome(before, path, check_inputs) + new = outcome(after, path, check_inputs) + require(old == new, f'conformance mismatch at {index}, {check_inputs}') + if index < len(valid): + require(new[0] == 'accept', 'positive fixture rejected') + elif check_inputs: + require(new[0] == 'reject', 'negative fixture accepted') + counts[new[0]] += 1 + witnesses.append([index, check_inputs, new]) + require(hashlib.sha256(path.read_bytes()).hexdigest() == digest, 'check wrote input') + return {**counts, 'fixtures': len(valid + invalid), + 'complete_outcomes_sha256': hashlib.sha256(canonical(witnesses).encode()).hexdigest()} + + +def measured_evidence(): + result = {} + scripts = [HERE / 'baseline/export_research_package.py', ROOT / 'scripts/export_research_package.py'] + for label in ('toolkit', 'small'): + value = json.loads((HERE / f'BENCHMARK-{label}.json').read_text()) + require(value['source_sha256'] == [hashlib.sha256(p.read_bytes()).hexdigest() for p in scripts], + 'timed source no longer matches') + require(value['pairs'] == 9 and value['commands_per_side_per_pair'] == 5, 'sample shape') + require(value['successful_commands'] == 90 and value['errors'] == 0, 'measurement failures') + runs = value['elapsed_ns'] + require(len(runs) == 2 and all(len(side) == 9 for side in runs), 'pair count') + require(all(len(row) == 5 for side in runs for row in side), 'batch count') + require(all(type(n) is int and 0 < n < 60_000_000_000 + for side in runs for row in side for n in row), 'invalid duration') + medians = [statistics.median([n for row in side for n in row]) for side in runs] + require(value['median_ns'] == medians, 'median arithmetic') + require(value['wall_time_reduction'] == 1 - medians[1] / medians[0], 'reduction arithmetic') + improvements = sum(statistics.median(b) < statistics.median(a) for a, b in zip(*runs)) + result[label] = {'median_ns': medians, 'improving_batches': improvements} + require(result['toolkit']['improving_batches'] == 9, 'primary batches do not all improve') + return result + + +def summary(): + bindings() + original_path = list(sys.path) + try: + sys.path.insert(0, str(ROOT / 'scripts')) + before = load(HERE / 'baseline/export_research_package.py', 'baseline_export') + after = load(ROOT / 'scripts/export_research_package.py', 'candidate_export') + with tempfile.TemporaryDirectory(prefix='export-conformance-') as directory: + result = {'conformance': conformance(before, after, directory), + 'recorded_measurements': measured_evidence()} + finally: + sys.path[:] = original_path + files = sorted(p for p in HERE.rglob('*') if p.is_file() + and p.name != 'RESULT.json' and '__pycache__' not in p.parts) + require(len(files) <= 24, 'package file limit') + result['package_sha256'] = {p.relative_to(HERE).as_posix(): hashlib.sha256(p.read_bytes()).hexdigest() + for p in files} + result['input_sha256'] = {name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest() + for name in json.loads((HERE / 'VERIFY.json').read_text())['inputs']} + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--emit-to', type=Path) + args = parser.parse_args() + result = summary() + if args.emit_to is None: + expected = json.loads((HERE / 'RESULT.json').read_text()) + require(canonical(result) == canonical(expected), 'generated result differs') + else: + target = args.emit_to.resolve() + require(not target.is_relative_to(ROOT), 'receipt output must be external') + with target.open('x') as stream: + json.dump(result, stream, sort_keys=True, indent=2) + stream.write('\n') + print('PASS_EXPORT_SINGLE_SCAN ' + canonical(result['conformance'])) + + +if __name__ == '__main__': + main() diff --git a/scripts/export_research_package.py b/scripts/export_research_package.py index 25b0a12a80..1e07a10cd1 100644 --- a/scripts/export_research_package.py +++ b/scripts/export_research_package.py @@ -4,7 +4,7 @@ Tekalign, with substantial OpenAI Codex assistance. Apache-2.0. """ import argparse -from contextlib import closing +from contextlib import closing, nullcontext import hashlib import io import json @@ -139,14 +139,17 @@ def export(root, package, ref, output, input_ref=None): return receipt -def check_declared_inputs(path, receipt, seen): +def check_declared_inputs(path, receipt, seen, *, archive=None): """Check the bundled manifest's file declarations without extraction or Git.""" name = receipt['package'] + '/VERIFY.json' - with tarfile.open(path, mode='r:') as archive: - item = archive.getmember(name) + # A completed integrity scan has already populated this archive's index. + # Borrow its handle without closing it; direct calls still own their handle. + context = tarfile.open(path, mode='r:') if archive is None else nullcontext(archive) + with context as bundle: + item = bundle.getmember(name) require(item.isfile() and item.size <= 16 * 1024 * 1024, 'invalid or oversized bundled manifest') - with archive.extractfile(item) as stream: + with bundle.extractfile(item) as stream: raw = stream.read(16 * 1024 * 1024 + 1) require(len(raw) == seen[name]['size'] and hashlib.sha256(raw).hexdigest() == seen[name]['sha256'], 'bundled manifest changed during declaration check') @@ -175,25 +178,29 @@ def unique(pairs): return len(inputs) -def check_bundle(path, check_inputs=False): - """Check declared inventory, modes and payload hashes; never extract or execute.""" +def read_bundle(archive): + """Read and hash each member, retaining the complete checked inventory.""" seen, receipt = {}, None - with tarfile.open(path, mode='r:') as archive: - for item in archive: - bundle_relative_path(item.name) - require(item.isfile(), 'non-regular archive member: ' + item.name) - require(item.name not in seen, 'duplicate archive member: ' + item.name) - require(item.mode in (0o644, 0o755), 'unsupported mode: ' + item.name) - digest = hashlib.sha256() - with archive.extractfile(item) as data: - if item.name == RECEIPT: - require(item.size <= 16 * 1024 * 1024, 'oversized export receipt') - receipt = json.load(data, object_pairs_hook=unique_json_keys) - else: - for block in iter(lambda: data.read(1024 * 1024), b''): - digest.update(block) - seen[item.name] = {'path': item.name, 'mode': item.mode, 'size': item.size, - 'sha256': digest.hexdigest()} + for item in archive: + bundle_relative_path(item.name) + require(item.isfile(), 'non-regular archive member: ' + item.name) + require(item.name not in seen, 'duplicate archive member: ' + item.name) + require(item.mode in (0o644, 0o755), 'unsupported mode: ' + item.name) + digest = hashlib.sha256() + with archive.extractfile(item) as data: + if item.name == RECEIPT: + require(item.size <= 16 * 1024 * 1024, 'oversized export receipt') + receipt = json.load(data, object_pairs_hook=unique_json_keys) + else: + for block in iter(lambda: data.read(1024 * 1024), b''): + digest.update(block) + seen[item.name] = {'path': item.name, 'mode': item.mode, 'size': item.size, + 'sha256': digest.hexdigest()} + return seen, receipt + + +def check_bundle_inventory(seen, receipt): + """Validate the receipt and remove its own entry from the payload inventory.""" require_file_tree(seen) require(isinstance(receipt, dict) and receipt.get('format') == FORMAT, 'missing or invalid export receipt') require(isinstance(receipt.get('commit'), str) and @@ -218,9 +225,17 @@ def check_bundle(path, check_inputs=False): del seen[RECEIPT] require(seen == expected, 'bundle inventory, mode, size or SHA256 mismatch') require(receipt['package'] + '/VERIFY.json' in seen, 'missing bundled manifest') - if check_inputs: - receipt = {**receipt, 'declared_inputs_checked': check_declared_inputs(path, receipt, seen)} - return receipt + + +def check_bundle(path, check_inputs=False): + """Check declared inventory, modes and payload hashes; never extract or execute.""" + with tarfile.open(path, mode='r:') as archive: + seen, receipt = read_bundle(archive) + check_bundle_inventory(seen, receipt) + if check_inputs: + count = check_declared_inputs(path, receipt, seen, archive=archive) + receipt = {**receipt, 'declared_inputs_checked': count} + return receipt def main(): diff --git a/tests/test_research_export_input_coverage.py b/tests/test_research_export_input_coverage.py index 6a7e5e08a4..d34397d2ad 100644 --- a/tests/test_research_export_input_coverage.py +++ b/tests/test_research_export_input_coverage.py @@ -8,6 +8,7 @@ import tarfile import tempfile import unittest +from unittest.mock import patch ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'scripts')) @@ -79,6 +80,67 @@ def test_real_optimized_cli_checks_declarations_without_git(self): self.assertEqual(prefix, 'PASS_RESEARCH_EXPORT') self.assertEqual(json.loads(payload)['declared_inputs_checked'], 1) + def test_complete_check_borrows_one_archive_and_closes_it(self): + self.bundle(['input.bin'], {'input.bin': b'data'}) + opened = [] + original = tarfile.open + + def tracked(*args, **kwargs): + archive = original(*args, **kwargs) + opened.append(archive) + return archive + + with patch.object(exporter.tarfile, 'open', tracked): + result = exporter.check_bundle(self.archive, check_inputs=True) + self.assertEqual(result['declared_inputs_checked'], 1) + self.assertEqual(len(opened), 1) + self.assertTrue(opened[0].closed) + + def test_declaration_failure_closes_the_single_archive(self): + self.bundle(['missing.bin']) + opened = [] + original = tarfile.open + + def tracked(*args, **kwargs): + archive = original(*args, **kwargs) + opened.append(archive) + return archive + + with patch.object(exporter.tarfile, 'open', tracked): + with self.assertRaisesRegex(ValueError, 'missing bundled declared input'): + exporter.check_bundle(self.archive, check_inputs=True) + self.assertEqual(len(opened), 1) + self.assertTrue(opened[0].closed) + + def test_borrowed_handle_stays_open_on_success_and_failure(self): + receipt = self.bundle() + seen = {row['path']: row for row in receipt['files']} + with tarfile.open(self.archive, 'r:') as archive: + self.assertEqual(exporter.check_declared_inputs( + self.archive, receipt, seen, archive=archive), 0) + self.assertFalse(archive.closed) + seen[self.package + '/VERIFY.json']['sha256'] = '0' * 64 + with self.assertRaisesRegex(ValueError, 'manifest changed during declaration check'): + exporter.check_declared_inputs(self.archive, receipt, seen, archive=archive) + self.assertFalse(archive.closed) + + def test_same_handle_reread_still_rejects_changed_manifest_bytes(self): + self.bundle(['input.bin'], {'input.bin': b'data'}) + with tarfile.open(self.archive, 'r:') as archive: + offset = archive.getmember(self.package + '/VERIFY.json').offset_data + original = exporter.check_bundle_inventory + + def change_after_hashing(seen, receipt): + original(seen, receipt) + raw = b'{"inputs": ["other.bin"]}' + with self.archive.open('r+b') as stream: + stream.seek(offset) + self.assertEqual(stream.write(raw), len(raw)) + + with patch.object(exporter, 'check_bundle_inventory', change_after_hashing): + with self.assertRaisesRegex(ValueError, 'manifest changed during declaration check'): + exporter.check_bundle(self.archive, check_inputs=True) + if __name__ == '__main__': unittest.main() From 9889f94b6ed045c1069b2e6f7e4471d0322f2f16 Mon Sep 17 00:00:00 2001 From: Nick Petalas Date: Thu, 29 Oct 2026 16:33:28 +0000 Subject: [PATCH 2/2] docs(research): preserve the predecessor credit as its exact login --- research/export-single-scan/README.md | 4 ++-- research/export-single-scan/RESULT.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/research/export-single-scan/README.md b/research/export-single-scan/README.md index a69d631f63..8bbb8882d2 100644 --- a/research/export-single-scan/README.md +++ b/research/export-single-scan/README.md @@ -124,8 +124,8 @@ Independent human and formal proof-kernel review are unclaimed. Credit Tekalign / PR3138 for the exporter; Ali Romman / PR5744 and PR5728 for declared coverage and typed metadata; roger-sato / PR3629 and PR3641 for path -and tree checks; Mihir Srivastava for portable fixtures; Jose Carlos Gonzalez -Roque / PR6189 for Windows publication; and Douglas Colkitt and the retained +and tree checks; Mihir Srivastava for portable fixtures; +josecarlosgonzalezroque-design / PR6189 for Windows publication; and Douglas Colkitt and the retained Git-reader contributors. Original notices and source bytes are unchanged. The new contribution is archive-index reuse, its lifecycle controls and measured complete-command effect. New work is Apache-2.0. diff --git a/research/export-single-scan/RESULT.json b/research/export-single-scan/RESULT.json index bd78c38ed9..8e287c1cf4 100644 --- a/research/export-single-scan/RESULT.json +++ b/research/export-single-scan/RESULT.json @@ -20,7 +20,7 @@ "BENCHMARK-small.json": "4f261a6709cb5cc33f97268b205357fc81804251b0268037b7dd6508f011eb5c", "BENCHMARK-toolkit.json": "78146a0d0905055d3629237404d1fa7efe8dd5aade69695b121927b3b9241d15", "PROFILE.json": "27a4b48bf480652ba8d27bfacfb4d2b7e7f7cff570b83683dd28adba8d1671d9", - "README.md": "8fd66e8a83c149bee89335f3281d061fedac49d2c4a0a6f104a41bee6bcdf28c", + "README.md": "15ddd5e5c5df48b6c972a7bbf1152c2244d5e1afe1551a1fd5c2e2b868ad1f9a", "SOURCES.json": "f699c23d9fd1ee46f4ded7a21b0f5ad65be292c1867b895e36a70cd312b90419", "TASK.json": "06638e8d16d96892ea8ac8a2eb3a3c32f97e2d725271f207b1e62efb082e6742", "VALIDATION.json": "24e959de5f350fc3e58cbe4195d7c5ea98a40142cf4b438830427539d4672f93",