From a6bee4472b4df4ff17510dba2b2bb06e5af0f029 Mon Sep 17 00:00:00 2001 From: Nick Petalas Date: Thu, 29 Oct 2026 16:26:16 +0000 Subject: [PATCH] perf(research): read draft catalogue once --- .../MEASUREMENTS.json | 361 ++++++++++++++++++ .../draft-catalogue-single-read/README.md | 143 +++++++ .../draft-catalogue-single-read/SOURCES.json | 36 ++ .../draft-catalogue-single-read/TASK.json | 79 ++++ .../draft-catalogue-single-read/VERIFY.json | 37 ++ .../draft-catalogue-single-read/baseline.py | 129 +++++++ .../draft-catalogue-single-read/benchmark.py | 116 ++++++ .../draft-catalogue-single-read/verify.py | 107 ++++++ scripts/new_research_package.py | 40 +- tests/test_research_scaffold_batch.py | 37 +- 10 files changed, 1068 insertions(+), 17 deletions(-) create mode 100644 research/draft-catalogue-single-read/MEASUREMENTS.json create mode 100644 research/draft-catalogue-single-read/README.md create mode 100644 research/draft-catalogue-single-read/SOURCES.json create mode 100644 research/draft-catalogue-single-read/TASK.json create mode 100644 research/draft-catalogue-single-read/VERIFY.json create mode 100644 research/draft-catalogue-single-read/baseline.py create mode 100644 research/draft-catalogue-single-read/benchmark.py create mode 100644 research/draft-catalogue-single-read/verify.py diff --git a/research/draft-catalogue-single-read/MEASUREMENTS.json b/research/draft-catalogue-single-read/MEASUREMENTS.json new file mode 100644 index 0000000000..9176fd7389 --- /dev/null +++ b/research/draft-catalogue-single-read/MEASUREMENTS.json @@ -0,0 +1,361 @@ +{ + "base": "a5bff3bf133e482541c09eba4e773884bdb2b1b7", + "benchmark_sha256": "ac7be62177e39e825e881e157ec2d04fe9970f59a6d4ef318a51b06f8e190449", + "cpu_count": 18, + "hashes": { + "baseline": "a9a66cb591239c27928047146040cb9adb8d138fdc327f2fc8cb873ef9892c27", + "candidate": "567b5300992b03aa685854a902445758ad8ee00f24ac1289f5ae711c7fb28542" + }, + "load_average": [ + 2.16015625, + 2.83056640625, + 2.48779296875 + ], + "platform": "macOS-27.0.1-arm64-arm-64bit", + "processor": "arm", + "python": "3.12.15 (main, Oct 14 2026, 00:55:37) [Clang 23.1.3 ]", + "schema_version": 1, + "scope": "Whole fresh Python CLI process through all five file writes; validation outside timer.", + "shared_reader_sha256": "e1256016e7f51827d04fa2179255989f639ae1d97afaa9c16265f85a0c6abd70", + "workloads": { + "default": { + "errors": 0, + "mismatches": 0, + "output_files": { + "README.md": { + "mode": 420, + "sha256": "1bd0b775cb03be5001b37be1f1e1abb382b7c45eaddd12fd2854bea3b9a93199" + }, + "SOURCES.json": { + "mode": 420, + "sha256": "7d090adbb78ade4e4342137ae3a6b63182732ca82736b54229de9b173da73329" + }, + "TASK.json": { + "mode": 420, + "sha256": "e847824b93e856117229985a91ff236ee80915c0c9b5619c93181b75a03ab33d" + }, + "VERIFY.json": { + "mode": 420, + "sha256": "e503155274b3a7903319fded3b231d8fe6c44cbbc881a08ee5e3a2e3f41b5e46" + }, + "verify.py": { + "mode": 420, + "sha256": "c4deebff0c93b8cb38d5c1971f0fab4115ea49f10e23e3682d2b989a40e4423f" + } + }, + "samples": [ + { + "baseline": 67232042, + "candidate": 59433083 + }, + { + "baseline": 73392625, + "candidate": 57786209 + }, + { + "baseline": 68104167, + "candidate": 58248292 + }, + { + "baseline": 65785666, + "candidate": 57447250 + }, + { + "baseline": 68043583, + "candidate": 57535875 + }, + { + "baseline": 72370500, + "candidate": 60789416 + }, + { + "baseline": 69809500, + "candidate": 60524041 + }, + { + "baseline": 66878208, + "candidate": 58239291 + }, + { + "baseline": 65925000, + "candidate": 57245125 + }, + { + "baseline": 69957583, + "candidate": 56795750 + }, + { + "baseline": 66108041, + "candidate": 56164416 + }, + { + "baseline": 66070125, + "candidate": 57150000 + }, + { + "baseline": 67116000, + "candidate": 56695542 + }, + { + "baseline": 67406291, + "candidate": 58990792 + }, + { + "baseline": 68468541, + "candidate": 57318959 + }, + { + "baseline": 68854042, + "candidate": 59568875 + }, + { + "baseline": 67706833, + "candidate": 58933958 + }, + { + "baseline": 66011708, + "candidate": 56991708 + }, + { + "baseline": 65803709, + "candidate": 56572292 + }, + { + "baseline": 66172334, + "candidate": 56569750 + }, + { + "baseline": 66400250, + "candidate": 57014750 + }, + { + "baseline": 69085625, + "candidate": 56501959 + }, + { + "baseline": 69499583, + "candidate": 56905000 + }, + { + "baseline": 69602959, + "candidate": 61209167 + }, + { + "baseline": 71290375, + "candidate": 61160167 + }, + { + "baseline": 68850125, + "candidate": 59655000 + }, + { + "baseline": 66043542, + "candidate": 56601500 + }, + { + "baseline": 66418250, + "candidate": 59745958 + }, + { + "baseline": 65928417, + "candidate": 59400500 + }, + { + "baseline": 66464083, + "candidate": 56685500 + }, + { + "baseline": 66522375, + "candidate": 57272208 + } + ], + "sources": [], + "summary": { + "baseline": { + "max_ns": 73392625, + "median_ns": 67232042, + "min_ns": 65785666 + }, + "candidate": { + "max_ns": 61209167, + "median_ns": 57447250, + "min_ns": 56164416 + } + } + }, + "eight_sources": { + "errors": 0, + "mismatches": 0, + "output_files": { + "README.md": { + "mode": 420, + "sha256": "1bd0b775cb03be5001b37be1f1e1abb382b7c45eaddd12fd2854bea3b9a93199" + }, + "SOURCES.json": { + "mode": 420, + "sha256": "69bbe190f4a57907616d541b18243b7509b50148e09e5946ee2ceb98c8e7602e" + }, + "TASK.json": { + "mode": 420, + "sha256": "e847824b93e856117229985a91ff236ee80915c0c9b5619c93181b75a03ab33d" + }, + "VERIFY.json": { + "mode": 420, + "sha256": "6d67caaf1d48c7e4fdcd179030b9cb12564797a8c3104ec07591ca7bb4e259f3" + }, + "verify.py": { + "mode": 420, + "sha256": "c4deebff0c93b8cb38d5c1971f0fab4115ea49f10e23e3682d2b989a40e4423f" + } + }, + "samples": [ + { + "baseline": 67949500, + "candidate": 60974750 + }, + { + "baseline": 66926375, + "candidate": 59475916 + }, + { + "baseline": 70149083, + "candidate": 60486167 + }, + { + "baseline": 67463709, + "candidate": 58926333 + }, + { + "baseline": 67335125, + "candidate": 58728959 + }, + { + "baseline": 68798458, + "candidate": 58780083 + }, + { + "baseline": 69787542, + "candidate": 62532750 + }, + { + "baseline": 67306333, + "candidate": 59138959 + }, + { + "baseline": 67516792, + "candidate": 60522375 + }, + { + "baseline": 66905833, + "candidate": 58450000 + }, + { + "baseline": 67642750, + "candidate": 58921750 + }, + { + "baseline": 69271208, + "candidate": 58826875 + }, + { + "baseline": 68879667, + "candidate": 58739792 + }, + { + "baseline": 71035625, + "candidate": 59587250 + }, + { + "baseline": 67739584, + "candidate": 58484709 + }, + { + "baseline": 68200166, + "candidate": 57687834 + }, + { + "baseline": 68700292, + "candidate": 61202834 + }, + { + "baseline": 68373584, + "candidate": 60160125 + }, + { + "baseline": 90029250, + "candidate": 60357042 + }, + { + "baseline": 69751000, + "candidate": 61530583 + }, + { + "baseline": 68429417, + "candidate": 61233125 + }, + { + "baseline": 69050458, + "candidate": 59041292 + }, + { + "baseline": 71924750, + "candidate": 59205500 + }, + { + "baseline": 67695750, + "candidate": 59300583 + }, + { + "baseline": 70237125, + "candidate": 59613333 + }, + { + "baseline": 70905750, + "candidate": 58821667 + }, + { + "baseline": 73523583, + "candidate": 62589542 + }, + { + "baseline": 76218375, + "candidate": 64784416 + }, + { + "baseline": 72079708, + "candidate": 62183709 + }, + { + "baseline": 74639875, + "candidate": 62388375 + }, + { + "baseline": 70836792, + "candidate": 62317042 + } + ], + "sources": [ + "AGENTS.md", + "CONTRIBUTING.md", + "autoresearch.md", + "docs/research/current-priorities.md", + "docs/research/agent-procedure.md", + "docs/research/research-package-guide.md", + "certificates/frontier6030-source.json", + "docs/research/pr-intake-evidence/pr6030-audit/PROOF-REVIEW.md" + ], + "summary": { + "baseline": { + "max_ns": 90029250, + "median_ns": 68879667, + "min_ns": 66905833 + }, + "candidate": { + "max_ns": 64784416, + "median_ns": 59587250, + "min_ns": 57687834 + } + } + } + } +} diff --git a/research/draft-catalogue-single-read/README.md b/research/draft-catalogue-single-read/README.md new file mode 100644 index 0000000000..1cb2e33f19 --- /dev/null +++ b/research/draft-catalogue-single-read/README.md @@ -0,0 +1,143 @@ +# Read the draft catalogue once + +Research-draft creation takes 14.55% less median wall time on the measured +default workload after removing one redundant Git process. This is an empirical +tooling improvement with a scoped source-level equivalence argument. It changes +no mathematical bound, supplier or selected frontier. + +The contribution was produced by Nick Petalas. New work is Apache-2.0. + +## Result and evidence + +The old command reads the pinned catalogue through `git show`, then reads it +again in the source-pin batch. The new command reads that catalogue first in +the batch and uses the same bytes for card selection and its source pin. It +sorts the completed inventory into the original order. On Git supporting the +NUL protocol this removes one of the three Git launches. The existing fallback +protocol and source-reader process ownership remain in the shared reader. + +| Complete CLI workload | Baseline median, range | Candidate median, range | Median reduction | +| --- | --- | --- | --- | +| Default, two provenance pins | 67.232 ms, 65.786 to 73.393 | 57.447 ms, 56.164 to 61.209 | 14.55% | +| Eight explicit sources, ten pins | 68.880 ms, 66.906 to 90.029 | 59.587 ms, 57.688 to 64.784 | 13.49% | + +Each row has 31 measured pairs after one warm-up pair. Pair order alternates +between baseline/candidate and candidate/baseline. Each candidate wins all 31 +pairs. All 128 complete commands, including warm-ups, succeed and produce the +same five file bytes and modes within each workload. The measured interval +starts before Python process creation and ends after exit, including Git reads +and all draft writes. Output comparison and moving each draft to scratch happen +after the timer. No source cache or precomputed manifest replaces either run. + +[MEASUREMENTS.json](MEASUREMENTS.json) is the generated raw receipt, with each +elapsed sample, output hashes, source hashes, interpreter and platform. The +machine has 18 logical CPUs; load was 3.76 before measurement and the receipt +records the load at completion. Background work was not stopped. Alternating +pair order gives both implementations the same warm filesystem and process-load +conditions. This is one macOS machine, CPython 3.12.15 and Git 2.54.0. + +An untimed `cProfile` run of the baseline takes 51 ms inside Python. Generator +creation takes 33 ms; the two ordinary Git calls take 19 ms together, and the +remaining batch path takes 11 ms. Import loading takes 18 ms. The removed +process accounts for about the observed 9.8 ms end-to-end saving. Interpreter +startup/imports and the remaining Git work limit the gain. These profile +durations explain the mechanism; they are not the reported benchmark samples. + +## Why generated output is preserved + +For a stable checkout and fixed resolved commit, `git show` and the existing +batched blob reader supply the same stored catalogue bytes. Both use the same +sanitized Git source environment. The selected card is unchanged. Every unique +source is still checked against its live file and receives the same blob ID, +SHA256 and usage. Sorting the inventory restores the old JSON order. The code +that renders and publishes the five draft files is unchanged. Therefore every +valid creation has the same generated bytes and file modes in that environment. + +`zip(..., strict=True)` exhausts the reader, so a final process-status failure +still prevents publication. Early rejection closes it through the existing +`closing` context. Output aliases and overwrites still reject before Git or +staging writes. Catalogue validation now precedes validation of the other live +files; when several inputs are invalid, the first reported diagnostic can +change. Rejection itself is preserved. Concurrent checkout mutation is outside +the inherited contract. + +The direct verifier compares all five files and modes on three additional +baseline/candidate cases, including binary/empty/Unicode sources and duplicated +explicit provenance paths. Nine focused tests cover exact pins, NUL/newline +names, source drift, untracked sources, late reader failure, unknown tasks, +early closure and linked output paths. The retained broader suite covers Git +replacement/root binding, overwrite rejection and the generator/validator +round trip. Locally, all 710 onboarding tests pass with 30 platform-specific +skips. No independent implementation or proof-kernel acceptance is claimed. + +## Reproduce + +From the repository root, with the local interpreter used for this result: + +```sh +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -W error -B research/draft-catalogue-single-read/verify.py +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -B scripts/research_package.py --package research/draft-catalogue-single-read +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -B scripts/check_research_commit.py --package research/draft-catalogue-single-read --check-source-history +``` + +The manifest runs normal and optimized verification, timed locally at 1.062 s +and 1.147 s. The verifier checks the recorded timing arithmetic and exact +measured source hashes; it does not remeasure performance or certify another +machine's timings. Git and the standard library suffice. + +To remeasure, choose a new nonexistent scratch subdirectory outside the checkout: + +```sh +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -W error -B research/draft-catalogue-single-read/benchmark.py --scratch /absolute/scratch/draft-benchmark --samples 31 +``` + +The benchmark temporarily creates an exclusively named draft in this worktree +through the real command, then moves it into scratch after each run, including +failure cleanup. The benchmark directory must be on the same filesystem for +these renames. It never overwrites an existing draft or uses the reference +checkout for writes. `baseline.py` retains the exact old generator bytes because +the benchmark executes both implementations; its original notice is unchanged. + +## Limits + +The measurements cover default and eight-source drafts on one environment. +Windows and older-Git performance were not measured. The shared reader's legacy +protocol is unchanged. Full selected scientific, native, compiler, precision, +BIT, outer-ledger, Lean and Docker replays are unrelated and were not run. +The five selected framework assumptions remain. + +Python's managed allocations are retained. The new source loop is bounded by +the finite deduplicated request list and retains one current blob, parsed +catalogue and output inventory. The existing long `create` function largely +contains unchanged templates; a template refactor would expand this scoped +change. The 21-line source-metadata helper isolates the new logic. Dynamic +loading in the verifier is limited to the two declared local generator files. +The benchmark bounds samples to 5 to 101 and each subprocess to 30 seconds. +No optional Python linter or type checker is installed; warning-strict normal +and optimized execution, AST parsing and the repository tests provide diagnostics. + +## Open next questions + +Does the gain persist for an actual source-heavy package with hundreds of pins? +Start with one named retained source inventory and the same complete command, +including exact output comparison. Materializing those inputs and a separate +measurement budget are the blockers. No further optimization is reserved. + +## Credit and overlap + +Douglas Colkitt supplies the generator and replay framework. ywz65's +[PR2539](https://github.com/CrocSwap/integer-mult-bounds/pull/2539) supplies batched +draft pins. Ali Romman's [PR5734](https://github.com/CrocSwap/integer-mult-bounds/pull/5734) +supplies output-path guards. The immutable-object, exact-path, Git-root, +process-ownership and source-authentication contributors retain their credit +in the shared reader and tests. This contribution removes the duplicate +catalogue fetch; it does not claim those predecessors' guarantees as new work. + +The [advance claim](https://github.com/CrocSwap/integer-mult-bounds/discussions/1166#discussioncomment-19272104) +records the bounded original performance question. Canonical main was +`a5bff3bf133e482541c09eba4e773884bdb2b1b7`, priority revision 8. Live PR/Discussion +checks, whole-main README searches and the account's recent work were checked. +The overlap cache materialized only 92 of 8,623 main notes; that cache alone is +not evidence of novelty. No matching active optimization was found in the +additional targeted searches. Independent human and formal proof-kernel review +are unclaimed. diff --git a/research/draft-catalogue-single-read/SOURCES.json b/research/draft-catalogue-single-read/SOURCES.json new file mode 100644 index 0000000000..61b717049e --- /dev/null +++ b/research/draft-catalogue-single-read/SOURCES.json @@ -0,0 +1,36 @@ +{ + "schema_version": 1, + "base_commit": "a5bff3bf133e482541c09eba4e773884bdb2b1b7", + "files": [ + { + "path": "certificates/selected-result.json", + "usage": "provenance", + "git_blob": "d59b5035e7b4cbaa9d688a1d32dd0d6eb79c466a", + "sha256": "9b8d0eea95b3bf0dfcb7938a732a6439b87a2685dec2a5ab9e525b3b34debbfa" + }, + { + "path": "docs/research/agent-task-catalog.json", + "usage": "provenance", + "git_blob": "3404fbdd5e7529a0674eef26114b5071e4586cec", + "sha256": "ead6cdc9d0af6d58e8a49744d8ec8c7c15e13d2ba93c276c434bcf9b51b93a81" + }, + { + "path": "scripts/new_research_package.py", + "usage": "provenance", + "git_blob": "1bd348ad3c27a8d0addac6156171e589edf4ff08", + "sha256": "a9a66cb591239c27928047146040cb9adb8d138fdc327f2fc8cb873ef9892c27" + }, + { + "path": "scripts/research_package.py", + "usage": "input", + "git_blob": "091b4c8f58bac9a9580d76025b887dc5c873d05f", + "sha256": "e1256016e7f51827d04fa2179255989f639ae1d97afaa9c16265f85a0c6abd70" + }, + { + "path": "tests/test_research_scaffold_aliases.py", + "usage": "input", + "git_blob": "2a72d9c4ba8510300dcb2fab726fd05195ad2b2e", + "sha256": "b262e4821540bf4f169da8d963fa4f2d7abf8e3719f5ae37011cbca5c940844b" + } + ] +} diff --git a/research/draft-catalogue-single-read/TASK.json b/research/draft-catalogue-single-read/TASK.json new file mode 100644 index 0000000000..70fe935c66 --- /dev/null +++ b/research/draft-catalogue-single-read/TASK.json @@ -0,0 +1,79 @@ +{ + "schema_version": 1, + "task_id": "replay-runtime", + "priority_revision": 8, + "base_commit": "a5bff3bf133e482541c09eba4e773884bdb2b1b7", + "original_card": { + "id": "replay-runtime", + "question": "Can one measured replay bottleneck be sped up without dropping coverage?", + "starting_points": [ + "scripts/run_isolated_tests.py", + "scripts/verify_replay_coverage.py", + "certificates/selected-result.json" + ], + "methods": [ + "Profile one isolated replay and optimize the measured bottleneck.", + "Design source-bound reuse with explicit dependency invalidation and negative controls." + ], + "acceptance_gate": "Report before/after wall time on the same fixture, retained checks and invalidation behavior.", + "first_step": { + "command": [ + "python3", + "-B", + "scripts/research_selected_baseline.py" + ], + "expected": "PASS_RESEARCH_BASELINE_BINDINGS_ONLY", + "inspect": "Resolve the current selected review, separate complex/ordinary BIT/admitted BIT/outer budgets, and compare the card against live handoffs before starting. Historical ceilings require exact source/model equivalence.", + "scope": "Source-binding orientation only; not a scientific replay or all-size proof. Materialize selected inputs if this checkout is sparse." + }, + "first_experiment": "Measure one unchanged fixture, preserve check coverage, and test one optimization with invalidation controls.", + "bottleneck_link": "Faster trustworthy replays increase review throughput and let mechanism candidates be checked sooner.", + "allowed_variations": [ + "Profile one stage or design one source-bound cache boundary." + ], + "scope_change_triggers": [ + "Skipping coverage or comparing different fixtures to claim speedup." + ], + "overlap_terms": [ + "replay runtime", + "cache invalidation", + "isolated tests" + ] + }, + "scope_status": "refined", + "actual_scope": "Remove the duplicate pinned-catalogue Git read from complete research-draft creation, preserving generated bytes and source/output rejection contracts.", + "scope_change_reason": "The replay-runtime card is narrowed to one original generator optimization after live overlap excluded recent-account mathematical follow-ups.", + "bottleneck_link": "Reduce onboarding overhead without reducing source binding or replay coverage; this is tooling and changes no multiplication bound.", + "budget_minutes": 60, + "stop_rule": "Ship the first complete-command gain beyond observed noise with exact five-file comparisons and rejecting controls; do not extend the optimization.", + "outcome": "completed", + "evidence": "MEASUREMENTS.json: 31 alternating pairs per workload, zero errors/mismatches, 14.55% and 13.49% median wall-time reductions. Direct verifier checks 3 differential cases and 9 guard tests.", + "author": "Nick Petalas", + "ai_assistance": "Independent human and formal proof-kernel review are unclaimed.", + "open_questions": [ + { + "question": "Does a workload with hundreds of explicit sources remain dominated by the batched reader rather than the removed catalogue read?", + "why_it_matters": "The measured result covers default and eight-source drafts; the relative gain may shrink for source-heavy packages.", + "first_step": "Select one actual large package source inventory, materialize its exact paths, and measure the same complete CLI with output equality.", + "blocked_by": "A separately budgeted representative workload and current overlap check." + } + ], + "claim_type": "empirical tooling improvement with a scoped source-level equivalence argument", + "claim_url": "https://github.com/CrocSwap/integer-mult-bounds/discussions/1166#discussioncomment-19272104", + "environment": { + "interpreter": "/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12", + "flags": [ + "-W", + "error", + "-B" + ], + "scientific_baseline": "a5bff3bf133e482541c09eba4e773884bdb2b1b7", + "priority_revision": 8 + }, + "overlap": { + "account_prs_since_2026_10_22": 389, + "local_notes_materialized": 92, + "local_notes_in_main": 8623, + "limitation": "Sparse local overlap cache is incomplete; whole-main README searches and targeted live all-state PR searches supplement it." + } +} diff --git a/research/draft-catalogue-single-read/VERIFY.json b/research/draft-catalogue-single-read/VERIFY.json new file mode 100644 index 0000000000..0fbd7bf45a --- /dev/null +++ b/research/draft-catalogue-single-read/VERIFY.json @@ -0,0 +1,37 @@ +{ + "schema_version": 1, + "status": "ready", + "python": "3.12", + "requirements": [], + "inputs": [ + "scripts/new_research_package.py", + "scripts/research_package.py", + "tests/test_research_scaffold_batch.py", + "tests/test_research_scaffold_aliases.py" + ], + "commands": [ + { + "argv": [ + "{python}", + "-W", + "error", + "-B", + "{package}/verify.py" + ], + "timeout_seconds": 120, + "checks": "Complete baseline/candidate draft bytes and modes, recorded timing/hash arithmetic, source and output rejection controls." + }, + { + "argv": [ + "{python}", + "-O", + "-W", + "error", + "-B", + "{package}/verify.py" + ], + "timeout_seconds": 120, + "checks": "The same evidence and rejecting controls with assertions disabled." + } + ] +} diff --git a/research/draft-catalogue-single-read/baseline.py b/research/draft-catalogue-single-read/baseline.py new file mode 100644 index 0000000000..1bd348ad3c --- /dev/null +++ b/research/draft-catalogue-single-read/baseline.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""Create a minimal draft package with exact source pins; never overwrite a package. + +Douglas Colkitt, with OpenAI Codex assistance. Apache-2.0. +""" +import argparse +from contextlib import closing +import hashlib +import json +from pathlib import Path +import re +import shutil +import subprocess +import tempfile + +from research_package import ROOT, git, git_blobs, relative_file, symlink_free_relative_path + + +def create(root, slug, task_id, author, sources=(), base="HEAD", budget=60): + if not re.fullmatch(r"[a-z0-9][a-z0-9-]{0,79}", slug): + raise ValueError("use a lowercase hyphenated package slug") + destination = symlink_free_relative_path(root, "research/" + slug) + if destination.exists(): + raise ValueError("package already exists; nothing overwritten") + if not author.strip() or budget <= 0: + raise ValueError("provide an author and positive time budget") + commit = git(root, "rev-parse", "--verify", base + "^{commit}").decode().strip() + catalog_path = "docs/research/agent-task-catalog.json" + catalog = json.loads(git(root, "show", commit + ":" + catalog_path)) + cards = [t for tr in catalog["tracks"] for t in tr["tasks"] if t["id"] == task_id] + if len(cards) != 1: + raise ValueError("unknown task in the pinned catalogue") + card = cards[0] + inventory = [] + names = sorted(set([catalog_path, "certificates/selected-result.json", *sources])) + with closing(git_blobs(root, [commit + ":" + name for name in names])) as objects: + for index, (blob, pinned) in enumerate(objects): + name = names[index] + live = relative_file(root, name).read_bytes() + if live != pinned: + raise ValueError("source has uncommitted changes or differs from chosen base: " + name) + inventory.append({"path": name, "usage": "input" if name in sources else "provenance", + "git_blob": blob, "sha256": hashlib.sha256(pinned).hexdigest()}) + task = {"schema_version": 1, "task_id": task_id, "priority_revision": catalog["priority_revision"], + "base_commit": commit, "original_card": card, "scope_status": "unchanged", + "actual_scope": card["question"], "scope_change_reason": None, + "bottleneck_link": "TODO: explain the causal connection or label this as a side result", + "budget_minutes": budget, "stop_rule": "TODO", "outcome": "in-progress", "evidence": "TODO", + "author": author, "ai_assistance": "TODO: disclose tools used", + "open_questions": [{"question": "TODO", "why_it_matters": "TODO", "first_step": "TODO", "blocked_by": "TODO"}]} + manifest = {"schema_version": 1, "status": "draft", "python": "3.12", "requirements": [], + "inputs": sorted(set(sources)), + "commands": [{"argv": ["{python}", "-B", "{package}/verify.py"], + "timeout_seconds": 120, "checks": "TODO: identity, positive fixture and failure controls"}]} + files = {"TASK.json": json.dumps(task, indent=2) + "\n", + "SOURCES.json": json.dumps({"schema_version": 1, "base_commit": commit, "files": inventory}, indent=2) + "\n", + "VERIFY.json": json.dumps(manifest, indent=2) + "\n", + "verify.py": '#!/usr/bin/env python3\n"""Replace this draft with a scoped verifier and meaningful controls."""\nraise SystemExit("DRAFT: no research result has been verified")\n', + "README.md": f'''# {slug} + +**Draft; no new bound claimed.** + +## Scope and evidence + +Original card: `{task_id}`. Complete [TASK.json](TASK.json), including any scope +change and its connection to the bottleneck. Explain the result and limitations here. +Source commits, Git blobs and SHA256 pins are in [SOURCES.json](SOURCES.json). +The selected record and catalogue are historical provenance, so later priority +updates do not invalidate this package. Explicit --source files are live inputs +and must match their pins. Full Git history is needed to resolve provenance. + +## Reproduce + +After completing the verifier and marking VERIFY.json ready, from repo root: + +```sh +python3 scripts/research_package.py --package research/{slug} --check-only +python3.12 scripts/research_package.py --package research/{slug} +``` + +Declare all external inputs and pinned dependencies in VERIFY.json. The generic +hook is bounded; native/Lean or long replays need an explicit dedicated workflow. + +## Open next questions + +Complete the structured `open_questions` list in TASK.json and explain it here: +the question, why it matters, a feasible first step and any blocker. If no next +question is justified, say so rather than inventing one. + +## Credit + +Prepared by {author}. Retain predecessor attribution and source-specific notices. +New work is Apache-2.0. Disclose substantial AI assistance in TASK.json. +'''} + destination.parent.mkdir(exist_ok=True) + temporary = Path(tempfile.mkdtemp(prefix=".research-draft-", dir=destination.parent)) + try: + for name, content in files.items(): + (temporary / name).write_text(content, encoding="utf-8", newline="\n") + # mkdir is the exclusive claim; rename alone can replace an empty + # directory that another agent created after the initial existence check. + destination.mkdir() + for child in temporary.iterdir(): + child.rename(destination / child.name) + finally: + if temporary.exists(): + shutil.rmtree(temporary) + return destination + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("slug") + ap.add_argument("--task", required=True) + ap.add_argument("--author", required=True) + ap.add_argument("--source", action="append", default=[]) + ap.add_argument("--base", default="HEAD") + ap.add_argument("--budget-minutes", type=int, default=60) + args = ap.parse_args() + try: + target = create(ROOT, args.slug, args.task, args.author, args.source, args.base, args.budget_minutes) + except (OSError, ValueError, KeyError, subprocess.SubprocessError) as exc: + ap.error(str(exc)) + print(str(target)) + print("DRAFT ONLY: complete scope, evidence, open questions and verifier; the stub deliberately fails.") + + +if __name__ == "__main__": + main() diff --git a/research/draft-catalogue-single-read/benchmark.py b/research/draft-catalogue-single-read/benchmark.py new file mode 100644 index 0000000000..1094c50b9b --- /dev/null +++ b/research/draft-catalogue-single-read/benchmark.py @@ -0,0 +1,116 @@ +"""Measure complete draft-creation commands; move every draft to the supplied scratch directory.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import platform +import statistics +import subprocess +import sys +import time + +PACKAGE = Path(__file__).resolve().parent +ROOT = PACKAGE.parents[1] +BASE = "a5bff3bf133e482541c09eba4e773884bdb2b1b7" +SOURCES = [ + "AGENTS.md", "CONTRIBUTING.md", "autoresearch.md", + "docs/research/current-priorities.md", "docs/research/agent-procedure.md", + "docs/research/research-package-guide.md", "certificates/frontier6030-source.json", + "docs/research/pr-intake-evidence/pr6030-audit/PROOF-REVIEW.md", +] + + +def digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def snapshot(directory): + files = sorted(directory.iterdir()) + if {p.name for p in files} != {"README.md", "TASK.json", "SOURCES.json", "VERIFY.json", "verify.py"}: + raise ValueError("the command did not produce exactly the five draft files") + return {p.name: {"sha256": digest(p), "mode": p.stat().st_mode & 0o777} for p in files} + + +def run_command(script, scratch, label, sources, slug): + target = ROOT / "research" / slug + if target.exists() or target.is_symlink(): + raise ValueError("benchmark output already exists: " + str(target)) + env = dict(os.environ, PYTHONPATH=str(ROOT / "scripts"), TMPDIR=str(scratch)) + env.pop("PYTHONOPTIMIZE", None) + argv = [sys.executable, "-W", "error", "-B", str(script), slug, + "--task", "replay-runtime", "--author", "Nick Petalas", "--base", BASE] + for name in sources: + argv.extend(["--source", name]) + start = time.perf_counter_ns() + try: + result = subprocess.run(argv, cwd=ROOT, env=env, capture_output=True, timeout=30) + elapsed = time.perf_counter_ns() - start + finally: + # The name was absent above and is owned by this invocation only. + # Move even a partial draft out of the checkout on failure/interruption. + if target.exists(): + target.rename(scratch / label) + if result.returncode != 0 or result.stderr: + raise ValueError("creation failed: " + result.stderr.decode(errors="replace")) + expected = (str(target) + "\nDRAFT ONLY: complete scope, evidence, open questions and verifier; " + "the stub deliberately fails.\n").encode() + if result.stdout != expected: + raise ValueError("unexpected complete-command stdout") + return elapsed, snapshot(scratch / label) + + +def measure(scratch, samples): + scripts = {"baseline": scratch / "baseline.py", "candidate": scratch / "candidate.py"} + originals = {"baseline": PACKAGE / "baseline.py", "candidate": ROOT / "scripts/new_research_package.py"} + for side, source in originals.items(): + scripts[side].write_bytes(source.read_bytes()) + slug = "draft-runtime-probe-" + scratch.name.lower().replace("_", "-")[-30:] + # Explicitly cap the generated slug independently of the caller's path spelling. + slug = "draft-runtime-probe-" + hashlib.sha256(slug.encode()).hexdigest()[:16] + results = {} + for workload, sources in (("default", []), ("eight_sources", SOURCES)): + reference = None + rows = [] + for index in range(samples + 1): + # One unreported warm-up pair, followed by alternating A/B and B/A pairs. + order = ("baseline", "candidate") if index % 2 == 0 else ("candidate", "baseline") + row = {} + for side in order: + label = f"{workload}-{index:03}-{side}" + elapsed, outputs = run_command(scripts[side], scratch, label, sources, slug) + if reference is None: + reference = outputs + if outputs != reference: + raise ValueError("five-file output/mode mismatch: " + label) + row[side] = elapsed + if index: + rows.append(row) + summaries = {} + for side in scripts: + values = [row[side] for row in rows] + summaries[side] = {"median_ns": statistics.median(values), "min_ns": min(values), "max_ns": max(values)} + results[workload] = {"sources": sources, "samples": rows, "summary": summaries, + "output_files": reference, "errors": 0, "mismatches": 0} + return {"schema_version": 1, "base": BASE, "python": sys.version, + "platform": platform.platform(), "processor": platform.processor(), "cpu_count": os.cpu_count(), + "load_average": os.getloadavg() if hasattr(os, "getloadavg") else None, + "hashes": {side: digest(path) for side, path in originals.items()}, + "benchmark_sha256": digest(Path(__file__)), "shared_reader_sha256": digest(ROOT / "scripts/research_package.py"), + "workloads": results, "scope": "Whole fresh Python CLI process through all five file writes; validation outside timer."} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--scratch", type=Path, required=True) + parser.add_argument("--samples", type=int, default=31) + args = parser.parse_args() + if not 5 <= args.samples <= 101: + parser.error("samples must be between 5 and 101") + scratch = args.scratch.resolve() + scratch.mkdir(parents=True, exist_ok=False) + print(json.dumps(measure(scratch, args.samples), indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/research/draft-catalogue-single-read/verify.py b/research/draft-catalogue-single-read/verify.py new file mode 100644 index 0000000000..e9221ae002 --- /dev/null +++ b/research/draft-catalogue-single-read/verify.py @@ -0,0 +1,107 @@ +"""Check complete draft equivalence, source binding, recorded timing arithmetic and rejection controls.""" +import hashlib +import json +from pathlib import Path +import runpy +import statistics +import subprocess +import sys +import tempfile +import unittest + +PACKAGE = Path(__file__).resolve().parent +ROOT = PACKAGE.parents[1] + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def timing_receipt(): + receipt = json.loads((PACKAGE / "MEASUREMENTS.json").read_bytes()) + paths = {"baseline": PACKAGE / "baseline.py", "candidate": ROOT / "scripts/new_research_package.py"} + for side, path in paths.items(): + require(receipt["hashes"][side] == digest(path), "recorded generator source changed: " + side) + require(receipt["benchmark_sha256"] == digest(PACKAGE / "benchmark.py"), "benchmark source changed") + require(receipt["shared_reader_sha256"] == digest(ROOT / "scripts/research_package.py"), "shared reader changed") + pins = json.loads((PACKAGE / "SOURCES.json").read_bytes())["files"] + old = next(row for row in pins if row["path"] == "scripts/new_research_package.py") + require(old["sha256"] == receipt["hashes"]["baseline"], "baseline provenance differs") + savings = {} + require(set(receipt["workloads"]) == {"default", "eight_sources"}, "workload inventory differs") + for name, workload in receipt["workloads"].items(): + require(type(workload["errors"]) is int and workload["errors"] == 0, "recorded creation errors") + require(type(workload["mismatches"]) is int and workload["mismatches"] == 0, "recorded output mismatches") + rows = workload["samples"] + require(len(rows) == 31, "expected 31 measured pairs") + for side in paths: + values = [row[side] for row in rows] + require(all(type(v) is int and v > 0 for v in values), "invalid elapsed nanoseconds") + summary = {"median_ns": statistics.median(values), "min_ns": min(values), "max_ns": max(values)} + require(workload["summary"][side] == summary, "recorded timing summary differs") + a = workload["summary"]["baseline"]["median_ns"] + b = workload["summary"]["candidate"]["median_ns"] + savings[name] = {"saved_ns": a - b, "baseline_ns": a} + return savings + + +def fixture(root): + data = { + "docs/research/agent-task-catalog.json": json.dumps({"priority_revision": 8, + "tracks": [{"tasks": [{"id": "fixture", "question": "Preserve exact output bytes"}]}]}).encode(), + "certificates/selected-result.json": b"{}\n", + "input/empty": b"", "input/binary": bytes(range(256)), "input/unicode-\u03ba": b"literal input\n", + } + for name, contents in data.items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(contents) + commands = [["init", "-q"], ["add", "."], ["-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", + "-c", "maintenance.auto=false", "-c", "gc.auto=0", "commit", "-qm", "fixture"]] + for args in commands: + subprocess.run(["git", *args], cwd=root, capture_output=True, check=True, timeout=30) + + +def compare_drafts(): + sys.path.insert(0, str(ROOT / "scripts")) + try: + baseline = runpy.run_path(str(PACKAGE / "baseline.py"))["create"] + candidate = runpy.run_path(str(ROOT / "scripts/new_research_package.py"))["create"] + finally: + sys.path.pop(0) + cases = [[], ["input/empty", "input/binary", "input/unicode-\u03ba"], + ["docs/research/agent-task-catalog.json"] * 2 + ["certificates/selected-result.json"]] + with tempfile.TemporaryDirectory(prefix="draft-equivalence-") as temporary: + root = Path(temporary) + fixture(root) + for index, sources in enumerate(cases): + outputs = [] + for side, create in (("baseline", baseline), ("candidate", candidate)): + target = create(root, "fixture", "fixture", "Nick Petalas", sources) + files = {p.name: (p.read_bytes(), p.stat().st_mode & 0o777) for p in target.iterdir()} + require(len(files) == 5, "expected exactly five generated files") + outputs.append(files) + target.rename(root / f"saved-{index}-{side}") + require(outputs[0] == outputs[1], "draft bytes or modes differ") + return len(cases) + + +def main(): + savings = timing_receipt() + cases = compare_drafts() + suite = unittest.defaultTestLoader.discover(str(ROOT / "tests"), pattern="test_research_scaffold*.py") + result = unittest.TextTestRunner(verbosity=1).run(suite) + require(result.wasSuccessful(), "draft rejection/conformance suite failed") + print(json.dumps({"status": "PASS_DRAFT_CATALOGUE_SINGLE_READ", "differential_cases": cases, + "generated_files_compared": cases * 5, "guard_tests": result.testsRun, + "recorded_savings": savings, "candidate_sha256": digest(ROOT / "scripts/new_research_package.py"), + "claim_type": "empirical tooling improvement with scoped output-equivalence evidence"}, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/scripts/new_research_package.py b/scripts/new_research_package.py index 1bd348ad3c..384995678e 100644 --- a/scripts/new_research_package.py +++ b/scripts/new_research_package.py @@ -16,6 +16,29 @@ from research_package import ROOT, git, git_blobs, relative_file, symlink_free_relative_path +def source_metadata(root, commit, task_id, sources): + catalog_path = "docs/research/agent-task-catalog.json" + # Read the catalogue first so its card and source pin share the same bytes. + names = [catalog_path] + sorted(set(["certificates/selected-result.json", *sources]) - {catalog_path}) + inventory = [] + with closing(git_blobs(root, [commit + ":" + name for name in names])) as objects: + # strict also exhausts the reader, including its final process-status check. + for name, (blob, pinned) in zip(names, objects, strict=True): + if name == catalog_path: + catalog = json.loads(pinned) + cards = [t for tr in catalog["tracks"] for t in tr["tasks"] if t["id"] == task_id] + if len(cards) != 1: + raise ValueError("unknown task in the pinned catalogue") + card = cards[0] + live = relative_file(root, name).read_bytes() + if live != pinned: + raise ValueError("source has uncommitted changes or differs from chosen base: " + name) + inventory.append({"path": name, "usage": "input" if name in sources else "provenance", + "git_blob": blob, "sha256": hashlib.sha256(pinned).hexdigest()}) + inventory.sort(key=lambda row: row["path"]) + return catalog, card, inventory + + def create(root, slug, task_id, author, sources=(), base="HEAD", budget=60): if not re.fullmatch(r"[a-z0-9][a-z0-9-]{0,79}", slug): raise ValueError("use a lowercase hyphenated package slug") @@ -25,22 +48,7 @@ def create(root, slug, task_id, author, sources=(), base="HEAD", budget=60): if not author.strip() or budget <= 0: raise ValueError("provide an author and positive time budget") commit = git(root, "rev-parse", "--verify", base + "^{commit}").decode().strip() - catalog_path = "docs/research/agent-task-catalog.json" - catalog = json.loads(git(root, "show", commit + ":" + catalog_path)) - cards = [t for tr in catalog["tracks"] for t in tr["tasks"] if t["id"] == task_id] - if len(cards) != 1: - raise ValueError("unknown task in the pinned catalogue") - card = cards[0] - inventory = [] - names = sorted(set([catalog_path, "certificates/selected-result.json", *sources])) - with closing(git_blobs(root, [commit + ":" + name for name in names])) as objects: - for index, (blob, pinned) in enumerate(objects): - name = names[index] - live = relative_file(root, name).read_bytes() - if live != pinned: - raise ValueError("source has uncommitted changes or differs from chosen base: " + name) - inventory.append({"path": name, "usage": "input" if name in sources else "provenance", - "git_blob": blob, "sha256": hashlib.sha256(pinned).hexdigest()}) + catalog, card, inventory = source_metadata(root, commit, task_id, sources) task = {"schema_version": 1, "task_id": task_id, "priority_revision": catalog["priority_revision"], "base_commit": commit, "original_card": card, "scope_status": "unchanged", "actual_scope": card["question"], "scope_change_reason": None, diff --git a/tests/test_research_scaffold_batch.py b/tests/test_research_scaffold_batch.py index ff25ec92da..02a7398236 100644 --- a/tests/test_research_scaffold_batch.py +++ b/tests/test_research_scaffold_batch.py @@ -52,7 +52,7 @@ def tracked(argv, *args, **kwargs): return child with patch.object(subprocess, 'Popen', side_effect=tracked): target = self.create(sources=self.inputs + self.inputs[:3]) - self.assertEqual(len(commands), 3, commands) + self.assertEqual(len(commands), 2, commands) self.assertTrue(all(child.poll() is not None for child in children)) rows = json.loads((target / 'SOURCES.json').read_bytes())['files'] expected = [] @@ -102,6 +102,41 @@ def fails_after_last_row(*args): self.assertEqual(closed, [True]) self.assertFalse((self.root / 'research/late-failure').exists()) + def test_catalogue_is_requested_once_even_when_explicitly_duplicated(self): + original = scaffold.git_blobs + requests = [] + def tracked(root, specs): + requests.extend(specs) + yield from original(root, specs) + name = 'docs/research/agent-task-catalog.json' + with patch.object(scaffold, 'git_blobs', side_effect=tracked): + target = self.create(sources=[name, name, self.inputs[0]]) + self.assertEqual(sum(spec.endswith(':' + name) for spec in requests), 1) + rows = json.loads((target / 'SOURCES.json').read_bytes())['files'] + self.assertEqual([row['path'] for row in rows], sorted(row['path'] for row in rows)) + self.assertEqual(next(row['usage'] for row in rows if row['path'] == name), 'input') + + def test_unknown_task_closes_reader_without_publishing(self): + original = scaffold.git_blobs + closed = [] + def tracked(*args): + try: + yield from original(*args) + finally: + closed.append(True) + with patch.object(scaffold, 'git_blobs', side_effect=tracked): + with self.assertRaisesRegex(ValueError, 'unknown task in the pinned catalogue'): + scaffold.create(self.root, 'unknown', 'missing', 'Fixture', self.inputs) + self.assertEqual(closed, [True]) + self.assertFalse((self.root / 'research/unknown').exists()) + + def test_live_catalogue_drift_rejects_without_publishing(self): + name = 'docs/research/agent-task-catalog.json' + (self.root / name).write_bytes(self.data[name] + b'\n') + with self.assertRaisesRegex(ValueError, 'source has uncommitted changes'): + self.create('catalogue-drift') + self.assertFalse((self.root / 'research/catalogue-drift').exists()) + if __name__ == '__main__': unittest.main()