From 637a0513b0ca75053fc62ddb99d1ac43eb1cb333 Mon Sep 17 00:00:00 2001 From: Nick Petalas Date: Thu, 29 Oct 2026 13:18:05 +0000 Subject: [PATCH] perf(research): defer unused current-picker validation --- .../BENCHMARK.json | 68 ++++++++ .../current-picker-lazy-validation/README.md | 134 +++++++++++++++ .../SOURCES.json | 30 ++++ .../current-picker-lazy-validation/TASK.json | 61 +++++++ .../VERIFY.json | 38 +++++ .../benchmark.py | 112 +++++++++++++ .../current-picker-lazy-validation/verify.py | 154 ++++++++++++++++++ scripts/choose_current_research_task.py | 2 +- 8 files changed, 598 insertions(+), 1 deletion(-) create mode 100644 research/current-picker-lazy-validation/BENCHMARK.json create mode 100644 research/current-picker-lazy-validation/README.md create mode 100644 research/current-picker-lazy-validation/SOURCES.json create mode 100644 research/current-picker-lazy-validation/TASK.json create mode 100644 research/current-picker-lazy-validation/VERIFY.json create mode 100644 research/current-picker-lazy-validation/benchmark.py create mode 100644 research/current-picker-lazy-validation/verify.py diff --git a/research/current-picker-lazy-validation/BENCHMARK.json b/research/current-picker-lazy-validation/BENCHMARK.json new file mode 100644 index 0000000000..34ae6f8efe --- /dev/null +++ b/research/current-picker-lazy-validation/BENCHMARK.json @@ -0,0 +1,68 @@ +{ + "after_sha256": "04e194b14a1f4157be5be83e8c13995d69c3efe8996773d918acba6bfcabb471", + "argv": [ + "--seed", + "00000000000000000000000000000000", + "--task", + "replay-runtime" + ], + "before_sha256": "51b9b217cf662557a8303136208ebd1625770dc8f4c8f5141a1fb849282825d4", + "benchmark_sha256": "19223030956bd2e8f01886d13e2ad11f9db56bd5b043f0e9b78d26508b1d424b", + "cpu_count": 18, + "direct_cli_controls": 1, + "errors": 0, + "fraction_less_wall_time": 0.04655790582170749, + "launcher": "import pathlib,sys; p=sys.argv.pop(1); f=sys.argv[0]=str(pathlib.Path.cwd()/'scripts/choose_current_research_task.py'); exec(compile(pathlib.Path(p).read_bytes(),f,'exec'),{'__name__':'__main__','__file__':f})", + "load_average": [ + 2.93798828125, + 2.8505859375, + 2.7275390625 + ], + "machine": "arm64", + "medians_seconds": { + "after": 0.09959755828604103, + "before": 0.10446104581933469 + }, + "pairs": 9, + "platform": "macOS-27.0.1-arm64-arm-64bit", + "python": "3.12.15 (main, Oct 14 2026, 00:55:37) [Clang 23.1.3 ]", + "ranges_seconds": { + "after": [ + 0.09396249998826534, + 0.10557451250497252 + ], + "before": [ + 0.10343423748854548, + 0.11021292919758707 + ] + }, + "repetitions_per_batch": 10, + "samples_seconds_per_command": { + "after": [ + 0.09959755828604103, + 0.10118455418851227, + 0.09396249998826534, + 0.099184066709131, + 0.10007922500371932, + 0.09903859999030828, + 0.09966542089823634, + 0.09871085421182216, + 0.10557451250497252 + ], + "before": [ + 0.10914635420776904, + 0.10519179170951247, + 0.10352727079298348, + 0.10446104581933469, + 0.10507754581049085, + 0.1042379666119814, + 0.10343423748854548, + 0.10353437920566648, + 0.11021292919758707 + ] + }, + "schema": "current-picker-complete-timing/1", + "scope": "Complete source-executed CLI, fresh interpreter per command; same launcher on both sides. Timing includes JSON decoding and output equality checks. No speed guarantee on other hosts or mapped historical routes.", + "stdout_sha256": "de87d316bc9eac6279769b9d6d42adcb001f49f90cff87ad404b9a61a33d125b", + "successful_timed_commands": 180 +} diff --git a/research/current-picker-lazy-validation/README.md b/research/current-picker-lazy-validation/README.md new file mode 100644 index 0000000000..b2b356df69 --- /dev/null +++ b/research/current-picker-lazy-validation/README.md @@ -0,0 +1,134 @@ +# Defer unused task-picker validation + +This contribution was produced by Nick Petalas. + +The current task picker loads the frozen legacy picker in the parent process +before launching that same picker in a child. Move the parent load into the +historical-route branch. Cards already carrying current commands need only the +child execution. The legacy subprocess, its catalogue checks and output, and +historical route validation retain their code. + +This is an empirical tooling improvement with a scoped equivalence argument +and finite regression evidence. It supplies no new mathematical bound, supplier +admission or kappa. [BENCHMARK.json](BENCHMARK.json) records the complete-command +measurements, source hashes, Python and host, raw timings and error count. +The recorded median falls from 104.461 ms to 99.598 ms, 4.66% less wall time. +Every one of the nine paired batches improves. Batch-average ranges are +103.434–110.213 ms before and 93.962–105.575 ms after; the paired decreases +range from 3.769 to 9.565 ms. All 180 timed commands succeed with identical +output, and the direct-CLI control agrees. The raw ranges overlap, so the +paired observations and host-specific scope matter. The profile attributes +about 4 ms of the original complete command to the removed parent load; +child startup and catalogue validation remain the main cost. + +## Why the checks survive + +Before the moved statement, the resolver identifies the exact historical +commands. If none occurs, it returns without using the legacy namespace. If a +historical command occurs, it loads the same exact source and obtains the same +validation function before validating or changing any card. Loading is not +cached between resolver calls. Subsequent calls therefore still read current +source. The CLI always runs its unchanged legacy subprocess before resolving +successful JSON output, and passes through its help and errors as before. + +This argument assumes the retained legacy module's top-level definitions and +imports. It is not an equivalence claim for arbitrary side-effectful replacements +of that module, imports by external code of the removed incidental module global, +or concurrent changes to the source tree. No public helper API is documented for +that global. Historical mapped routes may pay the same load on every resolver +call; no performance gain is claimed there. + +[verify.py](verify.py) reconstructs the exact pre-change source and checks its +pinned SHA256, authenticates the unchanged legacy source, checks loading at the +actual resolver boundary, compares all 22 live first-step cards and both frozen +route shapes, and exercises seven source/route rejection controls. Explicit +exceptions keep checks active under optimized Python. The existing routing suite +also covers CLI help/errors, exact matching, atomic failure, sparse references, +staged replacements and historical output preservation. + +## Reproduction + +Use CPython 3.12 and its standard library from the repository root. The local +interpreter was `/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12`. + +```sh +python3.12 -I -W error -B research/current-picker-lazy-validation/verify.py +python3.12 -I -O -W error -B research/current-picker-lazy-validation/verify.py +python3.12 -B scripts/research_package.py --package research/current-picker-lazy-validation +python3.12 -B scripts/check_research_commit.py --package research/current-picker-lazy-validation --check-source-history --check-modes +python3.12 -W error -B -m unittest discover -s tests -p test_research_current_assignment.py -v +``` + +Expected marker: `PASS_CURRENT_PICKER_LAZY_VALIDATION`. Registered checks are +bounded correctness checks; they do not assert a host-independent timing ratio. +To repeat the timing experiment, choose a scratch directory and a new receipt +path. The writer refuses an existing receipt. + +```sh +python3.12 -W error -B research/current-picker-lazy-validation/benchmark.py --pairs 9 --repetitions 10 --scratch /tmp --write /tmp/new-picker-timings.json +``` + +Both sides execute the entire CLI in fresh interpreters, using the real script +filename and root with different authenticated source bytes. The identical +launcher reads and compiles one temporary source file per command. It retains +the legacy child, all its checks and Git queries. Timings include output capture, +JSON decoding and equality checks. Every complete `(status, stdout, stderr)` +triple must agree, and a separate direct CLI invocation must match the candidate. +The script does not write or replace checkout source files. Nine alternating +batches contain ten complete commands per side, 180 timed commands in total. +The measurement is of this source-executed CLI, not a subtraction of isolated +import times. Concurrent host load and the launcher's common overhead limit +transfer of the measured ratio to other environments. + +## Scope and next questions + +The original `replay-runtime` card is changed from scientific replay to research +onboarding startup. This small improvement helps repeated coordination commands; +it does not accelerate the selected scientific replay. The first improving +implementation is frozen. Canonical base is +`3f0152a32eeebac0df11946dbd50cabe5a7f7839`, priority revision 8. +[The advance claim](https://github.com/CrocSwap/integer-mult-bounds/discussions/1166#discussioncomment-19267797) +records the scope before implementation. + +The existing historical picker, catalogue, selected frontier and predecessor +packages are unchanged. No historical scientific suite, native replay, Lean, +Docker, formal proof kernel or independent human review is claimed. + +## Open next questions + +Does startup loading still matter for the historical mapped routes, where the +resolver genuinely needs validation? Profile those complete commands separately +before changing their execution boundary. Preserving child failure propagation, +source freshness and sparse-reference rejection is the blocker to a broader +startup change. This package supplies no such change. + +## Code constraints + +The implementation moves one load and adds no cache, recursive call or loop. +The package's bounded verifier handles at most 1,000 cards, four route references, +two route shapes and three source mutants. Benchmark options cap work at 31 pairs +and 100 commands per batch, with a 20-second timeout on each child. Functions are +below 60 physical lines. + +Power of Ten deviations apply to the package's Python-managed allocation and +explicit trusted-source execution. Live memory is bounded by the pinned scripts, +the checked catalogue limit and one captured command output at a time. The +trusted picker emits a finite catalogue report; the benchmark rejects output +above 1 MiB after capture. It does not claim a hard capture-memory limit for +an arbitrarily changed child. Temporary fixtures have four files and are +removed by their context manager on both success and failure. Dynamic +execution is needed to compare actual source versions with the same filename; +both versions are authenticated before execution. No arbitrary submitted program +is accepted. The benchmark additionally uses the fixed standard-library module +loader for the local verifier. Replacing these mechanisms would require a new +benchmark contract; no shared runtime or build configuration is relaxed. + +## Credit + +Ali Romman / PR5378 supplied the current-command wrapper. Mihir Srivastava / +PR4906 and PR5520 supplied startup guidance and historical-ceiling routing. +Douglas Colkitt and the retained sparse-reference contributors supplied the +frozen picker and its reference checks. Deferred initialization is standard +programming practice. This contribution is its measured application at this +specific CLI boundary, with exact-source conformance and reproduction. Existing +source credit lines retain their bytes. New work is Apache-2.0. diff --git a/research/current-picker-lazy-validation/SOURCES.json b/research/current-picker-lazy-validation/SOURCES.json new file mode 100644 index 0000000000..c28f83ad96 --- /dev/null +++ b/research/current-picker-lazy-validation/SOURCES.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "base_commit": "3f0152a32eeebac0df11946dbd50cabe5a7f7839", + "files": [ + { + "path": "certificates/selected-result.json", + "usage": "provenance", + "git_blob": "d59b5035e7b4cbaa9d688a1d32dd0d6eb79c466a", + "sha256": "9b8d0eea95b3bf0dfcb7938a732a6439b87a2685dec2a5ab9e525b3b34debbfa" + }, + { + "path": "docs/research/agent-task-catalog.json", + "usage": "provenance", + "git_blob": "3404fbdd5e7529a0674eef26114b5071e4586cec", + "sha256": "ead6cdc9d0af6d58e8a49744d8ec8c7c15e13d2ba93c276c434bcf9b51b93a81" + }, + { + "path": "scripts/choose_research_task.py", + "usage": "input", + "git_blob": "5c2ed0bab22d1e93c8fd80245b64a0fe89cee32c", + "sha256": "484f553c488c6f427ca81f19f3ee2720399dcc28ce61a3886d50bdf28ca72d12" + }, + { + "path": "scripts/choose_current_research_task.py", + "usage": "provenance", + "git_blob": "0402d37f4f65de0f9a84fc7a7209a9f2e934efb1", + "sha256": "51b9b217cf662557a8303136208ebd1625770dc8f4c8f5141a1fb849282825d4" + } + ] +} diff --git a/research/current-picker-lazy-validation/TASK.json b/research/current-picker-lazy-validation/TASK.json new file mode 100644 index 0000000000..09e341c390 --- /dev/null +++ b/research/current-picker-lazy-validation/TASK.json @@ -0,0 +1,61 @@ +{ + "schema_version": 1, + "task_id": "replay-runtime", + "priority_revision": 8, + "base_commit": "3f0152a32eeebac0df11946dbd50cabe5a7f7839", + "original_card": { + "id": "replay-runtime", + "question": "Can one measured replay bottleneck be sped up without dropping coverage?", + "starting_points": [ + "scripts/run_isolated_tests.py", + "scripts/verify_replay_coverage.py", + "certificates/selected-result.json" + ], + "methods": [ + "Profile one isolated replay and optimize the measured bottleneck.", + "Design source-bound reuse with explicit dependency invalidation and negative controls." + ], + "acceptance_gate": "Report before/after wall time on the same fixture, retained checks and invalidation behavior.", + "first_step": { + "command": [ + "python3", + "-B", + "scripts/research_selected_baseline.py" + ], + "expected": "PASS_RESEARCH_BASELINE_BINDINGS_ONLY", + "inspect": "Resolve the current selected review, separate complex/ordinary BIT/admitted BIT/outer budgets, and compare the card against live handoffs before starting. Historical ceilings require exact source/model equivalence.", + "scope": "Source-binding orientation only; not a scientific replay or all-size proof. Materialize selected inputs if this checkout is sparse." + }, + "first_experiment": "Measure one unchanged fixture, preserve check coverage, and test one optimization with invalidation controls.", + "bottleneck_link": "Faster trustworthy replays increase review throughput and let mechanism candidates be checked sooner.", + "allowed_variations": [ + "Profile one stage or design one source-bound cache boundary." + ], + "scope_change_triggers": [ + "Skipping coverage or comparing different fixtures to claim speedup." + ], + "overlap_terms": [ + "replay runtime", + "cache invalidation", + "isolated tests" + ] + }, + "scope_status": "changed", + "actual_scope": "Defer unused parent-side legacy validation loading in the current task-picker CLI, preserving exact complete outputs, source freshness and historical-route rejection.", + "scope_change_reason": "The first BIT suggestion was not assigned by the donor. Live and recent-account overlap excluded the mathematical tour candidate before research. This uses the current verifier-engineering priority for a distinct original onboarding bottleneck rather than auditing a predecessor.", + "bottleneck_link": "Repeated coordination commands load the same frozen picker twice even when no historical first-step rewrite is needed. Removing the unused load reduces this bounded startup cost; it changes no supplier or scientific replay.", + "budget_minutes": 60, + "stop_rule": "Ship the first measured complete-command improvement after exact conformance and retained rejection checks. Do not expand the optimization.", + "outcome": "completed", + "evidence": "The exact-source conformance checker passes all 22 live cards, two historical route shapes and seven source/route rejection controls. The existing 12 routing tests pass. Generated BENCHMARK.json supplies alternating complete-command wall times, every successful command and source hashes. The recorded nine paired batches of ten fresh commands each have medians 104.461 ms before and 99.598 ms after, 4.66% less wall time, with all nine paired decreases positive and zero errors. The ratio is empirical and host-specific.", + "author": "Nick Petalas", + "ai_assistance": "Independent human and formal proof-kernel review are unclaimed.", + "open_questions": [ + { + "question": "Does startup loading remain a material bottleneck for historical mapped routes?", + "why_it_matters": "Those routes still require the legacy validation load, so this optimization does not accelerate that branch.", + "first_step": "Profile complete historical mapped CLI commands before changing execution boundaries.", + "blocked_by": "Any replacement must preserve source freshness, exact error propagation and sparse-reference rejection." + } + ] +} diff --git a/research/current-picker-lazy-validation/VERIFY.json b/research/current-picker-lazy-validation/VERIFY.json new file mode 100644 index 0000000000..83df3d4a0f --- /dev/null +++ b/research/current-picker-lazy-validation/VERIFY.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "status": "ready", + "python": "3.12", + "requirements": [], + "inputs": [ + "scripts/choose_research_task.py", + "scripts/choose_current_research_task.py", + "docs/research/agent-task-catalog.json" + ], + "commands": [ + { + "argv": [ + "{python}", + "-I", + "-W", + "error", + "-B", + "{package}/verify.py" + ], + "timeout_seconds": 120, + "checks": "Exact baseline reconstruction, fresh source loading, unchanged live-card outputs and historical-route rejection controls." + }, + { + "argv": [ + "{python}", + "-I", + "-W", + "error", + "-O", + "-B", + "{package}/verify.py" + ], + "timeout_seconds": 120, + "checks": "Exact baseline reconstruction, fresh source loading, unchanged live-card outputs and historical-route rejection controls." + } + ] +} diff --git a/research/current-picker-lazy-validation/benchmark.py b/research/current-picker-lazy-validation/benchmark.py new file mode 100644 index 0000000000..dc607763f8 --- /dev/null +++ b/research/current-picker-lazy-validation/benchmark.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Fresh-process complete CLI timing; produced by Nick Petalas, Apache-2.0.""" +import argparse +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import platform +import statistics +import subprocess +import sys +import tempfile +import time + +PACKAGE = Path(__file__).resolve().parent +ROOT = PACKAGE.parents[1] +SCRIPT = ROOT / "scripts/choose_current_research_task.py" +ARGS = ["--seed", "0" * 32, "--task", "replay-runtime"] +# Both sides execute the complete CLI in a fresh process with the real __file__. +# Only the source bytes differ. The legacy subprocess and Git queries still run. +LAUNCH = ( + "import pathlib,sys; p=sys.argv.pop(1); " + "f=sys.argv[0]=str(pathlib.Path.cwd()/'scripts/choose_current_research_task.py'); " + "exec(compile(pathlib.Path(p).read_bytes(),f,'exec')," + "{'__name__':'__main__','__file__':f})" +) + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def source_pair(): + spec = importlib.util.spec_from_file_location("picker_check", PACKAGE / "verify.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.sources() + + +def invoke(source): + command = [sys.executable, "-B", "-c", LAUNCH, str(source), *ARGS] + result = subprocess.run(command, cwd=ROOT, capture_output=True, timeout=20) + require(result.returncode == 0, "complete command failed: " + repr(result.stderr)) + require(len(result.stdout) <= 1024 * 1024, "unexpected output size") + output = (result.returncode, result.stdout, result.stderr) + report = json.loads(result.stdout) + require(report["suggestions"][0]["task_id"] == "replay-runtime", "wrong work") + return output + + +def measure(pairs, repetitions, scratch): + before, after = source_pair() + samples = {"before": [], "after": []} + outputs = {} + with tempfile.TemporaryDirectory(dir=scratch) as directory: + sources = {} + for side, source in (("before", before), ("after", after)): + path = Path(directory) / (side + ".py") + require(path.write_text(source, encoding="utf-8") == len(source), "short source write") + sources[side] = path + for pair in range(pairs): + order = ("before", "after") if pair % 2 == 0 else ("after", "before") + for side in order: + started = time.perf_counter() + for _ in range(repetitions): + output = invoke(sources[side]) + outputs.setdefault(side, output) + require(output == outputs[side], "unstable complete output") + samples[side].append((time.perf_counter() - started) / repetitions) + require(outputs["before"] == outputs["after"], "changed complete output") + direct = subprocess.run([sys.executable, "-B", str(SCRIPT), *ARGS], cwd=ROOT, + capture_output=True, timeout=20) + require((direct.returncode, direct.stdout, direct.stderr) == outputs["after"], + "launcher differs from direct CLI") + medians = {side: statistics.median(values) for side, values in samples.items()} + return { + "schema": "current-picker-complete-timing/1", "pairs": pairs, + "repetitions_per_batch": repetitions, "successful_timed_commands": 2 * pairs * repetitions, + "errors": 0, "direct_cli_controls": 1, "argv": ARGS, "launcher": LAUNCH, + "python": sys.version, "platform": platform.platform(), "machine": platform.machine(), + "cpu_count": os.cpu_count(), "load_average": os.getloadavg() if hasattr(os, "getloadavg") else None, + "samples_seconds_per_command": samples, "medians_seconds": medians, + "ranges_seconds": {side: [min(v), max(v)] for side, v in samples.items()}, + "fraction_less_wall_time": 1 - medians["after"] / medians["before"], + "stdout_sha256": hashlib.sha256(outputs["after"][1]).hexdigest(), + "before_sha256": hashlib.sha256(before.encode()).hexdigest(), + "after_sha256": hashlib.sha256(after.encode()).hexdigest(), + "benchmark_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "scope": "Complete source-executed CLI, fresh interpreter per command; same launcher on both sides. Timing includes JSON decoding and output equality checks. No speed guarantee on other hosts or mapped historical routes.", + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--pairs", type=int, default=9) + parser.add_argument("--repetitions", type=int, default=10) + parser.add_argument("--scratch", type=Path) + parser.add_argument("--write", type=Path) + args = parser.parse_args() + require(5 <= args.pairs <= 31 and 1 <= args.repetitions <= 100, "timing bounds") + result = measure(args.pairs, args.repetitions, args.scratch) + text = json.dumps(result, indent=2, sort_keys=True) + "\n" + if args.write: + with args.write.open("x", encoding="utf-8") as output: + require(output.write(text) == len(text), "short receipt write") + print(text, end="") + + +if __name__ == "__main__": + main() diff --git a/research/current-picker-lazy-validation/verify.py b/research/current-picker-lazy-validation/verify.py new file mode 100644 index 0000000000..cffa1df7b7 --- /dev/null +++ b/research/current-picker-lazy-validation/verify.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Exact current-picker conformance; produced by Nick Petalas, Apache-2.0.""" +import copy +import hashlib +import json +from pathlib import Path +import runpy +import statistics +import tempfile +from unittest.mock import patch + +PACKAGE = Path(__file__).resolve().parent +ROOT = PACKAGE.parents[1] +SCRIPT = "scripts/choose_current_research_task.py" +LEGACY = "scripts/choose_research_task.py" +LOAD = 'check_catalog_files = runpy.run_path(str(LEGACY))["check_catalog_files"]\n' +ANCHOR = 'LEGACY = ROOT / "scripts/choose_research_task.py"\n' +REQUIRED = ( + "scripts/research_selected_baseline.py", "scripts/research_package.py", + "research/historical-complex-ceiling-replay/VERIFY.json", + "research/historical-complex-ceiling-replay/verify.py", +) + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def reconstruct(after): + require(after.count(" " + LOAD) == 1, "missing unique deferred load") + require(after.count(ANCHOR) == 1, "missing unique legacy path") + require("\n" + LOAD not in after, "eager legacy load remains") + return after.replace(" " + LOAD, "").replace(ANCHOR, ANCHOR + LOAD) + + +def sources(): + pins = json.loads((PACKAGE / "SOURCES.json").read_bytes())["files"] + expected = {row["path"]: row["sha256"] for row in pins} + legacy = (ROOT / LEGACY).read_bytes() + require(digest(legacy) == expected[LEGACY], "legacy source drift") + after = (ROOT / SCRIPT).read_text(encoding="utf-8") + before = reconstruct(after) + require(digest(before.encode()) == expected[SCRIPT], "baseline source drift") + return before, after + + +def namespace(source): + result = {"__name__": "picker_conformance", "__file__": str(ROOT / SCRIPT)} + exec(compile(source, str(ROOT / SCRIPT), "exec"), result) + return result + + +def rejected(call): + try: + call() + except ValueError as error: + return str(error) + raise ValueError("negative control was accepted") + + +def suggestion(command): + return {"suggestions": [{"first_step": {"command": command}}]} + + +def route_controls(old, new): + controls = 0 + commands = (["python3", "-B", "scripts/research_baseline.py"], + ["python3", "-B", "research/complex-local-word-ceiling/verify.py"]) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name in REQUIRED: + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("inert entry reference\n", encoding="utf-8") + for command in commands: + left, right = suggestion(command), suggestion(command) + require(old(left, root) == new(right, root), "route omissions differ") + require(left == right, "historical route output differs") + for absent in REQUIRED: + path = root / absent + path.unlink() + original = suggestion(commands[1]) + left, right = copy.deepcopy(original), copy.deepcopy(original) + a = rejected(lambda: old(left, root)) + b = rejected(lambda: new(right, root)) + require(a == b and left == right == original, "route rejection drift") + path.write_text("inert entry reference\n", encoding="utf-8") + controls += 1 + return controls + + +def timing_receipt(before, after): + record = json.loads((PACKAGE / "BENCHMARK.json").read_bytes()) + require(record["before_sha256"] == digest(before.encode()) and + record["after_sha256"] == digest(after.encode()), "timing source mismatch") + require(record["benchmark_sha256"] == digest((PACKAGE / "benchmark.py").read_bytes()), + "timing driver mismatch") + require(type(record["errors"]) is int and record["errors"] == 0, "timing failures") + require(record["pairs"] == 9 and record["repetitions_per_batch"] == 10 and + record["successful_timed_commands"] == 180, "timing work count") + samples = record["samples_seconds_per_command"] + for side in ("before", "after"): + values = samples[side] + require(len(values) == 9 and all(type(x) is float and 0 < x < 20 for x in values), + "invalid timing samples") + require(statistics.median(values) == record["medians_seconds"][side] and + [min(values), max(values)] == record["ranges_seconds"][side], + "timing summary mismatch") + medians = record["medians_seconds"] + require(record["fraction_less_wall_time"] == 1 - medians["after"] / medians["before"], + "timing ratio mismatch") + return sum(a > b for a, b in zip(samples["before"], samples["after"])) + + +def check(): + before, after = sources() + paired_wins = timing_receipt(before, after) + with patch("runpy.run_path", wraps=runpy.run_path) as calls: + old = namespace(before)["resolve_current_first_steps"] + require(calls.call_count == 1, "baseline did not load eagerly") + calls.reset_mock() + new = namespace(after)["resolve_current_first_steps"] + require(calls.call_count == 0, "candidate loaded legacy eagerly") + unchanged = suggestion(["python3", "-B", "scripts/research_selected_baseline.py"]) + original = copy.deepcopy(unchanged) + require(new(unchanged, ROOT) == [] and unchanged == original, + "current route changed") + require(calls.call_count == 0, "unused validation loaded legacy") + catalog = json.loads((ROOT / "docs/research/agent-task-catalog.json").read_bytes()) + cards = [card for track in catalog["tracks"] for card in track["tasks"]] + require(1 <= len(cards) <= 1000, "catalog size outside verifier bound") + for card in cards: + left = {"suggestions": [{"first_step": copy.deepcopy(card["first_step"])}]} + right = copy.deepcopy(left) + require(old(left, ROOT) == new(right, ROOT) and left == right, + "live first-step output differs") + controls = route_controls(old, new) + for mutant in (after.replace(" " + LOAD, ""), after + "\n" + LOAD, + after.replace(ANCHOR, ANCHOR + "# changed\n")): + rejected(lambda: require(digest(reconstruct(mutant).encode()) == digest(before.encode()), + "baseline source drift")) + controls += 1 + return {"status": "PASS_CURRENT_PICKER_LAZY_VALIDATION", "live_cards": len(cards), + "route_fixtures": 2, "negative_controls": controls, "paired_timing_wins": paired_wins, + "before_sha256": digest(before.encode()), "after_sha256": digest(after.encode())} + + +if __name__ == "__main__": + print(json.dumps(check(), sort_keys=True)) diff --git a/scripts/choose_current_research_task.py b/scripts/choose_current_research_task.py index 0402d37f4f..b45cbe7bb4 100644 --- a/scripts/choose_current_research_task.py +++ b/scripts/choose_current_research_task.py @@ -13,7 +13,6 @@ ROOT = Path(__file__).resolve().parents[1] LEGACY = ROOT / "scripts/choose_research_task.py" -check_catalog_files = runpy.run_path(str(LEGACY))["check_catalog_files"] def resolve_current_first_steps(result, root): @@ -35,6 +34,7 @@ def resolve_current_first_steps(result, root): for name in (runner, package + "/VERIFY.json", package + "/verify.py")] # Validate the whole route before changing any suggestion. These are entry # references only; the registered replay authenticates its own input closure. + check_catalog_files = runpy.run_path(str(LEGACY))["check_catalog_files"] omitted = check_catalog_files(root, required) for card in cards: original = card["first_step"]