From 1278601c6832bbecb47c2207240eede23f99a2fe Mon Sep 17 00:00:00 2001 From: Nick Petalas Date: Wed, 28 Oct 2026 16:28:58 +0000 Subject: [PATCH] perf(research): skip impossible overlap reference scans --- .../BENCHMARK.json | 68 ++++++++++ .../CHECK_TIMES.json | 49 +++++++ .../overlap-reference-prefilter/README.md | 128 ++++++++++++++++++ .../overlap-reference-prefilter/SOURCES.json | 30 ++++ .../overlap-reference-prefilter/TASK.json | 61 +++++++++ .../overlap-reference-prefilter/VERIFY.json | 57 ++++++++ .../overlap-reference-prefilter/benchmark.py | 99 ++++++++++++++ .../overlap-reference-prefilter/verify.py | 107 +++++++++++++++ scripts/find_overlap.py | 4 + tests/test_research_overlap_terms.py | 25 ++++ 10 files changed, 628 insertions(+) create mode 100644 research/overlap-reference-prefilter/BENCHMARK.json create mode 100644 research/overlap-reference-prefilter/CHECK_TIMES.json create mode 100644 research/overlap-reference-prefilter/README.md create mode 100644 research/overlap-reference-prefilter/SOURCES.json create mode 100644 research/overlap-reference-prefilter/TASK.json create mode 100644 research/overlap-reference-prefilter/VERIFY.json create mode 100644 research/overlap-reference-prefilter/benchmark.py create mode 100644 research/overlap-reference-prefilter/verify.py diff --git a/research/overlap-reference-prefilter/BENCHMARK.json b/research/overlap-reference-prefilter/BENCHMARK.json new file mode 100644 index 0000000000..9898032aba --- /dev/null +++ b/research/overlap-reference-prefilter/BENCHMARK.json @@ -0,0 +1,68 @@ +{ + "base": "721c03822c421b7f7fcbb2598993b947c4d88419", + "python": "3.12.15 (main, Oct 14 2026, 00:55:37) [Clang 23.1.3 ]", + "platform": "macOS-27.0.1-arm64-arm-64bit", + "cache_sha256": "38181ef5eec12ac98d9d101e444913354bc1e76e89ca67a9292a40ea9fdb5bd0", + "cache_bytes": 16945386, + "baseline_source_sha256": "ffec011c704ff78b53097487b1dce926b1cb31ccb1124b5eaf63405c7e80d4f3", + "candidate_source_sha256": "c48d121d1fe4996a4c232b0d713d9163a1e47777ea3efe01e39bc36cec9aa053", + "samples_per_side": 7, + "errors": 0, + "compared_complete_reports": 14, + "coverage": { + "local": { + "head": "721c03822c421b7f7fcbb2598993b947c4d88419", + "records": 72, + "head_records": 8348, + "complete_head_inventory": false, + "unsearched_head_paths_count": 8278 + }, + "open_prs": { + "fetched_at": "2026-10-28T16:12:06.310964+00:00", + "complete_pagination": true, + "records": 407 + }, + "discussions": { + "fetched_at": "2026-10-28T16:07:35.266732+00:00", + "complete_pagination": true, + "records": 5816 + } + }, + "matches_total": 1580, + "normalized_report_sha256": "f336dd7d58808dc5c59973db44dc0e225f6ee072297c149f129b63b12866dda1", + "timing": { + "baseline": { + "seconds": [ + 0.9949404580984265, + 1.0183407498989254, + 1.0090217918623239, + 1.0870726669672877, + 1.0270258330274373, + 1.002464500023052, + 1.0151045839302242 + ], + "median": 1.0151045839302242, + "minimum": 0.9949404580984265, + "maximum": 1.0870726669672877 + }, + "candidate": { + "seconds": [ + 0.5106415001209825, + 0.5257168330717832, + 0.5250933750066906, + 0.5147430829238147, + 0.521243458846584, + 0.5136652081273496, + 0.5357615831308067 + ], + "median": 0.521243458846584, + "minimum": 0.5106415001209825, + "maximum": 0.5357615831308067 + } + }, + "reduction": 0.48651255535812554, + "claim_type": "Empirical verifier-engineering performance result with exact report equivalence.", + "cache_distribution": "Public GitHub snapshot is not vendored. Exact historical timing is scoped to this hash; rerun the harness on an external overlap cache.", + "measurement_scope": "Complete offline command subprocess, including interpreter startup, source loading, imports, local inventory, cached JSON parsing, search and JSON report output. Report comparison and cache hashing are outside the timed interval.", + "profile_note": "Separate untimed profiles identify numeric-reference regex scans as the removed cost. Local inventory and Unicode excerpt offsets remain." +} diff --git a/research/overlap-reference-prefilter/CHECK_TIMES.json b/research/overlap-reference-prefilter/CHECK_TIMES.json new file mode 100644 index 0000000000..8d188ad918 --- /dev/null +++ b/research/overlap-reference-prefilter/CHECK_TIMES.json @@ -0,0 +1,49 @@ +{ + "python": "3.12.15 (main, Oct 14 2026, 00:55:37) [Clang 23.1.3 ]", + "commands": [ + { + "argv": [ + "{python}", + "-I", + "-W", + "error", + "-B", + "{package}/verify.py" + ], + "seconds": 0.12617745902389288, + "exit_code": 0 + }, + { + "argv": [ + "{python}", + "-I", + "-O", + "-W", + "error", + "-B", + "{package}/verify.py" + ], + "seconds": 0.19773212517611682, + "exit_code": 0 + }, + { + "argv": [ + "{python}", + "-I", + "-W", + "error", + "-B", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-p", + "test_research_overlap_terms.py", + "-v" + ], + "seconds": 0.7096671247854829, + "exit_code": 0 + } + ] +} diff --git a/research/overlap-reference-prefilter/README.md b/research/overlap-reference-prefilter/README.md new file mode 100644 index 0000000000..a8a972615d --- /dev/null +++ b/research/overlap-reference-prefilter/README.md @@ -0,0 +1,128 @@ +# Skip impossible numeric-reference scans in research overlap searches + +The complete offline `joint-query-controller` overlap command takes **48.7% less +wall time** on the recorded workload. Seven alternating runs per side give +medians **1.015105 s before and 0.521243 s after**. Baseline range is +0.994940–1.087073 s; candidate range is 0.510642–0.535762 s. All fourteen +complete JSON reports agree after removing only `searched_at`, with zero errors. + +Claim type: empirical tooling improvement with a scoped equivalence argument +and finite regression evidence. No new kappa or selected-frontier change. +This contribution was produced by Nick Petalas. + +## Change and correctness + +`scripts/find_overlap.py` builds numeric-reference patterns from an optional PR +prefix, the escaped literal number, and boundary assertions. Every successful +match contains that literal number. If it is absent from the lowercased text, +returning `-1` is therefore exact. If it is present, the original regex still +runs on the entire text. Its original start offset, including any PR prefix, +is preserved. Explicit literal queries still use the original `str.find` path. + +The change adds one substring predicate. It does not alter query construction, +reference boundaries, scoring, excerpts, ordering, local/remote inventories, +cache freshness, pagination, or warnings. This proof concerns the patterns +constructed by `search`; arbitrary unrelated patterns supplied directly to the +internal helper are outside its contract. + +The baseline profile put 0.559 s in reference regex scans in a 1.652 s profiled +command. A separate candidate profile puts 0.035 s there; local Git inventory +and Unicode excerpt-offset conversion remain substantial costs. Profiling runs +are excluded from the timing samples. The first measured improvement was frozen. + +## Evidence and reproduction + +[BENCHMARK.json](BENCHMARK.json) is generated from measured output. It records all +samples, source and cache SHA256 values, environment and coverage. The external +16,945,386-byte cache contains 407 open PRs and 5,816 Discussion records. The +sparse checkout searches 72 local notes and explicitly reports 8,278 omitted +HEAD notes. The command returns 1,580 matches. Every full warning and omitted +path list participates in report comparison, even though the benchmark summary +prints only the omitted-path count. + +CPython 3.12.15, macOS 27.0.1 arm64, standard library only. The machine has +18 logical CPUs; load averages at intake were 4.62, 3.74 and 3.02. Other work +was not stopped. Both sides use the same interpreter, flags, immutable cache, +checkout, process wrapper and seven alternating pairs. The timed interval +includes startup, imports, local inventory, JSON parsing, search and output. +Comparison and workload hashing occur after timing. The gain is consistent +with removal of the profiled scans and exceeds the observed ranges. + +From the repository root: + +```sh +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -I -W error -B research/overlap-reference-prefilter/verify.py +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -B scripts/research_package.py --package research/overlap-reference-prefilter +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -B scripts/check_research_commit.py --package research/overlap-reference-prefilter --check-source-history +``` + +Another CPython 3.12 installation can replace the absolute path. Both registered +normal and optimized checks enumerate 39,216 numeric offsets and the same number +of literal offsets. Eight complete search comparisons use 289 constructed +title/body pairs, including Unicode expansion, decimal/hash false positives, +explicit numeric overrides, duplicate terms, AND/OR modes and late excerpts. +Work controls require zero regex calls on an absent literal and retain the +boundary checks on present but invalid numbers. The actual CLI regression suite +also runs, including its unchanged numeric-reference fixtures. +[CHECK_TIMES.json](CHECK_TIMES.json) records the three registered commands at +0.126, 0.198 and 0.710 seconds locally, each below its 120-second limit. + +To repeat the performance experiment with an external overlap cache: + +```sh +/Users/nick/git/integer-mult-bounds/.venv/bin/python3.12 -I -W error -B research/overlap-reference-prefilter/benchmark.py --cache /absolute/path/overlap.json --samples 7 +``` + +The harness retrieves the predecessor script from pinned reachable main +`721c03822c421b7f7fcbb2598993b947c4d88419`. It executes both complete commands +with the current shared dependencies and same stable checkout, compares every +report, and verifies the cache was unchanged. Keep the checkout stable while it +runs. It writes temporary baseline source beside the external cache and removes +it on exit. Full Git history and Git on PATH are required for this optional +benchmark. No network access is used. Timing is not a CI acceptance threshold. + +## Limitations + +The historical cache is hash-identified, not vendored. A fresh cache is a new +workload; the exact recorded timing cannot be reconstructed from this package +alone after that cache is removed. The verifier and actual CLI tests are fully +reproducible without it. The measured gain applies to a sparse checkout and a +reference-sparse task query. Literal-only queries receive no prefilter. A corpus +containing the reference number in nearly every text can pay an extra substring +scan, so no universal speedup is claimed. + +## Open next questions + +Does the same gain hold for other cards and a complete checkout? Run the same +harness on the intended immutable cache before changing another mechanism. +No additional optimization is part of this contribution. Native operators, +supplier correctness, compiler closure, ownership, precision, tape costs and +the complete multiplication theorem are not audited. Independent human and +formal proof-kernel review are unclaimed. + +## Credit and implementation constraints + +The existing overlap search retains Douglas Colkitt and its other contributors' +credit. Nick Petalas's [PR3002](https://github.com/CrocSwap/integer-mult-bounds/pull/3002) +supplies the numeric-reference rules; Ali Romman's +[PR5736](https://github.com/CrocSwap/integer-mult-bounds/pull/5736) supplies Unicode +excerpt positioning. Their source notices remain unchanged. Literal prefiltering is a standard optimization; +the contribution is its integration, complete-command measurement and controls. +New work is Apache-2.0. + +Build the Lever produced `benchmark.py` and `verify.py`. Prove It Works required +complete report equality rather than matching only result counts. Explain the +Number required separate profiles, seven alternating pairs, zero-error checks +and the complete command as the timed unit. The autonomous local run playbook +stopped optimization at the first measured improvement. + +Power of Ten rule 3 deviation: Python allocates managed objects. The deterministic +verifier is bounded by six word lengths over seven symbols, two terms and 289 +records. The optional benchmark caps its cache at 64 MiB, samples at fifteen and +each subprocess at sixty seconds. Rule 8/9 deviations are limited to loading the +one declared source module, patching its one helper for comparison, and executing +the two explicit complete command sources. Their names and lifetimes are fixed. +All added functions fit within sixty lines, use bounded iterations, and retain +explicit checks under `-O`. Unchanged discovery/network functions are outside +the modified path. Warning-strict checks run in both modes; no compiler or new +global analysis configuration is introduced. diff --git a/research/overlap-reference-prefilter/SOURCES.json b/research/overlap-reference-prefilter/SOURCES.json new file mode 100644 index 0000000000..ed1ce54eb4 --- /dev/null +++ b/research/overlap-reference-prefilter/SOURCES.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "base_commit": "721c03822c421b7f7fcbb2598993b947c4d88419", + "files": [ + { + "path": "certificates/selected-result.json", + "usage": "provenance", + "git_blob": "d59b5035e7b4cbaa9d688a1d32dd0d6eb79c466a", + "sha256": "9b8d0eea95b3bf0dfcb7938a732a6439b87a2685dec2a5ab9e525b3b34debbfa" + }, + { + "path": "docs/research/agent-task-catalog.json", + "usage": "provenance", + "git_blob": "3404fbdd5e7529a0674eef26114b5071e4586cec", + "sha256": "ead6cdc9d0af6d58e8a49744d8ec8c7c15e13d2ba93c276c434bcf9b51b93a81" + }, + { + "path": "scripts/find_overlap.py", + "usage": "provenance", + "git_blob": "b9269aadda13bcd081ef0cabe1e3a44d19ad73c2", + "sha256": "ffec011c704ff78b53097487b1dce926b1cb31ccb1124b5eaf63405c7e80d4f3" + }, + { + "path": "tests/test_research_overlap_terms.py", + "usage": "provenance", + "git_blob": "ed2f4bde8d7e25349eb274ba2516474c528d7447", + "sha256": "a6aff1c016b332216007bdde813b3ed14a80368a031a00fe29acfc1abbba88bf" + } + ] +} diff --git a/research/overlap-reference-prefilter/TASK.json b/research/overlap-reference-prefilter/TASK.json new file mode 100644 index 0000000000..16d7144b0d --- /dev/null +++ b/research/overlap-reference-prefilter/TASK.json @@ -0,0 +1,61 @@ +{ + "schema_version": 1, + "task_id": "replay-runtime", + "priority_revision": 8, + "base_commit": "721c03822c421b7f7fcbb2598993b947c4d88419", + "original_card": { + "id": "replay-runtime", + "question": "Can one measured replay bottleneck be sped up without dropping coverage?", + "starting_points": [ + "scripts/run_isolated_tests.py", + "scripts/verify_replay_coverage.py", + "certificates/selected-result.json" + ], + "methods": [ + "Profile one isolated replay and optimize the measured bottleneck.", + "Design source-bound reuse with explicit dependency invalidation and negative controls." + ], + "acceptance_gate": "Report before/after wall time on the same fixture, retained checks and invalidation behavior.", + "first_step": { + "command": [ + "python3", + "-B", + "scripts/research_selected_baseline.py" + ], + "expected": "PASS_RESEARCH_BASELINE_BINDINGS_ONLY", + "inspect": "Resolve the current selected review, separate complex/ordinary BIT/admitted BIT/outer budgets, and compare the card against live handoffs before starting. Historical ceilings require exact source/model equivalence.", + "scope": "Source-binding orientation only; not a scientific replay or all-size proof. Materialize selected inputs if this checkout is sparse." + }, + "first_experiment": "Measure one unchanged fixture, preserve check coverage, and test one optimization with invalidation controls.", + "bottleneck_link": "Faster trustworthy replays increase review throughput and let mechanism candidates be checked sooner.", + "allowed_variations": [ + "Profile one stage or design one source-bound cache boundary." + ], + "scope_change_triggers": [ + "Skipping coverage or comparing different fixtures to claim speedup." + ], + "overlap_terms": [ + "replay runtime", + "cache invalidation", + "isolated tests" + ] + }, + "scope_status": "changed", + "actual_scope": "Skip impossible numeric-reference regex scans in the complete research overlap command, retaining exact reports and coverage.", + "scope_change_reason": "This is coordination tooling rather than a mathematical supplier replay. Intake profiles show reference regex scans dominate a real offline overlap query.", + "bottleneck_link": "Reduce the time spent checking live research overlap without hiding claims or weakening sparse/remote coverage warnings. No mathematical bound changes.", + "budget_minutes": 60, + "stop_rule": "Ship the first verified complete-command reduction after exact report comparison; do not optimize another mechanism.", + "outcome": "completed", + "evidence": "README.md; BENCHMARK.json; verify.py; benchmark.py; tests/test_research_overlap_terms.py", + "author": "Nick Petalas", + "ai_assistance": "Independent human and formal proof-kernel review are unclaimed.", + "open_questions": [ + { + "question": "Does the same gain hold for other task cards and a fully materialized checkout?", + "why_it_matters": "The measured sparse checkout spends substantial remaining time on local inventory and excerpts.", + "first_step": "Run benchmark.py on an immutable cache and the intended checkout before changing another mechanism.", + "blocked_by": "Workload-dependent timing; no further performance change is claimed." + } + ] +} diff --git a/research/overlap-reference-prefilter/VERIFY.json b/research/overlap-reference-prefilter/VERIFY.json new file mode 100644 index 0000000000..441758853b --- /dev/null +++ b/research/overlap-reference-prefilter/VERIFY.json @@ -0,0 +1,57 @@ +{ + "schema_version": 1, + "status": "ready", + "python": "3.12", + "requirements": [], + "inputs": [ + "scripts/find_overlap.py", + "scripts/research_package.py", + "tests/test_research_overlap_terms.py" + ], + "commands": [ + { + "argv": [ + "{python}", + "-I", + "-W", + "error", + "-B", + "{package}/verify.py" + ], + "timeout_seconds": 120, + "checks": "39,216 exact offset comparisons; eight complete search comparisons; literal, boundary, prefix and work-elision controls." + }, + { + "argv": [ + "{python}", + "-I", + "-O", + "-W", + "error", + "-B", + "{package}/verify.py" + ], + "timeout_seconds": 120, + "checks": "The same deterministic comparisons and controls with optimization enabled." + }, + { + "argv": [ + "{python}", + "-I", + "-W", + "error", + "-B", + "-m", + "unittest", + "discover", + "-s", + "tests", + "-p", + "test_research_overlap_terms.py", + "-v" + ], + "timeout_seconds": 120, + "checks": "Eleven actual CLI and predicate tests covering numeric references, literal overrides, scoring, excerpts, Unicode and all-term search." + } + ] +} diff --git a/research/overlap-reference-prefilter/benchmark.py b/research/overlap-reference-prefilter/benchmark.py new file mode 100644 index 0000000000..4f2001fc2f --- /dev/null +++ b/research/overlap-reference-prefilter/benchmark.py @@ -0,0 +1,99 @@ +"""Time complete offline commands against the pinned predecessor; write only stdout.""" +import argparse +import hashlib +import json +from pathlib import Path +import platform +import statistics +import subprocess +import sys +import tempfile +import time + +ROOT = Path(__file__).resolve().parents[2] +BASE = "721c03822c421b7f7fcbb2598993b947c4d88419" +SCRIPT = ROOT / "scripts/find_overlap.py" +WRAPPER = """import pathlib,sys +source = pathlib.Path(sys.argv.pop(1)).read_bytes() +script = sys.argv.pop(1) +sys.argv[0] = script +exec(compile(source, script, 'exec'), {'__name__':'__main__', '__file__':script}) +""" + + +def command(source, cache): + return [sys.executable, "-I", "-W", "error", "-B", "-c", WRAPPER, + str(source), str(SCRIPT), "--task", "joint-query-controller", + "--offline", "--json", "--limit", "3", "--cache", str(cache)] + + +def run(source, cache): + started = time.perf_counter() + result = subprocess.run(command(source, cache), cwd=ROOT, capture_output=True, + check=True, timeout=60) + elapsed = time.perf_counter() - started + report = json.loads(result.stdout) + del report["searched_at"] + if result.stderr: + raise ValueError("unexpected stderr: " + result.stderr.decode()) + if report["status"] != "POSSIBLE_OVERLAP_NOT_A_NOVELTY_CERTIFICATE": + raise ValueError("wrong command status") + return elapsed, report + + +def measure(cache, samples): + if cache.stat().st_size > 64 * 1024 * 1024: + raise ValueError("cache exceeds the 64 MiB benchmark limit") + before = hashlib.sha256(cache.read_bytes()).hexdigest() + baseline = subprocess.run( + ["git", "--no-replace-objects", "show", BASE + ":scripts/find_overlap.py"], + cwd=ROOT, capture_output=True, check=True, timeout=30).stdout + times = {"baseline": [], "candidate": []} + expected = None + with tempfile.TemporaryDirectory(prefix="overlap-benchmark-", dir=cache.parent) as tmp: + source = Path(tmp) / "baseline.py" + source.write_bytes(baseline) + for _ in range(samples): + for label, path in (("baseline", source), ("candidate", SCRIPT)): + elapsed, report = run(path, cache) + if expected is None: + expected = report + if report != expected: + raise ValueError("complete reports differ beyond searched_at") + times[label].append(elapsed) + if hashlib.sha256(cache.read_bytes()).hexdigest() != before: + raise ValueError("workload cache changed") + summary = {label: {"seconds": rows, "median": statistics.median(rows), + "minimum": min(rows), "maximum": max(rows)} + for label, rows in times.items()} + coverage = dict(expected["coverage"]) + local = dict(coverage["local"]) + missing = local.pop("unsearched_head_paths") + local["unsearched_head_paths_count"] = None if missing is None else len(missing) + coverage["local"] = local + return {"base": BASE, "python": sys.version, "platform": platform.platform(), + "cache_sha256": before, "cache_bytes": cache.stat().st_size, + "baseline_source_sha256": hashlib.sha256(baseline).hexdigest(), + "candidate_source_sha256": hashlib.sha256(SCRIPT.read_bytes()).hexdigest(), + "samples_per_side": samples, "errors": 0, + "compared_complete_reports": samples * 2, + "coverage": coverage, "matches_total": expected["matches_total"], + "normalized_report_sha256": hashlib.sha256(json.dumps( + expected, sort_keys=True, ensure_ascii=True).encode()).hexdigest(), + "timing": summary, + "reduction": 1 - summary["candidate"]["median"] / summary["baseline"]["median"]} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cache", required=True, type=Path) + parser.add_argument("--samples", type=int, default=7, choices=range(5, 16)) + args = parser.parse_args() + cache = args.cache.resolve(strict=True) + if cache.is_relative_to(ROOT): + parser.error("keep the cache outside the checkout") + print(json.dumps(measure(cache, args.samples), indent=2)) + + +if __name__ == "__main__": + main() diff --git a/research/overlap-reference-prefilter/verify.py b/research/overlap-reference-prefilter/verify.py new file mode 100644 index 0000000000..96385d78c5 --- /dev/null +++ b/research/overlap-reference-prefilter/verify.py @@ -0,0 +1,107 @@ +"""Exact search-equivalence and work controls for the numeric-reference prefilter.""" +import hashlib +import importlib.util +from itertools import product +import json +from pathlib import Path +import re +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[2] +PACKAGE = Path(__file__).resolve().parent + + +def need(ok, message): + if not ok: + raise ValueError(message) + + +def legacy_position(text, term, pattern): + if pattern is None: + return text.find(term) + match = pattern.search(text) + return match.start() if match else -1 + + +def expression(term): + return re.compile(r"(?