From e91ee36ad2db9d6c41d48997506a3fb149934ec5 Mon Sep 17 00:00:00 2001 From: Alex Gorevski Date: Thu, 24 Sep 2026 14:14:04 -0700 Subject: [PATCH 1/4] Improve TAC fidelity and decompiler evaluation methodology Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .gitignore | 1 + docs/data-format.md | 22 + docs/runbook.md | 85 +++- download_hf_contracts.py | 123 ++++- learnings/learnings.md | 90 +++- run_eval_gate_suite_for_model.sh | 15 +- run_train_qwen_qlora_500.sh | 53 +- run_train_qwen_qlora_full_body_balanced.sh | 164 +++--- scripts/benchmark_runtime_controls.py | 375 ++++++++++++++ scripts/benchmark_tac_fidelity.py | 414 +++++++++++++++ scripts/build_curriculum_dataset.py | 47 +- scripts/compare_eval_runs.py | 173 ++++++- scripts/eval_gate_suite.py | 12 +- scripts/gate_dataset.py | 451 +++++++++++++++++ src/bytecode_analyzer.py | 241 ++++++--- src/contract_reconstruction.py | 7 + src/dataset_export_primitives.py | 90 +++- src/dataset_pipeline.py | 41 +- src/evaluation_report.py | 194 +++++-- src/inference.py | 136 ++++- src/model_setup.py | 57 ++- src/replication_metrics.py | 88 +++- src/training_pipeline.py | 280 +++++++++-- tests/test_benchmark_runtime_controls.py | 93 ++++ tests/test_benchmark_tac_fidelity.py | 109 ++++ tests/test_build_curriculum_dataset.py | 26 +- tests/test_bytecode_analyzer.py | 169 +++++++ tests/test_compare_eval_runs.py | 135 +++++ tests/test_contract_reconstruction.py | 12 + tests/test_data_generation_export_issues.py | 377 ++++++++++++++ tests/test_dataset_pipeline.py | 73 ++- tests/test_decompile_cli.py | 2 + tests/test_eval_gate_suite.py | 53 ++ tests/test_evaluation_quality_metrics.py | 320 ++++++++++++ tests/test_evaluation_report.py | 174 ++++++- tests/test_gate_dataset.py | 529 ++++++++++++++++++++ tests/test_inference.py | 121 +++++ tests/test_model_training_issues.py | 58 ++- tests/test_prompt_metadata.py | 69 +++ tests/test_replication_metrics.py | 67 +++ tests/test_train_cli_issues.py | 129 ++++- tests/test_web_app.py | 52 ++ train.py | 30 +- web/app.py | 45 +- web/static/app.js | 7 +- 45 files changed, 5336 insertions(+), 473 deletions(-) create mode 100644 scripts/benchmark_runtime_controls.py create mode 100644 scripts/benchmark_tac_fidelity.py create mode 100644 scripts/gate_dataset.py create mode 100644 tests/test_benchmark_runtime_controls.py create mode 100644 tests/test_benchmark_tac_fidelity.py create mode 100644 tests/test_gate_dataset.py create mode 100644 tests/test_inference.py diff --git a/.gitignore b/.gitignore index d4e11ca..99e3d00 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ data/*.log data/split_manifest.json data/manifests/ data/preflight_cache/ +data/audit_*/ scripts/data/*.db* # Keep demo dataset for reference diff --git a/docs/data-format.md b/docs/data-format.md index f03d5b3..e2a311a 100644 --- a/docs/data-format.md +++ b/docs/data-format.md @@ -185,6 +185,28 @@ Splits are leakage-connected components, not independent rows: rows sharing any leakage key stay in the same split. If fewer than three groups exist, all rows are written to train and validation/test are left empty to avoid leakage. +## Fixed evaluation gates + +`run_train_qwen_qlora_500.sh` and `run_train_qwen_qlora_full_body_balanced.sh` +exclude the six fixed gate datasets before selecting training rows. Exclusion +matches recomputed normalized Solidity bodies (including missing or stale +`metadata.body_hash`), exact TAC/target text, and source/contract identity. +`EVAL_EXCLUDE_DATASETS` adds extra gates; it does not replace the fixed gates. +`scripts/build_curriculum_dataset.py` uses the same exclusion keys when +constructing heldout-clean curricula. Selected rows receive a recomputed +`metadata.body_hash` so grouped train/val/test splitting also keeps normalized +target duplicates together when source metadata was missing or stale. +Previously selected datasets may be reused only when their source, exclusions, +parameters, and output match the selection manifest; otherwise set +`RECREATE_DATASET=1` to rebuild them. + +`run_eval_gate_suite_for_model.sh` requires a completed model training-input +manifest and verified train/val/test split artifacts without fixed-gate overlap. +`scripts/compare_eval_runs.py` rejects candidate improvements unless baseline +and candidate evaluate the same indexed rows with the same decode settings and +complete gate metrics. Sub-30-row slices can veto a regression but cannot by +themselves promote a candidate. + ## Quality checks and rejects Export-time rejects are written to `.rejects.jsonl`. Reject rows include diff --git a/docs/runbook.md b/docs/runbook.md index ad9f200..1bf0678 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -168,6 +168,10 @@ uv run python download_hf_contracts.py \ # Full Hugging Face-backed generation uv run python download_hf_contracts.py +# Enforce the training token budget during export (tokenizer must be cached locally) +uv run python download_hf_contracts.py --export-only \ + --length-tokenizer Qwen/Qwen2.5-Coder-7B-Instruct + # Keep all phase manifests together uv run python download_hf_contracts.py --limit 1000 --manifest-dir data/manifests ``` @@ -202,6 +206,7 @@ Useful flags: | `--export-only` | — | Export existing pairs to JSONL | | `--output PATH` | `data/hf_training_dataset.jsonl` | Export JSONL target | | `--max-seq-length N` | 8192 | Export-time prompt/target length budget | +| `--length-tokenizer PATH` | none | Use a locally cached training tokenizer for exact prompt/target/EOS length filtering; without it the export reports estimated counts and train-time preflight must still check lengths | | `--no-filter-overlength` | off | Keep rows over `--max-seq-length` instead of quarantining them | | `--rejects-output PATH` | `.rejects.jsonl` | JSONL quarantine for overlength, duplicate, quality, or auxiliary-contract rejects | | `--export-selectors PATH` | — | Export selector registry JSON | @@ -318,7 +323,7 @@ rows, and 1,890 test rows. The inspected worktree also has a nonstandard previous artifact at `models/final_model_378/`; pass that path explicitly for evaluation/inference until a fresh default `models/final_model/` exists. The splitter preserves leakage-connected groups (source hash, contract address, -contract+selector/signature, exact input hash, exact output hash), validates no +contract+selector/signature, normalized output body, exact input/output hashes), validates no overlap across train/val/test, and reports holdout coverage by compiler version, optimizer, visibility, source, length bucket, and function family. Production inference only has Etherscan **Contract > Bytecode**, so training prompts @@ -330,6 +335,15 @@ uv run python train.py \ --dataset data/hf_training_dataset.jsonl ``` +Direct `train.py` splitting does not reserve the six fixed evaluation gates. +Use `./run_train_qwen_qlora_full_body_balanced.sh` for an eval-clean full-data +experiment. With its default `RUN_GATES=1`, the runner now checks that all six +baseline JSONs exist and match the bundled-only selector policy, gate datasets, +decode settings, zero prompt truncation, and behavior-only metric **before** +spending GPU time. +Regenerate historical baselines first; `RUN_GATES=0` allows a diagnostic run +without baseline acceptance but is not evidence of improvement. + `--no-compiler-metadata` is retained as a deprecated no-op for old scripts. If an older JSONL embeds `// Compiler:` or source/ABI annotations inside the TAC `input`, the training prompt builder strips those lines before tokenization; @@ -380,6 +394,12 @@ databases: - Event/log and revert counts. - Bytecode length, instruction count, and function count. +The model prompt currently uses signature guesses only from the bundled +selector map. A local `contracts.db` compiled from the evaluation source is +not an independent inference-time resource: using its signatures in test +prompts leaks reference labels. Re-evaluate historical selector-assisted +results after changing this policy; the old scores are not comparable. + For inference, omit compiler flags and inspect the bytecode-derived TAC/analysis: ```bash @@ -397,6 +417,12 @@ sanitization still runs. Legacy inference flags such as `--compiler-version`, `--optimizer-enabled`, `--optimizer-runs`, and `--evm-version` are deprecated no-ops and are ignored. +Inference reports `analysis_failed` when no function boundary is recoverable, +instead of sending whole-contract TAC to a function-level model. When a malformed +dispatcher target is rejected but another function or fallback remains, the +output is explicitly `partial_analysis` with the missing selector in +`analysis.rejected_dispatcher_targets` and an unresolved-chunk warning. + ### Oracle-only compiler metadata study `scripts/run_compiler_metadata_ablation.py` is deprecated for production prompt @@ -568,17 +594,62 @@ Key metrics: | `edit_distance_mean` | lower is better | | `replication_precision_micro` | higher is better; measures recovered facts that are correct | | `replication_recall_micro` | higher is better; measures ground-truth facts recovered | -| `replication_f1_micro` | higher is better; balanced structured replication score | -| `solidity_valid_mean` | higher is better; generated Solidity passed compiler/AST validation when local solc was available | -| `bytecode_semantic_checked_mean` | should be 100%; rows need opcode/runtime/compiler evidence to count as bytecode-grounded | -| `bytecode_deployable_mean` | should be 100%; scaffold-only syntax checks are not considered deployable | +| `replication_f1_micro` | higher is better; structured fact score, including ABI facts that can dominate the aggregate | +| `replication_behavior_only_f1_micro` | source-extracted calls, control flow, events, guards, returns, and state writes only; not execution equivalence | +| `solidity_valid_mean` | higher is better; syntax/scaffold validity for function fragments, including context-limited compiler failures, not full contract compilability | +| `solidity_ast_valid_mean` | generated fragment passed compiler AST validation with available context | +| `bytecode_semantic_checked_mean` | coverage of the evaluator's available evidence; do not read it as runtime-equivalence coverage | +| `bytecode_runtime_checked_mean` | proportion with an actual runtime-bytecode comparison; zero means no measured runtime equivalence | +| `bytecode_deployable_mean` | generated Solidity compiled into bytecode in the harness; not proof of chain deployment or runtime equivalence | The replication metrics compare structured Solidity facts extracted from the ground-truth function and generated function: ABI/function facts, visibility, mutability, modifiers, guards, events, calls, state writes, returns, and control flow. The evaluation JSON also includes `replication_by_category_micro` so you can see whether failures are concentrated in ABI recovery, state writes, guards, -calls, or other categories. +calls, or other categories. The aggregate may look reassuring while behavior +categories fail: in `latest_results.txt`, all-fact F1 was 0.6281 while the +behavior-only subset was 0.2245 (ABI F1 was 0.9588, call F1 0.2531, guard +F1 0.1454, and state-write F1 0.1912). The reported bytecode +semantic score is a proxy built from available evidence, not proof that the +generated source executes equivalently to the original bytecode; check runtime +coverage separately before making an equivalence claim. Paired gate comparisons +now require a non-regressing behavior-only F1 and a matching +`selector_signature_prompt_policy`; historical reports lacking that policy must +be regenerated instead of compared as though their prompts were identical. +Baselines and candidates must also measure zero prompt truncation for a keep +decision. + +Exact runtime-bytecode comparison is opt-in: provide a complete named contract +as both reference and generated source, a real reference runtime bytecode, and +`metadata.runtime_comparison` with matching `contract_name`, `compiler_version`, +`optimizer_enabled`, and `optimizer_runs` (optionally `source_name`). The +evaluator recompiles the reference locally with those settings first; mismatched +reference bytecode, missing local solc, fragments, constructors, inherited +contracts, or unsupported deployment context remain **unchecked**, with skip +reasons reported separately. Exact runtime equality is stricter than behavioral +equivalence and does not prove deployment or execution equivalence. Inspect +checked-row counts and equality *among checked rows* rather than treating +unchecked rows as matches or mismatches. + +Run model-free controls before comparing trained adapters: + +```bash +python -m scripts.benchmark_tac_fidelity +python -m scripts.benchmark_runtime_controls +``` + +The first benchmark compiles 36 distinct selector-bearing functions with local +solc 0.8.20 (getter, storage write, guard, external call, event, and branch). +It checks ABI selector recovery and source-mapped opcode reachability in each +function's TAC: the measured coverage improved from 18/36 to 36/36 after the +parameter-decoder CFG fix. Opcode coverage is **not** semantic correctness. +The second compiles 30 complete contracts and checks identical, modified, +fragment, and unsupported-import controls. Its 121 controls yielded 61 checked +(30 byte-equal, 31 byte-unequal) and 60 correctly unchecked. A comment-only +change is byte-unequal despite unchanged executable prefix, illustrating why +exact bytecode equality is not a behavioral-equivalence score. Both benchmarks +use installed compilers only, write JSON to stdout, and do not use a model. Evaluation helpers now use true normalized Levenshtein distance for `edit_distance_mean` instead of a `difflib.SequenceMatcher` ratio. Compiler/AST @@ -800,7 +871,7 @@ not read `src/settings.yaml`. | `WEB_DEFAULT_MAX_NEW_TOKENS` | `1024` | UI/API default generation cap | | `WEB_DEFAULT_TEMPERATURE` | `0.1` | UI/API default | | `WEB_DEFAULT_DO_SAMPLE` | `false` | UI/API default | -| `WEB_DEFAULT_REPETITION_PENALTY` | `1.15` | UI/API default | +| `WEB_DEFAULT_REPETITION_PENALTY` | `1.05` | UI/API default | | `WEB_ENABLE_REMOTE_SELECTOR_LOOKUP` / `--remote-selector-lookup` | `false` | Enables 4byte.directory lookups | | `WEB_TAC_LOOKUP_ENABLED` | `true` | Enables TAC exact-match lookup unless request lookup config disables it | | `WEB_READYZ_PUBLIC` | `false` | Makes `/readyz` public; keep false on shared hosts | diff --git a/download_hf_contracts.py b/download_hf_contracts.py index 8f7391e..3d609da 100644 --- a/download_hf_contracts.py +++ b/download_hf_contracts.py @@ -9,7 +9,7 @@ for analysis/filtering/manifests. Deduplication strategy (multi-layer): - 1. Contract-level: address PRIMARY KEY + source_hash dedup + 1. Contract-level: address PRIMARY KEY + source/target/compiler-config dedup 2. Pair-level exact: hash(TAC + body) UNIQUE in function_pairs 3. Pair-level semantic: pair_norm_hash(normalized_TAC + normalized_body) UNIQUE -- catches identical bytecode from different compilers @@ -26,6 +26,7 @@ python download_hf_contracts.py --workers 8 # set parallelism python download_hf_contracts.py --max-body-dupes 5 # cap duplicates python download_hf_contracts.py --min-body-length 50 # quality filter + python download_hf_contracts.py --export-only --length-tokenizer ./models/tokenizer python download_hf_contracts.py --validate-jsonl data/hf_training_dataset.jsonl """ @@ -90,6 +91,7 @@ parse_etherscan_source, parse_pragma, select_compilation_configs, + _normalize_version, ) try: @@ -656,13 +658,24 @@ def count_function_pairs(db_path: Path = DB_PATH) -> int: return _count_query(db_path, "SELECT COUNT(*) FROM function_pairs") -def get_existing_source_hashes(db_path: Path = DB_PATH) -> Set[str]: - """Load all existing source_hash values for fast dedup lookups.""" +def get_existing_contract_keys(db_path: Path = DB_PATH) -> Set[Tuple[str, str, str, bool, int]]: + """Load source/target/compiler identities already queued for compilation.""" with _db_connection(db_path) as conn: rows = conn.execute( - "SELECT source_hash FROM contracts WHERE source_hash IS NOT NULL" + "SELECT source_hash, contract_name, compiler_version, " + "optimization_enabled, optimization_runs " + "FROM contracts WHERE source_hash IS NOT NULL" ).fetchall() - return {r[0] for r in rows} + return { + ( + row[0], + str(row[1] or "").strip().lower(), + _normalize_version(row[2]) or "", + _parse_opt_used(row[3]), + _parse_runs(row[4]), + ) + for row in rows + } def get_body_hash_counts(db_path: Path = DB_PATH) -> Dict[str, int]: @@ -1128,7 +1141,7 @@ def download_contracts( command_args: Optional[List[str]] = None, parquet_batch_size: int = PARQUET_READ_BATCH_SIZE, ) -> int: - """Download contracts from HuggingFace with source-code-level dedup. + """Download contracts from HuggingFace with source/target/config dedup. Parquet files are cached locally (default: ~/.cache/huggingface/hub/). On rerun, cached files are reused without re-downloading. @@ -1171,8 +1184,8 @@ def download_contracts( return 0 logger.info(f"Found {len(parquet_files)} Parquet file(s)") - existing_hashes = get_existing_source_hashes(db_path) - logger.info(f"Loaded {len(existing_hashes)} existing source hashes for dedup") + existing_keys = get_existing_contract_keys(db_path) + logger.info("Loaded %s existing source/target/compiler identities for dedup", len(existing_keys)) conn = sqlite3.connect(db_path) cur = conn.cursor() @@ -1240,18 +1253,28 @@ def download_contracts( continue src_hash = hash_source_code(src) - if src_hash in existing_hashes: + contract_name = str(_row_get(row, "contract_name", "") or "").strip() + compiler_version = str(_row_get(row, "compiler_version", "") or "") + opt_used = _parse_opt_used(_row_get(row, "optimization_used", False)) + runs = _parse_runs(_row_get(row, "runs", 200)) + contract_key = ( + src_hash, + contract_name.lower(), + _normalize_version(compiler_version) or "", + opt_used, + runs, + ) + if contract_key in existing_keys: deduped += 1 continue - existing_hashes.add(src_hash) + existing_keys.add(contract_key) address = str(_row_get(row, "contract_address", "") or "").strip() if not address: - address = f"hf_{src_hash}" - - compiler_version = str(_row_get(row, "compiler_version", "") or "") - opt_used = _parse_opt_used(_row_get(row, "optimization_used", False)) - runs = _parse_runs(_row_get(row, "runs", 200)) + key_hash = hashlib.sha256( + json.dumps(contract_key, separators=(",", ":")).encode("utf-8") + ).hexdigest() + address = f"hf_{key_hash}" batch.append( ( @@ -1262,7 +1285,7 @@ def download_contracts( opt_used, runs, str(_row_get(row, "abi", "") or ""), - str(_row_get(row, "contract_name", "") or ""), + contract_name, "huggingface", src_hash, ) @@ -1313,8 +1336,8 @@ def download_contracts( max_rss_mb, ) logger.info( - f" Running: {inserted} inserted, {ignored} ignored, " - f"{deduped} deduped, {skipped} skipped, {total_seen} seen" + " Running: %s inserted, %s ignored, %s source/target/config deduped, " + "%s skipped, %s seen", inserted, ignored, deduped, skipped, total_seen, ) if limit_reached: logger.info(f"Reached download limit of {limit}") @@ -1325,8 +1348,8 @@ def download_contracts( conn.close() logger.info( - f"Download complete: {inserted} stored, {ignored} ignored, " - f"{deduped} source-deduped, {skipped} skipped, {total_seen} total" + "Download complete: %s stored, %s ignored, %s source/target/config deduped, " + "%s skipped, %s total", inserted, ignored, deduped, skipped, total_seen, ) duration_seconds = time.perf_counter() - start_time @@ -1348,6 +1371,10 @@ def download_contracts( "limit": limit, "cache_dir": cache_dir, "parquet_batch_size": parquet_batch_size, + "dedup_key": [ + "source_hash", "contract_name", "compiler_version", + "optimization_enabled", "optimization_runs", + ], }, "artifacts": {"database": _artifact_summary(db_path)}, "row_counts": { @@ -1478,6 +1505,7 @@ def _compile_one_job( pairs: List[Dict] = [] analysis_errors: List[str] = [] drop_counts: Counter = Counter() + ambiguous_selector_values: Set[str] = set() compiled_contract_counts: Counter = Counter() target_name = str(target_contract_name or "").strip() for cname, compiled in comp.contracts.items(): @@ -1508,7 +1536,15 @@ def _compile_one_job( f for f in solidity_functions if f.get("contract_name", "") == cname ] or solidity_functions - for m in _match_functions(contract_sol_funcs, bytecode_functions, analyzer): + ambiguous_selectors: Set[str] = set() + matches = _match_functions( + contract_sol_funcs, bytecode_functions, analyzer, + ambiguous_selectors=ambiguous_selectors, + ) + if ambiguous_selectors: + ambiguous_selector_values.update(ambiguous_selectors) + drop_counts["ambiguous_selector_alignment"] += len(ambiguous_selectors) + for m in matches: sol_body = m["solidity_function"]["body"] if len(sol_body.strip()) < min_body_length: continue @@ -1548,6 +1584,17 @@ def _compile_one_job( "drop_counts": dict(drop_counts), "compiled_contract_counts": dict(compiled_contract_counts), } + if drop_counts.get("ambiguous_selector_alignment"): + return { + "pairs": [], + "status": "ambiguous_selector_alignment", + "error": ( + "conflicting source targets or runtime entries for selectors: " + + ", ".join(sorted(ambiguous_selector_values)[:10]) + ), + "drop_counts": dict(drop_counts), + "compiled_contract_counts": dict(compiled_contract_counts), + } return { "pairs": [], "status": "no_pairs", @@ -2032,13 +2079,17 @@ def _match_functions( optimizer_enabled: bool = False, abi_enricher: Optional[Any] = None, storage_resolver: Optional[Any] = None, + ambiguous_selectors: Optional[Set[str]] = None, ) -> List[Dict]: """Match Solidity labels to bytecode functions by selector. Source/ABI/compiler arguments are accepted for backwards compatibility but are intentionally ignored when constructing TAC prompt text. """ - return match_functions_by_selector(solidity_functions, bytecode_functions, analyzer) + return match_functions_by_selector( + solidity_functions, bytecode_functions, analyzer, + ambiguous_selectors=ambiguous_selectors, + ) def _ensure_tac_integrated(analyzer) -> None: @@ -2309,6 +2360,7 @@ def export_training_data( max_seq_length: int = DEFAULT_EXPORT_MAX_SEQ_LENGTH, filter_overlength: bool = True, rejects_path: Optional[Path] = None, + length_tokenizer: Optional[Any] = None, ) -> str: """Export deterministic JSONL with bytecode-only inputs and metadata. @@ -2451,7 +2503,7 @@ def export_training_data( target_contract_name, compiled_contract, ) - length_report = _export_length_report(record, max_seq_length) + length_report = _export_length_report(record, max_seq_length, length_tokenizer) length_reasons = length_report["reasons"] if filter_overlength else [] row_hash = final_row_hash(record["input"], record["output"]) duplicate_reasons: List[str] = [] @@ -2569,6 +2621,12 @@ def export_training_data( "validate_body_dupes": validate_body_dupes, "max_seq_length": max_seq_length, "filter_overlength": filter_overlength, + "token_count_method": "tokenizer" if length_tokenizer is not None else "estimate", + "length_tokenizer": ( + getattr(length_tokenizer, "name_or_path", None) + if length_tokenizer is not None + else None + ), "final_row_duplicate_policy": "quarantine", "tac_quality_policy": "quarantine", "record_quality_policy": "quarantine", @@ -2644,6 +2702,7 @@ def export_training_data( "status": "filtered" if overlength_row_rejects else "passed", "max_seq_length": max_seq_length, "filter_overlength": filter_overlength, + "token_count_method": "tokenizer" if length_tokenizer is not None else "estimate", "reject_count": overlength_row_rejects, "reason_counts": dict(sorted(overlength_counts.items())), "reject_samples": reject_samples, @@ -2746,6 +2805,16 @@ def main(): action="store_true", help="Do not quarantine rows that exceed --max-seq-length.", ) + parser.add_argument( + "--length-tokenizer", + type=str, + default=None, + help=( + "Locally available tokenizer path or model ID for exact export-time " + "length filtering; without it, counts are estimates and training " + "preflight can still reject rows." + ), + ) parser.add_argument( "--rejects-output", type=str, @@ -2904,6 +2973,13 @@ def main(): logger.info(f"Total function pairs in DB: {total}") if total > 0: + length_tokenizer = None + if args.length_tokenizer: + from transformers import AutoTokenizer + + length_tokenizer = AutoTokenizer.from_pretrained( + args.length_tokenizer, local_files_only=True + ) out = export_training_data( output_path=args.output, max_body_dupes=args.max_body_dupes, @@ -2914,6 +2990,7 @@ def main(): max_seq_length=args.max_seq_length, filter_overlength=not args.no_filter_overlength, rejects_path=Path(args.rejects_output) if args.rejects_output else None, + length_tokenizer=length_tokenizer, ) logger.info(f"Training data written to: {out}") else: diff --git a/learnings/learnings.md b/learnings/learnings.md index 74f90ef..aef56ee 100644 --- a/learnings/learnings.md +++ b/learnings/learnings.md @@ -4,7 +4,13 @@ This file is the persistent research log for TAC-to-Solidity training runs. It r ## Current verdict -**Model quality issue; raw all-row scale-up is diagnostic but not an accepted improvement.** Overfit sanity checks can memorize exactly up to at least 30 seeded examples, which validates prompt/label wiring, adapter save/load, checkpoint evaluation, and multi-GPU execution at small scale. The largest measured model-side gain still comes from exposing locally resolved selector-signature provenance in prompt metadata: on a 30-row non-overlap eval, the old 100-row adapter improved from 0/30 to 19/30 strict signature matches and from 0.3022 to 0.6788 semantic similarity. The 2026-07-02 raw full-source `run_train_qwen_qlora_500.sh` run completed on all 31,577 rows for 1 epoch, but its 1,890-row built-in heldout eval underperformed the earlier default body-balanced full run on F1/semantic/bytecode, and its fixed gate-suite pass is contaminated because every fixed gate dataset overlapped the raw train/val/test split. The eval-clean body-balanced run still has no final model/results, so clean full-run acceptance remains open. Remaining failures are still model/data quality, dominated by calls, state writes, selectors, guards, returns, events, hallucinated facts, and long-output scaffold runaways. +**Model quality issue with previously untrustworthy methodology; no accepted clean full-run result.** Same-row overfits memorized 30/30, but they do not establish heldout or deployed-bytecode recovery. The old 30-row selector-on result (19/30 strict signatures versus 0/30 selector-off) used a local registry capable of reading signatures compiled from the evaluation source; only 7/30 broad selectors and 1/6 historical heldout selectors are in the bundled inference-time map. That ablation is **not evidence of a bytecode-only improvement** until rerun with comparable prompts. The 2026-07-02 raw 31,577-row run's 1,890-row built-in test reported all-fact F1 0.6281 but behavior-only source-fact F1 0.2245; its bytecode-semantic 0.5083 is a source-fact proxy with 0% runtime comparisons, and every fixed gate overlapped its split. Data alignment, prompt provenance, duplicate-body splitting, gate exclusions, and evaluator labeling were corrected in the 2026-09-24 audit, but **no model was retrained or re-evaluated**. The August checkpoint-11500 result remains a 15-row first-N smoke check, and no final artifact for the earlier eval-clean run is available here. Calls, guards, state writes, hallucinations, and full-context deployability remain open. + +**Follow-up (2026-09-24, model-free):** A compiler-produced optimized getter lost a reachable RETURN because the CFG inferred a consumed PUSH0 as its jump target; a bounded stack-aware fix restores the edge. Training-absent contract-wide metadata was removed from inference prompts. Invalid dispatcher targets now fail or mark a fallback-only reconstruction partial across shared inference and web. Exact full-contract runtime comparison is available as an opt-in *identity* check on compiler-verified examples, not as a behavioral-equivalence claim. A source-fact parser fix removes false ABI matches for nested mappings and false calls from type casts, so even the derived historical behavior-only F1=0.2245 must be recomputed before comparison. No trained model or large current dataset is present to measure downstream model-quality gains. + +**Further checkpoint:** A deterministic solc 0.8.20 source-map benchmark found ABI selectors in all 36 locally compiled functions but reached source-mapped behavior opcodes in only 18/36; the 18 parameterized functions lost decoder-return paths. Bounded stack-context traversal now reaches 36/36, with no selector/CFG failures in this corpus. A separate 30-contract runtime control measured 61 checked (30 exact matches, 31 mismatches) and 60 intentionally unchecked cases; one comment-only change yielded unequal full bytecode while its executable prefix was unchanged. These are model-free *structural and identity* controls, not proof of semantic recovery or a new trained-model result. + +**Bounded real-data audit (2026-09-24):** Regenerated 2,000 public verified-contract records locally with the exact cached Qwen tokenizer and source-aligned single-compiler policy. Of 1,027 unique-body candidate rows after other filters, 618 (60.2%) exceeded 8,192 true training tokens and only 409 exported. Rejected prompts had median 50,224 context tokens (max 204,976), mostly parameterized (475/618), despite the small 36-function benchmark's 36/36 source-map witness coverage. The 409-row dataset is an *audit artifact*, not a clean model run or acceptance result. Prioritize compact, behavior-preserving per-function TAC or hierarchical retrieval; do not silently truncate or train only the easy minority and claim broad decompilation quality. ## Iteration log @@ -56,6 +62,15 @@ This file is the persistent research log for TAC-to-Solidity training runs. It r | 43. Mid-run CPU smoke eval of checkpoint-11500 (fresh-data run) | checkpoint `models/qwen_qlora_full_fresh_20260801-140727/checkpoints/checkpoint-11500/`, staged eval dir `/tmp/ckpt11500_eval`, eval `results/eval_1785710884.json`, log `/tmp/cpu_smoke_eval.log` | While the 4-GPU run was still training (step ~11,836/17,472, epoch 2.03), all four GPUs held 42.8-47.7 GiB of 48 GiB, leaving too little headroom for a GPU eval without risking an OOM in the live trainer. Ran the eval entirely on CPU instead with `CUDA_VISIBLE_DEVICES=""`, `--no-quantization --precision fp32`, 15 first-N test rows, `eval_max_new_tokens=512`, repetition penalty 1.05. Staging the adapter required copying base tokenizer/config files next to `adapter_model.safetensors`, otherwise preflight fails with `Unrecognized model ... model_type`. | 15/15 succeeded, failure rate 0. Semantic 0.8557, normalized edit 0.2815, replication F1 micro 0.8112 (precision 0.8467, recall 0.7785), bytecode semantic 0.6901 (checked 1.0), Solidity valid 1.0, AST/deployable 0.0667, signature match 0.80, visibility match 0.8667, structural preservation 0.8967. Prompt truncation 0; prompt tokens 608-1181 vs 8192 window; generated tokens 19-114 with zero rows hitting the 512 cap. Split audit: 0/1702 test rows share an input or output hash with train, but test itself holds only 1,314 unique outputs across 1,702 rows, so duplicates cluster inside the split (group-aware splitting working as intended). | Directionally the strongest heldout numbers this repository has recorded (F1 micro 0.8112 vs 0.6612 best prior, bytecode 0.6901 vs 0.5425), but n=15 is smoke-only per the 30-sample rule and the first-N slice contained 5 duplicate-output pairs, so it measures roughly 10 distinct targets. Do not treat as acceptance evidence. Also discovered the run was launched WITHOUT LoRA flags and silently used train.py defaults r=16/alpha=32/dropout=0.1 instead of the intended r=32/alpha=64/dropout=0. | +| 44. Methodology audit (no new model run; 2026-09-24) | Historical report `latest_results.txt` (2026-07-02, n=1,890); prior selector ablations `results/eval_1782608489.json`, `results/eval_1782608594.json`; regression tests in `tests/test_prompt_metadata.py`, `tests/test_dataset_pipeline.py`, `tests/test_train_cli_issues.py`, `tests/test_gate_dataset.py`, `tests/test_evaluation_report.py`, `tests/test_bytecode_analyzer.py` | Reproduced a compiled heldout selector leaking into prompts, conflicting ERC20/ERC721 targets choosing the last body, normalized duplicate targets crossing a direct `train.py` split, a 60-selector dispatcher losing its last 10 selectors after instruction 256, and unchecked/compiler-only results appearing bytecode-verified. Fixed prompt provenance (bundled signatures only), ambiguous alignment quarantine, EOS-aware exact tokenizer filtering, split/cache and gate/paired-eval checks, selector scan, runtime provenance, and report labels. | Historical report: all-fact F1 0.6281 vs behavior-only source-fact F1 0.2245 (TP=1,732, FP=5,476, FN=6,488); source-fact proxy 0.5083, runtime-checked 0%, AST/bytecode-compiler-valid 2.43%, hallucination rate 35.37%, prompt truncation 0/1,890. Only 7/30 broad and 1/6 historical heldout selectors are bundled; old selector-on gains may reflect evaluation-source registry entries, but the effect size needs a clean rerun. Detailed eval JSON and model artifacts are unavailable here. | Old selector ablations and fixed-gate baselines are not comparable to the revised bytecode-only prompt and split policy. Rebuild clean datasets and baselines; use behavior-only source-fact F1 with runtime/differential coverage, never proxy/unchecked scores as equivalence. | + +| 45. Model-free decompiler quality checkpoint (2026-09-24) | `src/bytecode_analyzer.py`, `src/inference.py`, `web/app.py`, `src/training_pipeline.py`, `src/replication_metrics.py`; compiler-produced fixture `tests/test_bytecode_analyzer.py`, focused `tests/test_inference.py`, `tests/test_web_app.py`, `tests/test_evaluation_quality_metrics.py`, `tests/test_replication_metrics.py` | On a solc 0.8.20 optimized getter, a consumed PUSH0 previously hid the reachable `block_004c` RETURN after `block_0045`; stack-aware CFG resolves 0x45 -> 0x4c without adding that block to the sibling function. An invalid dispatcher target formerly caused whole-contract TAC to be sent to a function model; missing targets now fail closed and fallback-only cases report `partial_analysis`. Inference no longer adds contract-wide instruction/function counts absent from training prompts. Source-fact parsing now handles nested mapping/function-typed parameters, excludes type casts from call facts, and does not borrow another function's body after a declaration. | Exact runtime comparison on a locally compiled full contract: 1/1 identical reference/candidate checked+equal; a changed return compiles but compares unequal; fragments, unverified bytecode, unsupported deployment contexts remain unchecked with recorded reasons. The earlier synthetic 60-selector dispatcher now retains all 60 valid selectors and diagnoses invalid ones. No trained adapter was available and no heldout model score was measured. | The TAC input and evaluator become more faithful, but this is not evidence the model improved. Build compiler-produced behavioral slices and clean, disjoint model evals before ranking any model; runtime *identity* and source-fact F1 are separate from execution equivalence. | + +| 46. Compiler source-mapped TAC fidelity benchmark and parameter-decoder fix | `scripts/benchmark_tac_fidelity.py`, `tests/test_benchmark_tac_fidelity.py`, `src/bytecode_analyzer.py`, `tests/test_bytecode_analyzer.py` | One installed solc 0.8.20 with optimizer enabled (200 runs), Shanghai EVM and metadata disabled, 36 distinct ABI functions across getters, setters, guards, calls, events, branches; compiler AST/source map establishes each function's expected RETURN/SSTORE/REVERT/CALL/LOG2/JUMPI witness PC. Measured selector presence, CFG reachability and TAC rendering separately before and after a bounded stack-context CFG traversal that tracks decoder return addresses. | Selector 36/36 before and after. Source-mapped opcode reachability and end-to-end TAC witness coverage **18/36 -> 36/36**; simple no-parameter cases 18/18 unchanged, parameterized 0/18 -> 18/18; each behavior stratum 3/6 -> 6/6. No invalid dispatcher targets, CFG successors, unresolved reachable jump blocks, or sample failures on this synthetic local corpus after the fix. | An earlier one-getter fix was not enough; parameterized ABI decoder handoff systematically hid the behavior-bearing body. Structural opcode occurrence is not semantic correctness, and larger real-contract/compiler slices are still needed. | +| 47. Complete-contract runtime identity control suite | `scripts/benchmark_runtime_controls.py`, `tests/test_benchmark_runtime_controls.py`, local solc 0.8.20 | Built 30 distinct full contracts with independent reference runtime compilation, then checked identical-source, changed-return/storage, source-comment-only, fragment, and unsupported-import candidates. Opt-in evaluator requires matched compiler provenance and reports why unsupported rows are not checked. | 121 controls in ~6 s: 61 checked (30 exact equal, 31 exact unequal), 60 unchecked (30 fragments, 30 unsupported imports), zero control failures. A comment-only candidate has the same executable prefix but a different metadata trailer and is exact-byte unequal. | Runtime identity is useful as a strict check but cannot rank behavioral equivalence by itself; report checked coverage and distinguish executable-code differences from source-metadata differences. | +| 48. Pre-training fixed-gate baseline validation | `run_train_qwen_qlora_full_body_balanced.sh`, `scripts/gate_dataset.py`, `tests/test_gate_dataset.py` | Moved all six baseline requirements ahead of dataset materialization and GPU training when `RUN_GATES=1`; verify dataset path and every row's input/output hashes, decode settings, `bundled_only_v1` policy, nonmissing behavior-only F1, and measured zero prompt truncation. Diagnostic `RUN_GATES=0` and `DRY_RUN=1` remain available. | Missing/old/mismatched gate baselines now fail before GPU time is spent; no full model run or acceptance comparison was attempted because current model/source and results artifacts are absent. | Historical baselines cannot be quietly reused after prompt/metric changes. Regenerate clean, comparable baselines first; otherwise run diagnostic training explicitly without an acceptance claim. | +| 49. Bounded public source-aligned regeneration with true Qwen length gate | Ignored local `data/audit_contracts.db`, `data/audit_hf_dataset.jsonl`, `data/audit_hf_dataset.rejects.jsonl`, manifests `data/audit_manifests/hf_{download,compile,export}_manifest.json`; cached tokenizer `data/audit_qwen_tokenizer/` | `download_hf_contracts.py --limit 2000 --workers 8 --max-body-dupes 1 --max-seq-length 8192 --length-tokenizer data/audit_qwen_tokenizer`; one source-aligned compiler job per prepared contract, using locally available solc versions where possible. No model was trained; generated data is ignored by Git. | Downloaded 2,000 source records, ran 1,957 compile jobs, generated 1,906 pairs (1,704 unique persisted), saw 1,027 unique normalized bodies, capped 677 duplicate-body rows, and **quarantined 618/1,027 unique bodies (60.2%) for actual Qwen context overlength**, leaving 409 exported rows. Rejects: context tokens min 8,144 / median 50,224 / p90 102,814 / max 204,976; target tokens median 61; 475/618 rejects had parameters. 370 compile failures and 2 ambiguous-alignment outcomes were recorded, not silently accepted. | Compiler-source-map witness coverage on 36 small functions does not imply 8k-trainable TAC on real contracts. Investigate long per-function CFG/shared-decoder duplication before scaling training or changing model capacity. | + ## Durable learnings 1. **The overfit sanity check is necessary and should run without train-time eval.** The earlier 10-row run stopped at epoch 9 because trainer eval enabled early stopping and selected an earlier checkpoint. Disabling train-time eval via `TRAIN_EVAL_STRATEGY=no` allowed the corrected 10-row run to reach 30 epochs and memorize exactly. @@ -65,7 +80,7 @@ This file is the persistent research log for TAC-to-Solidity training runs. It r 5. **The old `function_signature_match` metric was invalid.** It only compared whether both strings contained a function keyword. This falsely gave `function_signature_match_mean=1.0` to outputs with wrong names like `setMaxTxAmount`, `setBaseURI`, or `getReward`. The strict signature metric now compares function name, parameter count/types, and return count/types. 6. **The model can memorize but is not yet learning robust TAC-conditioned generalization.** The corrected 10-row and 30-row overfits show exact same-row fit, while 300-row and 100-row heldout results still generate unrelated but plausible Solidity functions. 7. **Heldout sample counts below 30 are smoke signals.** The 100-row run had only 6 heldout test examples, so its heldout metrics should not be overinterpreted. Its train-row eval is still useful for memorization diagnostics. -8. **Selector signatures are a major recoverable context feature.** The local selector registry recovered 100% of the iteration-4 split signatures, and adding `selector_signature=...` to prompt metadata improved a 30-row non-overlap eval from 0 to 19 strict signature matches without retraining. +8. **Superseded as bytecode-only evidence: selector signatures appeared recoverable from the local registry.** That registry also contained signatures compiled from heldout source; the 0-to-19/30 strict-match ablation cannot establish production-reproducible gains. Recheck with bundled-only or independently sourced signatures (learning 40). 9. **Selector metadata does not solve bytecode-grounded semantics by itself.** The retrained selector-metadata model improved broad-30 replication F1 micro to ~0.595 and signatures to 24/30, but call, storage-write, guard, selector, compiler/deployability, and syntax/scaffold buckets still dominate. 10. **Deployability remains an evaluator/model-context issue for function fragments.** Exact or near-exact train-row outputs still score 0 deployability because function-level snippets are compiled without contract state/context. 11. **`solidity_valid` should mean syntax/scaffold validity for function fragments, not full deployability.** Local solc often reports undeclared identifiers or missing modifiers when a correct function body references contract-level state; those are now classified as context-limited compiler results while `bytecode_deployable` remains false. @@ -90,14 +105,29 @@ This file is the persistent research log for TAC-to-Solidity training runs. It r 30. **Raw full-source runs through `run_train_qwen_qlora_500.sh` contaminate fixed gates.** The all-row run included every fixed gate body hash somewhere in train/val/test, with most in train, so a passing gate suite after raw full-source training is diagnostic rather than trustworthy heldout evidence. 31. **Duplicate-heavy raw all-row training did not beat body-balanced full training.** The raw 31,577-row run scored lower than the earlier 16,759-identity body-balanced run on comparable built-in test metrics: F1 0.6281 vs 0.6612, semantic 0.7734 vs 0.7840, and bytecode 0.5083 vs 0.5425. 32. **Do not generate training rows by sweeping multiple compiler versions per source.** The prompt is bytecode-only and compiler metadata is oracle-only, while prior data audits show heavy duplicate bodies; source-aligned single compiler configs avoid reweighting identical Solidity targets and preserve compile budget for unique bodies. -33. **Export-time length filtering is not tokenizer-accurate.** `src/dataset_export_primitives._token_count_estimate` uses `max(whitespace_tokens, chars/4)`, which under-measures TAC text; a fresh export capped at `--max-seq-length 8192` still left 2,326/25,507 train rows over budget under the real Qwen tokenizer (max context 17,452 tokens). Refilter the source with `train._preflight_prompt_parts` plus the actual tokenizer before splitting to keep prompt truncation at zero. +33. **Heuristic export-time length filtering was not tokenizer-accurate; exact filtering is now opt-in.** `_token_count_estimate` under-measured TAC, and a prior export left 2,326/25,507 train rows over the Qwen 8192-token budget. The exporter now uses the training prompt and can count target EOS with `--length-tokenizer` (locally cached); without that flag it records `token_count_method=estimate`. Train-time preflight remains mandatory. 34. **Never run single-process `train.py` with multiple GPUs visible.** Hugging Face Trainer wraps the model in DataParallel, which fails with a device-side `nll_loss` assert (`t >= 0 && t < n_classes`) when unquantized and `CUDA error: an illegal memory access` when quantized, despite labels being verifiably in range. `CUDA_VISIBLE_DEVICES=0` makes the same smoke test pass; multi-GPU must use torchrun/DDP. 35. **A mid-training eval can run on CPU when GPUs are saturated.** With all four RTX 8000s at 42-47 GiB of 48 GiB, a 4-bit 7B eval could not fit without risking an OOM in the live trainer. `CUDA_VISIBLE_DEVICES="" --no-quantization --precision fp32` evaluates on CPU at roughly 28 s/sample for short functions, giving zero-risk directional signal. Staged adapter dirs must also contain the base `config.json`/`tokenizer*` files or preflight aborts with `Unrecognized model in `. 36. **Checkpoint dirs are LoRA adapters only and are not directly evaluable.** `checkpoint-*/` holds `adapter_config.json` + `adapter_model.safetensors` + optimizer/scheduler state, with no tokenizer or base config; copy the base model's `config.json`, `generation_config.json`, `tokenizer.json`, `tokenizer_config.json`, `vocab.json`, and `merges.txt` alongside the adapter before running `--eval-only`. 37. **`train.py` silently falls back to r=16/alpha=32/dropout=0.1 when LoRA flags are omitted.** The fresh-data run `qwen_qlora_full_fresh_20260801-140727` was launched without `--lora-rank/--lora-alpha/--lora-dropout` and trained at half the intended rank with nonzero dropout, unlike every prior successful run (r=32/alpha=64/dropout=0). Always assert LoRA hyperparameters from `training_input_manifest.json` or `adapter_config.json` after launch rather than trusting the command line. -38. **Identical selectors can be genuinely ambiguous, so selector metadata cannot disambiguate every prompt.** `transferFrom(address,address,uint256)` is selector `0x23b872dd` for both ERC20 and ERC721. On that test row the model emitted memorized OpenZeppelin ERC721 boilerplate (`_isApprovedOrOwner`, `tokenId`, no `returns (bool)`) for an ERC20 target, scoring semantic 0.668 / F1 0.583 / bytecode 0.412. This is prior-collapse on the TAC body, not a selector-coverage gap, and refines learning 8: selector metadata is necessary but not sufficient where selectors collide. +38. **Identical selectors can be genuinely ambiguous, so selector metadata cannot disambiguate every prompt.** `transferFrom(address,address,uint256)` is selector `0x23b872dd` for both ERC20 and ERC721. On that test row the model emitted memorized OpenZeppelin ERC721 boilerplate (`_isApprovedOrOwner`, `tokenId`, no `returns (bool)`) for an ERC20 target, scoring semantic 0.668 / F1 0.583 / bytecode 0.412. This is prior-collapse on the TAC body, not a selector-coverage gap. Learning 40 further supersedes the earlier assertion that all locally resolved selector metadata is independently available at inference. 39. **Under-generation and truncation are ruled out for this run.** Prompt tokens were 608-1181 against an 8192 window with truncation count 0, and generated tokens ranged 19-114 with no row reaching the 512 cap. Per learning 33's prior, do not spend effort on context-window or decode-budget work for this configuration. +40. **Production selector metadata must not be populated from evaluation-source registries.** A compiled heldout signature for `0x5312ea8e` entered a test prompt before the fix; only 7/30 broad and 1/6 historical heldout selectors are in the bundled map. `src/model_setup.py` now accepts only bundled signatures, and `train.py` records `selector_signature_prompt_policy=bundled_only_v1`; old ablations/baselines without this marker are incomparable. +41. **A selector does not uniquely determine the target Solidity function.** ERC20 and ERC721 `transferFrom(address,address,uint256)` share `0x23b872dd`; the exporter formerly chose the last conflicting body. `match_functions_by_selector` now quarantines conflicting bodies or runtime entries and records ambiguity instead of silently mislabeling rows. Compiler ABI can narrow return types but cannot prove which body produced the runtime. +42. **Direct splitting and runner gate exclusion must recompute normalized body identities.** A stale/missing `metadata.body_hash` previously let equivalent targets cross a direct `train.py` train/test split and evade fixed-gate exclusions. The splitter now groups by output-derived body hash and invalidates old split cache schema; runners exclude all six fixed gates using recomputed bodies and other leakage keys before selecting rows. +43. **The historical bytecode semantic score is not a behavioral-equivalence test.** `src/training_pipeline.py` primarily compares facts extracted from two Solidity strings. In `latest_results.txt`, `bytecode_semantic_checked_mean=1.0` coexisted with `bytecode_runtime_checked_mean=0.0`; 0% runtime match means *not evaluated*. Unchecked rows can no longer score 1, reference bytecode is not recycled as generated runtime, and reports distinguish the source-fact proxy from actual runtime comparisons. +44. **Aggregate F1 hides behavioral failures.** The July 1,890-row report's all-fact F1 was 0.6281, but restricting counts to calls, control flow, events, guards, member calls, returns, and state writes yielded pre-fix F1 0.2245 (1,732 TP, 5,476 FP, 6,488 FN). New reports display behavior-only *source-fact* micro F1, and the paired gate rejects its regression; its historical value needs recomputation after learning 48's parser fix. +45. **Fixed instruction-count limits can silently drop large-contract functions.** A 60-selector synthetic dispatcher lost its last 10 selectors at the old 256-instruction scan cap; `src/bytecode_analyzer.py` now scans to the dispatcher terminator and includes all 60 in per-function TAC. This is a deterministic analyzer fix, not measured model improvement. +46. **One incorrect CFG jump can remove an entire reachable Solidity behavior.** On a solc 0.8.20 optimized getter, the old nearest-PUSH heuristic targeted consumed PUSH0 instead of the destination: `block_0045` omitted `block_004c` and its RETURN. A bounded stack simulation now resolves the real destination and isolates the sibling function; other cross-block dynamic jumps still need a broader fidelity study. +47. **Never turn failed function segmentation into successful source generation.** An invalid selector target (`0x60003560e01c80631234567814602057`) formerly produced a fake `contract` TAC prompt; shared inference, CLI, and web now report `analysis_failed` with no model call. If a real fallback remains, they retain it but report `partial_analysis`, rejected selector/target diagnostics, and unresolved quality instead of claiming full coverage. +48. **The source-fact extractor can overstate both ABI and call recovery.** `function f(mapping(address => uint256) storage ledger)` vs a bool-mapping target previously scored ABI F1=1.0 because a regex stopped at the inner `)` and split `=>` at `=`. Balanced signatures now distinguish types; `bytes32(...)`/`uint8(...)` casts are not calls, and a bodyless declaration does not inherit the next function's body. Recompute old F1 and hallucination buckets before comparing metrics. +49. **Inference metadata must match the trained prompt, not merely be bytecode-derived.** Contract-wide instruction/function counts were being added to every inference function despite being absent from sampled training rows; `src/inference.py` no longer injects them. These facts remain in contract-level analysis and reconstruction diagnostics. +50. **Runtime bytecode equality needs independently verified compiler provenance and explicit coverage.** `metadata.runtime_comparison` opts in complete named reference/generated contracts with installed matching solc settings; the reference source must reproduce the provided runtime before generated code is compared. Record checked n, equal n, skip reasons, and equality among checked rows; 0 checked does not mean 0% match. Exact bytecode identity is not deployment or behavioral equivalence, and unsupported constructor/inheritance/function-fragment contexts stay unchecked. +51. **Parameterized function bodies were systematically absent despite perfect selector recall.** The 36-case compiler-source-map benchmark initially found 36/36 selectors but only 18/36 reachable behavior opcode witnesses: 0/18 parameterized vs 18/18 simple. Bounded stack-context traversal across ABI decoder returns restored all 18 parameterized witnesses without regressing simple cases. A selector or TAC header alone does not establish behavioral input coverage. +52. **Exact runtime equality is too strict to stand in for semantic equivalence.** In 30 full-contract controls, 30 identical candidates were byte-equal and 30 return/storage mutations byte-unequal, but a comment-only source change also produced byte-unequal runtime due to Solidity metadata despite an identical executable prefix. Preserve the exact-byte label and seek independent differential/semantic controls. +53. **Full-run gates must reject stale baselines before expensive training.** The full body-balanced runner now validates all six fixed-gate baseline JSONs for row identities, settings, prompt policy, behavior-only metric and zero truncation before training, rather than failing hours later. `RUN_GATES=0` is a diagnostic override, not an acceptance decision. +54. **Actual-token length is the immediate real-data bottleneck after CFG reachability repair.** In a bounded 2,000-source regeneration, 618/1,027 unique-body candidates (60.2%) exceeded the real Qwen 8,192-token prompt+target cap; rejected prompt median was 50,224 tokens despite target median only 61. Long samples can contain 2,417 TAC instruction lines at the median; 475/618 rejects are parameterized. Do not treat 409 surviving short rows as representative heldout decompilation. ## Hypotheses @@ -106,7 +136,7 @@ This file is the persistent research log for TAC-to-Solidity training runs. It r 3. **Medium-high confidence: current data scale/coverage is insufficient for heldout generalization.** Evidence: iteration 4 achieved strong train memorization on many examples but 0/6 strict signature matches on heldout rows, and iteration 5 showed the same pipeline can memorize 30/30 identical train/test rows. 4. **Medium confidence: target/source quality or prompt metadata may be too weak for function identity.** Evidence: the model often generates plausible ABI-compatible Solidity unrelated to the TAC, suggesting it is relying on memorized priors rather than grounding in TAC details. 5. **Medium confidence: more updates help train fit, but alone will overfit without solving generalization.** Evidence: 10 epochs on 83 train rows drove training loss down to ~0.474 overall and final logged losses near 0.01, yet heldout remained poor. -6. **High confidence: selector-signature metadata should stay enabled for QLoRA runs.** Evidence: the broad-30 paired ablation improved semantic similarity by +0.3766 and replication F1 micro by +0.3380 on the same model and rows. +6. **Superseded pending a clean rerun: selector-signature metadata should stay enabled for QLoRA runs.** The broad-30 paired ablation improved semantic similarity by +0.3766 and replication F1 micro by +0.3380, but compiled-source registry access could provide heldout signatures unavailable at deployment. Test bundled-only metadata against selector-off before concluding it helps. 7. **Medium confidence: selector-metadata retraining helps broad generalization modestly, but the current 100-row scale is still too small.** Evidence: broad-30 old-adapter selector-on F1 was ~0.5524, while the selector-metadata retrain reached ~0.5954; exact correctness remained 0/30. 8. **Medium-high confidence: decoding was slightly over-penalizing repetition at 1.15.** Evidence: lowering eval repetition penalty to 1.05 improved broad-30 semantic/edit/F1/bytecode and validity without changing model weights. 9. **High confidence: calls are the first curriculum bottleneck.** Evidence: the calls slice has 43 missing call facts and replication F1 micro 0.5422 under the best decode setting. @@ -126,6 +156,13 @@ This file is the persistent research log for TAC-to-Solidity training runs. It r 23. **Medium-high confidence: duplicate-heavy raw scale dilutes useful TAC-conditioned signal compared with body balancing.** Evidence: the raw run used all 31,577 rows and 6,431 steps but underperformed the 16,759-row body-balanced run on the built-in heldout test while still failing behavior targets. 24. **High confidence: the raw full-source gate pass is contamination, not clean generalization.** Evidence: all fixed gate body hashes overlapped the raw split, and heavily overlapped slices such as holdout64 achieved very high scores after 63/64 identities landed in train. 25. **High confidence: compiler-version augmentation is lower value than unique-body coverage.** Evidence: the source export had 14,818 duplicate-body rows before body balancing, and the best current full-run path already caps by `metadata.body_hash`; multi-version rows mostly add alternate TAC for the same target body rather than new semantic supervision. +26. **Medium-high confidence: evaluation-source selector lookup inflated at least some previous identity scores.** A regression reproduces a heldout compiled-source signature entering the prompt; only 7/30 broad selectors are bundled. The size of its impact remains unknown without a clean paired rerun. +27. **Medium confidence: clean body-balanced training may still improve source-fact behavior, but prior gates cannot decide acceptance.** Fixed-gate overlap and changed prompt policy invalidate previous gate passes; an eval-clean run with fresh paired baselines is required. The August 15-row CPU smoke was first-N with duplicate targets. +28. **High confidence: improved TAC reachability is a better near-term input-conditioning experiment than increasing context length.** The compiled getter lacked a reachable RETURN while prompt truncation remained zero in historical evals; quantify missing returns/calls/guards across compiler-produced slices before another training scale-up. +29. **Medium confidence: exact runtime checks on a well-selected full-contract slice can calibrate the source-fact proxy.** The opt-in harness differentiates one identical vs one changed-return complete contract, but current function-fragment datasets cannot supply comparable full-context rows without additional ground truth. +30. **High confidence: stack-aware decoder-return tracking improves the available TAC signal on parameterized behavior families.** On one local solc/compiler/optimizer configuration, all 18 parameterized source-map witnesses changed from unreachable to reached; test other compiler versions, loops, and complex contracts before assuming generalized 100% coverage. +31. **High confidence: strict runtime byte equality has metadata-induced false negatives as a semantic proxy.** The comment-only control changes full runtime bytes while leaving executable prefix unchanged. Recompilation with matched source metadata helps distinguish identity from behavior, but cannot replace differential execution. +32. **High confidence: most current unique-body function rows are too long at 8k without loss-aware TAC reduction.** The bounded regenerated corpus's exact tokenizer quarantined 60.2% for context overlength, whereas no target was quarantined for overlength. Unknown whether per-function shared-code inclusion, compiler source diversity, or legitimate complexity dominates; measure block overlap and behavior-witness retention before changing context length. ## Invalidated assumptions @@ -142,6 +179,17 @@ This file is the persistent research log for TAC-to-Solidity training runs. It r 11. **Invalidated: "The default full body-balanced gate suite was cleanly heldout."** The full body-balanced split was built from the same source as the fixed gates and included many gate body hashes; the runner now excludes those identities by default. 12. **Invalidated: "Using all source rows through `run_train_qwen_qlora_500.sh` makes fixed gates trustworthy."** The raw all-row split overlapped 100% of the fixed gate identities, so fixed-gate improvements from that run cannot be treated as heldout evidence. 13. **Invalidated: "Three compiler versions per training point are useful default augmentation."** Current evidence favors source-aligned, body-deduplicated coverage; the generator now treats multi-version expansion as removed rather than a default or recommended path. +14. **Invalidated: "A locally resolved selector is independently available from deployed bytecode."** `contracts.db` contained compiled evaluation-source signatures. Production prompts now admit only bundled signatures, and historical selector-assisted evals need reruns. +15. **Invalidated: "`bytecode_semantic_checked_mean=1` means runtime behavior was compared."** The July report had 100% legacy checked but 0% generated/reference runtime comparisons; the score is primarily source-fact overlap. +16. **Invalidated: "A passing all-fact F1 indicates guards/calls/storage were recovered."** The same July report had all-fact F1 0.6281 but behavior-only source-fact F1 0.2245; inspect category slices and actual runtime coverage. +17. **Invalidated: "All gate exclusions and cached splits were already evaluation-clean."** Missing/stale body hashes and raw sampler cache reuse could leak normalized-equivalent rows; runners and direct splitter now recompute identities and validate provenance. +18. **Invalidated: "A parsed selector necessarily yields recoverable function TAC."** Invalid targets previously fell back to the entire contract and looked model-generated; now missing selectors are rejected and surfaced even if a fallback is valid. +19. **Invalidated: "Bytecode-derived prompt counts are always helpful to add at inference."** Training lacked the contract-wide instruction/function count fields; inference now matches training rather than presenting an untrained distribution. +20. **Invalidated: "Source-extracted call/ABI buckets are reliable for nested types and casts."** The old regex gave a wrong mapping type ABI F1=1 and treated `bytes32(...)` as a call; the corrected extractor requires regenerated baselines. +21. **Invalidated: "Finding all ABI selectors means function TAC contains their behavior."** Before decoder-return repair, 36/36 selectors existed but the 18 parameterized bodies lost compiler-mapped opcodes. +22. **Invalidated: "Exact runtime byte mismatch proves a behavioral difference."** One comment-only source change yielded unequal bytecode because solc appended different source metadata; the executable prefix was unchanged. +23. **Invalidated: "Missing gate baselines can be diagnosed after a full training run."** Post-training rejection wastes GPU time and cannot rescue a contaminated acceptance gate; the runner now checks all baselines first. +24. **Invalidated: "The small synthetic 36/36 TAC benchmark implies real examples fit training context."** On 2,000 independently sourced records, 618 of 1,027 unique-body candidates exceeded 8,192 Qwen tokens despite 36/36 small-function witness coverage. Coverage and trainability must be reported separately. ## Suggestions and next experiments @@ -168,9 +216,21 @@ This file is the persistent research log for TAC-to-Solidity training runs. It r 21. **Train/evaluate the fallback balanced300 curriculum before any 1k random run.** Candidate source: `data/curriculum/fallback_balanced300_iter32.jsonl`. Success criterion: broad30 F1 > 0.6049, holdout64 bytecode/validity do not regress, large192 validity >= 0.9219, and pure-negative unsupported calls/state writes do not increase. 22. **Rerun the full body-balanced experiment with gate exclusions.** The first full run was contaminated by gate overlap, and the patched runner now excludes broad/calls/state/holdout64/pure-negative/large192 body identities before body balancing. Success criterion: the eval-clean full run improves true-heldout F1/bytecode/semantic without scaffold-valid regressions and without increasing pure-negative hallucinations. 23. **Checkpoint-3000 is rejected, but it narrows the failure mode.** It preserves validity on broad/calls/state and improves pure-negative64, but holdout64 and large192 still have real scaffold runaways. The 1024-token sweep only extended the same malformed rows to the larger cap. Success criterion to reopen checkpoint-3000: a data or stopping change must eliminate holdout/large malformed cap rows without reducing behavior metrics. -24. **Do not use the raw all-row sampled runner as an acceptance path unless gate exclusions are added.** If `run_train_qwen_qlora_500.sh` is reused for full-source training, success must come from its generated heldout test plus an external clean gate suite, not the existing fixed gates that are embedded in the source. Success criterion: no fixed gate body hashes in train/val/test, or use a separately heldout eval source. -25. **Prefer body-balanced/eval-clean full runs over raw duplicate-heavy full runs.** The next clean full experiment should either resume/rerun `run_train_qwen_qlora_full_body_balanced.sh` with exclusions or add equivalent body-hash capping/exclusion support to the sampled runner. Success criterion: beat raw full test F1 0.6281 and body-balanced test F1 0.6612 while keeping Solidity validity >= 0.9619 and reducing hallucination rate below 35.37%. +24. **The raw all-row sampled runner now excludes fixed gates, so do not request every source row.** `run_train_qwen_qlora_500.sh` excludes all six fixed gates before sampling and fails when `SAMPLE_COUNT` exceeds the available rows. Success criterion: zero overlap for every train/val/test row using output-derived identities, and fresh comparable, bundled-only gate baselines. +25. **Prefer body-balanced/eval-clean full runs over raw duplicate-heavy full runs, but supersede the old numerical keep thresholds.** Both runners now exclude fixed gates; raw full-test F1 0.6281, body-balanced F1 0.6612, and the old validity/hallucination targets were measured before the prompt-policy and metric corrections. Establish fresh paired, clean baselines and require behavior-only F1, full-context validity, and negative-hallucination improvements without regression. 26. **When regenerating `data/hf_training_dataset.jsonl`, use source-aligned single-compiler generation.** Success criterion: compile manifests show `compiler_config_policy=single_source_aligned`, compile jobs are about one per prepared contract, and exported duplicate-body pressure drops without reducing behavior coverage. +27. **Priority: regenerate clean baselines under `bundled_only_v1`.** Evaluate the same fixed gate rows with old and proposed adapters using identical prompts, selector policy, decoding (512 tokens, repetition penalty 1.05), and artifacts. Success criterion: zero shared source/body/input identities with training, at least 30 distinct body identities on primary heldout (prefer large192), and comparable rows/settings before any keep decision. +28. **Ablate only independently available selector context.** On 30+ non-overlap rows compare bundled-only selector-on vs selector-off with the *same* adapter and settings; count actual bundled signature coverage. Success criterion: strict ABI signatures and behavior-only source-fact F1 improve on comparable rows without increased hallucination; otherwise retire the old selector-augmentation priority. +29. **Rebuild data with exact local tokenizer checks and ambiguity quarantine.** Use `download_hf_contracts.py --export-only --length-tokenizer Qwen/Qwen2.5-Coder-7B-Instruct` after caching that tokenizer; inspect manifests for `token_count_method=tokenizer` and `ambiguous_selector_alignment`. Success criterion: all exported train/val/test rows pass 8192-token preflight including EOS, with no ambiguous source-body labels. +30. **Add independently checked runtime/differential behavior evaluation.** Compile outputs with matched contract context/compiler and compare generated/reference runtime or execute representative calls, state writes, guards, returns, and events on at least 30 unambiguous, fully compilable heldout rows. Success criterion: nonzero `bytecode_runtime_checked_mean` with the match rate *among checked rows* reported separately; do not set an all-row equivalence target when most fragments lack context. +31. **Keep the accepted overfit sanity control, but do not use it to validate generalization.** Train/evaluate exactly the same 1-5 rows after rebuilding prompts and labels, then use paired clean 30+ heldout slices and 128-256 diverse large-gate rows for acceptance. Success criterion: near-exact same-row signatures/F1 plus rising behavior-only source-fact F1 on untouched heldout rows without compiler/AST regressions. +32. **Benchmark analyzer fidelity before retraining.** Compile at least 30 diverse getters/setters/calls/guards/events/storage-heavy contracts under source-aligned solc settings; assert expected selectors, valid entry blocks, reachable RETURN/LOG/SSTORE/CALL blocks, and no unsupported whole-contract-as-function prompts. Success criterion: zero silently rejected selectors and measured per-slice TAC/block coverage, with failures isolated by compiler/optimizer setting. +33. **Create a full-context runtime-equality control slice.** Store independent reference runtime and exact compilation settings for at least 30 simple fully compilable contracts, run identical-source and deliberately changed-return/state-write controls, then sample generated complete contracts. Success criterion: all controls are classified as checked/equal or checked/unequal appropriately, every other row reports why it is unchecked, and no fragment is labeled runtime-verified. +34. **Regenerate every quality baseline affected by fact extraction and prompt changes.** The July F1=0.6281, behavior-only F1=0.2245, call hallucinations=2,193, and earlier selector ablations were computed with now-superseded logic or prompts. Success criterion: same clean heldout rows/settings under the current extractor and `bundled_only_v1`, no gate overlap, plus a separate runtime-checked subset before selecting a larger QLoRA run. +35. **Extend the TAC witness benchmark to independently sourced contracts.** Current 36/36 structural coverage is six local families compiled together with solc 0.8.20, optimizer enabled and Shanghai. Success criterion: at least 30 distinct real, eval-clean contract identities per compiler/optimizer/behavior slice; report selector presence, source-map witness reachability, rendered TAC, bounded-traversal truncation, and sample failures separately. +36. **Add execution-level or metadata-normalized controls before promoting runtime identity to a model gate.** Keep 30/30 identical and 30/30 behavioral mutations correctly distinguished while recognizing the comment-only runtime identity false negative. Success criterion: executable-prefix or differential-call checks classify comment-only as same observable behavior without excusing altered returns or storage, with fully matched deployment state/compiler settings. +37. **Regenerate model/data artifacts before any claimed quality gain (partially addressed).** A 409-row exact-token audit export and Qwen tokenizer now exist only as ignored `data/audit_*` assets, but the pretrained 7B weights, accepted clean full dataset, model results, and six refreshed gate baselines are absent. Success criterion: first fix the 60.2% real-data overlength loss, then run a 1-5-row overfit sanity control plus 30+ nonoverlap and large192 paired heldout evals with behavior-only F1, hallucination, AST/bytecode validity, zero truncation and a runtime-checked subset. +38. **Measure and reduce true-token TAC blowup without erasing bytecode behavior.** On the ignored 2,000-source audit export, compare `function_pairs` TAC length to reachable CFG block counts, function count, shared decoder/helper block overlap and compiler source-map witnesses for both 409 kept and 618 quarantined unique bodies. Success criterion: materially lower overlength loss (aim <10% on the comparable short-target slice under 8k) while preserving selector and behavior-opcode witness coverage and maintaining zero prompt truncation; do not simply clip TAC or raise decode tokens. ## Artifact map @@ -235,3 +295,19 @@ This file is the persistent research log for TAC-to-Solidity training runs. It r - Strict signature metric implementation: `src/training_pipeline.py` - Selector prompt metadata implementation: `src/model_setup.py` - Evaluation metric tests: `tests/test_evaluation_quality_metrics.py` +- 2026-09-24 audit code: `src/dataset_export_primitives.py`, `src/model_setup.py`, `train.py`, `src/training_pipeline.py`, `src/bytecode_analyzer.py`, `scripts/gate_dataset.py`, `scripts/compare_eval_runs.py`, `src/evaluation_report.py` +- 2026-09-24 audit tests: `tests/test_data_generation_export_issues.py`, `tests/test_dataset_pipeline.py`, `tests/test_model_training_issues.py`, `tests/test_train_cli_issues.py`, `tests/test_prompt_metadata.py`, `tests/test_bytecode_analyzer.py`, `tests/test_compare_eval_runs.py`, `tests/test_eval_gate_suite.py`, `tests/test_evaluation_quality_metrics.py`, `tests/test_evaluation_report.py` +- 2026-09-24 model-free fidelity tests: optimized getter and invalid dispatcher fixtures in `tests/test_bytecode_analyzer.py`; train/inference parity and partial-coverage checks in `tests/test_inference.py`, `tests/test_decompile_cli.py`, `tests/test_web_app.py`; full-contract runtime controls in `tests/test_evaluation_quality_metrics.py`; nested ABI/cast regression in `tests/test_replication_metrics.py` +- 36-case compiler-source-map benchmark: `scripts/benchmark_tac_fidelity.py`, `tests/test_benchmark_tac_fidelity.py` +- 30-contract/121-control runtime identity benchmark: `scripts/benchmark_runtime_controls.py`, `tests/test_benchmark_runtime_controls.py` +- 2,000-record bounded exact-token generation audit (local ignored assets): `data/audit_manifests/hf_download_manifest.json`, `data/audit_manifests/hf_compile_manifest.json`, `data/audit_manifests/hf_export_manifest.json`; 409 accepted `data/audit_hf_dataset.jsonl`, 618 length rejects `data/audit_hf_dataset.rejects.jsonl` + +## Path reflection (2026-09-24) + +The previous recommendations to increase context length, relax decoding, or scale random rows remain low priority: July prompt truncation was 0/1,890, the August 15-row smoke did not hit its 512-token cap, and prior random curricula failed behavior gates. The eval-clean body-balanced path remains the best *experiment*, but its old selector-assisted baselines and fixed-gate results cannot be compared to the bundled-only prompt policy; refresh split lineage, dedup identities, tokenizer counts, and baselines before spending GPU time. Selector metadata is not retired entirely: restrict it to bundled or independently collected signatures, measure coverage by slice, and retain it only if a clean paired ablation improves strict signatures and behavior-only F1. No model-quality claim changes from this code-only audit; the final status of the August long-running experiment is unknown because its model/results are absent in this worktree. + +The follow-up model-free checkpoint promotes **analyzer and evaluator control slices** ahead of more decoding sweeps: a real optimized getter lost RETURN before the CFG fix, and a wrong mapping type scored ABI F1=1 before the parser fix. The guarded full-contract runtime check is a narrow measurement path worth expanding only with independently compiled references; do not treat exact bytecode inequality as proof of behavioral mismatch. An eval-clean full run remains worth pursuing after these controls and fresh baselines, not as an acceptance claim from the old reports. + +The 36-case benchmark **promotes parameter-decoder TAC fidelity** from a conjectured bottleneck to a measured 18/36 -> 36/36 structural gain on one compiler setting; this is the first code-level input-conditioning change to prioritize when models/data are restored. The runtime comment-only negative control **demotes exact byte identity as a stand-alone keep gate**: its false negative reflects metadata, not changed executable prefix. Continue clean model training only after baselines pass fail-fast provenance/truncation checks. No model-quality or world-leading claim is warranted without restored model weights, dataset, and independent heldout/differential evidence. + +The bounded 2,000-source regeneration **changes the immediate priority again**: actual-token length is now a measured 60.2% unique-body loss, not a speculation about prompt truncation on the short rows that survived earlier exports. Keep earlier zero-truncation findings scoped to *evaluated survivors*. First audit whether shared decoder/internal function blocks swell real per-function TAC and implement loss-aware compression or a longer-context training design before spending GPU time. The small cached tokenizer and 409 audit rows do not replace a representative accepted training set or establish model gains. diff --git a/run_eval_gate_suite_for_model.sh b/run_eval_gate_suite_for_model.sh index 5ba60f8..cfe6229 100755 --- a/run_eval_gate_suite_for_model.sh +++ b/run_eval_gate_suite_for_model.sh @@ -13,6 +13,8 @@ if [[ ! -d "${MODEL_PATH}" ]]; then echo "MODEL_PATH does not exist or is not a directory: ${MODEL_PATH}" >&2 exit 1 fi +MODEL_PATH="$(realpath "${MODEL_PATH}")" +cd "${SCRIPT_DIR}" LABEL="${LABEL:-$(basename "${MODEL_PATH}")}" NUM_GPUS="${NUM_GPUS:-4}" @@ -21,7 +23,6 @@ EVAL_MAX_NEW_TOKENS="${EVAL_MAX_NEW_TOKENS:-512}" EVAL_REPETITION_PENALTY="${EVAL_REPETITION_PENALTY:-1.05}" GATE_DIR="${GATE_DIR:-${SCRIPT_DIR}/models/eval_gate_${LABEL}_$(date +%Y%m%d-%H%M%S)}" -TRAIN_DATASET="${TRAIN_DATASET:-${SCRIPT_DIR}/data/qwen_qlora_full_body_balanced_20260628-162859/splits/train_dataset.jsonl}" BROAD_DATASET="${BROAD_DATASET:-${SCRIPT_DIR}/data/loop_iter10_selector_prompt_broad_eval/broad30_excluding_iter4.jsonl}" CALLS_DATASET="${CALLS_DATASET:-${SCRIPT_DIR}/data/eval_failure_slices/broad30_reppenalty_1_05_fixed_extractor/calls.jsonl}" STATE_DATASET="${STATE_DATASET:-${SCRIPT_DIR}/data/eval_failure_slices/broad30_reppenalty_1_05_fixed_extractor/state_writes.jsonl}" @@ -37,7 +38,7 @@ PURE_NEGATIVE_BASELINE="${PURE_NEGATIVE_BASELINE:-${SCRIPT_DIR}/results/eval_178 LARGE192_BASELINE="${LARGE192_BASELINE:-${SCRIPT_DIR}/results/eval_1782687160.json}" for required in \ - "${TRAIN_DATASET}" "${BROAD_DATASET}" "${CALLS_DATASET}" "${STATE_DATASET}" \ + "${BROAD_DATASET}" "${CALLS_DATASET}" "${STATE_DATASET}" \ "${HOLDOUT64_DATASET}" "${PURE_NEGATIVE_DATASET}" "${LARGE192_DATASET}" \ "${BROAD_BASELINE}" "${CALLS_BASELINE}" "${STATE_BASELINE}" \ "${HOLDOUT64_BASELINE}" "${PURE_NEGATIVE_BASELINE}" "${LARGE192_BASELINE}"; do @@ -47,6 +48,14 @@ for required in \ fi done +FIXED_EVAL_DATASETS="${BROAD_DATASET}:${CALLS_DATASET}:${STATE_DATASET}:${HOLDOUT64_DATASET}:${PURE_NEGATIVE_DATASET}:${LARGE192_DATASET}" +VERIFIED_TRAIN_DATASET="$(python -m scripts.gate_dataset verify-model "${MODEL_PATH}" "${FIXED_EVAL_DATASETS}")" +if [[ -n "${TRAIN_DATASET:-}" && "$(realpath "${TRAIN_DATASET}")" != "$(realpath "${VERIFIED_TRAIN_DATASET}")" ]]; then + echo "TRAIN_DATASET does not match verified model training input: ${VERIFIED_TRAIN_DATASET}" >&2 + exit 1 +fi +TRAIN_DATASET="${VERIFIED_TRAIN_DATASET}" + mkdir -p "${GATE_DIR}" EVAL_MAP="${GATE_DIR}/eval_paths.tsv" : > "${EVAL_MAP}" @@ -75,6 +84,8 @@ run_eval() { "$@" local eval_json eval_json="$(newest_eval_json)" + python -m scripts.gate_dataset verify-eval "${eval_json}" "${MODEL_PATH}" "${dataset_path}" \ + --max-new-tokens "${EVAL_MAX_NEW_TOKENS}" --repetition-penalty "${EVAL_REPETITION_PENALTY}" printf '%s\t%s\t%s\t%s\n' "${label}" "${MODEL_PATH}" "${dataset_path}" "${eval_json}" | tee -a "${EVAL_MAP}" } diff --git a/run_train_qwen_qlora_500.sh b/run_train_qwen_qlora_500.sh index 4fc6cba..693fd1f 100755 --- a/run_train_qwen_qlora_500.sh +++ b/run_train_qwen_qlora_500.sh @@ -14,6 +14,9 @@ # LORA_RANK=32 LORA_ALPHA=64 LORA_DROPOUT=0 TRAIN_EVAL_STRATEGY=no ./run_train_qwen_qlora_500.sh # RESUME=/path/to/checkpoint ./run_train_qwen_qlora_500.sh # DRY_RUN=1 ./run_train_qwen_qlora_500.sh +# +# Fixed eval slices are excluded before sampling, including aliases of their +# body hashes. SAMPLE_COUNT must fit the remaining eval-clean source. set -euo pipefail @@ -54,6 +57,15 @@ GRADIENT_CHECKPOINTING="${GRADIENT_CHECKPOINTING:-true}" RECREATE_DATASET="${RECREATE_DATASET:-false}" DRY_RUN="${DRY_RUN:-false}" +BROAD_DATASET="${BROAD_DATASET:-${SCRIPT_DIR}/data/loop_iter10_selector_prompt_broad_eval/broad30_excluding_iter4.jsonl}" +CALLS_DATASET="${CALLS_DATASET:-${SCRIPT_DIR}/data/eval_failure_slices/broad30_reppenalty_1_05_fixed_extractor/calls.jsonl}" +STATE_DATASET="${STATE_DATASET:-${SCRIPT_DIR}/data/eval_failure_slices/broad30_reppenalty_1_05_fixed_extractor/state_writes.jsonl}" +HOLDOUT64_DATASET="${HOLDOUT64_DATASET:-${SCRIPT_DIR}/data/curriculum_eval/calls_state64_holdout_nonoverlap.jsonl}" +PURE_NEGATIVE_DATASET="${PURE_NEGATIVE_DATASET:-${SCRIPT_DIR}/data/curriculum_negative/no_calls_no_state_simple64_nonoverlap.jsonl}" +LARGE192_DATASET="${LARGE192_DATASET:-${SCRIPT_DIR}/data/curriculum_eval/large_stratified192_nonoverlap_iter32.jsonl}" +FIXED_EVAL_DATASETS="${BROAD_DATASET}:${CALLS_DATASET}:${STATE_DATASET}:${HOLDOUT64_DATASET}:${PURE_NEGATIVE_DATASET}:${LARGE192_DATASET}" +EVAL_EXCLUDE_DATASETS="${FIXED_EVAL_DATASETS}${EVAL_EXCLUDE_DATASETS:+:${EVAL_EXCLUDE_DATASETS}}" + if [[ ! -f "${SOURCE_DATASET}" ]]; then echo "Source dataset not found: ${SOURCE_DATASET}" >&2 echo "Set SOURCE_DATASET=/path/to/dataset.jsonl or generate data first." >&2 @@ -67,42 +79,12 @@ fi mkdir -p "${DATA_DIR}" "${OUTPUT_DIR}" -if [[ ! -s "${SAMPLED_DATASET}" || "${RECREATE_DATASET}" == "true" || "${RECREATE_DATASET}" == "1" ]]; then - echo "Sampling ${SAMPLE_COUNT} rows from ${SOURCE_DATASET} -> ${SAMPLED_DATASET}" - uv run python - "${SOURCE_DATASET}" "${SAMPLED_DATASET}" "${SAMPLE_COUNT}" "${SEED}" <<'PY' -import json -import random -import sys -from pathlib import Path - -source = Path(sys.argv[1]) -target = Path(sys.argv[2]) -sample_count = int(sys.argv[3]) -seed = int(sys.argv[4]) - -rows = [] -with source.open("r", encoding="utf-8") as handle: - for line in handle: - line = line.strip() - if line: - rows.append(line) - -if len(rows) < sample_count: - raise SystemExit(f"{source} only has {len(rows)} non-empty rows; need {sample_count}") - -rng = random.Random(seed) -indices = list(range(len(rows))) -rng.shuffle(indices) - -target.parent.mkdir(parents=True, exist_ok=True) -with target.open("w", encoding="utf-8") as handle: - for index in indices[:sample_count]: - json.loads(rows[index]) - handle.write(rows[index] + "\n") -PY -else - echo "Using existing sampled dataset: ${SAMPLED_DATASET}" +SAMPLE_ARGS=() +if [[ "${RECREATE_DATASET}" == "true" || "${RECREATE_DATASET}" == "1" ]]; then + SAMPLE_ARGS+=(--recreate) fi +python -m scripts.gate_dataset sample "${SOURCE_DATASET}" "${SAMPLED_DATASET}" "${EVAL_EXCLUDE_DATASETS}" \ + --count "${SAMPLE_COUNT}" --seed "${SEED}" "${SAMPLE_ARGS[@]}" if [[ "${GRADIENT_CHECKPOINTING}" == "false" || "${GRADIENT_CHECKPOINTING}" == "0" ]]; then GRADIENT_CHECKPOINTING_ARG="--no-gradient-checkpointing" @@ -175,6 +157,7 @@ fi echo "=== Basic Qwen QLoRA training ===" echo " Sampled dataset: ${SAMPLED_DATASET}" echo " Sample count: ${SAMPLE_COUNT}" +echo " Fixed eval exclusions: ${EVAL_EXCLUDE_DATASETS}" echo " Epochs: ${EPOCHS}" echo " Model: ${MODEL}" echo " QLoRA 4-bit: enabled" diff --git a/run_train_qwen_qlora_full_body_balanced.sh b/run_train_qwen_qlora_full_body_balanced.sh index 5bc6ef7..ccd59ca 100755 --- a/run_train_qwen_qlora_full_body_balanced.sh +++ b/run_train_qwen_qlora_full_body_balanced.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash # Full-data, body-balanced Qwen2.5-Coder QLoRA training run. # -# This runner first materializes a dataset capped by metadata.body_hash so +# This runner first materializes a dataset capped by normalized Solidity body so # optimizer/compiler variants of the same Solidity body do not dominate the -# training signal. By default it keeps one row per body_hash, trains for one +# training signal. By default it keeps one row per body, trains for one # epoch, then evaluates the resulting adapter against the current gate suite. # # Common overrides: @@ -11,7 +11,7 @@ # RUN_ID=full_body_balanced_v1 ./run_train_qwen_qlora_full_body_balanced.sh # CAP_PER_BODY=2 EPOCHS=1 ./run_train_qwen_qlora_full_body_balanced.sh # MAX_ROWS=5000 ./run_train_qwen_qlora_full_body_balanced.sh -# EVAL_EXCLUDE_DATASETS= ./run_train_qwen_qlora_full_body_balanced.sh +# EVAL_EXCLUDE_DATASETS=other_gate.jsonl ./run_train_qwen_qlora_full_body_balanced.sh # RUN_GATES=0 ./run_train_qwen_qlora_full_body_balanced.sh set -euo pipefail @@ -56,7 +56,6 @@ DRY_RUN="${DRY_RUN:-false}" RUN_GATES="${RUN_GATES:-true}" GATE_DIR="${GATE_DIR:-${OUTPUT_DIR}/eval_gates}" -BASELINE_MODEL="${BASELINE_MODEL:-${SCRIPT_DIR}/models/qwen2_5_coder_7b_qlora_500_loop_iter12_selector_metadata_100/final_model}" BROAD_DATASET="${BROAD_DATASET:-${SCRIPT_DIR}/data/loop_iter10_selector_prompt_broad_eval/broad30_excluding_iter4.jsonl}" CALLS_DATASET="${CALLS_DATASET:-${SCRIPT_DIR}/data/eval_failure_slices/broad30_reppenalty_1_05_fixed_extractor/calls.jsonl}" @@ -65,7 +64,7 @@ HOLDOUT64_DATASET="${HOLDOUT64_DATASET:-${SCRIPT_DIR}/data/curriculum_eval/calls PURE_NEGATIVE_DATASET="${PURE_NEGATIVE_DATASET:-${SCRIPT_DIR}/data/curriculum_negative/no_calls_no_state_simple64_nonoverlap.jsonl}" LARGE192_DATASET="${LARGE192_DATASET:-${SCRIPT_DIR}/data/curriculum_eval/large_stratified192_nonoverlap_iter32.jsonl}" DEFAULT_EVAL_EXCLUDE_DATASETS="${BROAD_DATASET}:${CALLS_DATASET}:${STATE_DATASET}:${HOLDOUT64_DATASET}:${PURE_NEGATIVE_DATASET}:${LARGE192_DATASET}" -EVAL_EXCLUDE_DATASETS="${EVAL_EXCLUDE_DATASETS:-${DEFAULT_EVAL_EXCLUDE_DATASETS}}" +EVAL_EXCLUDE_DATASETS="${DEFAULT_EVAL_EXCLUDE_DATASETS}${EVAL_EXCLUDE_DATASETS:+:${EVAL_EXCLUDE_DATASETS}}" BROAD_BASELINE="${BROAD_BASELINE:-${SCRIPT_DIR}/results/eval_1782624189.json}" CALLS_BASELINE="${CALLS_BASELINE:-${SCRIPT_DIR}/results/eval_1782624247.json}" @@ -96,9 +95,40 @@ if [[ -n "${EVAL_EXCLUDE_DATASETS}" ]]; then done fi +require_baseline() { + local label="$1" + local path="$2" + local dataset="$3" + if [[ ! -f "${path}" ]]; then + echo "Required ${label} baseline eval not found: ${path}; regenerate under bundled_only_v1 before training." >&2 + exit 1 + fi + python -m scripts.gate_dataset verify-baseline "${path}" "${dataset}" \ + --max-new-tokens "${EVAL_MAX_NEW_TOKENS}" \ + --repetition-penalty "${EVAL_REPETITION_PENALTY}" +} + +if [[ "${RUN_GATES}" != "false" && "${RUN_GATES}" != "0" && + "${DRY_RUN}" != "true" && "${DRY_RUN}" != "1" ]]; then + require_baseline "broad30" "${BROAD_BASELINE}" "${BROAD_DATASET}" + require_baseline "calls23" "${CALLS_BASELINE}" "${CALLS_DATASET}" + require_baseline "state17" "${STATE_BASELINE}" "${STATE_DATASET}" + require_baseline "holdout64" "${HOLDOUT64_BASELINE}" "${HOLDOUT64_DATASET}" + require_baseline "pure_negative64" "${PURE_NEGATIVE_BASELINE}" "${PURE_NEGATIVE_DATASET}" + require_baseline "large192" "${LARGE192_BASELINE}" "${LARGE192_DATASET}" +fi + mkdir -p "${DATA_DIR}" "${OUTPUT_DIR}" -if [[ ! -s "${BALANCED_DATASET}" || "${RECREATE_DATASET}" == "true" || "${RECREATE_DATASET}" == "1" ]]; then +if [[ -e "${BALANCED_DATASET}" && "${RECREATE_DATASET}" != "true" && "${RECREATE_DATASET}" != "1" ]]; then + CACHE_ARGS=(--manifest "${BALANCED_MANIFEST}" --cap-per-body "${CAP_PER_BODY}" --seed "${SEED}") + if [[ -n "${MAX_ROWS}" ]]; then + CACHE_ARGS+=(--max-rows "${MAX_ROWS}") + fi + python -m scripts.gate_dataset verify-balanced-cache \ + "${SOURCE_DATASET}" "${BALANCED_DATASET}" "${EVAL_EXCLUDE_DATASETS}" "${CACHE_ARGS[@]}" + echo "Using verified body-balanced dataset: ${BALANCED_DATASET}" +else echo "Building body-balanced dataset from ${SOURCE_DATASET}" echo " cap per body_hash: ${CAP_PER_BODY}" if [[ -n "${MAX_ROWS}" ]]; then @@ -107,7 +137,7 @@ if [[ ! -s "${BALANCED_DATASET}" || "${RECREATE_DATASET}" == "true" || "${RECREA if [[ -n "${EVAL_EXCLUDE_DATASETS}" ]]; then echo " excluding eval datasets: ${EVAL_EXCLUDE_DATASETS}" fi - uv run python - "${SOURCE_DATASET}" "${BALANCED_DATASET}" "${BALANCED_MANIFEST}" "${CAP_PER_BODY}" "${SEED}" "${MAX_ROWS}" "${EVAL_EXCLUDE_DATASETS}" <<'PY' + python - "${SOURCE_DATASET}" "${BALANCED_DATASET}" "${BALANCED_MANIFEST}" "${CAP_PER_BODY}" "${SEED}" "${MAX_ROWS}" "${EVAL_EXCLUDE_DATASETS}" <<'PY' from __future__ import annotations import hashlib @@ -119,6 +149,15 @@ from pathlib import Path from statistics import mean from typing import Any, Mapping +from scripts.gate_dataset import ( + body_identity, + canonicalize_body_hash, + exclude_eval_rows, + file_sha256, + load_jsonl, + selection_fingerprint, + selection_inputs, +) from src.replication_metrics import extract_solidity_facts @@ -137,71 +176,39 @@ exclude_paths = [ rng = random.Random(seed) -def load_jsonl(path: Path) -> list[dict[str, Any]]: - rows: list[dict[str, Any]] = [] - with path.open("r", encoding="utf-8") as handle: - for line_number, line in enumerate(handle, start=1): - raw = line.strip() - if not raw: - continue - row = json.loads(raw) - if not isinstance(row, dict): - raise ValueError(f"{path}:{line_number} is not a JSON object") - rows.append(row) - return rows - - -def body_identity(row: Mapping[str, Any]) -> tuple[str, bool]: - metadata = row.get("metadata") - if isinstance(metadata, Mapping): - body_hash = metadata.get("body_hash") - if body_hash: - return f"body_hash:{body_hash}", True - parts = [ - metadata.get("contract_address"), - metadata.get("selector"), - metadata.get("function_signature"), - metadata.get("compiler_version"), - ] - if any(parts): - return "metadata:" + "|".join(str(part or "") for part in parts), False - digest = hashlib.sha256( - (str(row.get("input", "")) + "\0" + str(row.get("output", ""))).encode("utf-8") - ).hexdigest() - return f"content:{digest}", False - - def fact_coverage(row: Mapping[str, Any]) -> dict[str, int]: facts = extract_solidity_facts(str(row.get("output", ""))) return {category: len(values) for category, values in facts.items() if values} +inputs = selection_inputs(source, exclude_paths, cap_per_body=cap_per_body, seed=seed, max_rows=max_rows) source_rows = load_jsonl(source) source_identity_counts: Counter[str] = Counter() for row in source_rows: - identity, _ = body_identity(row) - source_identity_counts[identity] += 1 + source_identity_counts[body_identity(row)] += 1 excluded_identities: set[str] = set() +gate_rows: list[dict[str, Any]] = [] excluded_dataset_rows = 0 for exclude_path in exclude_paths: for row in load_jsonl(exclude_path): - identity, _ = body_identity(row) - excluded_identities.add(identity) + excluded_identities.add(body_identity(row)) + gate_rows.append(row) excluded_dataset_rows += 1 groups: dict[str, list[dict[str, Any]]] = defaultdict(list) -identity_uses_body_hash: dict[str, bool] = {} -missing_body_hash_count = 0 -excluded_source_rows = 0 +missing_body_hash_count = sum( + not (isinstance(row.get("metadata"), Mapping) and row["metadata"].get("body_hash")) + for row in source_rows +) +available_rows, excluded_source_rows = exclude_eval_rows(source_rows, gate_rows) +if not available_rows: + raise ValueError("All source rows overlap fixed eval gates; refusing empty training dataset") +available_ids = {id(row) for row in available_rows} for source_index, row in enumerate(source_rows): - identity, used_body_hash = body_identity(row) - if identity in excluded_identities: - excluded_source_rows += 1 + if id(row) not in available_ids: continue - identity_uses_body_hash[identity] = used_body_hash - if not used_body_hash: - missing_body_hash_count += 1 + identity = body_identity(row) enriched = dict(row) enriched["_body_balance_source_index"] = source_index enriched["_body_balance_tie"] = rng.random() @@ -231,13 +238,12 @@ visibilities: Counter[str] = Counter() input_chars: list[int] = [] output_chars: list[int] = [] for row in selected: - identity, _ = body_identity(row) - identity_counts[identity] += 1 - clean = { + identity_counts[body_identity(row)] += 1 + clean = canonicalize_body_hash({ key: value for key, value in row.items() if not key.startswith("_body_balance_") - } + }) clean_rows.append(clean) input_text = str(clean.get("input", "")) output_text = str(clean.get("output", "")) @@ -258,13 +264,14 @@ with output.open("w", encoding="utf-8") as handle: json.dump(row, handle, sort_keys=True) handle.write("\n") -selected_identity_digest = hashlib.sha256( - "\n".join(sorted(identity_counts)).encode("utf-8") -).hexdigest() +selected_identity_digest = hashlib.sha256("\n".join(sorted(identity_counts)).encode("utf-8")).hexdigest() source_body_duplicate_rows = sum(max(0, count - 1) for count in source_identity_counts.values()) available_body_duplicate_rows = sum(max(0, len(rows) - 1) for rows in groups.values()) manifest = { "manifest_kind": "body_balanced_dataset", + "selection_inputs": inputs, + "selection_fingerprint": selection_fingerprint(inputs), + "output_sha256": file_sha256(output), "source_dataset": str(source), "output_dataset": str(output), "cap_per_body": cap_per_body, @@ -285,8 +292,8 @@ manifest = { "selected_identity_sha256": selected_identity_digest, "missing_body_hash_source_rows": missing_body_hash_count, "selection_policy": ( - "group by metadata.body_hash when present; otherwise use metadata/content fallback; " - "select up to cap_per_body deterministic seeded variants per identity" + "group by recomputed normalized Solidity output body; exclude all fixed gate " + "body, input, output, source, and contract identities before deterministic selection" ), "input_chars": { "mean": mean(input_chars) if input_chars else 0, @@ -307,8 +314,6 @@ with manifest_path.open("w", encoding="utf-8") as handle: print(json.dumps(manifest, indent=2, sort_keys=True)) PY -else - echo "Using existing body-balanced dataset: ${BALANCED_DATASET}" fi if [[ "${GRADIENT_CHECKPOINTING}" == "false" || "${GRADIENT_CHECKPOINTING}" == "0" ]]; then @@ -419,6 +424,7 @@ if [[ ! -d "${FINAL_MODEL}" ]]; then exit 1 fi +TRAIN_DATASET="$(python -m scripts.gate_dataset verify-model "${FINAL_MODEL}" "${DEFAULT_EVAL_EXCLUDE_DATASETS}")" mkdir -p "${GATE_DIR}" EVAL_MAP="${GATE_DIR}/eval_paths.tsv" : > "${EVAL_MAP}" @@ -452,34 +458,12 @@ run_eval() { "$@" local eval_json eval_json="$(newest_eval_json)" + python -m scripts.gate_dataset verify-eval "${eval_json}" "${model_path}" "${dataset_path}" \ + --max-new-tokens "${EVAL_MAX_NEW_TOKENS}" --repetition-penalty "${EVAL_REPETITION_PENALTY}" printf '%s\t%s\t%s\t%s\n' "${label}" "${model_path}" "${dataset_path}" "${eval_json}" | tee -a "${EVAL_MAP}" } -require_baseline() { - local label="$1" - local path="$2" - if [[ ! -f "${path}" ]]; then - echo "Required ${label} baseline eval not found: ${path}" >&2 - exit 1 - fi -} - -require_baseline "broad30" "${BROAD_BASELINE}" -require_baseline "calls23" "${CALLS_BASELINE}" -require_baseline "state17" "${STATE_BASELINE}" -require_baseline "holdout64" "${HOLDOUT64_BASELINE}" -require_baseline "pure_negative64" "${PURE_NEGATIVE_BASELINE}" - -if [[ ! -f "${LARGE192_BASELINE}" ]]; then - if [[ ! -d "${BASELINE_MODEL}" ]]; then - echo "Large192 baseline is missing and BASELINE_MODEL does not exist: ${BASELINE_MODEL}" >&2 - exit 1 - fi - run_eval "large192_baseline" "${BASELINE_MODEL}" "${LARGE192_DATASET}" "${GATE_DIR}/latest_results_large192_baseline.txt" - LARGE192_BASELINE="$(tail -1 "${EVAL_MAP}" | cut -f4)" -fi - -run_eval "train_first30" "${FINAL_MODEL}" "${DATA_DIR}/splits/train_dataset.jsonl" "${GATE_DIR}/latest_results_train_first30.txt" --eval-limit 30 --eval-first-n +run_eval "train_first30" "${FINAL_MODEL}" "${TRAIN_DATASET}" "${GATE_DIR}/latest_results_train_first30.txt" --eval-limit 30 --eval-first-n run_eval "broad30" "${FINAL_MODEL}" "${BROAD_DATASET}" "${GATE_DIR}/latest_results_broad30.txt" run_eval "calls23" "${FINAL_MODEL}" "${CALLS_DATASET}" "${GATE_DIR}/latest_results_calls23.txt" run_eval "state17" "${FINAL_MODEL}" "${STATE_DATASET}" "${GATE_DIR}/latest_results_state17.txt" diff --git a/scripts/benchmark_runtime_controls.py b/scripts/benchmark_runtime_controls.py new file mode 100644 index 0000000..8ff2e99 --- /dev/null +++ b/scripts/benchmark_runtime_controls.py @@ -0,0 +1,375 @@ +#!/usr/bin/env python3 +"""Offline controls for opt-in, exact full-contract runtime bytecode comparison. + +This measures identity of compiled runtime bytes, *not* execution equivalence. +Solidity's trailing source metadata can make identical executable code compare +unequal; a comment-only control makes that limitation observable. No model or +dataset is loaded, and no compiler is installed or downloaded. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import sys +import time +from collections import Counter +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable + + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from src.training_pipeline import ( # noqa: E402 + _compile_named_runtime, + evaluate_bytecode_semantics, + validate_generated_solidity, +) + + +SOURCE_NAME = "contract.sol" +OPTIMIZER_ENABLED = False +OPTIMIZER_RUNS = 200 +EXPECTED = { + "identical_source": (True, True, None), + "return_changed": (True, False, None), + "storage_changed": (True, False, None), + "fragment": (False, None, "candidate_fragment"), + "unsupported_import": (False, None, "candidate_imports_unsupported"), + "comment_only": (True, False, None), +} + + +@dataclass(frozen=True) +class ContractCase: + name: str + source: str + changed_source: str + changed_control: str + + +def select_solc_version(installed: Iterable[object]) -> str: + """Prefer 0.8.20, otherwise the lowest locally installed 0.8.x >= 0.8.20.""" + versions = { + tuple(int(part) for part in text.split(".")) + for version in installed + if re.fullmatch(r"0\.8\.\d+", text := str(version)) and int(text.split(".")[2]) >= 20 + } + if not versions: + raise RuntimeError("Local solc 0.8.20 or newer 0.8.x is required (no downloads attempted)") + chosen = (0, 8, 20) if (0, 8, 20) in versions else min(versions) + return ".".join(map(str, chosen)) + + +def build_contracts() -> list[ContractCase]: + """Thirty distinct, self-contained sources with executable return/storage mutations.""" + cases: list[ContractCase] = [] + for index in range(1, 6): + constant = 100 + index * 11 + name = f"ConstantReturn{index}" + source = ( + "pragma solidity ^0.8.0;\n" + f"contract {name} {{\n" + f" function value() external pure returns (uint256) {{ return {constant}; }}\n" + "}\n" + ) + cases.append( + ContractCase( + name, + source, + source.replace(f"return {constant};", f"return {constant + 1};"), + "return_changed", + ) + ) + + offset = 3 + index * 7 + name = f"OffsetReturn{index}" + source = ( + "pragma solidity ^0.8.0;\n" + f"contract {name} {{\n" + " function value(uint256 input) external pure returns (uint256) " + f"{{ return input + {offset}; }}\n" + "}\n" + ) + cases.append( + ContractCase( + name, + source, + source.replace(f"input + {offset};", f"input + {offset + 1};"), + "return_changed", + ) + ) + + factor = 2 + index * 3 + name = f"ProductReturn{index}" + source = ( + "pragma solidity ^0.8.0;\n" + f"contract {name} {{\n" + " function value(uint256 input) external pure returns (uint256) " + f"{{ return input * {factor}; }}\n" + "}\n" + ) + cases.append( + ContractCase( + name, + source, + source.replace(f"input * {factor};", f"input * {factor + 1};"), + "return_changed", + ) + ) + + increment = 1 + index * 5 + name = f"AccumulatedStorage{index}" + source = ( + "pragma solidity ^0.8.0;\n" + f"contract {name} {{\n" + " uint256 public total;\n" + " function add(uint256 amount) external " + f"{{ total = total + amount + {increment}; }}\n" + "}\n" + ) + cases.append( + ContractCase( + name, + source, + source.replace(f"amount + {increment};", f"amount + {increment + 1};"), + "storage_changed", + ) + ) + + credit = 2 + index * 9 + name = f"MappedStorage{index}" + source = ( + "pragma solidity ^0.8.0;\n" + f"contract {name} {{\n" + " mapping(address => uint256) public credits;\n" + " function credit(uint256 amount) external " + f"{{ credits[msg.sender] += amount + {credit}; }}\n" + "}\n" + ) + cases.append( + ContractCase( + name, + source, + source.replace(f"amount + {credit};", f"amount + {credit + 1};"), + "storage_changed", + ) + ) + + threshold = 10 + index * 13 + name = f"ConditionalStorage{index}" + source = ( + "pragma solidity ^0.8.0;\n" + f"contract {name} {{\n" + " uint256 public latest;\n" + " function set(uint256 amount) external " + f"{{ if (amount > {threshold}) latest = amount; }}\n" + "}\n" + ) + cases.append( + ContractCase( + name, + source, + source.replace(f"amount > {threshold}", f"amount > {threshold + 1}"), + "storage_changed", + ) + ) + if len(cases) != 30 or len({case.name for case in cases}) != 30: + raise AssertionError("Benchmark needs 30 separately named contracts") + if len({case.source for case in cases}) != 30 or any( + case.source == case.changed_source for case in cases + ): + raise AssertionError("Benchmark sources or mutations must be distinct") + return cases + + +def runtime_body(runtime: str) -> str: + """Return executable prefix, excluding the solc CBOR metadata length/trailer. + + Used only to check that mutations touch executable bytes, not to assert + behavioral equivalence from equal prefixes. + """ + if not re.fullmatch(r"[0-9a-fA-F]+", runtime) or len(runtime) < 6 or len(runtime) % 2: + raise ValueError("Expected complete compiled runtime hex") + metadata_bytes = int(runtime[-4:], 16) + start = len(runtime) - 2 * (metadata_bytes + 2) + if start < 0 or not 0xA0 <= int(runtime[start : start + 2], 16) <= 0xBF: + raise ValueError("Missing Solidity CBOR metadata trailer") + return runtime[:start].lower() + + +def check_outcomes(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Treat control expectations and all skip reasons as testable assertions.""" + failures = [] + for row in rows: + label = row["control"] + expected_checked, expected_equal, expected_reason = EXPECTED[label] + actual = ( + row["checked"], + row["equal"], + row["skip_reason"], + ) + if actual != (expected_checked, expected_equal, expected_reason): + failures.append( + { + "contract": row["contract"], + "control": label, + "expected": { + "checked": expected_checked, + "equal": expected_equal, + "skip_reason": expected_reason, + }, + "actual": { + "checked": row["checked"], + "equal": row["equal"], + "skip_reason": row["skip_reason"], + }, + } + ) + return failures + + +def _compile(source: str, name: str, version: str) -> tuple[str | None, str | None]: + return _compile_named_runtime( + source, name, version, OPTIMIZER_ENABLED, OPTIMIZER_RUNS, SOURCE_NAME + ) + + +def run_benchmark() -> dict[str, Any]: + import solcx + + version = select_solc_version(solcx.get_installed_solc_versions()) + started = time.perf_counter() + rows: list[dict[str, Any]] = [] + failures: list[dict[str, Any]] = [] + runtime_hashes: set[str] = set() + runtime_bodies: set[str] = set() + cases = build_contracts() + for case in cases: + reference_runtime, reason = _compile(case.source, case.name, version) + if reference_runtime is None: + raise RuntimeError(f"{case.name}: reference runtime could not compile: {reason}") + runtime_hashes.add(hashlib.sha256(bytes.fromhex(reference_runtime)).hexdigest()) + reference_body = runtime_body(reference_runtime) + runtime_bodies.add(reference_body) + metadata = { + "runtime_bytecode": "0x" + reference_runtime, + "compiler_version": version, + "optimizer_enabled": OPTIMIZER_ENABLED, + "optimizer_runs": OPTIMIZER_RUNS, + "runtime_comparison": { + "contract_name": case.name, + "compiler_version": version, + "optimizer_enabled": OPTIMIZER_ENABLED, + "optimizer_runs": OPTIMIZER_RUNS, + "source_name": SOURCE_NAME, + }, + } + controls = ( + ("identical_source", case.source), + (case.changed_control, case.changed_source), + ("fragment", "function value() external pure returns (uint256) { return 7; }"), + ("unsupported_import", 'import "unsupported_dependency.sol";\n' + case.source), + ) + if case is cases[0]: + controls += (("comment_only", case.source + "// harmless source comment\n"),) + for label, candidate in controls: + result = evaluate_bytecode_semantics( + case.source, + candidate, + metadata, + solidity_validity=validate_generated_solidity(candidate, allow_compiler=False), + ) + rows.append( + { + "contract": case.name, + "control": label, + "checked": result.runtime_bytecode_checked, + "equal": result.runtime_bytecode_match, + "skip_reason": result.runtime_bytecode_skip_reason, + "comparison_kind": result.to_dict()["runtime_comparison_kind"], + "score_kind": result.to_dict()["score_kind"], + } + ) + if label in ("return_changed", "storage_changed", "comment_only"): + candidate_runtime, candidate_reason = _compile(candidate, case.name, version) + if candidate_runtime is None: + failures.append( + { + "contract": case.name, + "control": label, + "failure": f"independent candidate compile: {candidate_reason}", + } + ) + else: + body_changed = runtime_body(candidate_runtime) != reference_body + if body_changed != (label != "comment_only"): + failures.append( + { + "contract": case.name, + "control": label, + "failure": "unexpected executable-prefix comparison", + "expected_body_changed": label != "comment_only", + "body_changed": body_changed, + } + ) + failures += check_outcomes(rows) + if len(runtime_hashes) != len(cases) or len(runtime_bodies) != len(cases): + failures.append( + { + "failure": "reference runtimes are not distinct, including executable prefixes", + "distinct_full_runtimes": len(runtime_hashes), + "distinct_executable_prefixes": len(runtime_bodies), + } + ) + equal = sum(row["equal"] is True for row in rows) + unequal = sum(row["checked"] and row["equal"] is False for row in rows) + unchecked = sum(not row["checked"] for row in rows) + return { + "solc_version": version, + "optimizer_enabled": OPTIMIZER_ENABLED, + "optimizer_runs": OPTIMIZER_RUNS, + "source_name": SOURCE_NAME, + "distinct_complete_contracts": len(cases), + "independently_compiled_reference_runtimes": len(runtime_hashes), + "distinct_executable_prefixes": len(runtime_bodies), + "control_counts": dict(sorted(Counter(row["control"] for row in rows).items())), + "coverage": { + "total": len(rows), + "checked": equal + unequal, + "equal": equal, + "unequal": unequal, + "unchecked": unchecked, + "skip_reasons": dict( + sorted(Counter(row["skip_reason"] for row in rows if not row["checked"]).items()) + ), + }, + "runtime_seconds": round(time.perf_counter() - started, 3), + "comparison_limit": ( + "Exact compiler-produced runtime byte identity, NOT semantic equivalence; " + "source metadata alone can change bytecode without changing executable code." + ), + "rows": rows, + "failures": failures, + } + + +def main() -> int: + try: + report = run_benchmark() + except (ImportError, OSError, RuntimeError, ValueError) as exc: + print(f"Runtime controls unavailable: {exc}", file=sys.stderr) + return 2 + summary = {key: value for key, value in report.items() if key != "rows"} + print(json.dumps(summary, indent=2, sort_keys=True)) + for failure in report["failures"]: + print(f"FAILURE SLICE: {json.dumps(failure, sort_keys=True)}", file=sys.stderr) + return 1 if report["failures"] else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/benchmark_tac_fidelity.py b/scripts/benchmark_tac_fidelity.py new file mode 100644 index 0000000..c39a074 --- /dev/null +++ b/scripts/benchmark_tac_fidelity.py @@ -0,0 +1,414 @@ +#!/usr/bin/env python3 +"""Model-free, compiler-source-mapped TAC opcode coverage benchmark. + +Run from the repository root: python scripts/benchmark_tac_fidelity.py +The JSON report goes to stdout. No model, selector registry, network, or +compiler installation is used. Source-mapped opcode coverage is a structural +check, NOT a proof of recovered Solidity semantics or runtime equivalence. +""" + +import hashlib +import json +import re +import subprocess +import sys +from collections import Counter +from pathlib import Path + +import solcx +from eth_utils import function_signature_to_4byte_selector +from solcx.install import get_executable + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from src.bytecode_analyzer import BytecodeAnalyzer # noqa: E402 + + +SOLC_VERSION = "0.8.20" +SOURCE_NAME = "TacFidelityBench.sol" +CASES_PER_STRATUM = 6 +OPCODES = { + "getter_return": "RETURN", + "setter_storage": "SSTORE", + "guard_revert": "REVERT", + "external_call": "CALL", + "event_log": "LOG2", + "branch": "JUMPI", +} +EVM_OPCODES = { + "RETURN": 0xF3, + "SSTORE": 0x55, + "REVERT": 0xFD, + "CALL": 0xF1, + "LOG2": 0xA2, + "JUMPI": 0x57, +} +RENDERED = { + "getter_return": re.compile(r"\breturn memory\["), + "setter_storage": re.compile(r"\bstorage\[[^\n]*\]\s*="), + "guard_revert": re.compile(r"\brevert\b"), + "external_call": re.compile(r"=\s*call\("), + "event_log": re.compile(r"\blog2\("), + "branch": re.compile(r"\bif\b[^\n]*\bgoto\b"), +} +SETTINGS = { + "optimizer": {"enabled": True, "runs": 200}, + "evmVersion": "shanghai", + "metadata": {"appendCBOR": False, "bytecodeHash": "none"}, + "outputSelection": { + "*": { + "": ["ast"], + "*": ["abi", "evm.deployedBytecode.object", "evm.deployedBytecode.sourceMap"], + } + }, +} + + +def _fixtures(): + cases = [] + for i in range(CASES_PER_STRATUM): + if i < 3: + getter = ( + f"uint256 private stored; function read{i}() external view returns (uint256) " + + ( + "{ return stored; }" + if i == 0 + else "{ return block.number; }" if i == 1 else "{ return 7; }" + ) + ) + setter = f"uint256 private stored; function write{i}() external {{ stored = {i + 1}; }}" + guard = ( + f"function guard{i}() external view " + f'{{ require(block.number > {i + 1}, "too small"); }}' + ) + call = ( + f"function invoke{i}() external returns (bool) " + f'{{ (bool ok,) = msg.sender.call{{gas: {30000 + i}}}(""); return ok; }}' + ) + event = ( + f"event Notice(uint256 indexed key, uint256 value); " + f"function emit{i}() external {{ emit Notice({i}, block.number); }}" + ) + branch = ( + f"function choose{i}() external view returns (uint256) " + f"{{ if (block.number > {i + 1}) return {i + 2}; return {i + 3}; }}" + ) + else: + getter = ( + f"uint256 private stored; function read{i}(uint256 x) external view " + f"returns (uint256) {{ return stored + x + {i}; }}" + ) + setter = ( + f"uint256 private stored; function write{i}(uint256 x) external " + f"{{ stored = x + {i}; }}" + ) + guard = ( + f"uint256 private stored; function guard{i}(uint256 x) external " + f'{{ require(x > {i + 1}, "too small"); stored = x; }}' + ) + call = ( + f"function invoke{i}(address target) external returns (bool) " + f'{{ (bool ok,) = target.call{{gas: {30000 + i}}}(""); return ok; }}' + ) + event = ( + f"event Notice(uint256 indexed key, uint256 value); " + f"function emit{i}(uint256 x) external {{ emit Notice({i}, x); }}" + ) + branch = ( + f"function choose{i}(uint256 x) external pure returns (uint256) " + f"{{ if (x > {i + 1}) return x; return {i + 1}; }}" + ) + cases.extend( + [ + ("getter_return", f"BGetter{i}", f"read{i}", getter), + ("setter_storage", f"BSetter{i}", f"write{i}", setter), + ("guard_revert", f"BGuard{i}", f"guard{i}", guard), + ("external_call", f"BCall{i}", f"invoke{i}", call), + ("event_log", f"BEvent{i}", f"emit{i}", event), + ("branch", f"BBranch{i}", f"choose{i}", branch), + ] + ) + source = ( + "// SPDX-License-Identifier: UNLICENSED\npragma solidity ^0.8.20;\n" + + "\n".join(f"contract {contract} {{ {body} }}" for _, contract, _, body in cases) + + "\n" + ) + return cases, source + + +def _opcodes_with_pcs(bytecode): + """Decode PC/opcode pairs independently of the analyzer; skip PUSH immediate bytes.""" + raw = bytes.fromhex(bytecode.removeprefix("0x")) + pc = 0 + while pc < len(raw): + opcode = raw[pc] + yield pc, opcode + pc += 1 + (opcode - 0x5F if 0x60 <= opcode <= 0x7F else 0) + + +def _source_mapped_opcode_pcs(bytecode, source_map, source_id, span, opcode): + instructions = list(_opcodes_with_pcs(bytecode)) + entries = source_map.split(";") + if len(instructions) != len(entries): + raise ValueError( + f"solc source-map/instruction mismatch: {len(entries)} vs {len(instructions)}" + ) + start, length = span + previous = ["", "", "-1", "", ""] + witnesses = [] + for (pc, raw_opcode), entry in zip(instructions, entries): + fields = entry.split(":") + for field_idx, field in enumerate(fields): + if field: + previous[field_idx] = field + if ( + raw_opcode == EVM_OPCODES[opcode] + and int(previous[2]) == source_id + and previous[0] + and previous[1] + ): + mapped_start = int(previous[0]) + mapped_end = mapped_start + int(previous[1]) + if start <= mapped_start and mapped_end <= start + length: + witnesses.append(pc) + return witnesses + + +def _function_span(ast, contract_name, function_name): + contract = next( + node + for node in ast["nodes"] + if node["nodeType"] == "ContractDefinition" and node["name"] == contract_name + ) + function = next( + node + for node in contract["nodes"] + if node["nodeType"] == "FunctionDefinition" and node["name"] == function_name + ) + start, length, _ = map(int, function["src"].split(":")) + return start, length + + +def _ratio(found, total): + return {"found": found, "total": total, "rate": found / total if total else None} + + +def _analyze_case(spec, compiled, ast, source_id): + stratum, contract, function_name, _ = spec + output = compiled["contracts"][SOURCE_NAME][contract] + abi = next( + item + for item in output["abi"] + if item["type"] == "function" and item["name"] == function_name + ) + signature = f"{function_name}({','.join(item['type'] for item in abi['inputs'])})" + selector = "0x" + function_signature_to_4byte_selector(signature).hex() + bytecode = output["evm"]["deployedBytecode"]["object"] + source_map = output["evm"]["deployedBytecode"]["sourceMap"] + opcode = OPCODES[stratum] + witness_pcs = _source_mapped_opcode_pcs( + bytecode, + source_map, + source_id, + _function_span(ast, contract, function_name), + opcode, + ) + if not witness_pcs: + raise ValueError(f"{contract}.{signature}: compiler emitted no source-mapped {opcode}") + + analyzer = BytecodeAnalyzer(bytecode) + per_function_tac = analyzer.generate_per_function_tac() + name = "function_" + selector + tac = per_function_tac.get(name, "") + function = analyzer.functions.get(name) + blocks = analyzer._blocks_for_function(function) if function and tac else [] + pc_to_block = { + instr.address: block + for block in blocks + for instr in block.metadata.get("raw_instructions", []) + } + all_pc_to_block = { + instr.address: block.id + for block in analyzer.basic_blocks.values() + for instr in block.metadata.get("raw_instructions", []) + } + reachable = [pc for pc in witness_pcs if pc in pc_to_block] + rendered = bool(RENDERED[stratum].search(tac)) + tac_pcs = [ + pc + for pc in reachable + if rendered + and any( + ins.metadata and ins.metadata.get("original_op") == opcode + for ins in pc_to_block[pc].instructions + ) + ] + bad_edges = sorted( + f"{block.id}->{successor}" + for block in analyzer.basic_blocks.values() + for successor in block.successors + if successor not in analyzer.basic_blocks + ) + unresolved_jumps = sorted( + block.id + for block in blocks + if not block.successors + and block.metadata.get("raw_instructions") + and block.metadata["raw_instructions"][-1].name == "JUMP" + ) + rejected = { + key: f"0x{value:x}" for key, value in sorted(analyzer.rejected_dispatcher_targets.items()) + } + selector_found = bool(tac) + reasons = [] + if not selector_found: + reasons.append("ABI selector has no per-function TAC") + if len(reachable) != len(witness_pcs): + reasons.append("compiler source-mapped opcode missing from function CFG reachability") + if reachable and len(tac_pcs) != len(reachable): + reasons.append("reachable source-mapped opcode missing from rendered per-function TAC") + if rejected: + reasons.append("invalid dispatcher target") + if bad_edges: + reasons.append("invalid CFG successor") + return { + "stratum": stratum, + "contract": contract, + "signature": signature, + "selector": selector, + "abi_parameter_count": len(abi["inputs"]), + "runtime_sha256": hashlib.sha256(bytes.fromhex(bytecode)).hexdigest(), + "runtime_bytes": len(bytecode) // 2, + "expected_opcode": opcode, + "source_mapped_opcode_pcs": witness_pcs, + "source_opcode_blocks": [all_pc_to_block.get(pc) for pc in witness_pcs], + "reachable_source_opcode_pcs": reachable, + "tac_witness_pcs": tac_pcs, + "selector_found": selector_found, + "rendered_tac_opcode": rendered, + "invalid_dispatcher_targets": rejected, + "invalid_cfg_successors": bad_edges, + "unresolved_reachable_jump_blocks": unresolved_jumps, + "failure_reasons": reasons, + } + + +def _summarize(cases): + found = sum(case["selector_found"] for case in cases) + reached = sum( + len(case["reachable_source_opcode_pcs"]) == len(case["source_mapped_opcode_pcs"]) + for case in cases + ) + reached_cases = [case for case in cases if case["reachable_source_opcode_pcs"]] + tac_found = sum( + len(case["tac_witness_pcs"]) == len(case["reachable_source_opcode_pcs"]) + for case in reached_cases + ) + end_to_end = sum( + len(case["tac_witness_pcs"]) == len(case["source_mapped_opcode_pcs"]) for case in cases + ) + return { + "cases": len(cases), + "selector_coverage": _ratio(found, len(cases)), + "source_mapped_opcode_reachability": _ratio(reached, len(cases)), + "reachable_tac_opcode_coverage": _ratio(tac_found, len(reached_cases)), + "end_to_end_tac_opcode_coverage": _ratio(end_to_end, len(cases)), + "invalid_dispatcher_targets": sum(len(c["invalid_dispatcher_targets"]) for c in cases), + "invalid_cfg_successors": sum(len(c["invalid_cfg_successors"]) for c in cases), + "unresolved_reachable_jump_blocks": sum( + len(c["unresolved_reachable_jump_blocks"]) for c in cases + ), + "failing_cases": sum(bool(c["failure_reasons"]) for c in cases), + } + + +def run_benchmark(): + if SOLC_VERSION not in {str(version) for version in solcx.get_installed_solc_versions()}: + raise RuntimeError(f"solc {SOLC_VERSION} must already be installed locally; no downloads") + binary = get_executable(SOLC_VERSION) + version = ( + subprocess.run([str(binary), "--version"], check=True, capture_output=True, text=True) + .stdout.strip() + .splitlines()[-1] + .removeprefix("Version: ") + ) + specs, source = _fixtures() + compilation = solcx.compile_standard( + { + "language": "Solidity", + "sources": {SOURCE_NAME: {"content": source}}, + "settings": SETTINGS, + }, + solc_binary=binary, + ) + ast = compilation["sources"][SOURCE_NAME]["ast"] + source_id = compilation["sources"][SOURCE_NAME]["id"] + cases = [_analyze_case(spec, compilation, ast, source_id) for spec in specs] + selectors = [case["selector"] for case in cases] + if len(selectors) < 30 or len(set(selectors)) != len(selectors): + raise ValueError("Expected at least 30 distinct ABI function selectors") + failures = [case for case in cases if case["failure_reasons"]] + return { + "schema_version": 1, + "lineage": { + "origin": "deterministically generated local Solidity; one ABI function per contract", + "source_name": SOURCE_NAME, + "source_sha256": hashlib.sha256(source.encode()).hexdigest(), + "compiler_version": version, + "compiler_binary_sha256": hashlib.sha256(Path(binary).read_bytes()).hexdigest(), + "compiler_settings": SETTINGS, + "runtime_metadata": "disabled", + "selector_reference": "ABI signatures from local solc output; not model prompt metadata", + }, + "interpretation": ( + "Compiler AST/source-map locations identify opcode witnesses attributed to each " + "function. Analyzer CFG reachability and TAC instructions are measured separately. " + "A source-mapped opcode or TAC occurrence alone does not establish execution, " + "semantic correctness, source recovery, or behavioral equivalence." + ), + "summary": { + **_summarize(cases), + "contracts": len(specs), + "unique_selectors": len(set(selectors)), + }, + "abi_parameter_slices": { + "no_parameters": _summarize( + [case for case in cases if not case["abi_parameter_count"]] + ), + "parameterized": _summarize([case for case in cases if case["abi_parameter_count"]]), + }, + "strata": { + name: { + "expected_opcode": opcode, + **_summarize([case for case in cases if case["stratum"] == name]), + } + for name, opcode in OPCODES.items() + }, + "cases": cases, + "sample_failures": [ + { + key: case[key] + for key in ( + "stratum", + "contract", + "signature", + "selector", + "expected_opcode", + "source_mapped_opcode_pcs", + "source_opcode_blocks", + "reachable_source_opcode_pcs", + "tac_witness_pcs", + "rendered_tac_opcode", + "unresolved_reachable_jump_blocks", + "failure_reasons", + ) + } + for case in failures[:12] + ], + "failure_reason_counts": dict( + sorted(Counter(reason for case in cases for reason in case["failure_reasons"]).items()) + ), + } + + +if __name__ == "__main__": + print(json.dumps(run_benchmark(), indent=2, sort_keys=True)) diff --git a/scripts/build_curriculum_dataset.py b/scripts/build_curriculum_dataset.py index 5b02695..97ca665 100755 --- a/scripts/build_curriculum_dataset.py +++ b/scripts/build_curriculum_dataset.py @@ -4,7 +4,6 @@ from __future__ import annotations import argparse -import hashlib import json import random import sys @@ -15,7 +14,15 @@ PROJECT_ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(PROJECT_ROOT)) -from src.replication_metrics import extract_solidity_facts +from src.replication_metrics import extract_solidity_facts # noqa: E402 + +from scripts.gate_dataset import ( # noqa: E402 + Key, + body_identity, + canonicalize_body_hash, + load_jsonl as load_gate_jsonl, + row_keys, +) FOCUS_CATEGORIES = { @@ -56,23 +63,7 @@ def load_jsonl(path: str | Path) -> list[dict[str, Any]]: def row_identity(row: Mapping[str, Any]) -> str: - metadata = row.get("metadata") - if isinstance(metadata, Mapping): - body_hash = metadata.get("body_hash") - if body_hash: - return f"body_hash:{body_hash}" - parts = [ - metadata.get("contract_address"), - metadata.get("selector"), - metadata.get("function_signature"), - metadata.get("compiler_version"), - ] - if any(parts): - return "metadata:" + "|".join(str(part or "") for part in parts) - digest = hashlib.sha256( - (str(row.get("input", "")) + "\0" + str(row.get("output", ""))).encode("utf-8") - ).hexdigest() - return f"content:{digest}" + return f"body_hash:{body_identity(row)}" def focus_categories(focus: str) -> tuple[str, ...]: @@ -84,11 +75,11 @@ def focus_categories(focus: str) -> tuple[str, ...]: return categories -def excluded_identities(paths: Iterable[str | Path]) -> set[str]: - identities: set[str] = set() +def excluded_identities(paths: Iterable[str | Path]) -> set[Key]: + identities: set[Key] = set() for path in paths: - for row in load_jsonl(path): - identities.add(row_identity(row)) + for row in load_gate_jsonl(path): + identities.update(row_keys(row)) return identities @@ -96,7 +87,7 @@ def select_curriculum_rows( rows: Sequence[dict[str, Any]], *, categories: Sequence[str], - exclude: set[str] | None = None, + exclude: set[Key] | None = None, max_rows: int = 64, min_focus_facts: int = 1, max_focus_facts: int | None = None, @@ -117,7 +108,7 @@ def select_curriculum_rows( candidates_by_identity: dict[str, CurriculumCandidate] = {} for index, row in enumerate(rows): identity = row_identity(row) - if identity in excluded: + if row_keys(row) & excluded: continue input_text = str(row.get("input", "")) output_text = str(row.get("output", "")) @@ -182,7 +173,7 @@ def write_curriculum( output.parent.mkdir(parents=True, exist_ok=True) with output.open("w", encoding="utf-8") as handle: for candidate in candidates: - json.dump(candidate.row, handle, sort_keys=True) + json.dump(canonicalize_body_hash(candidate.row), handle, sort_keys=True) handle.write("\n") manifest = { @@ -225,7 +216,9 @@ def main() -> None: parser.add_argument("--source-dataset", required=True, help="Source JSONL dataset") parser.add_argument("--output", required=True, help="Output curriculum JSONL path") parser.add_argument("--manifest", help="Output manifest path") - parser.add_argument("--focus", default="calls", help="Known focus or comma-separated categories") + parser.add_argument( + "--focus", default="calls", help="Known focus or comma-separated categories" + ) parser.add_argument("--exclude-dataset", action="append", default=[], help="Dataset to exclude") parser.add_argument("--max-rows", type=int, default=64, help="Maximum rows to write") parser.add_argument("--min-focus-facts", type=int, default=1) diff --git a/scripts/compare_eval_runs.py b/scripts/compare_eval_runs.py index b279557..d0cb5d2 100755 --- a/scripts/compare_eval_runs.py +++ b/scripts/compare_eval_runs.py @@ -4,7 +4,10 @@ from __future__ import annotations import argparse +import hashlib import json +import math +import re import sys from collections import Counter from pathlib import Path @@ -23,6 +26,7 @@ SUMMARY_GATE_METRICS = ( "replication_f1_micro", + "replication_behavior_only_f1_micro", "bytecode_semantic_score_mean", "semantic_similarity_mean", "solidity_valid_mean", @@ -33,6 +37,25 @@ "semantic_similarity", "solidity_valid", ) +COMPARABLE_SETTINGS = ( + "eval_batch_size", + "eval_max_new_tokens", + "eval_repetition_penalty", + "include_selector_signature_metadata", + "selector_signature_prompt_policy", + "eval_sampling_strategy", + "eval_sample_indices", +) +IDENTITY_METADATA = ( + "body_hash", + "contract_address", + "source_hash", + "selector", + "function_signature", + "compiler_version", + "optimizer_enabled", +) +_HASH_RE = re.compile(r"^[0-9a-fA-F]{64}$") def load_eval(path: str | Path) -> dict[str, Any]: @@ -41,7 +64,7 @@ def load_eval(path: str | Path) -> dict[str, Any]: if not isinstance(payload, Mapping): raise ValueError(f"{path} does not contain a JSON object") details = payload.get("details") or payload.get("detailed_results") - if not isinstance(details, list): + if not isinstance(details, list) or any(not isinstance(item, Mapping) for item in details): raise ValueError(f"{path} does not contain details/detailed_results") summary = payload.get("summary") or payload.get("aggregate_statistics") or {} if not isinstance(summary, Mapping): @@ -50,7 +73,10 @@ def load_eval(path: str | Path) -> dict[str, Any]: def _numeric(value: Any) -> float | None: - return float(value) if isinstance(value, (int, float, bool)) else None + if not isinstance(value, (int, float, bool)): + return None + number = float(value) + return number if math.isfinite(number) else None def _detail_metric(detail: Mapping[str, Any], metric: str) -> float | None: @@ -60,12 +86,111 @@ def _detail_metric(detail: Mapping[str, Any], metric: str) -> float | None: return _numeric(metrics.get(metric)) -def _details_by_index(details: Sequence[Mapping[str, Any]]) -> dict[int, Mapping[str, Any]]: +def _details_by_index( + details: Sequence[Mapping[str, Any]], label: str +) -> tuple[dict[int, Mapping[str, Any]], list[str]]: indexed: dict[int, Mapping[str, Any]] = {} - for fallback_index, detail in enumerate(details): - dataset_index = int(detail.get("dataset_index", fallback_index)) + errors: list[str] = [] + for position, detail in enumerate(details): + dataset_index = detail.get("dataset_index") + if type(dataset_index) is not int or dataset_index < 0: + errors.append(f"{label} detail {position}: missing or invalid dataset_index") + continue + if dataset_index in indexed: + errors.append(f"{label}: duplicate dataset_index {dataset_index}") + continue indexed[dataset_index] = detail - return indexed + return indexed, errors + + +def _identity_hash(detail: Mapping[str, Any], field: str) -> str: + declared = detail.get(f"{field}_hash") + text = detail.get("original" if field == "output" else "input") + if declared is None: + if not isinstance(text, str): + raise ValueError(f"missing {field}_hash and full {field} text") + return hashlib.sha256(text.encode("utf-8")).hexdigest() + if not isinstance(declared, str) or not _HASH_RE.fullmatch(declared): + raise ValueError(f"invalid {field}_hash") + if ( + isinstance(text, str) + and hashlib.sha256(text.encode("utf-8")).hexdigest() != declared.lower() + ): + raise ValueError(f"{field}_hash does not match full {field} text") + return declared.lower() + + +def _detail_identity(detail: Mapping[str, Any]) -> tuple[Any, ...]: + metadata = detail.get("metadata") + if not isinstance(metadata, Mapping): + raise ValueError("missing row metadata") + return ( + _identity_hash(detail, "input"), + _identity_hash(detail, "output"), + tuple((key, json.dumps(metadata.get(key), sort_keys=True)) for key in IDENTITY_METADATA), + ) + + +def _comparability_errors( + baseline_summary: Mapping[str, Any], + candidate_summary: Mapping[str, Any], + baseline_details: Sequence[Mapping[str, Any]], + candidate_details: Sequence[Mapping[str, Any]], + baseline_by_index: Mapping[int, Mapping[str, Any]], + candidate_by_index: Mapping[int, Mapping[str, Any]], + gate_metrics: Sequence[str], + paired_metrics: Sequence[str], +) -> tuple[list[str], list[int]]: + errors: list[str] = [] + for label, summary, details in ( + ("baseline", baseline_summary, baseline_details), + ("candidate", candidate_summary, candidate_details), + ): + if type(summary.get("num_evaluated")) is not int or summary["num_evaluated"] != len( + details + ): + errors.append(f"{label}: num_evaluated does not match details") + if type(summary.get("prompt_truncation_count")) is not int or ( + summary["prompt_truncation_count"] != 0 + ): + errors.append(f"{label}: prompt truncation must be measured and zero") + for metric in gate_metrics: + value = summary.get(metric) + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + ): + errors.append(f"{label}: missing or non-finite gate metric {metric}") + for setting in COMPARABLE_SETTINGS: + if setting not in baseline_summary or setting not in candidate_summary: + errors.append(f"missing comparable eval setting: {setting}") + elif baseline_summary[setting] != candidate_summary[setting]: + errors.append(f"eval setting differs: {setting}") + + if set(baseline_by_index) != set(candidate_by_index): + errors.append("evaluated dataset_index sets differ") + matched = [] + for index in sorted(set(baseline_by_index) & set(candidate_by_index)): + try: + if _detail_identity(baseline_by_index[index]) != _detail_identity( + candidate_by_index[index] + ): + errors.append(f"dataset_index {index}: different source row") + else: + matched.append(index) + for label, detail in ( + ("baseline", baseline_by_index[index]), + ("candidate", candidate_by_index[index]), + ): + for metric in paired_metrics: + if _detail_metric(detail, metric) is None: + errors.append( + f"{label} dataset_index {index}: missing or non-finite {metric}" + ) + except ValueError as exc: + errors.append(f"dataset_index {index}: {exc}") + return errors, matched def _counter_delta( @@ -125,9 +250,19 @@ def compare_eval_runs( elif delta > tolerance: improvements.append(metric) - baseline_by_index = _details_by_index(baseline_details) - candidate_by_index = _details_by_index(candidate_details) - paired_indices = sorted(set(baseline_by_index) & set(candidate_by_index)) + baseline_by_index, baseline_errors = _details_by_index(baseline_details, "baseline") + candidate_by_index, candidate_errors = _details_by_index(candidate_details, "candidate") + comparable_errors, paired_indices = _comparability_errors( + baseline_summary, + candidate_summary, + baseline_details, + candidate_details, + baseline_by_index, + candidate_by_index, + gate_metrics, + paired_metrics, + ) + comparable_errors = baseline_errors + candidate_errors + comparable_errors paired_results = {} row_deltas = [] for metric in paired_metrics: @@ -165,9 +300,12 @@ def compare_eval_runs( } row_deltas.sort(key=lambda item: (item["delta"], item["dataset_index"])) - candidate_rows = int(candidate_summary.get("num_evaluated") or len(candidate_details)) - baseline_rows = int(baseline_summary.get("num_evaluated") or len(baseline_details)) - if candidate_rows < min_rows or baseline_rows < min_rows: + candidate_rows = len(candidate_details) + baseline_rows = len(baseline_details) + if comparable_errors: + decision = "reject" + reason = "incomparable evaluations: " + "; ".join(comparable_errors[:3]) + elif candidate_rows < min_rows or baseline_rows < min_rows: decision = "smoke_only" reason = f"fewer than {min_rows} rows in baseline or candidate" elif regressions: @@ -186,6 +324,7 @@ def compare_eval_runs( "baseline_rows": baseline_rows, "candidate_rows": candidate_rows, "paired_rows": len(paired_indices), + "comparability_errors": comparable_errors, "decision": decision, "decision_reason": reason, "summary_deltas": summary_deltas, @@ -263,7 +402,15 @@ def format_markdown_report(comparison: Mapping[str, Any]) -> str: ("Hallucination bucket deltas", "hallucination_bucket_deltas"), ("Missing fact deltas", "missing_fact_deltas"), ): - lines.extend(["", f"## {title}", "", "| bucket | baseline | candidate | delta |", "| --- | ---: | ---: | ---: |"]) + lines.extend( + [ + "", + f"## {title}", + "", + "| bucket | baseline | candidate | delta |", + "| --- | ---: | ---: | ---: |", + ] + ) rows = sorted( comparison[key].items(), key=lambda item: (-abs(item[1]["delta"]), item[0]), diff --git a/scripts/eval_gate_suite.py b/scripts/eval_gate_suite.py index a796d7e..ea93bea 100755 --- a/scripts/eval_gate_suite.py +++ b/scripts/eval_gate_suite.py @@ -32,15 +32,19 @@ def evaluate_gate_suite( comparisons.append(comparison) decisions = [comparison["decision"] for comparison in comparisons] + trustworthy = [comparison for comparison in comparisons if comparison["paired_rows"] >= 30] if "reject" in decisions: suite_decision = "reject" reason = "one or more required comparisons regressed" elif "smoke_only" in decisions: suite_decision = "smoke_only" reason = "one or more required comparisons has too few rows" - elif "keep_candidate" in decisions: + elif not trustworthy: + suite_decision = "smoke_only" + reason = "no comparison has at least 30 paired rows" + elif any(comparison["decision"] == "keep_candidate" for comparison in trustworthy): suite_decision = "keep_candidate" - reason = "all required comparisons passed and at least one improved" + reason = "all required comparisons passed and at least one 30+ row comparison improved" else: suite_decision = "no_change" reason = "all required comparisons passed but none improved" @@ -76,9 +80,7 @@ def format_markdown_report(suite: Mapping[str, Any]) -> str: decision=comparison["decision"], rows=comparison["candidate_rows"], f1=_format_delta(_metric_delta(comparison, "replication_f1_micro")), - bytecode=_format_delta( - _metric_delta(comparison, "bytecode_semantic_score_mean") - ), + bytecode=_format_delta(_metric_delta(comparison, "bytecode_semantic_score_mean")), semantic=_format_delta(_metric_delta(comparison, "semantic_similarity_mean")), valid=_format_delta(_metric_delta(comparison, "solidity_valid_mean")), reason=comparison["decision_reason"], diff --git a/scripts/gate_dataset.py b/scripts/gate_dataset.py new file mode 100644 index 0000000..d933102 --- /dev/null +++ b/scripts/gate_dataset.py @@ -0,0 +1,451 @@ +#!/usr/bin/env python3 +"""Prepare eval-clean training samples and verify fixed-gate exclusions.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import random +from pathlib import Path +from typing import Any, Mapping, Sequence + +from src.dataset_export_primitives import hash_normalized_body + +Key = tuple[str, str | tuple[str, str]] + + +def load_jsonl(path: str | Path, *, allow_empty: bool = False) -> list[dict[str, Any]]: + rows = [] + with Path(path).open("r", encoding="utf-8") as handle: + for line_number, line in enumerate(handle, 1): + if not line.strip(): + continue + row = json.loads(line) + if not isinstance(row, dict): + raise ValueError(f"{path}:{line_number} is not a JSON object") + for field in ("input", "output"): + if not isinstance(row.get(field), str) or not row[field].strip(): + raise ValueError(f"{path}:{line_number} has no usable {field}") + rows.append(row) + if not rows and not allow_empty: + raise ValueError(f"{path} has no rows; gate exclusion cannot be verified") + return rows + + +def _metadata_value(row: Mapping[str, Any], *names: str) -> str | None: + metadata = row.get("metadata") + nested = [] + for container in (row, metadata): + if not isinstance(container, Mapping): + continue + for name in ( + "decontamination", + "decontamination_keys", + "split_keys", + "dedup_keys", + "quality_keys", + ): + value = container.get(name) + if isinstance(value, Mapping): + nested.append(value) + for name in names: + for container in (row, metadata, *nested): + if isinstance(container, Mapping) and container.get(name) not in (None, ""): + value = str(container[name]).strip().lower() + if value: + return value + return None + + +def body_identity(row: Mapping[str, Any]) -> str: + """Use the target itself: missing or stale metadata cannot split duplicate bodies.""" + return hash_normalized_body(str(row["output"])) + + +def canonicalize_body_hash(row: Mapping[str, Any]) -> dict[str, Any]: + selected = dict(row) + metadata = row.get("metadata") + selected["metadata"] = { + **(metadata if isinstance(metadata, Mapping) else {}), + "body_hash": body_identity(row), + } + return selected + + +def row_keys(row: Mapping[str, Any]) -> set[Key]: + keys: set[Key] = {("body_hash", body_identity(row))} + for category, names in ( + ( + "source_hash", + ( + "source_hash", + "source_code_hash", + "contract_source_hash", + "decontamination_source_hash", + ), + ), + ("contract_address", ("contract_address", "address", "deployed_address")), + ( + "body_hash", + ("body_hash", "function_body_hash", "normalized_body_hash", "solidity_body_hash"), + ), + ("input_hash", ("input_hash", "exact_input_hash", "tac_hash", "bytecode_hash")), + ("output_hash", ("output_hash", "exact_output_hash", "solidity_hash")), + ): + value = _metadata_value(row, *names) + if value: + keys.add((category, value)) + address = _metadata_value(row, "contract_address", "address", "deployed_address") + if address: + for names in ( + ("function_selector", "selector", "method_id", "4byte_selector"), + ("function_signature", "signature", "canonical_signature", "method_signature"), + ): + value = _metadata_value(row, *names) + if value: + keys.add(("contract_function", (address, value))) + for field in ("input", "output"): + keys.add((f"{field}_hash", hashlib.sha256(row[field].encode("utf-8")).hexdigest())) + return keys + + +def exclude_eval_rows( + source_rows: Sequence[dict[str, Any]], gate_rows: Sequence[dict[str, Any]] +) -> tuple[list[dict[str, Any]], int]: + if not gate_rows: + raise ValueError("No fixed eval rows; cannot verify exclusion") + gate_keys = set().union(*(row_keys(row) for row in gate_rows)) + selected = [row for row in source_rows if not (row_keys(row) & gate_keys)] + return selected, len(source_rows) - len(selected) + + +def file_sha256(path: str | Path) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def selection_inputs( + source: Path, exclude_paths: Sequence[Path], **parameters: Any +) -> dict[str, Any]: + return { + "selection_schema_version": 2, + "source": {"path": str(source.resolve()), "sha256": file_sha256(source)}, + "exclusions": [ + {"path": str(path.resolve()), "sha256": file_sha256(path)} for path in exclude_paths + ], + **parameters, + } + + +def selection_fingerprint(inputs: Mapping[str, Any]) -> str: + return hashlib.sha256(json.dumps(inputs, sort_keys=True).encode("utf-8")).hexdigest() + + +def verify_cache(output: Path, manifest_path: Path, inputs: Mapping[str, Any]) -> bool: + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + return ( + output.is_file() + and manifest.get("selection_fingerprint") == selection_fingerprint(inputs) + and manifest.get("output_sha256") == file_sha256(output) + ) + except (OSError, ValueError, TypeError, AttributeError): + return False + + +def verify_model_gate_exclusion(model_path: Path, gate_paths: Sequence[Path]) -> Path: + if not gate_paths: + raise ValueError("No fixed eval datasets were provided") + manifest_path = model_path / "training_input_manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if manifest.get("manifest_kind") != "training_inputs" or manifest.get("status") != "completed": + raise ValueError(f"{manifest_path} is not a completed training-input manifest") + train_artifact = manifest["datasets"]["train"]["artifact"] + train_path = Path(train_artifact["path"]) + split_manifest_path = train_path.parent / "split_manifest.json" + split_manifest = json.loads(split_manifest_path.read_text(encoding="utf-8")) + if ( + split_manifest.get("manifest_kind") != "dataset_split" + or split_manifest.get("leakage_validation", {}).get("status") != "passed" + or split_manifest.get("split_quality", {}).get("status") != "passed" + ): + raise ValueError(f"{split_manifest_path}: split validation has not passed") + eval_dataset = manifest["datasets"].get("eval", {}) + if isinstance(eval_dataset, Mapping): + eval_artifact = eval_dataset.get("artifact") + if isinstance(eval_artifact, Mapping) and eval_artifact.get("path"): + val_artifact = split_manifest["outputs"]["val"] + if Path(eval_artifact["path"]).resolve() != Path(val_artifact["path"]).resolve(): + raise ValueError(f"{manifest_path}: eval dataset differs from verified val split") + if eval_artifact.get("sha256") != val_artifact.get("sha256"): + raise ValueError(f"{manifest_path}: eval artifact hash differs from val split") + + gates = [row for path in gate_paths for row in load_jsonl(path)] + gate_keys = set().union(*(row_keys(row) for row in gates)) + seen_split_keys: dict[Key, str] = {} + for name in ("train", "val", "test"): + artifact = split_manifest["outputs"][name] + path = Path(artifact["path"]) + if name == "train" and path.resolve() != train_path.resolve(): + raise ValueError(f"{manifest_path}: train split differs from {split_manifest_path}") + if not path.is_file() or artifact.get("sha256") != file_sha256(path): + raise ValueError(f"{path}: split artifact is missing or changed") + if name == "train" and train_artifact.get("sha256") != file_sha256(path): + raise ValueError(f"{manifest_path}: model train artifact hash differs") + for row in load_jsonl(path, allow_empty=name != "train"): + keys = row_keys(row) + if keys & gate_keys: + raise ValueError(f"{name} split overlaps fixed evaluation gates: {path}") + for key in keys: + if key in seen_split_keys and seen_split_keys[key] != name: + raise ValueError(f"{name} split overlaps {seen_split_keys[key]} split: {path}") + seen_split_keys[key] = name + return train_path + + +def verify_eval_artifact( + eval_path: Path, + model_path: Path, + dataset_path: Path, + max_new_tokens: int, + repetition_penalty: float, +) -> None: + payload = json.loads(eval_path.read_text(encoding="utf-8")) + summary = payload.get("summary") + if not isinstance(summary, Mapping): + raise ValueError(f"{eval_path}: missing eval summary") + if ( + not summary.get("model_path") + or Path(summary["model_path"]).resolve() != model_path.resolve() + or not summary.get("test_dataset") + or Path(summary["test_dataset"]).resolve() != dataset_path.resolve() + or summary.get("eval_max_new_tokens") != max_new_tokens + or summary.get("eval_repetition_penalty") != repetition_penalty + ): + raise ValueError( + f"{eval_path}: eval output does not match requested model, dataset or decoding" + ) + + +def verify_baseline_artifact( + eval_path: Path, + dataset_path: Path, + max_new_tokens: int, + repetition_penalty: float, +) -> None: + """Reject stale or incomparable baselines before an expensive training run.""" + from scripts.compare_eval_runs import ( + SUMMARY_GATE_METRICS, + _detail_identity, + _details_by_index, + ) + + payload = json.loads(eval_path.read_text(encoding="utf-8")) + summary = payload.get("summary") + if not isinstance(summary, Mapping) or not summary.get("model_path"): + raise ValueError(f"{eval_path}: missing baseline model/evaluation summary") + verify_eval_artifact( + eval_path, + Path(str(summary["model_path"])), + dataset_path, + max_new_tokens, + repetition_penalty, + ) + rows = load_jsonl(dataset_path) + row_count = len(rows) + details = payload.get("details") or payload.get("detailed_results") + if ( + type(summary.get("num_evaluated")) is not int + or summary["num_evaluated"] != row_count + or not isinstance(details, list) + or len(details) != row_count + ): + raise ValueError(f"{eval_path}: baseline must evaluate every fixed-gate row") + if summary.get("selector_signature_prompt_policy") != "bundled_only_v1": + raise ValueError(f"{eval_path}: regenerate baseline with bundled_only_v1 prompts") + if type(summary.get("prompt_truncation_count")) is not int or ( + summary["prompt_truncation_count"] != 0 + ): + raise ValueError(f"{eval_path}: baseline prompt truncation must be measured and zero") + indexed, identity_errors = _details_by_index(details, "baseline") + if identity_errors or set(indexed) != set(range(row_count)): + raise ValueError(f"{eval_path}: baseline row indices are missing or duplicated") + for index, row in enumerate(rows): + try: + input_hash, output_hash, _metadata = _detail_identity(indexed[index]) + except ValueError as exc: + raise ValueError( + f"{eval_path}: baseline row {index} has invalid identity: {exc}" + ) from exc + if input_hash != hashlib.sha256(row["input"].encode("utf-8")).hexdigest() or ( + output_hash != hashlib.sha256(row["output"].encode("utf-8")).hexdigest() + ): + raise ValueError(f"{eval_path}: baseline row {index} differs from current gate dataset") + for metric in SUMMARY_GATE_METRICS: + value = summary.get(metric) + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + ): + raise ValueError(f"{eval_path}: missing or non-finite baseline gate metric {metric}") + + +def sample_dataset( + source: Path, + target: Path, + count: int, + seed: int, + exclude_paths: Sequence[Path], + *, + recreate: bool = False, +) -> dict[str, Any]: + if count < 1: + raise ValueError("sample count must be positive") + inputs = selection_inputs(source, exclude_paths, sample_count=count, seed=seed) + manifest_path = target.with_suffix(".manifest.json") + if not recreate and target.exists(): + if not verify_cache(target, manifest_path, inputs): + raise ValueError( + f"Sample cache is unverified or stale: {target}; set RECREATE_DATASET=1" + ) + return json.loads(manifest_path.read_text(encoding="utf-8")) + + rows = load_jsonl(source) + gates = [row for path in exclude_paths for row in load_jsonl(path)] + available, excluded_count = exclude_eval_rows(rows, gates) + if len(available) < count: + raise ValueError( + f"{source} has only {len(available)} eval-clean rows (excluded {excluded_count}); " + f"need {count}" + ) + indices = list(range(len(available))) + random.Random(seed).shuffle(indices) + target.parent.mkdir(parents=True, exist_ok=True) + with target.open("w", encoding="utf-8") as handle: + for index in indices[:count]: + json.dump(canonicalize_body_hash(available[index]), handle) + handle.write("\n") + manifest = { + "manifest_kind": "eval_clean_sample", + "selection_inputs": inputs, + "selection_fingerprint": selection_fingerprint(inputs), + "output_sha256": file_sha256(target), + "source_rows": len(rows), + "excluded_source_rows": excluded_count, + "selected_rows": count, + } + manifest_path.write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return manifest + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "action", + choices=( + "sample", + "verify-balanced-cache", + "verify-model", + "verify-eval", + "verify-baseline", + ), + ) + parser.add_argument("source", type=Path) + parser.add_argument("output", type=Path, nargs="?") + parser.add_argument("exclusions", nargs="?", help="Colon-separated fixed eval JSONL paths") + parser.add_argument("--count", type=int) + parser.add_argument("--cap-per-body", type=int) + parser.add_argument("--max-rows", type=int) + parser.add_argument("--seed", type=int) + parser.add_argument("--max-new-tokens", type=int) + parser.add_argument("--repetition-penalty", type=float) + parser.add_argument("--manifest", type=Path) + parser.add_argument("--recreate", action="store_true") + args = parser.parse_args() + if args.action == "verify-baseline": + if args.output is None or args.max_new_tokens is None or args.repetition_penalty is None: + parser.error("verify-baseline requires EVAL_JSON DATASET_PATH and decode settings") + try: + verify_baseline_artifact( + args.source, args.output, args.max_new_tokens, args.repetition_penalty + ) + except (OSError, ValueError, KeyError, TypeError) as exc: + parser.error(str(exc)) + return + if args.action == "verify-eval": + if ( + args.output is None + or args.exclusions is None + or args.max_new_tokens is None + or args.repetition_penalty is None + ): + parser.error( + "verify-eval requires EVAL_JSON MODEL_PATH DATASET_PATH and decode settings" + ) + try: + verify_eval_artifact( + args.source, + args.output, + Path(args.exclusions), + args.max_new_tokens, + args.repetition_penalty, + ) + except (OSError, ValueError, KeyError, TypeError) as exc: + parser.error(str(exc)) + return + # In verify-model mode, SOURCE is the model directory and OUTPUT is the gate list. + gate_list = str(args.output) if args.action == "verify-model" else args.exclusions + exclude_paths = [Path(path) for path in (gate_list or "").split(":") if path] + if not exclude_paths: + parser.error("At least one fixed eval dataset must be excluded") + if args.action == "verify-model": + try: + print(verify_model_gate_exclusion(args.source, exclude_paths)) + except (OSError, ValueError, KeyError, TypeError) as exc: + parser.error(f"Cannot verify eval-clean model lineage: {exc}") + elif args.action == "sample": + if args.count is None or args.seed is None: + parser.error("--count and --seed are required for sample") + if args.output is None: + parser.error("OUTPUT is required for sample") + print( + json.dumps( + sample_dataset( + args.source, + args.output, + args.count, + args.seed, + exclude_paths, + recreate=args.recreate, + ) + ) + ) + else: + if ( + args.manifest is None + or args.cap_per_body is None + or args.seed is None + or args.output is None + ): + parser.error("--manifest, --cap-per-body, --seed and OUTPUT are required") + inputs = selection_inputs( + args.source, + exclude_paths, + cap_per_body=args.cap_per_body, + seed=args.seed, + max_rows=args.max_rows, + ) + if not verify_cache(args.output, args.manifest, inputs): + parser.error("Body-balanced cache is unverified or stale; set RECREATE_DATASET=1") + + +if __name__ == "__main__": + main() diff --git a/src/bytecode_analyzer.py b/src/bytecode_analyzer.py index c49cc0c..1196521 100644 --- a/src/bytecode_analyzer.py +++ b/src/bytecode_analyzer.py @@ -16,6 +16,7 @@ """ import logging +from collections import deque from dataclasses import dataclass, field from enum import Enum from typing import Any, Dict, List, Optional, Tuple, Union @@ -234,6 +235,7 @@ def __init__(self, bytecode: str, abi_enricher: Any = None) -> None: self.instructions: List = [] self.basic_blocks: Dict[str, BasicBlock] = {} self.functions: Dict[str, Function] = {} + self.rejected_dispatcher_targets: Dict[str, int] = {} self.variable_counter: int = 0 self.logger = logging.getLogger(__name__) self.abi_enricher = abi_enricher # Optional ABIEnricher for custom error decoding @@ -538,6 +540,75 @@ def _analyze_block_relationships(self, blocks: Dict[str, BasicBlock]) -> None: pc_to_block: Dict[int, str] = {b.start_address: b.id for b in blocks.values()} for block in blocks.values(): self._analyze_single_block_relationships(block, blocks, pc_to_block) + if blocks: + self._trace_stack_paths(blocks, self._entry_blocks(blocks)[0], add_edges=True) + + def _trace_stack_paths( + self, blocks: Dict[str, BasicBlock], entry: str, *, add_edges: bool = False + ) -> set: + """Track the top 64 stack slots through bounded paths, without guessing jumps.""" + if entry not in blocks: + return set() + destinations = { + block.start_address: block.id + for block in blocks.values() + if block.metadata.get("raw_instructions") + and self._get_instruction_name(block.metadata["raw_instructions"][0]) == "JUMPDEST" + } + starts = {block.start_address: block.id for block in blocks.values()} + initial_stack = () if add_edges else (None,) * 16 + worklist = deque([(entry, initial_stack)]) + seen: set = set() + reachable: set = set() + max_states = min(4096, max(128, len(blocks) * 32)) + + while worklist and len(seen) < max_states: + bid, stack_state = worklist.popleft() + state = (bid, stack_state) + if state in seen: + continue + seen.add(state) + reachable.add(bid) + raw = blocks[bid].metadata.get("raw_instructions", []) + if not raw or len(raw) > 4096: + continue + + simulator = self._StackSimulator() + simulator.stack = list(stack_state) + for i, instr in enumerate(raw[:-1]): + simulator.process_instruction(instr, i) + + last = raw[-1] + name = self._get_instruction_name(last) + last_pc = self._get_pc(last, blocks[bid].end_address) + successors: List[str] = [] + if name in ("JUMP", "JUMPI"): + target = simulator.get_stack_top_value() + condition = simulator.stack[-2] if len(simulator.stack) >= 2 else None + if name == "JUMP" or condition is None or condition != 0: + if target in destinations: + successors.append(destinations[target]) + if name == "JUMPI" and (condition is None or condition == 0): + ft = self._get_next_instruction_pc(last_pc) + if ft in starts: + successors.append(starts[ft]) + elif name not in ("RETURN", "REVERT", "STOP", "SELFDESTRUCT", "INVALID"): + ft = self._get_next_instruction_pc(last_pc) + if ft in starts: + successors.append(starts[ft]) + + simulator.process_instruction(last, len(raw) - 1) + next_stack = tuple(simulator.stack[-64:]) + for successor in successors: + if add_edges: + self._add_edge(bid, successor, blocks) + next_state = (successor, next_stack) + if next_state not in seen: + worklist.append(next_state) + + if worklist: + logger.debug("Stopped stack-aware CFG traversal after %d contexts", max_states) + return reachable def _analyze_single_block_relationships( self, @@ -572,16 +643,20 @@ def _analyze_single_block_relationships( self._add_edge(block.id, pc_to_block[ft], all_blocks) def _get_jump_targets_from_block(self, instructions: List) -> set: - """Return the nearest PUSH operand preceding the terminating jump.""" - targets: set = set() - for i in range(len(instructions) - 2, -1, -1): - instr = instructions[i] - if self._get_instruction_name(instr).startswith("PUSH"): - val = self._parse_operand_as_int(self._get_operand(instr)) - if val is not None: - targets.add(val) - break - return targets + """Resolve a local jump only when its actual stack operand is a JUMPDEST.""" + if not instructions or len(instructions) > 4096: + return set() + + simulator = self._StackSimulator() + simulator.stack = [None] * 16 + for i, instr in enumerate(instructions[:-1]): + simulator.process_instruction(instr, i) + + target = simulator.get_stack_top_value() + idx = self._pc_to_index.get(target) + if idx is not None and self._get_instruction_name(self.instructions[idx]) == "JUMPDEST": + return {target} + return set() def _get_next_instruction_pc(self, current_pc: int) -> Optional[int]: """Return the PC of the instruction after *current_pc* (O(1) lookup).""" @@ -726,14 +801,20 @@ def _fallback_control_flow_analysis(self) -> Dict[str, BasicBlock]: def identify_functions(self) -> Dict[str, Function]: """Identify functions by scanning the dispatcher for ``PUSH4 … EQ … JUMPI`` patterns.""" functions: Dict[str, Function] = {} + self.rejected_dispatcher_targets = {} logger.info("Identifying functions from bytecode dispatcher") - for pattern in self._find_dispatcher_patterns(): + dispatcher_patterns = self._find_dispatcher_patterns() + for pattern in dispatcher_patterns: selector = pattern["selector"] jump_target = pattern["target"] + entry = self._jumpdest_block_at_address(jump_target) + if entry is None: + self.rejected_dispatcher_targets[selector] = jump_target + logger.warning("Skipping selector %s with invalid destination %04x", selector, jump_target) + continue fname = f"function_{selector}" - entry = self._block_at_address(jump_target) or f"block_{jump_target:04x}" functions[fname] = Function( name=fname, selector=selector, basic_blocks=[], entry_block=entry, ) @@ -759,7 +840,7 @@ def identify_functions(self) -> Dict[str, Function]: if internal: self.logger.info("Detected %d internal function(s)", len(internal)) - if not functions and self.basic_blocks: + if not functions and self.basic_blocks and not dispatcher_patterns: entry = next(iter(self.basic_blocks)) functions["fallback"] = Function( name="fallback", selector=None, @@ -796,8 +877,8 @@ def _detect_receive_function(self, dispatcher_targets: set) -> Optional[Function self._get_operand(self.instructions[k]) ) elif kname == "JUMPI" and target is not None: - entry = self._block_at_address(target) or f"block_{target:04x}" - if entry not in dispatcher_targets: + entry = self._jumpdest_block_at_address(target) + if entry is not None and entry not in dispatcher_targets: return Function( name="receive", selector=None, @@ -920,7 +1001,6 @@ def _find_dispatcher_patterns(self) -> List[Dict[str, Any]]: """Find selector comparisons fed by calldata selector extraction.""" patterns: List[Dict[str, Any]] = [] stack: List[Any] = [] - max_scan = min(len(self.instructions), 256) def pop() -> Any: return stack.pop() if stack else ("unknown",) @@ -949,8 +1029,7 @@ def selector_comparison(left: Any, right: Any) -> Optional[Tuple[str, int]]: def is_selector_divisor(value: Optional[int]) -> bool: return value == (1 << 224) - for i in range(max_scan): - instr = self.instructions[i] + for i, instr in enumerate(self.instructions): name = self._get_instruction_name(instr) if patterns and name == "JUMPDEST": @@ -1099,6 +1178,16 @@ def _block_at_address(self, addr: int) -> Optional[str]: return bid return None + def _jumpdest_block_at_address(self, addr: int) -> Optional[str]: + """Return a real JUMPDEST block at *addr*, never a synthetic entry.""" + bid = self._block_at_address(addr) + if bid is None: + return None + raw = self.basic_blocks[bid].metadata.get("raw_instructions", []) + if raw and self._get_instruction_name(raw[0]) == "JUMPDEST": + return bid + return None + # ------------------------------------------------------------------ # # TAC Conversion — Public API # ------------------------------------------------------------------ # @@ -1611,6 +1700,10 @@ def __init__(self) -> None: def process_instruction(self, instr: Any, index: int) -> None: name = self._get_name(instr) + def ensure_depth(depth: int) -> None: + if len(self.stack) < depth: + self.stack[:0] = [None] * (depth - len(self.stack)) + if name == "PUSH0": self.stack.append(0) self.stack_values[index] = 0 @@ -1628,37 +1721,34 @@ def process_instruction(self, instr: Any, index: int) -> None: self.stack.append(None) elif name == "POP": - if self.stack: - self.stack.pop() + ensure_depth(1) + self.stack.pop() elif name.startswith("DUP"): d = int(name[3:]) if len(name) > 3 else 1 - self.stack.append(self.stack[-d] if len(self.stack) >= d else None) + ensure_depth(d) + self.stack.append(self.stack[-d]) elif name.startswith("SWAP"): d = int(name[4:]) if len(name) > 4 else 1 - if len(self.stack) > d: - self.stack[-1], self.stack[-1 - d] = self.stack[-1 - d], self.stack[-1] - - elif name in ("ADDMOD", "MULMOD"): - for _ in range(min(3, len(self.stack))): - self.stack.pop() - self.stack.append(None) - - elif name in ("ISZERO", "NOT"): - if self.stack: - self.stack[-1] = None - - elif name in _BINARY_OPS: - if len(self.stack) >= 2: - self.stack.pop() - self.stack[-1] = None - - else: - effects = _EVM_STACK_EFFECTS.get(name) - if effects: + ensure_depth(d + 1) + self.stack[-1], self.stack[-1 - d] = self.stack[-1 - d], self.stack[-1] + + elif name != "JUMPDEST": + effects = ( + (3, 1) if name in ("ADDMOD", "MULMOD") + else (2, 1) if name in _BINARY_OPS + else (1, 1) if name in ("ISZERO", "NOT", "MLOAD", "SLOAD") + else (2, 0) if name in ("MSTORE", "SSTORE", "JUMPI") + else (1, 0) if name == "JUMP" + else _EVM_STACK_EFFECTS.get(name) + ) + if effects is None: + self.stack = [None] * len(self.stack) + else: pops, pushes = effects - for _ in range(min(pops, len(self.stack))): + ensure_depth(pops) + for _ in range(pops): self.stack.pop() self.stack.extend([None] * pushes) @@ -1697,12 +1787,15 @@ def generate_function_tac(self, func: Function) -> str: :meth:`identify_functions`. Returns: - A human-readable TAC string for *func*. + A human-readable TAC string for *func*, or an empty string if + its entry block is absent. """ if self.basic_blocks and not any(b.instructions for b in self.basic_blocks.values()): self._convert_and_integrate_tac() - blocks = self._blocks_for_function(func, fallback_to_all=True) + blocks = self._blocks_for_function(func) + if not blocks: + return "" lines: List[str] = [] lines.append(f"function {func.name}:") @@ -1711,33 +1804,17 @@ def generate_function_tac(self, func: Function) -> str: lines.append(f" // Entry block: {func.entry_block}") lines.append("") + included = {block.id for block in blocks} for b in blocks: - self._format_block_tac(lines, b, indent=" ") + self._format_block_tac(lines, b, indent=" ", allowed_blocks=included) return "\n".join(lines) def _blocks_for_function( self, func: Function, fallback_to_all: bool = False ) -> List[BasicBlock]: - """Return blocks reachable from a function entry without global duplication.""" - reachable_ids: set = set() - - def _walk(bid: str) -> None: - if bid in reachable_ids or bid not in self.basic_blocks: - return - reachable_ids.add(bid) - for s in self.basic_blocks[bid].successors: - _walk(s) - - entry = func.entry_block - _walk(entry) - - # If walk found nothing (entry not in basic_blocks), use func.basic_blocks - if not reachable_ids and func.basic_blocks: - reachable_ids = {b.id for b in func.basic_blocks} - - if fallback_to_all and not reachable_ids: - reachable_ids = set(self.basic_blocks.keys()) + """Return reachable blocks; retain the old argument without fabricating blocks.""" + reachable_ids = self._trace_stack_paths(self.basic_blocks, func.entry_block) blocks = [ self.basic_blocks[bid] @@ -1761,7 +1838,9 @@ def generate_per_function_tac(self) -> Dict[str, str]: result: Dict[str, str] = {} for fname, func in self.functions.items(): - result[fname] = self.generate_function_tac(func) + tac = self.generate_function_tac(func) + if tac: + result[fname] = tac return result # ------------------------------------------------------------------ # @@ -2019,13 +2098,14 @@ def _format_function_tac(self, lines: List[str], fname: str, func: Function) -> lines.append(" // View: true") lines.append("") - blocks = self._blocks_for_function(func, fallback_to_all=False) + blocks = self._blocks_for_function(func) if not blocks: lines.append(" // No reachable basic blocks found for this function") lines.append("") return + included = {block.id for block in blocks} for b in sorted(blocks, key=lambda x: x.start_address): - self._format_block_tac(lines, b, indent=" ") + self._format_block_tac(lines, b, indent=" ", allowed_blocks=included) def _format_all_blocks_tac(self, lines: List[str]) -> None: lines.append("main:") @@ -2034,15 +2114,30 @@ def _format_all_blocks_tac(self, lines: List[str]) -> None: for b in sorted(self.basic_blocks.values(), key=lambda x: x.start_address): self._format_block_tac(lines, b, indent=" ") - def _format_block_tac(self, lines: List[str], block: BasicBlock, indent: str = "") -> None: + def _format_block_tac( + self, lines: List[str], block: BasicBlock, indent: str = "", + allowed_blocks: Optional[set] = None, + ) -> None: lines.append(f"{indent}{block.id}:") lines.append(f"{indent} // Address range: {block.start_address:04x} - {block.end_address:04x}") - if block.predecessors: - lines.append(f"{indent} // Predecessors: {', '.join(block.predecessors)}") - if block.successors: - lines.append(f"{indent} // Successors: {', '.join(block.successors)}") + predecessors = [ + bid for bid in block.predecessors if allowed_blocks is None or bid in allowed_blocks + ] + successors = [ + bid for bid in block.successors if allowed_blocks is None or bid in allowed_blocks + ] + if predecessors: + lines.append(f"{indent} // Predecessors: {', '.join(predecessors)}") + if successors: + lines.append(f"{indent} // Successors: {', '.join(successors)}") if "block_type" in block.metadata: - lines.append(f"{indent} // Block type: {block.metadata['block_type']}") + block_type = block.metadata["block_type"] + if allowed_blocks is not None: + block_type = ( + "exit" if not successors else "sequential" if len(successors) == 1 + else "conditional" if len(successors) == 2 else "complex" + ) + lines.append(f"{indent} // Block type: {block_type}") if block.metadata.get("is_loop_header"): lines.append(f"{indent} // Loop header") if block.metadata.get("is_dead_code"): diff --git a/src/contract_reconstruction.py b/src/contract_reconstruction.py index 07142d1..62d31bb 100644 --- a/src/contract_reconstruction.py +++ b/src/contract_reconstruction.py @@ -496,6 +496,7 @@ def build_contract_quality( function_results: list[Mapping[str, Any]] | None = None, source_summary: Mapping[str, Any] | None = None, reconstruction_plan: Mapping[str, Any] | None = None, + rejected_dispatcher_targets: Mapping[str, int] | None = None, ) -> dict[str, Any]: """Return whole-contract quality/severity without conflating selector confidence.""" validation = validation if isinstance(validation, Mapping) else {} @@ -515,6 +516,10 @@ def build_contract_quality( for item in function_results if item.get("status") == "error" or item.get("source") == "error" ] + for selector in rejected_dispatcher_targets or {}: + name = f"function_{selector}" + if name not in unresolved: + unresolved.append(name) truncated = [ str(item.get("name")) for item in function_results @@ -536,6 +541,8 @@ def build_contract_quality( actions: list[str] = [] if unresolved: actions.append("Resolve failed chunks before using the reconstructed contract.") + if rejected_dispatcher_targets: + actions.append("Inspect invalid dispatcher targets before trusting contract coverage.") if not validation_valid: actions.append("Fix Solidity validation errors and rerun compiler validation.") if scaffold_only: diff --git a/src/dataset_export_primitives.py b/src/dataset_export_primitives.py index 1392e86..2c5518a 100644 --- a/src/dataset_export_primitives.py +++ b/src/dataset_export_primitives.py @@ -672,22 +672,33 @@ def _token_count_estimate(text: Any) -> int: def export_prompt_parts(record: Dict[str, Any]) -> Tuple[str, str, str]: - try: - import train + import train - return train._preflight_prompt_parts( - record, - include_bytecode_metadata=True, - template_format="alpaca", - ) - except Exception: - return record.get("input", ""), record.get("output", ""), "" + return train._preflight_prompt_parts( + record, + include_bytecode_metadata=True, + include_selector_signature_metadata=True, + template_format="alpaca", + ) -def export_length_report(record: Dict[str, Any], max_seq_length: int) -> Dict[str, Any]: +def export_length_report( + record: Dict[str, Any], max_seq_length: int, tokenizer: Any = None +) -> Dict[str, Any]: + """Count the training prompt and target, exactly when a tokenizer is supplied.""" prefix, target, suffix = export_prompt_parts(record) - context_tokens = _token_count_estimate(prefix) - target_tokens = _token_count_estimate(f"{target}{suffix}") + if tokenizer is None: + context_tokens = _token_count_estimate(prefix) + target_tokens = _token_count_estimate(f"{target}{suffix}") + else: + from .model_setup import _append_eos_if_supported, _tokenize_to_ids + + context_tokens = len(_tokenize_to_ids(tokenizer, prefix)) + target_tokens = len( + _append_eos_if_supported( + tokenizer, _tokenize_to_ids(tokenizer, f"{target}{suffix}") + ) + ) total_tokens = context_tokens + target_tokens reasons: List[str] = [] if target_tokens >= max_seq_length: @@ -699,6 +710,7 @@ def export_length_report(record: Dict[str, Any], max_seq_length: int) -> Dict[st "target_tokens": target_tokens, "total_tokens": total_tokens, "max_seq_length": max_seq_length, + "token_count_method": "tokenizer" if tokenizer is not None else "estimate", "reasons": reasons, } @@ -780,21 +792,51 @@ def match_functions_by_selector( solidity_functions: List[Dict[str, Any]], bytecode_functions: Dict[Any, Any], analyzer: Any, + *, + ambiguous_selectors: Optional[set[str]] = None, ) -> List[Dict[str, Any]]: + """Match only selectors with an unambiguous Solidity target and runtime entry.""" ensure_tac_integrated(analyzer) - sol_by_sel = {f["selector"]: f for f in solidity_functions if f.get("selector")} - bc_by_sel = {f.selector: f for f in bytecode_functions.values() if getattr(f, "selector", None)} + sol_by_sel: Dict[str, List[Dict[str, Any]]] = defaultdict(list) + for function in solidity_functions: + if function.get("selector"): + sol_by_sel[function["selector"]].append(function) + bc_by_sel: Dict[str, List[Any]] = defaultdict(list) + for function in bytecode_functions.values(): + if getattr(function, "selector", None): + bc_by_sel[function.selector].append(function) matches: List[Dict[str, Any]] = [] - for selector, sol_func in sol_by_sel.items(): - if selector in bc_by_sel: - tac = extract_tac_for_function(bc_by_sel[selector], analyzer) - matches.append( - { - "solidity_function": sol_func, - "bytecode_function": bc_by_sel[selector], - "tac": tac, - "selector": selector, - } + for selector, source_candidates in sol_by_sel.items(): + runtime_candidates = bc_by_sel.get(selector) + if not runtime_candidates: + continue + target = str(source_candidates[0].get("body") or "").strip() + if ( + len(source_candidates) > 1 + and ( + not target + or any(str(f.get("body") or "").strip() != target for f in source_candidates[1:]) ) + ) or ( + len(runtime_candidates) > 1 + and ( + any(not getattr(f, "entry_block", None) for f in runtime_candidates) + or len({str(f.entry_block) for f in runtime_candidates}) > 1 + ) + ): + if ambiguous_selectors is not None: + ambiguous_selectors.add(selector) + continue + sol_func = source_candidates[0] + bytecode_func = runtime_candidates[0] + tac = extract_tac_for_function(bytecode_func, analyzer) + matches.append( + { + "solidity_function": sol_func, + "bytecode_function": bytecode_func, + "tac": tac, + "selector": selector, + } + ) return matches diff --git a/src/dataset_pipeline.py b/src/dataset_pipeline.py index f5972f4..85a9b06 100644 --- a/src/dataset_pipeline.py +++ b/src/dataset_pipeline.py @@ -1315,10 +1315,26 @@ def add_diagnostic(**kwargs: Any) -> None: f for f in solidity_with_selectors if f.get("contract_name", "") == cname ] or solidity_with_selectors + ambiguous_selectors: set[str] = set() matched = self._match_functions_by_selector( - contract_sol_funcs, bytecode_functions, analyzer + contract_sol_funcs, bytecode_functions, analyzer, + ambiguous_selectors=ambiguous_selectors, ) - if not matched: + if ambiguous_selectors: + address_failures.append( + ("ambiguous_selector_alignment", ", ".join(sorted(ambiguous_selectors))) + ) + add_diagnostic( + stage="match", + contract_address=addr, + compiler_version=ver, + optimizer_enabled=opt, + optimization_runs=runs, + contract_name=cname, + status="ambiguous_selector_alignment", + error=f"conflicting source or runtime candidates: {', '.join(sorted(ambiguous_selectors))}", + ) + if not matched and not ambiguous_selectors: address_failures.append(("no_selector_matches", cname)) add_diagnostic( stage="match", @@ -1825,6 +1841,8 @@ def _match_functions_by_selector( solidity_functions: List[Dict], bytecode_functions: Dict, analyzer: BytecodeAnalyzer, + *, + ambiguous_selectors: Optional[set[str]] = None, ) -> List[Dict]: """Match Solidity functions with bytecode functions by selector. @@ -1836,12 +1854,17 @@ def _match_functions_by_selector( Returns: List of matched function dicts. """ - matches = match_functions_by_selector(solidity_functions, bytecode_functions, analyzer) + matches = match_functions_by_selector( + solidity_functions, bytecode_functions, analyzer, + ambiguous_selectors=ambiguous_selectors, + ) matched_selectors = {m["selector"] for m in matches} for sol_func in solidity_functions: selector = sol_func.get("selector") if selector in matched_selectors: logger.debug(f"Matched function {sol_func['name']} with selector {selector}") + elif ambiguous_selectors is not None and selector in ambiguous_selectors: + logger.debug("Skipped ambiguous selector %s for %s", selector, sol_func["name"]) elif selector: logger.debug(f"No bytecode match for {sol_func['name']} (selector: {selector})") return matches @@ -2216,6 +2239,7 @@ def export_dataset( filter_overlength: bool = True, rejects_path: Optional[str] = None, write_manifest: bool = True, + length_tokenizer: Any = None, ) -> str: """Export the dataset in the specified format. @@ -2230,6 +2254,8 @@ def export_dataset( filter_overlength: Quarantine rows exceeding ``max_seq_length``. rejects_path: Optional JSONL path for rejected main-dataset rows. write_manifest: Write an export manifest next to the main artifact. + length_tokenizer: Optional training tokenizer for exact length + filtering; without one, token lengths are only estimates. Returns: Path to exported file. @@ -2346,7 +2372,7 @@ def row_to_record(row: Any) -> Dict[str, Any]: record = row_to_record(row) quality_report = training_record_quality_report(record) reasons = list(quality_report["reasons"]) - length_report = export_length_report(record, max_seq_length) + length_report = export_length_report(record, max_seq_length, length_tokenizer) length_reasons = length_report["reasons"] if filter_overlength else [] reasons.extend(length_reasons) row_hash = record_final_row_hash(record) @@ -2456,6 +2482,12 @@ def row_to_record(row: Any) -> Dict[str, Any]: "include_partial": include_partial, "max_seq_length": max_seq_length, "filter_overlength": filter_overlength, + "token_count_method": "tokenizer" if length_tokenizer is not None else "estimate", + "length_tokenizer": ( + getattr(length_tokenizer, "name_or_path", None) + if length_tokenizer is not None + else None + ), "final_row_duplicate_policy": "quarantine", "tac_quality_policy": "quarantine", "record_quality_policy": "quarantine", @@ -2495,6 +2527,7 @@ def row_to_record(row: Any) -> Dict[str, Any]: "status": "filtered" if overlength_row_rejects else "passed", "max_seq_length": max_seq_length, "filter_overlength": filter_overlength, + "token_count_method": "tokenizer" if length_tokenizer is not None else "estimate", "reject_count": overlength_row_rejects, "reason_counts": dict(sorted(overlength_counts.items())), }, diff --git a/src/evaluation_report.py b/src/evaluation_report.py index 2414148..d57385d 100644 --- a/src/evaluation_report.py +++ b/src/evaluation_report.py @@ -12,6 +12,8 @@ from pathlib import Path from typing import Any, Dict, Mapping, Optional, Sequence +from .replication_metrics import BEHAVIOR_FACT_CATEGORIES, behavior_only_replication_micro + DEFAULT_LATEST_RESULTS_PATH = "latest_results.txt" @@ -131,7 +133,7 @@ def metric(name: str, *aliases: str) -> Optional[float]: evidence=evidence, likely_root_causes=[ "Training data may under-represent the failing opcode/control-flow slices.", - "The model may be learning surface syntax while missing bytecode-grounded behavior.", + "High source-text similarity can hide incorrect guards, calls, or state effects.", "Learning rate, LoRA rank, or epoch count may not provide enough capacity for semantic recovery.", ], suggested_experiments=[ @@ -216,6 +218,18 @@ def metric(name: str, *aliases: str) -> Optional[float]: "replication_metrics.micro.precision", "aggregate_statistics.replication_metrics.micro.precision", ) + behavior_micro = _behavior_only_replication_micro(summary) + if behavior_micro is not None: + behavior_f1 = behavior_micro["f1"] + metrics_used["replication_behavior_only_f1_micro"] = _json_safe_number(behavior_f1) + if replication_f1 is not None and replication_f1 - behavior_f1 >= 0.10: + caveats.append( + f"All-fact replication F1={replication_f1:.4f} exceeds behavior-only source-fact F1={behavior_f1:.4f}; ABI-heavy counts can mask missing behavior facts. Neither score verifies bytecode behavior." + ) + else: + caveats.append( + "Behavior-only F1 compares source-extracted call, control-flow, event, guard, return, and state-write facts; it does not verify bytecode behavior." + ) category_gaps = _replication_category_gaps(summary) replication_issue = _below_any((replication_f1, 0.75), (replication_recall, 0.75)) or bool( category_gaps @@ -344,6 +358,15 @@ def metric(name: str, *aliases: str) -> Optional[float]: ) solidity_valid = metric("solidity_valid_mean") + solidity_ast_valid = metric("solidity_ast_valid_mean") + if ( + solidity_valid is not None + and solidity_ast_valid is not None + and solidity_ast_valid < solidity_valid + ): + caveats.append( + "solidity_valid_mean includes scaffold or missing-context checks; solidity_ast_valid_mean is the separate compiler AST-valid fraction, not a deployment test." + ) if solidity_valid is not None and solidity_valid < 0.95: issues.append( _diagnostic_issue( @@ -377,26 +400,42 @@ def metric(name: str, *aliases: str) -> Optional[float]: bytecode_deployable = metric("bytecode_deployable_mean") runtime_checked = metric("bytecode_runtime_checked_mean") runtime_match = metric("bytecode_runtime_match_mean") + runtime_match_rate = _checked_runtime_match_rate(runtime_checked, runtime_match) + if runtime_match_rate is not None: + metrics_used["bytecode_runtime_match_rate_checked"] = _json_safe_number(runtime_match_rate) + if bytecode_score is not None: + caveats.append( + "bytecode_semantic_score_mean is a source-fact overlap proxy with bytecode/opcode hints, not a verified bytecode or behavioral match." + ) + if bytecode_checked is not None: + caveats.append( + "bytecode_semantic_checked_mean is a legacy flag: historical runs also counted compiler-only checks, so aggregates alone cannot prove reference bytecode coverage." + ) + if runtime_checked == 0: + caveats.append( + "No generated/reference runtime comparisons were performed; bytecode_semantic_checked_mean is not behavioral verification." + ) if bytecode_checked is not None and bytecode_checked < 0.95: issues.append( _diagnostic_issue( issue_id="bytecode_grounding_coverage_gap", category="evaluation_data", severity="medium", - title="Not enough rows have bytecode-grounded semantic checks.", + title="The legacy bytecode-check flag is below target; reference evidence needs verification.", evidence=[ _target_evidence( "bytecode_semantic_checked_mean", bytecode_checked, ">=", 0.95, percent=True ) ], likely_root_causes=[ - "Evaluation rows may be missing runtime bytecode, opcode analysis, compiler version, or differential-call evidence.", + "Evaluation rows may be missing reference bytecode, TAC, or opcode metadata.", + "Older evaluators also counted compiler-only checks, so this flag does not prove reference coverage.", "Dataset export may not be carrying bytecode metadata into the evaluation artifact.", ], suggested_experiments=[ _experiment( "eval-metadata-backfill", - "Backfill bytecode/opcode/runtime metadata for the holdout set before using bytecode metrics as model gates.", + "Backfill reference bytecode/opcode metadata for the holdout before using the source-fact proxy as a model gate.", "data/evaluation", "bytecode_semantic_checked_mean reaches 95%+", ) @@ -409,31 +448,60 @@ def metric(name: str, *aliases: str) -> Optional[float]: issue_id="bytecode_semantic_gap", category="model_behavior", severity="high" if bytecode_score < 0.60 else "medium", - title="Bytecode-grounded behavior does not match reference behavior often enough.", + title="The source-fact overlap proxy is below target; bytecode behavior is unverified.", evidence=[ _target_evidence("bytecode_semantic_score_mean", bytecode_score, ">=", 0.80) ], likely_root_causes=[ - "The model may recover text that looks plausible but changes guards, returns, calls, or state effects.", - "Training may need examples that emphasize bytecode behavior over lexical similarity.", + "The generated source differs in extracted guards, returns, calls, ABI, or state effects.", + "The source-fact parser can miss behavior not expressible as extracted facts.", ], suggested_experiments=[ _experiment( - "bytecode-mismatch-slice", - "Fine-tune or evaluate separately on rows with bytecode mismatch buckets before broad retraining.", - "model/training", + "source-fact-mismatch-slice", + "Inspect source-fact mismatch buckets separately from compiled runtime or differential-call checks.", + "evaluation/model", "bytecode_semantic_score_mean improves and replication_f1_micro does not regress", ) ], ) ) + if bytecode_checked is not None and runtime_checked == 0: + issues.append( + _diagnostic_issue( + issue_id="bytecode_runtime_coverage_gap", + category="evaluation_data", + severity="medium", + title="No generated/reference runtime bytecode comparisons were performed.", + evidence=[ + _diagnostic_metric_text( + "bytecode_runtime_checked_mean", runtime_checked, percent=True + ), + _diagnostic_metric_text( + "bytecode_semantic_checked_mean", bytecode_checked, percent=True + ), + ], + likely_root_causes=[ + "Reference bytecode alone does not supply a generated runtime for comparison.", + "Function fragments may not compile without their contract-level context.", + ], + suggested_experiments=[ + _experiment( + "paired-runtime-check", + "Compile generated source with matched contract context and compiler settings, then compare its runtime with reference runtime on evaluable rows.", + "evaluation", + "bytecode_runtime_checked_mean rises above zero on a comparable heldout set", + ) + ], + ) + ) if bytecode_deployable is not None and bytecode_deployable < 0.95: issues.append( _diagnostic_issue( issue_id="deployability_gap", category="syntax_and_deployability", severity="medium", - title="Generated Solidity is not consistently deployable in bytecode-aware validation.", + title="Generated source rarely compiles to bytecode in the evaluation harness.", evidence=[ _target_evidence( "bytecode_deployable_mean", bytecode_deployable, ">=", 0.95, percent=True @@ -453,21 +521,20 @@ def metric(name: str, *aliases: str) -> Optional[float]: ], ) ) - if ( - runtime_checked is not None - and runtime_checked > 0 - and runtime_match is not None - and runtime_match < 0.95 - ): + if runtime_match_rate is not None and runtime_match_rate < 0.95: issues.append( _diagnostic_issue( issue_id="runtime_bytecode_mismatch", category="model_behavior", - severity="high" if runtime_match < 0.50 else "medium", + severity="high" if runtime_match_rate < 0.50 else "medium", title="Generated runtime bytecode does not match the reference runtime when checked.", evidence=[ _target_evidence( - "bytecode_runtime_match_mean", runtime_match, ">=", 0.95, percent=True + "bytecode_runtime_match_rate_checked", + runtime_match_rate, + ">=", + 0.95, + percent=True, ), _diagnostic_metric_text( "bytecode_runtime_checked_mean", runtime_checked, percent=True @@ -482,7 +549,7 @@ def metric(name: str, *aliases: str) -> Optional[float]: "runtime-diff-triage", "Compare normalized runtime diffs for the lowest bytecode_semantic_score samples.", "model/evaluation", - "bytecode_runtime_match_mean improves on checked rows", + "bytecode_runtime_match_rate_checked improves on checked rows", ) ], ) @@ -642,6 +709,7 @@ def format_latest_results_report( f"Evaluation duration: {_format_duration(duration)}", f"Distributed world size: {world_size}", f"Eval limit: {eval_limit if eval_limit is not None else 'none'}", + f"Selector signature prompt policy: {summary.get('selector_signature_prompt_policy', 'unknown')}", "", "Model", "-----", @@ -677,9 +745,8 @@ def format_latest_results_report( ] ) - diagnostics = summary.get("evaluation_diagnostics") - if not isinstance(diagnostics, Mapping): - diagnostics = build_evaluation_diagnostics(summary) + # Historical summaries can embed diagnostics generated with older metric labels. + diagnostics = build_evaluation_diagnostics(summary) _append_evaluation_diagnostics_report(lines, diagnostics) lines.extend( @@ -704,6 +771,8 @@ def format_latest_results_report( f"Replication precision micro: {_metric(summary, 'replication_precision_micro')}", f"Replication recall micro: {_metric(summary, 'replication_recall_micro')}", f"Replication F1 micro: {_metric(summary, 'replication_f1_micro')}", + f"Behavior-only source-fact F1 micro: {_format_number((_behavior_only_replication_micro(summary) or {}).get('f1'))}", + f"Behavior facts: {', '.join(BEHAVIOR_FACT_CATEGORIES)} (excludes ABI, modifier, mutability, visibility; not bytecode equivalence)", f"Functions > 0.8 replication F1: {_percent_metric(summary, 'pct_above_0.8_replication_f1')}", ] ) @@ -718,7 +787,7 @@ def format_latest_results_report( ): lines.extend( [ - f"Solidity valid outputs: {_percent_metric(summary, 'solidity_valid_mean')}", + f"Solidity best-effort valid (may be scaffold/context-limited): {_percent_metric(summary, 'solidity_valid_mean')}", f"Solidity compiler-checked outputs: {_percent_metric(summary, 'solidity_compiler_checked_mean')}", f"Solidity AST-valid outputs: {_percent_metric(summary, 'solidity_ast_valid_mean')}", ] @@ -736,13 +805,23 @@ def format_latest_results_report( ): lines.extend( [ - f"Bytecode semantic score mean: {_metric(summary, 'bytecode_semantic_score_mean')}", - f"Bytecode semantic checked outputs: {_percent_metric(summary, 'bytecode_semantic_checked_mean')}", - f"Bytecode deployable outputs: {_percent_metric(summary, 'bytecode_deployable_mean')}", + f"Source-fact overlap proxy mean: {_metric(summary, 'bytecode_semantic_score_mean')}", + f"Legacy bytecode semantic checked flag: {_percent_metric(summary, 'bytecode_semantic_checked_mean')}", + f"Compiler-valid with generated bytecode (legacy deployable metric): {_percent_metric(summary, 'bytecode_deployable_mean')}", f"Runtime bytecode checked outputs: {_percent_metric(summary, 'bytecode_runtime_checked_mean')}", - f"Runtime bytecode matches: {_percent_metric(summary, 'bytecode_runtime_match_mean')}", + f"Runtime bytecode match rate (checked only): {_runtime_match_rate(summary)}", ] ) + runtime_counts = _runtime_comparison_counts(summary) + if runtime_counts is not None: + checked, equal, total = runtime_counts + lines.extend( + [ + f"Exact full-contract runtime bytecode checked: {checked} / {total}", + f"Exact runtime bytecode equal (checked only): {equal} / {checked}", + "Runtime equality is bytecode identity, not constructor, deployment, or behavioral equivalence.", + ] + ) lines.extend( [ @@ -1142,6 +1221,48 @@ def _percent_metric(summary: Mapping[str, Any], key: str) -> str: return f"{float(value) * 100:.2f}%" +def _checked_runtime_match_rate( + checked: Optional[float], matched: Optional[float] +) -> Optional[float]: + if checked is None or checked <= 0 or matched is None: + return None + return max(0.0, min(1.0, matched / checked)) + + +def _runtime_comparison_counts( + summary: Mapping[str, Any], +) -> Optional[tuple[int, int, int]]: + comparison = _mapping_path_value( + summary, ("aggregate_statistics", "runtime_bytecode_comparison") + ) + if not isinstance(comparison, Mapping): + return None + checked, equal, total = ( + comparison.get("checked_n"), + comparison.get("equal_n"), + comparison.get("total_n"), + ) + if not all(type(value) is int for value in (checked, equal, total)): + return None + if not 0 <= equal <= checked <= total: + return None + return checked, equal, total + + +def _runtime_match_rate(summary: Mapping[str, Any]) -> str: + counts = _runtime_comparison_counts(summary) + if counts is not None: + checked, equal, _ = counts + return f"{equal / checked * 100:.2f}%" if checked else "n/a (no comparisons)" + checked = _coerce_float(summary.get("bytecode_runtime_checked_mean")) + if checked == 0: + return "n/a (no comparisons)" + rate = _checked_runtime_match_rate( + checked, _coerce_float(summary.get("bytecode_runtime_match_mean")) + ) + return f"{rate * 100:.2f}%" if rate is not None else "n/a" + + def _format_number(value: Any) -> str: if not isinstance(value, (int, float)) or math.isnan(float(value)): return "n/a" @@ -1350,6 +1471,19 @@ def _replication_category_gaps(summary: Mapping[str, Any]) -> list[Dict[str, Any return gaps +def _behavior_only_replication_micro(summary: Mapping[str, Any]) -> Optional[Dict[str, Any]]: + category_scores = summary.get("replication_by_category_micro") + if not isinstance(category_scores, Mapping): + category_scores = _mapping_path_value(summary, ("replication_metrics", "by_category_micro")) + if not isinstance(category_scores, Mapping): + category_scores = _mapping_path_value( + summary, ("aggregate_statistics", "replication_metrics", "by_category_micro") + ) + if not isinstance(category_scores, Mapping): + return None + return behavior_only_replication_micro(category_scores) + + def _hallucination_rate(summary: Mapping[str, Any]) -> Optional[float]: direct = _first_numeric_summary_value( summary, @@ -1462,7 +1596,7 @@ def _add_missing_metric_caveats(summary: Mapping[str, Any], caveats: list[str]) ), ( ("bytecode_semantic_checked_mean",), - "Bytecode-grounded metrics are missing; semantic conclusions rely mainly on text similarity.", + "Legacy bytecode semantic checked flag is missing; reference bytecode coverage cannot be established from the aggregate.", ), ] for keys, caveat in checks: @@ -1782,7 +1916,7 @@ def _append_benchmark_suite_report( return lines.extend(["", "Benchmark Suites", "----------------"]) lines.append( - "suite | count | semantic | edit distance | replication F1 | bytecode semantic | Solidity valid" + "suite | count | text similarity | edit distance | replication F1 | source-fact proxy | best-effort Solidity valid" ) lines.append("--- | ---: | ---: | ---: | ---: | ---: | ---:") for suite, suite_summary in sorted(benchmark_suites.items()): diff --git a/src/inference.py b/src/inference.py index 11db8e0..d5ec62e 100644 --- a/src/inference.py +++ b/src/inference.py @@ -101,7 +101,7 @@ def analyze_bytecode_tac(bytecode: str): analyzer = BytecodeAnalyzer(bytecode) func_tac = analyzer.generate_per_function_tac() if not func_tac: - func_tac = {"contract": analyzer.generate_tac_representation()} + return analyzer, {}, analyzer.generate_tac_representation() combined = "\n\n".join(func_tac.values()) return analyzer, func_tac, combined @@ -113,7 +113,7 @@ def build_function_metadata( tac_text: str, base_metadata: Mapping[str, Any] | None = None, ) -> Dict[str, Any]: - """Build inference metadata using only bytecode/analyzer-derived facts.""" + """Build function prompt metadata without adding training-absent contract counts.""" metadata = dict(base_metadata or {}) hex_body = bytecode[2:] if str(bytecode).lower().startswith("0x") else str(bytecode) func_obj = getattr(analyzer, "functions", {}).get(fname) @@ -124,9 +124,7 @@ def build_function_metadata( { "bytecode_hex_length": len(hex_body), "bytecode_byte_length": len(hex_body) // 2, - "bytecode_instruction_count": len(getattr(analyzer, "instructions", []) or []), "basic_block_count": len(getattr(analyzer, "basic_blocks", {}) or {}), - "function_count": len(getattr(analyzer, "functions", {}) or {}), "tac_line_count": len(str(tac_text or "").splitlines()), "tac_char_count": len(str(tac_text or "")), } @@ -412,7 +410,10 @@ def run_bytecode_inference( "bytecode analysis", operation_runner, ) - func_names = list(func_tac_map.keys()) + no_function_boundaries = not func_tac_map or ( + set(func_tac_map) == {"contract"} and not getattr(analyzer, "functions", {}) + ) + func_names = [] if no_function_boundaries else list(func_tac_map.keys()) if max_functions is not None and len(func_names) > max_functions: raise InferenceWorkLimitError( f"too many functions detected ({len(func_names)}); maximum is {max_functions}" @@ -422,12 +423,88 @@ def run_bytecode_inference( analysis_base = { "num_instructions": len(getattr(analyzer, "instructions", []) or []), "num_basic_blocks": len(getattr(analyzer, "basic_blocks", {}) or {}), - "num_functions": len(func_tac_map), + "num_functions": len(func_names), "tac_generation_time_s": round(tac_time, 3), } + rejected_targets = dict(getattr(analyzer, "rejected_dispatcher_targets", {}) or {}) + if rejected_targets: + analysis_base["rejected_dispatcher_targets"] = rejected_targets + incomplete_reason = ( + "Some dispatcher selectors have invalid targets: " + + ", ".join( + f"{selector} -> 0x{target:x}" for selector, target in sorted(rejected_targets.items()) + ) + if rejected_targets + else None + ) trace["analysis"].update(analysis_base) _trace_event(trace, "analysis_done", **analysis_base) + if no_function_boundaries: + error = ( + "No recoverable function boundaries in bytecode; refusing to infer a " + "function from whole-contract TAC." + ) + diagnostic_tac = f"// Analysis failed: {error}\n{combined_tac}" + source_summary = {"exact_match": 0, "model_inference": 0, "error": 0, "unknown": 0} + reconstruction = build_reconstruction_plan(bytecode, analyzer, {}) + reconstruction["strategy"] = "no_recoverable_function_chunks" + reconstruction["description"] = error + quality = build_contract_quality( + {"valid": False, "method": "not_run"}, + [], + source_summary, + reconstruction, + rejected_dispatcher_targets=rejected_targets, + ) + quality["recommended_actions"].insert(0, "Inspect runtime dispatcher targets.") + analysis = { + **analysis_base, + "error": error, + "function_sources": {}, + "function_errors": {}, + "function_results": [], + "source_summary": source_summary, + "reconstruction": reconstruction, + "quality": quality, + } + trace["analysis"].update(analysis) + trace["reconstruction"] = reconstruction + _trace_event(trace, "analysis_failed", error=error) + _finish_trace(trace, started_at, "failed", error) + return { + "success": False, + "partial_success": False, + "decompilation_status": "analysis_failed", + "error": error, + "model_path": model_path, + "model_config": model_config_dict, + "generation_config": generation, + "effective_generation_config": generation, + "lookup_config": lookup, + "lookup": { + "enabled": bool(lookup.get("enabled")), + "available": bool(tac_lookup is not None and getattr(tac_lookup, "available", True)), + "hits": 0, + "misses": 0, + "provenance": {}, + }, + "tac": diagnostic_tac, + "tac_per_function": {}, + "functions": {}, + "function_results": [], + "source_summary": source_summary, + "solidity": "", + "validation": {}, + "function_validation": {}, + "selector_map": {}, + "reconstruction": reconstruction, + "quality": quality, + "analysis": analysis, + "trace": trace, + "trace_path": None, + } + selector_map: Dict[str, Any] = {} for fname in func_names: selector = _selector_for_function(analyzer, fname) @@ -527,7 +604,10 @@ def run_bytecode_inference( if tac_only: source_summary = {"exact_match": 0, "model_inference": 0, "error": 0, "unknown": 0} - quality = build_contract_quality({}, [], source_summary, reconstruction_plan) + quality = build_contract_quality( + {}, [], source_summary, reconstruction_plan, + rejected_dispatcher_targets=rejected_targets, + ) analysis = { **analysis_base, "solidity_generation_time_s": 0.0, @@ -551,11 +631,14 @@ def run_bytecode_inference( "quality": quality, } trace["analysis"].update(analysis) - _finish_trace(trace, started_at, "tac_only") + _finish_trace( + trace, started_at, "partial" if incomplete_reason else "tac_only", incomplete_reason + ) return { - "success": True, - "partial_success": False, - "decompilation_status": "tac_only_no_model", + "success": incomplete_reason is None, + "partial_success": incomplete_reason is not None, + "decompilation_status": "partial_analysis" if incomplete_reason else "tac_only_no_model", + "error": incomplete_reason, "model_path": model_path, "model_config": model_config_dict, "generation_config": generation, @@ -678,18 +761,29 @@ def run_bytecode_inference( lookup_provenance=lookup_provenance, ) source_summary = _source_summary(function_sources, function_errors, function_results) - quality = build_contract_quality(validation, function_results, source_summary, reconstruction_plan) + quality = build_contract_quality( + validation, function_results, source_summary, reconstruction_plan, + rejected_dispatcher_targets=rejected_targets, + ) failure_count = len(function_errors) validation_failed = not bool(validation.get("valid")) - success = failure_count == 0 and not validation_failed and not (decompiler is None and unresolved_fnames) - partial_success = (failure_count > 0 or validation_failed) and any( - source != "error" for source in function_sources.values() + success = ( + failure_count == 0 + and not validation_failed + and incomplete_reason is None + and not (decompiler is None and unresolved_fnames) ) - decompilation_status = ( - "partial_error" - if function_errors - else "validation_failed" if validation_failed else "model_generated" + partial_success = (failure_count > 0 or validation_failed or incomplete_reason is not None) and any( + source != "error" for source in function_sources.values() ) + if function_errors: + decompilation_status = "partial_error" + elif validation_failed: + decompilation_status = "validation_failed" + elif incomplete_reason: + decompilation_status = "partial_analysis" + else: + decompilation_status = "model_generated" analysis = { **analysis_base, "solidity_generation_time_s": round(gen_time, 3), @@ -722,13 +816,15 @@ def run_bytecode_inference( trace, started_at, "success" if success else "partial" if partial_success else "failed", - None if success or partial_success else "Solidity validation failed" if validation_failed else None, + incomplete_reason + or (None if success or partial_success else "Solidity validation failed" if validation_failed else None), ) return { "success": success, "partial_success": partial_success, "decompilation_status": decompilation_status, + "error": incomplete_reason, "model_path": model_path, "model_config": model_config_dict, "generation_config": generation, diff --git a/src/model_setup.py b/src/model_setup.py index 8510750..49bc78e 100644 --- a/src/model_setup.py +++ b/src/model_setup.py @@ -52,7 +52,8 @@ logger = logging.getLogger(__name__) -TOKENIZATION_CACHE_VERSION = 2 +TOKENIZATION_CACHE_VERSION = 3 +SELECTOR_SIGNATURE_PROMPT_POLICY = "bundled_only_v1" DEFAULT_REPETITION_PENALTY = 1.05 SOLIDITY_RESERVED_WORDS = { @@ -495,7 +496,7 @@ def _format_count_part(label: str, value: Optional[int]) -> Optional[str]: def resolve_selector_signature_for_prompt(selector: Optional[str]) -> Optional[str]: - """Resolve a selector locally for prompt context without remote lookups.""" + """Use only bundled selector signatures available without the training database.""" normalized = _normalize_selector(selector) if not normalized: return None @@ -508,7 +509,7 @@ def resolve_selector_signature_for_prompt(selector: Optional[str]) -> Optional[s return None best = getattr(result, "best_match", None) - if best is None or getattr(best, "source", None) == "unknown": + if best is None or getattr(best, "source", None) != "builtin": return None try: confidence = float(getattr(best, "confidence", 0.0) or 0.0) @@ -1381,31 +1382,33 @@ def _tokenize_item(self, idx: int) -> Dict[str, List[Any]]: raise ValueError(f"Training example {idx} has an empty tokenized target") if len(target_ids) >= self.max_length: - input_ids = target_ids[: self.max_length] - prefix_len = 0 - else: - prefix_before_tac, tac_text, prefix_after_tac, _target, _suffix = ( - self._format_prompt_components(item["input"], output, item.get("metadata", {})) + raise ValueError( + f"Training example {idx} target requires {len(target_ids)} tokens " + f"(including EOS), leaving no context under max_length={self.max_length}" ) - header_ids = _tokenize_to_ids(self.tokenizer, prefix_before_tac) - tac_ids = _tokenize_to_ids(self.tokenizer, tac_text) - footer_ids = _tokenize_to_ids(self.tokenizer, prefix_after_tac) - - prefix_budget = self.max_length - len(target_ids) - fixed_prefix_len = len(header_ids) + len(footer_ids) - if fixed_prefix_len > prefix_budget: - footer_budget = min(len(footer_ids), prefix_budget) - header_budget = max(0, prefix_budget - footer_budget) - header_ids = header_ids[:header_budget] - footer_ids = footer_ids[-footer_budget:] if footer_budget else [] - tac_ids = [] - else: - tac_budget = prefix_budget - fixed_prefix_len - tac_ids = tac_ids[:tac_budget] - prefix_ids = header_ids + tac_ids + footer_ids - prefix_len = len(prefix_ids) - input_ids = prefix_ids + target_ids + prefix_before_tac, tac_text, prefix_after_tac, _target, _suffix = ( + self._format_prompt_components(item["input"], output, item.get("metadata", {})) + ) + header_ids = _tokenize_to_ids(self.tokenizer, prefix_before_tac) + tac_ids = _tokenize_to_ids(self.tokenizer, tac_text) + footer_ids = _tokenize_to_ids(self.tokenizer, prefix_after_tac) + + prefix_budget = self.max_length - len(target_ids) + fixed_prefix_len = len(header_ids) + len(footer_ids) + if fixed_prefix_len > prefix_budget: + footer_budget = min(len(footer_ids), prefix_budget) + header_budget = max(0, prefix_budget - footer_budget) + header_ids = header_ids[:header_budget] + footer_ids = footer_ids[-footer_budget:] if footer_budget else [] + tac_ids = [] + else: + tac_budget = prefix_budget - fixed_prefix_len + tac_ids = tac_ids[:tac_budget] + + prefix_ids = header_ids + tac_ids + footer_ids + prefix_len = len(prefix_ids) + input_ids = prefix_ids + target_ids labels = [-100] * prefix_len + target_ids[: len(input_ids) - prefix_len] tokenized = { @@ -2304,6 +2307,7 @@ def train( tokenizer, max_length=self.config.max_sequence_length, include_bytecode_metadata=self.config.include_bytecode_metadata, + include_selector_signature_metadata=self.config.include_selector_signature_metadata, tokenization_cache=tokenization_cache_config, ) @@ -2321,6 +2325,7 @@ def train( tokenizer, max_length=self.config.max_sequence_length, include_bytecode_metadata=self.config.include_bytecode_metadata, + include_selector_signature_metadata=self.config.include_selector_signature_metadata, tokenization_cache=tokenization_cache_config, ) if len(eval_dataset) == 0: diff --git a/src/replication_metrics.py b/src/replication_metrics.py index 976b8ce..3b7a8f3 100644 --- a/src/replication_metrics.py +++ b/src/replication_metrics.py @@ -26,7 +26,7 @@ "view", "virtual", } -_PARAM_QUALIFIERS = {"memory", "storage", "calldata", "indexed"} +_PARAM_QUALIFIERS = {"memory", "storage", "calldata", "indexed", "payable"} _CONTROL_KEYWORDS = {"if", "for", "while"} _CALL_EXCLUSIONS = { "assert", @@ -59,6 +59,7 @@ "uint", "uint256", } +_TYPE_CONVERSION_RE = re.compile(r"(?:u?int(?:\d{1,3})?|bytes(?:\d{1,2})?)\Z") _HALLUCINATION_BUCKET_BY_CATEGORY = { "abi": "unsupported_abi_elements", "visibility": "unsupported_abi_elements", @@ -72,6 +73,15 @@ "return": "unsupported_return_expressions", "control_flow": "unsupported_control_flow", } +BEHAVIOR_FACT_CATEGORIES = ( + "call", + "control_flow", + "event", + "guard", + "member_call", + "return", + "state_write", +) @dataclass(frozen=True) @@ -228,6 +238,27 @@ def evaluate_replication( ) +def behavior_only_replication_micro( + by_category: Mapping[str, Mapping[str, Any]], +) -> Optional[Dict[str, Any]]: + """Micro F1 for extracted behavior facts, excluding ABI and modifiers. + + This compares source-extracted facts, not runtime or bytecode behavior. + """ + counts = {"true_positives": 0, "false_positives": 0, "false_negatives": 0} + for category in BEHAVIOR_FACT_CATEGORIES: + category_score = by_category.get(category) + if not isinstance(category_score, Mapping): + continue + for key in counts: + counts[key] += int(category_score.get(key, 0)) + if not any(counts.values()): + return None + return PrecisionRecallF1.from_counts( + counts["true_positives"], counts["false_positives"], counts["false_negatives"] + ).to_dict() + + def aggregate_replication_scores(metrics: Iterable[Mapping[str, Any]]) -> Dict[str, Any]: """Aggregate per-example replication metrics into mean and micro scores.""" metric_rows = list(metrics) @@ -298,6 +329,10 @@ def aggregate_replication_scores(metrics: Iterable[Mapping[str, Any]]) -> Dict[s ).to_dict() for category, counts in sorted(category_counts.items()) } + behavior_micro = behavior_only_replication_micro(summary["by_category_micro"]) + if behavior_micro is not None: + summary["behavior_only_micro"] = behavior_micro + summary["behavior_only_categories"] = list(BEHAVIOR_FACT_CATEGORIES) summary["category_gap_summary"] = _category_gap_summary(category_counts) hallucination_total = sum(hallucination_counts.values()) @@ -441,11 +476,12 @@ def _mask_string_literals(code: str) -> str: def _extract_function_signature(code: str) -> Dict[str, Any]: match = re.search( - r"\bfunction\s+([A-Za-z_][A-Za-z0-9_]*)?\s*" r"\((?P.*?)\)\s*(?P[^{;]*)", + r"\bfunction\s+([A-Za-z_][A-Za-z0-9_]*)?\s*\(", code, flags=re.DOTALL, ) - if not match: + params_end = _find_matching_paren(code, match.end() - 1) if match else None + if not match or params_end is None: return { "function_name": None, "params": {"types": [], "names": []}, @@ -457,19 +493,20 @@ def _extract_function_signature(code: str) -> Dict[str, Any]: "match_end": 0, } - tail = match.group("tail") or "" - returns_match = re.search(r"\breturns\s*\((?P.*?)\)", tail, re.DOTALL) - returns = ( - _parse_parameter_list(returns_match.group("returns")) - if returns_match - else { - "types": [], - "names": [], - } - ) - tail_without_returns = ( - tail[: returns_match.start()] + tail[returns_match.end() :] if returns_match else tail - ) + tail_start = params_end + 1 + tail_boundary = re.search(r"[{;]", code[tail_start:]) + tail_end = tail_start + tail_boundary.start() if tail_boundary else len(code) + tail = code[tail_start:tail_end] + returns_match = re.search(r"\breturns\s*\(", tail, re.DOTALL) + returns = {"types": [], "names": []} + tail_without_returns = tail + if returns_match: + returns_end = _find_matching_paren(tail, returns_match.end() - 1) + if returns_end is not None: + returns = _parse_parameter_list(tail[returns_match.end() : returns_end]) + tail_without_returns = tail[: returns_match.start()] + tail[returns_end + 1 :] + else: + tail_without_returns = tail[: returns_match.start()] tail_tokens = [ token.lower() if token.lower() in _SIGNATURE_QUALIFIERS else token for token in re.findall(r"[A-Za-z_][A-Za-z0-9_]*", tail_without_returns) @@ -485,13 +522,13 @@ def _extract_function_signature(code: str) -> Dict[str, Any]: return { "function_name": match.group(1), - "params": _parse_parameter_list(match.group("params")), + "params": _parse_parameter_list(code[match.end() : params_end]), "returns": returns["types"], "visibility": visibility, "mutability": mutability, "is_payable": "payable" in tail_tokens, "modifiers": sorted(set(modifiers)), - "match_end": match.end(), + "match_end": tail_end, } @@ -527,8 +564,11 @@ def _add_signature_facts(facts: FactMap, signature: Mapping[str, Any]): def _extract_function_body(code: str, match_end: int) -> str: start = code.find("{", max(0, match_end - 1)) + declaration_end = code.find(";", max(0, match_end - 1)) + if declaration_end >= 0 and (start < 0 or declaration_end < start): + return "" if start < 0: - return code + return "" end = _find_matching_brace(code, start) return code[start + 1 : end] if end is not None else code[start + 1 :] @@ -583,7 +623,7 @@ def _split_top_level_commas(text: str) -> List[str]: def _canonical_type(parameter: str) -> str: - parameter = parameter.split("=")[0].strip() + parameter = parameter.strip() parameter = re.sub( rf"\b({'|'.join(sorted(_PARAM_QUALIFIERS))})\b", " ", @@ -601,10 +641,10 @@ def _canonical_type(parameter: str) -> str: def _parameter_name(parameter: str) -> Optional[str]: - tokens = re.findall(r"[A-Za-z_][A-Za-z0-9_]*", parameter) - if len(tokens) < 2: + match = re.search(r"\s+([A-Za-z_][A-Za-z0-9_]*)\s*$", parameter) + if not match: return None - candidate = tokens[-1] + candidate = match.group(1) if candidate.lower() in _PARAM_QUALIFIERS: return None return candidate @@ -640,6 +680,8 @@ def _add_call_facts(facts: FactMap, body: str, function_name: Optional[str]): continue if normalized in _RESERVED_IDENTIFIERS: continue + if _TYPE_CONVERSION_RE.fullmatch(normalized): + continue if function_name and normalized == _normalize_identifier(function_name): continue _add_fact(facts, "call", normalized) diff --git a/src/training_pipeline.py b/src/training_pipeline.py index 1a2fd91..6f55342 100644 --- a/src/training_pipeline.py +++ b/src/training_pipeline.py @@ -379,7 +379,13 @@ def to_dict(self) -> Dict[str, Any]: @dataclass(frozen=True) class BytecodeSemanticResult: - """Bytecode-grounded semantic/deployability checks for one generated output.""" + """Source-fact proxy with reference-bytecode hints and optional runtime checks. + + ``checked`` means reference bytecode/opcode evidence is present, not that + generated bytecode was compared or that behavior was verified. + ``runtime_bytecode_match`` means exact full-contract bytecode identity, + not semantic or deployment equivalence. + """ checked: bool score: float @@ -387,6 +393,7 @@ class BytecodeSemanticResult: deployable: bool runtime_bytecode_checked: bool runtime_bytecode_match: Optional[bool] + runtime_bytecode_skip_reason: Optional[str] = None opcode_groups: List[str] = field(default_factory=list) control_flow_buckets: List[str] = field(default_factory=list) mismatch_buckets: Dict[str, List[str]] = field(default_factory=dict) @@ -394,11 +401,17 @@ class BytecodeSemanticResult: def to_dict(self) -> Dict[str, Any]: return { "checked": self.checked, + "checked_kind": "reference_bytecode_or_opcode_evidence", "score": self.score, + "score_kind": "source_fact_overlap_proxy", "deployability_checked": self.deployability_checked, "deployable": self.deployable, "runtime_bytecode_checked": self.runtime_bytecode_checked, "runtime_bytecode_match": self.runtime_bytecode_match, + "runtime_bytecode_skip_reason": self.runtime_bytecode_skip_reason, + "runtime_comparison_kind": ( + "exact_full_contract_runtime_bytecode" if self.runtime_bytecode_checked else None + ), "opcode_groups": list(self.opcode_groups), "control_flow_buckets": list(self.control_flow_buckets), "mismatch_buckets": { @@ -459,11 +472,13 @@ def evaluate_bytecode_semantics( solidity_validity: Optional[SolidityValidityResult] = None, replication: Optional[Any] = None, ) -> BytecodeSemanticResult: - """Evaluate generated Solidity against bytecode-grounded semantic signals. + """Score source-fact overlap with available reference bytecode/TAC diagnostics. - The check is deterministic without solc: it uses bytecode/TAC opcode evidence - from metadata plus structured Solidity facts. When local solc compilation is - available, compiled runtime bytecode is compared to expected runtime bytecode. + This is not a bytecode equivalence or execution test: the primary score + compares facts extracted from reference and generated Solidity. Without + reference bytecode/opcode evidence, the score is zero and unchecked. When + Runtime comparison is opt-in for full contracts with verified reference + bytecode and matching local compiler settings; fragments remain unchecked. """ metadata = metadata or {} if solidity_validity is None: @@ -526,26 +541,13 @@ def evaluate_bytecode_semantics( reasons = solidity_validity.compiler_errors or solidity_validity.scaffold_errors mismatch_buckets["deployability_failure"].extend(reasons or ["not_deployable"]) - expected_runtime = _first_bytecode_value(metadata, ("runtime_bytecode", "deployed_bytecode")) - compiled_runtime = ( - _first_bytecode_value( - metadata, - ( - "candidate_runtime_bytecode", - "generated_runtime_bytecode", - "compiled_runtime_bytecode", - ), - ) - or solidity_validity.compiled_runtime_bytecode + runtime_match, runtime_skip_reason = _compare_grounded_runtime_bytecode( + reference_code, candidate_code, metadata ) - runtime_checked = bool(expected_runtime and compiled_runtime) - runtime_match: Optional[bool] = None - if runtime_checked: - runtime_match = normalize_evm_bytecode(expected_runtime) == normalize_evm_bytecode( - compiled_runtime - ) + runtime_checked = runtime_match is not None + if runtime_match is not None: if not runtime_match: - mismatch_buckets["runtime_bytecode_mismatch"].append("normalized_runtime_differs") + mismatch_buckets["runtime_bytecode_mismatch"].append("compiled_runtime_differs") for failure in _differential_call_failures(metadata): mismatch_buckets["differential_call_failure"].append(failure) @@ -565,14 +567,13 @@ def evaluate_bytecode_semantics( if mismatch_buckets: score = min(score, 1.0 - min(0.9, 0.1 * len(mismatch_buckets))) - has_bytecode_evidence = bool( - opcode_slices["opcode_groups"] or expected_runtime or compiled_runtime - ) checked = bool( - has_bytecode_evidence - or solidity_validity.compiler_checked - or solidity_validity.bytecode_checked + _first_bytecode_value(metadata) + or _extract_result_opcodes({"metadata": metadata}) + or opcode_slices["opcode_groups"] ) + if not checked: + score = 0.0 deployable = bool(solidity_validity.deployable) return BytecodeSemanticResult( @@ -584,6 +585,7 @@ def evaluate_bytecode_semantics( deployable=deployable, runtime_bytecode_checked=runtime_checked, runtime_bytecode_match=runtime_match, + runtime_bytecode_skip_reason=runtime_skip_reason, opcode_groups=opcode_slices["opcode_groups"], control_flow_buckets=opcode_slices["control_flow"], mismatch_buckets={key: sorted(set(values)) for key, values in mismatch_buckets.items()}, @@ -611,6 +613,175 @@ def normalize_evm_bytecode(bytecode: Optional[str]) -> str: return text +def _exact_evm_bytecode(bytecode: Any) -> Optional[str]: + """Accept complete hex bytecode, without silently dropping non-hex characters.""" + if not isinstance(bytecode, str): + return None + text = bytecode.removeprefix("0x").removeprefix("0X") + if not text or len(text) % 2 or re.fullmatch(r"[0-9a-fA-F]+", text) is None: + return None + return text.lower() + + +def _compare_grounded_runtime_bytecode( + reference_code: str, candidate_code: str, metadata: Mapping[str, Any] +) -> Tuple[Optional[bool], Optional[str]]: + """Compare named, self-contained contracts only when their reference runtime is verified. + + ``runtime_comparison`` opts in with contract_name, compiler_version, + optimizer_enabled, optimizer_runs, and optionally source_name (default: + contract.sol, the local dataset compiler's source unit). The metadata + ``runtime_bytecode``/``deployed_bytecode`` or ``bytecode`` must reproduce + from the complete reference source with those exact settings. No creation + bytecode, provided candidate runtime, or fragment harness is accepted. + Returns exact bytecode equality and a per-row reason when unchecked. + """ + if "runtime_comparison" not in metadata: + return _runtime_comparison_skipped("not_opted_in") + settings = metadata["runtime_comparison"] + if not isinstance(settings, Mapping): + return _runtime_comparison_skipped("invalid_compiler_settings") + contract_name = settings.get("contract_name") + version = settings.get("compiler_version") + optimized = settings.get("optimizer_enabled") + runs = settings.get("optimizer_runs") + source_name = settings.get("source_name", "contract.sol") + if ( + not isinstance(contract_name, str) + or re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", contract_name) is None + or not isinstance(version, str) + or re.fullmatch(r"\d+\.\d+\.\d+", version) is None + or type(optimized) is not bool + or type(runs) is not int + or runs < 0 + or not isinstance(source_name, str) + or re.fullmatch(r"[A-Za-z_][A-Za-z0-9_.-]*\.sol", source_name) is None + ): + return _runtime_comparison_skipped("invalid_compiler_settings") + if ( + (_metadata_compiler_version(metadata) not in (None, version)) + or ("optimizer_enabled" in metadata and metadata["optimizer_enabled"] is not optimized) + or ("optimizer_runs" in metadata and metadata["optimizer_runs"] != runs) + ): + return _runtime_comparison_skipped("conflicting_compiler_settings") + expected = _first_bytecode_value(metadata, ("runtime_bytecode", "deployed_bytecode")) + if not expected: + expected = metadata.get("bytecode") + expected_hex = _exact_evm_bytecode(expected) + if expected_hex is None: + return _runtime_comparison_skipped("missing_or_invalid_reference_runtime") + + reference_runtime, reference_reason = _compile_named_runtime( + reference_code, contract_name, version, optimized, runs, source_name + ) + if reference_runtime is None: + return _runtime_comparison_skipped(f"reference_{reference_reason}") + if reference_runtime != expected_hex: + return _runtime_comparison_skipped("reference_runtime_mismatch") + candidate_runtime, candidate_reason = _compile_named_runtime( + candidate_code, contract_name, version, optimized, runs, source_name + ) + if candidate_runtime is None: + return _runtime_comparison_skipped(f"candidate_{candidate_reason}") + return candidate_runtime == expected_hex, None + + +def _runtime_comparison_skipped(reason: str) -> Tuple[None, str]: + if reason != "not_opted_in": + logger.info("Exact runtime bytecode comparison skipped: %s", reason) + return None, reason + + +def _compile_named_runtime( + source_code: str, + contract_name: str, + version: str, + optimized: bool, + runs: int, + source_name: str, +) -> Tuple[Optional[str], Optional[str]]: + """Compile a full, constructor-free contract without installing solc.""" + masked_source = _mask_solidity_comments_and_strings(source_code or "") + if re.search(r"\bimport\s", masked_source): + return None, "imports_unsupported" + if not re.search(rf"\bcontract\s+{re.escape(contract_name)}\b", masked_source): + reason = ( + "target_contract_missing" + if re.search(r"\bcontract\s+[A-Za-z_][A-Za-z0-9_]*\b", masked_source) + else "fragment" + ) + return None, reason + try: + import solcx + from solcx.exceptions import SolcError, SolcInstallationError, SolcNotInstalled + except ImportError: + return None, "solcx_unavailable" + except OSError as exc: + logger.warning("Local solcx import failed for runtime comparison: %s", exc) + return None, "solcx_import_io_error" + + try: + if version not in {str(installed) for installed in solcx.get_installed_solc_versions()}: + return None, "solc_version_not_installed" + except OSError as exc: + logger.warning("Local solc discovery failed for runtime comparison: %s", exc) + return None, "compiler_discovery_io_error" + + try: + output = solcx.compile_standard( + { + "language": "Solidity", + "sources": {source_name: {"content": source_code}}, + "settings": { + "optimizer": {"enabled": optimized, "runs": runs}, + "outputSelection": { + "*": { + "": ["ast"], + "*": ["evm.deployedBytecode.object"], + } + }, + }, + }, + solc_version=version, + allow_paths=[], + ) + except SolcError: + return None, "compiler_rejected_source" + except (SolcNotInstalled, SolcInstallationError): + return None, "compiler_unavailable" + except OSError as exc: + logger.warning("Local solc failed for runtime comparison: %s", exc) + return None, "compiler_io_error" + + ast = output.get("sources", {}).get(source_name, {}).get("ast", {}) + contracts = [ + node + for node in ast.get("nodes", []) + if node.get("nodeType") == "ContractDefinition" and node.get("name") == contract_name + ] + if len(contracts) != 1 or contracts[0].get("contractKind") != "contract": + return None, "target_contract_missing" + contract = contracts[0] + if contract.get("abstract") or contract.get("baseContracts"): + return None, "abstract_or_inherited_contract" + for node in contract.get("nodes", []): + if node.get("nodeType") == "FunctionDefinition" and node.get("kind") == "constructor": + return None, "constructor_or_state_initializer" + if node.get("nodeType") == "VariableDeclaration" and node.get("stateVariable"): + if node.get("value") is not None or node.get("mutability") == "immutable": + return None, "constructor_or_state_initializer" + runtime = ( + output.get("contracts", {}) + .get(source_name, {}) + .get(contract_name, {}) + .get("evm", {}) + .get("deployedBytecode", {}) + .get("object") + ) + compiled = _exact_evm_bytecode(runtime) + return (compiled, None) if compiled is not None else (None, "compiled_runtime_unavailable") + + def extract_evm_opcodes(bytecode_or_opcode_text: Any) -> List[str]: """Extract opcode names from bytecode hex, opcode lists, or TAC-like text.""" if isinstance(bytecode_or_opcode_text, Sequence) and not isinstance( @@ -758,16 +929,12 @@ def _first_bytecode_value( direct = _first_bytecode_value(nested, keys) if direct: return direct - bytecode = nested.get("bytecode") - if isinstance(bytecode, Mapping): - obj = bytecode.get("object") - if isinstance(obj, str) and normalize_evm_bytecode(obj): - return obj - deployed = nested.get("deployedBytecode") or nested.get("deployed_bytecode") - if isinstance(deployed, Mapping): - obj = deployed.get("object") - if isinstance(obj, str) and normalize_evm_bytecode(obj): - return obj + if "runtime_bytecode" in keys or "deployed_bytecode" in keys: + deployed = nested.get("deployedBytecode") or nested.get("deployed_bytecode") + if isinstance(deployed, Mapping): + obj = deployed.get("object") + if isinstance(obj, str) and normalize_evm_bytecode(obj): + return obj return None @@ -2393,6 +2560,11 @@ def _split_dataset(self, dataset_path: str) -> Tuple[str, str, str]: "include_bytecode_metadata", True, ), + include_selector_signature_metadata=getattr( + self.config.model_config, + "include_selector_signature_metadata", + True, + ), skip=not self.config.run_data_preflight, allow_tokenizer_download=self.config.preflight_tokenizer_download, allow_whitespace_fallback=self.config.allow_whitespace_preflight_fallback, @@ -2637,6 +2809,36 @@ def _compute_aggregate_statistics( "95th": float(np.percentile(values, 95)), } + runtime_checked_n = sum( + result["metrics"].get("bytecode_runtime_checked") is True for result in results + ) + runtime_equal_n = sum( + result["metrics"].get("bytecode_runtime_checked") is True + and result["metrics"].get("bytecode_runtime_match") is True + for result in results + ) + runtime_skip_reasons: Counter[str] = Counter() + for result in results: + metrics = result["metrics"] + details = metrics.get("metadata") + if metrics.get("bytecode_runtime_checked") is True or not isinstance(details, Mapping): + continue + bytecode_details = details.get("bytecode_semantics") + if isinstance(bytecode_details, Mapping): + reason = bytecode_details.get("runtime_bytecode_skip_reason") + if isinstance(reason, str) and reason: + runtime_skip_reasons[reason] += 1 + stats["runtime_bytecode_comparison"] = { + "checked_n": runtime_checked_n, + "equal_n": runtime_equal_n, + "total_n": len(results), + "equality_rate_checked": ( + runtime_equal_n / runtime_checked_n if runtime_checked_n else None + ), + "skip_reasons": dict(sorted(runtime_skip_reasons.items())), + "scope": "exact_full_contract_runtime_bytecode_not_deployment_or_behavior", + } + # Add paper-specific metrics if "semantic_similarity" in metrics_data: semantic_values = metrics_data["semantic_similarity"] diff --git a/tests/test_benchmark_runtime_controls.py b/tests/test_benchmark_runtime_controls.py new file mode 100644 index 0000000..0f52b02 --- /dev/null +++ b/tests/test_benchmark_runtime_controls.py @@ -0,0 +1,93 @@ +"""Local-solc regression controls for the opt-in runtime identity comparator.""" + +from collections import Counter + +import pytest + +from scripts.benchmark_runtime_controls import ( + build_contracts, + check_outcomes, + run_benchmark, + select_solc_version, +) + + +def test_deterministic_local_compiler_selection_without_downloads(): + assert select_solc_version(["0.8.33", "0.8.20", "0.8.21"]) == "0.8.20" + assert select_solc_version(["0.7.6", "0.8.25", "0.8.22"]) == "0.8.22" + with pytest.raises(RuntimeError, match="no downloads attempted"): + select_solc_version(["0.8.19", "0.7.6"]) + + +def test_contract_sources_are_distinct_full_contracts_with_mutations(): + cases = build_contracts() + assert len(cases) == len({case.name for case in cases}) == 30 + assert len({case.source for case in cases}) == 30 + assert Counter(case.changed_control for case in cases) == { + "return_changed": 15, + "storage_changed": 15, + } + for case in cases: + assert f"contract {case.name} " in case.source + assert "pragma solidity" in case.source + assert case.source != case.changed_source + assert "import " not in case.source + + +@pytest.fixture(scope="module") +def local_report(): + solcx = pytest.importorskip("solcx") + try: + select_solc_version(solcx.get_installed_solc_versions()) + except RuntimeError as exc: + pytest.skip(str(exc)) + return run_benchmark() + + +def test_real_solc_controls_are_grounded_and_labels_match(local_report): + assert local_report["failures"] == [] + assert local_report["distinct_complete_contracts"] == 30 + assert local_report["independently_compiled_reference_runtimes"] == 30 + assert local_report["distinct_executable_prefixes"] == 30 + assert local_report["control_counts"] == { + "comment_only": 1, + "fragment": 30, + "identical_source": 30, + "return_changed": 15, + "storage_changed": 15, + "unsupported_import": 30, + } + assert local_report["coverage"] == { + "total": 121, + "checked": 61, + "equal": 30, + "unequal": 31, + "unchecked": 60, + "skip_reasons": {"candidate_fragment": 30, "candidate_imports_unsupported": 30}, + } + for row in local_report["rows"]: + assert row["score_kind"] == "source_fact_overlap_proxy" + assert row["comparison_kind"] == ( + "exact_full_contract_runtime_bytecode" if row["checked"] else None + ) + assert check_outcomes(local_report["rows"]) == [] + + +def test_label_mismatches_fail_with_contract_and_skip_reason(local_report): + identical = next(row for row in local_report["rows"] if row["control"] == "identical_source") + fragment = next(row for row in local_report["rows"] if row["control"] == "fragment") + comment_only = next(row for row in local_report["rows"] if row["control"] == "comment_only") + failures = check_outcomes( + [ + {**identical, "equal": False}, + {**fragment, "checked": True, "equal": False, "skip_reason": None}, + {**comment_only, "equal": True}, + ] + ) + assert [failure["control"] for failure in failures] == [ + "identical_source", + "fragment", + "comment_only", + ] + assert failures[1]["expected"]["skip_reason"] == "candidate_fragment" + assert all(failure["contract"] for failure in failures) diff --git a/tests/test_benchmark_tac_fidelity.py b/tests/test_benchmark_tac_fidelity.py new file mode 100644 index 0000000..5dcfbeb --- /dev/null +++ b/tests/test_benchmark_tac_fidelity.py @@ -0,0 +1,109 @@ +"""Local-solc fidelity benchmark checks; no model or network access.""" + +from collections import Counter + +import pytest + +from scripts import benchmark_tac_fidelity as benchmark + + +@pytest.fixture(scope="module") +def report(): + return benchmark.run_benchmark() + + +def test_fixture_has_unique_functions_in_each_stratum(): + specs, source = benchmark._fixtures() + assert len(specs) == 36 + assert Counter(spec[0] for spec in specs) == {name: 6 for name in benchmark.OPCODES} + assert len({(contract, name) for _, contract, name, _ in specs}) == 36 + assert source == benchmark._fixtures()[1] + assert "pragma solidity ^0.8.20;" in source + + +def test_source_map_uses_opcode_pcs_not_push_data_or_dispatcher(): + # 0xf3 inside PUSH1 is not a RETURN; the actual RETURN is at PC 2. + assert benchmark._source_mapped_opcode_pcs("60f3f3", "0:3:0;0:3:0", 0, (0, 3), "RETURN") == [2] + assert benchmark._source_mapped_opcode_pcs("60f3f3", "0:3:0;4:3:0", 0, (0, 3), "RETURN") == [] + with pytest.raises(ValueError, match="source-map/instruction mismatch"): + benchmark._source_mapped_opcode_pcs("60f3f3", "0:3:0", 0, (0, 3), "RETURN") + + +def test_missing_local_compiler_refuses_download(monkeypatch): + monkeypatch.setattr(benchmark.solcx, "get_installed_solc_versions", lambda: []) + + def no_compilation(*_args, **_kwargs): + pytest.fail("must not invoke compilation or download for a missing compiler") + + monkeypatch.setattr(benchmark.solcx, "compile_standard", no_compilation) + with pytest.raises(RuntimeError, match="must already be installed locally"): + benchmark.run_benchmark() + + +def test_report_compares_abi_selectors_with_source_mapped_tac(report): + summary = report["summary"] + assert report["schema_version"] == 1 + assert report["lineage"]["compiler_version"].startswith("0.8.20+commit.") + assert report["lineage"]["compiler_settings"]["evmVersion"] == "shanghai" + assert report["lineage"]["compiler_settings"]["optimizer"] == {"enabled": True, "runs": 200} + assert len(report["lineage"]["source_sha256"]) == 64 + assert len(report["lineage"]["compiler_binary_sha256"]) == 64 + assert summary["contracts"] == summary["cases"] == 36 + assert summary["unique_selectors"] == len({case["selector"] for case in report["cases"]}) == 36 + assert summary["selector_coverage"] == {"found": 36, "total": 36, "rate": 1} + assert summary["source_mapped_opcode_reachability"] == { + "found": 36, + "total": 36, + "rate": 1, + } + assert summary["reachable_tac_opcode_coverage"] == { + "found": 36, + "total": 36, + "rate": 1, + } + assert summary["end_to_end_tac_opcode_coverage"] == { + "found": 36, + "total": 36, + "rate": 1, + } + assert summary["failing_cases"] == 0 + assert report["sample_failures"] == [] + assert set(report["strata"]) == set(benchmark.OPCODES) + for name, stratum in report["strata"].items(): + assert stratum["cases"] == stratum["selector_coverage"]["total"] == 6 + assert stratum["expected_opcode"] == benchmark.OPCODES[name] + assert stratum["source_mapped_opcode_reachability"]["total"] == 6 + assert stratum["end_to_end_tac_opcode_coverage"]["found"] == 6 + assert all(len(case["source_mapped_opcode_pcs"]) > 0 for case in report["cases"]) + assert all(len(case["runtime_sha256"]) == 64 for case in report["cases"]) + assert sum(s["failing_cases"] for s in report["strata"].values()) == summary["failing_cases"] + assert "does not establish" in report["interpretation"] + + +def test_source_mapped_witnesses_are_not_hidden_by_unrelated_tac_opcodes(report): + cases = {case["signature"]: case for case in report["cases"]} + simple_getter = cases["read0()"] + assert ( + simple_getter["source_mapped_opcode_pcs"] + == (simple_getter["reachable_source_opcode_pcs"]) + == simple_getter["tac_witness_pcs"] + ) + assert not simple_getter["failure_reasons"] + + for signature in ("read3(uint256)", "guard3(uint256)"): + case = cases[signature] + assert case["source_mapped_opcode_pcs"] + assert case["source_opcode_blocks"] + assert set(case["tac_witness_pcs"]).issubset(case["reachable_source_opcode_pcs"]) + if case["reachable_source_opcode_pcs"] == case["source_mapped_opcode_pcs"]: + assert case["tac_witness_pcs"] == case["source_mapped_opcode_pcs"] + assert not case["failure_reasons"] + else: + assert "compiler source-mapped opcode missing from function CFG reachability" in ( + case["failure_reasons"] + ) + + guard = cases["guard3(uint256)"] + if not guard["reachable_source_opcode_pcs"]: + assert guard["rendered_tac_opcode"] # ABI decoder REVERT is not the source guard. + assert not guard["tac_witness_pcs"] diff --git a/tests/test_build_curriculum_dataset.py b/tests/test_build_curriculum_dataset.py index 3c5a9be..d771fc7 100644 --- a/tests/test_build_curriculum_dataset.py +++ b/tests/test_build_curriculum_dataset.py @@ -11,9 +11,9 @@ ) -def _row(body_hash, output, input_text="function selector_deadbeef:\n CALL"): +def _row(body_hash, output, input_text=None): return { - "input": input_text, + "input": input_text or f"function selector_{body_hash}:\n CALL", "metadata": { "body_hash": body_hash, "function_signature": f"function f_{body_hash}()", @@ -34,7 +34,7 @@ def test_select_curriculum_rows_focuses_calls_and_excludes_identities(tmp_path): _row("excluded", "function excluded() public { token.transfer(msg.sender, 1); }"), _row( "best", - "function duplicateLong() public { token.balanceOf(msg.sender); token.transfer(owner, 1); }", + "FUNCTION best() public { /* same body */ token.balanceOf(msg.sender); token.transfer(owner, 1); }", input_text="function selector_deadbeef:\n CALL\n" + (" JUMP\n" * 100), ), _row( @@ -58,6 +58,22 @@ def test_select_curriculum_rows_focuses_calls_and_excludes_identities(tmp_path): assert selected[0].focus_counts == {"call": 2, "member_call": 2} +def test_curriculum_excludes_missing_hash_duplicates_and_keeps_distinct_bodies(tmp_path): + gate = _row("declared", "function a() public { token.transfer(owner, 1); }") + gate_path = tmp_path / "gate.jsonl" + _write_jsonl(gate_path, [gate]) + without_hash = _row(None, "FUNCTION a() public { /* comment */ token.transfer(owner, 1); }") + stale_hash = _row("declared", "function other() public { token.transfer(owner, 1); }") + distinct = _row(None, "function different() public { token.transfer(owner, 1); }") + selected = select_curriculum_rows( + [without_hash, stale_hash, distinct], + categories=focus_categories("calls"), + exclude=excluded_identities([gate_path]), + ) + + assert [candidate.row for candidate in selected] == [distinct] + + def test_write_curriculum_outputs_dataset_and_manifest(tmp_path): source_path = tmp_path / "source.jsonl" output_path = tmp_path / "calls.jsonl" @@ -86,6 +102,10 @@ def test_write_curriculum_outputs_dataset_and_manifest(tmp_path): assert output_path.exists() assert manifest_path.exists() assert len(load_jsonl(output_path)) == 1 + assert ( + load_jsonl(output_path)[0]["metadata"]["body_hash"] + == row_identity(source_rows[0]).split(":", 1)[1] + ) assert manifest["selected_rows"] == 1 assert manifest["rows"][0]["focus_counts"]["member_call"] == 1 diff --git a/tests/test_bytecode_analyzer.py b/tests/test_bytecode_analyzer.py index 5282ffa..12cf9ea 100644 --- a/tests/test_bytecode_analyzer.py +++ b/tests/test_bytecode_analyzer.py @@ -55,6 +55,41 @@ "634300080a0033" ) +# solc 0.8.20, optimizer enabled (200 runs): public stored getter + update helper. +OPTIMIZED_GETTER_BYTECODE = ( + "0x6080604052348015600e575f80fd5b50600436106030575f3560e01c806382ab890a" + "146034578063e582dd31146045575b5f80fd5b6043603f366004607c565b605e565b" + "005b604c5f5481565b60405190815260200160405180910390f35b5f606682606c565b" + "5f555050565b5f60768260016092565b92915050565b5f60208284031215608b575f80" + "fd5b5035919050565b80820180821115607657634e487b7160e01b5f526011600452" + "60245ffdfea2646970667358221220716cbfad7bed540550d4fcbeb66cd0fc5110f7e" + "9b20a9d062f6a203614de258c64736f6c63430008140033" +) + +# solc 0.8.20, optimizer 200, Shanghai, metadata disabled: read3(uint256) +# returns stored + x + 3; write3(uint256) stores x + 3. +PARAMETERIZED_GETTER_BYTECODE = ( + "0x6080604052348015600e575f80fd5b50600436106026575f3560e01c80630342c79d" + "14602a575b5f80fd5b603960353660046066565b604b565b604051908152602001604051" + "80910390f35b5f815f5460579190607c565b6060906003607c565b92915050565b5f60" + "2082840312156075575f80fd5b5035919050565b80820180821115606057634e487b71" + "60e01b5f52601160045260245ffd" +) +PARAMETERIZED_SETTER_BYTECODE = ( + "0x6080604052348015600e575f80fd5b50600436106026575f3560e01c80631e9e15ab" + "14602a575b5f80fd5b603960353660046049565b603b565b005b6044816003605f565b" + "5f5550565b5f602082840312156058575f80fd5b5035919050565b8082018082111560" + "7d57634e487b7160e01b5f52601160045260245ffd5b9291505056" +) +SHARED_PARAMETER_DECODER_BYTECODE = ( + "0x6080604052348015600e575f80fd5b50600436106030575f3560e01c80633f81a2c0" + "1460345780634d0392a8146045575b5f80fd5b6043603f3660046086565b6066565b" + "005b605460503660046086565b6074565b60405190815260200160405180910390f35b60" + "6f816001609c565b5f5550565b5f815f5460809190609c565b92915050565b5f602082" + "840312156095575f80fd5b5035919050565b80820180821115608057634e487b7160e0" + "1b5f52601160045260245ffd" +) + def _make_dict_instructions(names): """Build a list of dict-format instructions with auto-incremented PCs.""" @@ -301,6 +336,77 @@ def test_shr_dispatcher_identifies_selector(self): functions = analyzer.identify_functions() assert "function_0x12345678" in functions + def test_large_dispatcher_identifies_selectors_beyond_256_instructions(self): + prefix = "60003560e01c" + entries = "".join(f"8063{selector:08x}1461029a57" for selector in range(1, 61)) + analyzer = BytecodeAnalyzer("0x" + prefix + entries + "5b00") + analyzer.analyze_control_flow() + functions = analyzer.identify_functions() + + assert len(analyzer.instructions) > 256 + assert sum(func.selector is not None for func in functions.values()) == 60 + assert "function_0x0000003c" in functions + assert "block_029a:" in analyzer.generate_per_function_tac()["function_0x0000003c"] + + def test_large_dispatcher_target_after_jumpdest_is_not_a_function(self): + prefix = "60003560e01c" + # The original 60-selector fixture pointed at STOP (0x29b), not JUMPDEST (0x29a). + entries = "".join(f"8063{selector:08x}1461029b57" for selector in range(1, 61)) + tac = BytecodeAnalyzer("0x" + prefix + entries + "5b00").generate_per_function_tac() + + assert set(tac) == {"fallback_function"} + assert "block_029a:" in tac["fallback_function"] + + @pytest.mark.parametrize("target", ["11", "20", "99"]) + def test_invalid_dispatcher_target_does_not_create_function(self, target): + # The fallback JUMPDEST is at 0x10; 0x11 is STOP, and 0x20/0x99 are absent. + analyzer = BytecodeAnalyzer("0x60003560e01c8063123456781460" + target + "575b00") + tac = analyzer.generate_per_function_tac() + + assert set(analyzer.basic_blocks) == {"block_0000", "block_0010"} + assert "function_0x12345678" not in tac + assert analyzer.rejected_dispatcher_targets == {"0x12345678": int(target, 16)} + assert set(tac) == {"fallback_function"} + assert "block_0010:" in tac["fallback_function"] + assert "block_0000:" not in tac["fallback_function"] + fake = Function( + name="function_0x12345678", selector="0x12345678", + basic_blocks=list(analyzer.basic_blocks.values()), entry_block=f"block_00{target}", + ) + assert analyzer.generate_function_tac(fake) == "" + assert analyzer._blocks_for_function(fake, fallback_to_all=True) == [] + + def test_valid_dispatcher_target_includes_only_its_reachable_block(self): + analyzer = BytecodeAnalyzer("0x60003560e01c806312345678146010575b00") + tac = analyzer.generate_per_function_tac()["function_0x12345678"] + + assert analyzer.functions["function_0x12345678"].entry_block == "block_0010" + assert [b.id for b in analyzer._blocks_for_function( + analyzer.functions["function_0x12345678"], fallback_to_all=False + )] == ["block_0010"] + assert "block_0010:" in tac + assert "block_0000:" not in tac + + def test_invalid_selector_does_not_hide_valid_receive_and_fallback(self): + # receive() at 0x17, fallback at 0x15; selector points outside code. + bytecode = "0x361560175760003560e01c8063123456781460ff575b005b005b00" + tac = BytecodeAnalyzer(bytecode).generate_per_function_tac() + + assert "function_0x12345678" not in tac + assert "block_0017:" in tac["receive"] + assert "block_0015:" in tac["fallback_function"] + + def test_invalid_selector_without_fallthrough_has_no_function_tac(self): + analyzer = BytecodeAnalyzer("0x60003560e01c80631234567814602057") + assert analyzer.generate_per_function_tac() == {} + + @pytest.mark.parametrize("target", ["05", "99"]) + def test_invalid_receive_target_does_not_create_receive_function(self, target): + invalid = BytecodeAnalyzer("0x361560" + target + "57005b00") + assert "receive" not in invalid.generate_per_function_tac() + valid = BytecodeAnalyzer("0x3615600657005b00") + assert "block_0006:" in valid.generate_per_function_tac()["receive"] + def test_legacy_div_dispatcher_identifies_selector(self): divisor = "01" + ("00" * 28) bytecode = "0x6000357c" + divisor + "0480631234567814602c575b00" @@ -969,6 +1075,69 @@ def test_full_pipeline_sample_contract(self): assert "0x893d20e8" in output assert "0xa6f9dae1" in output + def test_optimized_getter_tracks_jump_target_below_consumed_push0(self): + analyzer = BytecodeAnalyzer(OPTIMIZED_GETTER_BYTECODE) + functions = analyzer.generate_per_function_tac() + + getter = functions["function_0xe582dd31"] + assert analyzer.basic_blocks["block_0045"].successors == ["block_004c"] + assert "block_004c:" in getter + assert "return memory[" in getter + assert "stack_underflow" not in getter + assert "block_004c:" not in functions["function_0x82ab890a"] + assert not any(line.strip().startswith("storage[") for line in getter.splitlines()) + assert any( + line.strip().startswith("storage[") + for line in functions["function_0x82ab890a"].splitlines() + ) + assert "stack_underflow" not in functions["function_0x82ab890a"] + + @pytest.mark.parametrize( + "bytecode,selector,handoff,witness,rendered", + [ + (PARAMETERIZED_GETTER_BYTECODE, "0x0342c79d", "block_0075", + "block_0039", "return memory["), + (PARAMETERIZED_SETTER_BYTECODE, "0x1e9e15ab", "block_0058", + "block_0044", "storage["), + ], + ) + def test_compiler_parameter_decoder_returns_to_function( + self, bytecode, selector, handoff, witness, rendered + ): + analyzer = BytecodeAnalyzer(bytecode) + tac = analyzer.generate_per_function_tac()["function_" + selector] + + assert analyzer.basic_blocks[handoff].successors == ["block_0035"] + assert witness + ":" in tac + assert rendered in tac + assert "stack_underflow" not in tac + for block in analyzer.basic_blocks.values(): + raw = block.metadata["raw_instructions"] + if analyzer._get_instruction_name(raw[-1]) == "JUMP": + assert all( + analyzer._get_instruction_name( + analyzer.basic_blocks[successor].metadata["raw_instructions"][0] + ) == "JUMPDEST" + for successor in block.successors + ) + + def test_shared_parameter_decoder_keeps_selector_paths_separate(self): + analyzer = BytecodeAnalyzer(SHARED_PARAMETER_DECODER_BYTECODE) + tac = analyzer.generate_per_function_tac() + setter = tac["function_0x3f81a2c0"] + getter = tac["function_0x4d0392a8"] + + assert set(analyzer.basic_blocks["block_0095"].successors) == { + "block_003f", "block_0050", + } + assert "block_003f:" in setter and "block_0050:" not in setter + assert "block_0050:" in getter and "block_003f:" not in getter + assert " // Successors: block_003f" in setter + assert " // Successors: block_0050" in getter + assert any(line.strip().startswith("storage[") for line in setter.splitlines()) + assert not any(line.strip().startswith("storage[") for line in getter.splitlines()) + assert "return memory[" in getter + def test_generate_tac_representation(self): analyzer = BytecodeAnalyzer(SAMPLE_OWNER_BYTECODE) output = analyzer.generate_tac_representation() diff --git a/tests/test_compare_eval_runs.py b/tests/test_compare_eval_runs.py index ad4d655..d9b5d2c 100644 --- a/tests/test_compare_eval_runs.py +++ b/tests/test_compare_eval_runs.py @@ -1,15 +1,41 @@ +import hashlib import json +import pytest + from scripts.compare_eval_runs import compare_eval_runs, format_markdown_report +SETTINGS = { + "eval_batch_size": 1, + "eval_max_new_tokens": 512, + "eval_repetition_penalty": 1.05, + "include_selector_signature_metadata": True, + "selector_signature_prompt_policy": "bundled_only_v1", + "prompt_truncation_count": 0, + "eval_sampling_strategy": "all", + "eval_sample_indices": None, +} + + def _write_json(path, payload): + for key, value in SETTINGS.items(): + payload["summary"].setdefault(key, value) + payload["summary"].setdefault( + "replication_behavior_only_f1_micro", + payload["summary"]["replication_f1_micro"], + ) path.write_text(json.dumps(payload), encoding="utf-8") def _detail(index, replication_f1, bytecode_score, semantic, valid=True, buckets=None): + source = f"tac {index}" + original = f"function f{index}() external {{}}" return { "dataset_index": index, + "input_hash": hashlib.sha256(source.encode()).hexdigest(), + "output_hash": hashlib.sha256(original.encode()).hexdigest(), + "original": original, "metadata": {"function_signature": f"function f{index}()"}, "metrics": { "replication_f1": replication_f1, @@ -77,6 +103,8 @@ def test_compare_eval_runs_marks_small_runs_smoke_only(tmp_path): "num_evaluated": 2, "replication_f1_micro": 0.4, "bytecode_semantic_score_mean": 0.2, + "semantic_similarity_mean": 0.5, + "solidity_valid_mean": 1.0, }, "details": [_detail(0, 0.4, 0.2, 0.5), _detail(1, 0.4, 0.2, 0.5)], }, @@ -88,6 +116,8 @@ def test_compare_eval_runs_marks_small_runs_smoke_only(tmp_path): "num_evaluated": 2, "replication_f1_micro": 0.5, "bytecode_semantic_score_mean": 0.3, + "semantic_similarity_mean": 0.6, + "solidity_valid_mean": 1.0, }, "details": [_detail(0, 0.5, 0.3, 0.6), _detail(1, 0.5, 0.3, 0.6)], }, @@ -97,3 +127,108 @@ def test_compare_eval_runs_marks_small_runs_smoke_only(tmp_path): assert comparison["decision"] == "smoke_only" assert comparison["paired_metric_deltas"]["replication_f1"]["improved_count"] == 2 + + +def test_comparison_rejects_behavior_regression_hidden_by_all_fact_gain(tmp_path): + baseline, candidate, baseline_data, candidate_data = _pair(tmp_path) + baseline_data["summary"]["replication_behavior_only_f1_micro"] = 0.5 + candidate_data["summary"]["replication_behavior_only_f1_micro"] = 0.4 + _write_json(baseline, baseline_data) + _write_json(candidate, candidate_data) + + comparison = compare_eval_runs(baseline, candidate) + + assert comparison["decision"] == "reject" + assert "replication_behavior_only_f1_micro" in comparison["decision_reason"] + + +def _pair(tmp_path): + baseline = tmp_path / "baseline.json" + candidate = tmp_path / "candidate.json" + + def payload(f1): + return { + "summary": { + "num_evaluated": 30, + "replication_f1_micro": f1, + "bytecode_semantic_score_mean": f1, + "semantic_similarity_mean": f1, + "solidity_valid_mean": 1.0, + }, + "details": [_detail(i, f1, f1, f1) for i in range(30)], + } + + return baseline, candidate, payload(0.5), payload(0.6) + + +@pytest.mark.parametrize( + "tamper,expected", + [ + (lambda data: data["details"][1].update({"dataset_index": 0}), "duplicate dataset_index"), + ( + lambda data: data["details"][0].update({"dataset_index": 99}), + "dataset_index sets differ", + ), + ( + lambda data: data["details"][0]["metadata"].update({"function_signature": "other()"}), + "different source row", + ), + (lambda data: data["details"][0].update({"input_hash": "0" * 64}), "different source row"), + (lambda data: data["details"][0].pop("input_hash"), "missing input_hash"), + (lambda data: data["summary"].update({"num_evaluated": 31}), "num_evaluated"), + (lambda data: data["summary"].pop("solidity_valid_mean"), "solidity_valid_mean"), + (lambda data: data["summary"].update({"eval_max_new_tokens": 256}), "eval_max_new_tokens"), + (lambda data: data["summary"].update({"prompt_truncation_count": 1}), "prompt truncation"), + ( + lambda data: data["details"][0]["metrics"].pop("replication_f1"), + "missing or non-finite replication_f1", + ), + ( + lambda data: data["details"][0]["metrics"].update({"replication_f1": float("nan")}), + "missing or non-finite replication_f1", + ), + ], +) +def test_comparison_rejects_unpaired_or_unverifiable_improvements(tmp_path, tamper, expected): + baseline, candidate, baseline_data, candidate_data = _pair(tmp_path) + _write_json(baseline, baseline_data) + tamper(candidate_data) + _write_json(candidate, candidate_data) + + comparison = compare_eval_runs(baseline, candidate) + + assert comparison["decision"] == "reject" + assert expected in comparison["decision_reason"] or any( + expected in error for error in comparison["comparability_errors"] + ) + + +def test_comparison_rejects_baseline_without_selector_provenance(tmp_path): + baseline, candidate, baseline_data, candidate_data = _pair(tmp_path) + _write_json(baseline, baseline_data) + _write_json(candidate, candidate_data) + historical = json.loads(baseline.read_text()) + del historical["summary"]["selector_signature_prompt_policy"] + baseline.write_text(json.dumps(historical)) + + comparison = compare_eval_runs(baseline, candidate) + + assert comparison["decision"] == "reject" + assert any( + "selector_signature_prompt_policy" in error for error in comparison["comparability_errors"] + ) + + +def test_comparison_rejects_swapped_rows_even_when_indices_and_scores_collide(tmp_path): + baseline, candidate, baseline_data, candidate_data = _pair(tmp_path) + candidate_data["details"][0]["input_hash"], candidate_data["details"][1]["input_hash"] = ( + candidate_data["details"][1]["input_hash"], + candidate_data["details"][0]["input_hash"], + ) + _write_json(baseline, baseline_data) + _write_json(candidate, candidate_data) + + comparison = compare_eval_runs(baseline, candidate) + + assert comparison["decision"] == "reject" + assert comparison["paired_rows"] == 28 diff --git a/tests/test_contract_reconstruction.py b/tests/test_contract_reconstruction.py index aef9e32..6ea8592 100644 --- a/tests/test_contract_reconstruction.py +++ b/tests/test_contract_reconstruction.py @@ -6,6 +6,18 @@ build_function_quality, build_reconstruction_plan, ) +from src.inference import analyze_bytecode_tac + + +def test_reconstruction_plan_keeps_reachable_blocks_from_real_function(): + bytecode = "0x60003560e01c806312345678146010575b00" + analyzer, function_tac, _ = analyze_bytecode_tac(bytecode) + + plan = build_reconstruction_plan(bytecode, analyzer, function_tac) + chunk = next(c for c in plan["semantic_chunks"] if c["selector"] == "0x12345678") + + assert chunk["basic_blocks"] == ["block_0010"] + assert chunk["instruction_count"] == 2 def test_reconstruction_plan_handles_contract_fallback_chunk(): diff --git a/tests/test_data_generation_export_issues.py b/tests/test_data_generation_export_issues.py index c7fdfcb..ddc24de 100644 --- a/tests/test_data_generation_export_issues.py +++ b/tests/test_data_generation_export_issues.py @@ -1,10 +1,18 @@ import json import sqlite3 +import sys from pathlib import Path import pytest +class _CharacterTokenizer: + name_or_path = "character-test-tokenizer" + + def __call__(self, text, **_kwargs): + return {"input_ids": list(range(len(text)))} + + def _make_pair(download_hf_contracts, idx, body): tac = f"function same:\n block_{idx}:\n temp = {idx}" return { @@ -199,6 +207,83 @@ def test_download_contracts_streams_parquet_batches_without_full_dataframe(tmp_p assert manifest["drop_counts"]["non_solidity"] == 1 +def test_download_preserves_distinct_targets_and_verified_compiler_configs(tmp_path, monkeypatch): + pa = pytest.importorskip("pyarrow") + pq = pytest.importorskip("pyarrow.parquet") + import download_hf_contracts + + source = ( + "pragma solidity ^0.8.0; " + "contract A { function a() public { emit Done(1); } } " + "contract B { function b() public { emit Done(2); } }" + ) + rows = [ + ("A", "v0.8.20", True, 200), + ("B", "v0.8.20", True, 200), + ("A", "v0.8.19", True, 200), + ("A", "v0.8.20", False, 200), + ("A", "v0.8.20+commit.a1b79de6", True, 200), + ("B", "v0.8.19", True, 200), + ("B", "v0.8.19", False, 200), + ] + table = pa.table( + { + "language": ["Solidity"] * len(rows), + "source_code": [source] * len(rows), + "contract_address": [ + f"0x{idx:040x}" if idx <= 5 else "" + for idx in range(1, len(rows) + 1) + ], + "contract_name": [row[0] for row in rows], + "compiler_version": [row[1] for row in rows], + "optimization_used": [row[2] for row in rows], + "runs": [row[3] for row in rows], + } + ) + parquet_path = tmp_path / "same-source.parquet" + pq.write_table(table, parquet_path) + monkeypatch.setattr( + download_hf_contracts, "_get_parquet_files", lambda *a, **k: ["data/train/0.parquet"] + ) + monkeypatch.setattr(download_hf_contracts, "_resolve_hf_revision", lambda _: "resolved") + monkeypatch.setattr( + download_hf_contracts, "hf_hub_download", lambda **_kwargs: str(parquet_path) + ) + + db_path = tmp_path / "contracts.db" + manifest_path = tmp_path / "download.manifest.json" + download_hf_contracts.init_database(db_path) + assert download_hf_contracts.download_contracts( + db_path=db_path, manifest_path=manifest_path + ) == 6 + with sqlite3.connect(db_path) as conn: + stored = conn.execute( + "SELECT address, contract_name, compiler_version, optimization_enabled " + "FROM contracts ORDER BY address" + ).fetchall() + named = [row[1:] for row in stored if row[0].startswith("0x")] + synthetic = [row for row in stored if row[0].startswith("hf_")] + assert named == [ + ("A", "v0.8.20", 1), + ("B", "v0.8.20", 1), + ("A", "v0.8.19", 1), + ("A", "v0.8.20", 0), + ] + assert len({row[0] for row in synthetic}) == 2 + assert {row[1:] for row in synthetic} == { + ("B", "v0.8.19", 1), ("B", "v0.8.19", 0) + } + assert download_hf_contracts.download_contracts( + db_path=db_path, manifest_path=manifest_path + ) == 0 + manifest = json.loads(manifest_path.read_text()) + assert manifest["drop_counts"]["source_deduped"] == 7 + assert manifest["parameters"]["dedup_key"] == [ + "source_hash", "contract_name", "compiler_version", + "optimization_enabled", "optimization_runs", + ] + + def test_prepare_contract_uses_single_source_aligned_compiler_config(): import download_hf_contracts @@ -233,6 +318,136 @@ def test_prepare_contract_uses_single_source_aligned_compiler_config(): assert "compatible_versions" not in prepared +def test_hf_compile_quarantines_ambiguous_selector_alignment(monkeypatch): + from types import SimpleNamespace + import download_hf_contracts + + selector = "0x23b872dd" + source_functions = [ + { + "name": "transferFrom", "selector": selector, "contract_name": "ERC20", + "body": "function transferFrom(address from, address to, uint256 amount) public returns (bool) { return true; }", + }, + { + "name": "transferFrom", "selector": selector, "contract_name": "ERC721", + "body": "function transferFrom(address from, address to, uint256 tokenId) public { ownerOf[tokenId] = to; }", + }, + ] + bytecode_function = SimpleNamespace( + selector=selector, entry_block="block_0", basic_blocks=[] + ) + + class FakeAnalyzer: + basic_blocks = {} + + def __init__(self, _bytecode): + pass + + def analyze_control_flow(self): + pass + + def identify_functions(self): + return {"transferFrom": bytecode_function} + + monkeypatch.setattr(download_hf_contracts, "install_solc_version", lambda _: True) + monkeypatch.setattr(download_hf_contracts, "BytecodeAnalyzer", FakeAnalyzer) + monkeypatch.setattr( + download_hf_contracts, "compile_source", + lambda *_args, **_kwargs: SimpleNamespace( + success=True, + contracts={ + "Derived": SimpleNamespace( + runtime_bytecode="6000600055", + abi=[{"name": "transferFrom", "type": "function"}], + ) + }, + ), + ) + outcome = download_hf_contracts._compile_one_job( + "0x" + "1" * 40, + { + "contract.sol": ( + "pragma solidity ^0.8.0; contract ERC20 { " + "function transferFrom(address from, address to, uint256 value) " + "public returns (bool) { return true; } } " + "contract Derived is ERC20 {}" + ) + }, + source_functions, + "0.8.20", + True, + 200, + min_body_length=0, + target_contract_name="Derived", + ) + assert outcome["pairs"] == [] + assert outcome["status"] == "ambiguous_selector_alignment" + assert outcome["drop_counts"]["ambiguous_selector_alignment"] == 1 + assert selector in outcome["error"] + + +def test_etherscan_compile_records_ambiguous_selector_diagnostic(tmp_path, monkeypatch): + import threading + from types import SimpleNamespace + from src import dataset_pipeline + + source = """ + pragma solidity ^0.8.0; + contract ERC20 { + function transferFrom(address from, address to, uint256 value) + public returns (bool) { return true; } + } + contract ERC721 { + function transferFrom(address from, address to, uint256 tokenId) + public { ownerOf[tokenId] = to; } + } + contract Derived is ERC20 {} + """ + + class FakeAnalyzer: + basic_blocks = {} + + def __init__(self, _bytecode): + pass + + def analyze_control_flow(self): + pass + + def identify_functions(self): + return { + "transferFrom": SimpleNamespace( + selector="0x23b872dd", entry_block="block_0", basic_blocks=[] + ) + } + + builder = dataset_pipeline.DatasetBuilder("dummy", output_dir=str(tmp_path / "builder")) + builder.etherscan.get_contract_source = lambda address: dataset_pipeline.ContractData( + address=address, source_code=source, bytecode="0x6000600055", + compiler_version="v0.8.20", optimization_enabled=True, optimization_runs=200, + ) + monkeypatch.setattr(dataset_pipeline, "install_solc_version", lambda _: True) + monkeypatch.setattr(dataset_pipeline, "BytecodeAnalyzer", FakeAnalyzer) + monkeypatch.setattr( + dataset_pipeline, "compile_source", + lambda *_args, **_kwargs: SimpleNamespace( + success=True, + contracts={ + "Derived": SimpleNamespace(runtime_bytecode="6000600055", abi=[]) + }, + ), + ) + outcome = builder._collect_compile_address( + "0x" + "1" * 40, max_compiler_configs=1, compiler_install_lock=threading.Lock() + ) + assert outcome.pairs == [] + assert outcome.status_update["status"] == "ambiguous_selector_alignment" + assert any( + diag["status"] == "ambiguous_selector_alignment" + and "0x23b872dd" in diag["error"] + for diag in outcome.diagnostics + ) + + def test_export_training_data_quarantines_overlength_rows(tmp_path): import download_hf_contracts @@ -277,6 +492,168 @@ def test_export_training_data_quarantines_overlength_rows(tmp_path): assert manifest["artifacts"]["rejects_jsonl"]["row_count"] == 2 +def test_export_length_report_uses_training_prompt_and_exact_tokenizer(): + import train + from src.dataset_export_primitives import export_length_report, export_prompt_parts + + record = { + "input": "function selector_12345678:\n block_0:\n temp_1 = ADD 1 2", + "output": "function add() public pure returns (uint256) { return 1 + 2; }", + "metadata": {"selector": "0x12345678"}, + } + expected_parts = train._preflight_prompt_parts( + record, + include_bytecode_metadata=True, + include_selector_signature_metadata=True, + template_format="alpaca", + ) + assert export_prompt_parts(record) == expected_parts + prefix, target, suffix = expected_parts + assert prefix.startswith("### Instruction:") + assert "Bytecode metadata:" in prefix + tokenizer = _CharacterTokenizer() + exact = export_length_report(record, len(prefix) + len(target + suffix) - 1, tokenizer) + assert exact["context_tokens"] == len(prefix) + assert exact["target_tokens"] == len(target + suffix) + assert exact["total_tokens"] == len(prefix) + len(target + suffix) + assert exact["token_count_method"] == "tokenizer" + assert exact["reasons"] == ["context_overlength"] + estimated = export_length_report(record, exact["max_seq_length"]) + assert estimated["token_count_method"] == "estimate" + assert estimated["total_tokens"] < exact["total_tokens"] + + +def test_export_exact_length_includes_training_eos(): + from src.dataset_export_primitives import export_length_report + + class EosCharacterTokenizer(_CharacterTokenizer): + eos_token_id = 999999 + + record = {"input": "function example:\n return", "output": "function example() {}"} + without_eos = export_length_report(record, 8192, _CharacterTokenizer()) + with_eos = export_length_report( + record, without_eos["total_tokens"], EosCharacterTokenizer() + ) + + assert with_eos["target_tokens"] == without_eos["target_tokens"] + 1 + assert with_eos["reasons"] == ["context_overlength"] + + +def test_both_export_paths_quarantine_exact_tokenizer_overlength(tmp_path): + import download_hf_contracts + from src.dataset_export_primitives import build_training_record, export_length_report + from src.dataset_pipeline import DatasetBuilder, FunctionPair + + tokenizer = _CharacterTokenizer() + short_body = "function short() public { uint256 x = 1; emit Done(x); }" + long_body = "function longer() public { uint256 x = 2; emit Done(x); }" + short_tac = "function same:\n block_1:\n temp = 1" + long_tac = "function same:\n block_2:\n temp = " + "ADD 1 2 " * 45 + budget = export_length_report( + build_training_record(short_tac, short_body), 8192, tokenizer + )["total_tokens"] + 1 + assert not export_length_report( + build_training_record(long_tac, long_body), budget + )["reasons"] + assert "context_overlength" in export_length_report( + build_training_record(long_tac, long_body), budget, tokenizer + )["reasons"] + + db_path = tmp_path / "hf.db" + output_path = tmp_path / "hf.jsonl" + manifest_path = tmp_path / "hf.manifest.json" + download_hf_contracts.init_database(db_path) + pairs = [ + {**_make_pair(download_hf_contracts, idx, body), "tac_representation": tac} + for idx, (body, tac) in enumerate( + [(short_body, short_tac), (long_body, long_tac)], start=1 + ) + ] + assert download_hf_contracts._store_pairs_batch(db_path, pairs) == 2 + download_hf_contracts.export_training_data( + str(output_path), + db_path=db_path, + manifest_path=manifest_path, + max_seq_length=budget, + length_tokenizer=tokenizer, + ) + assert len(output_path.read_text().splitlines()) == 1 + hf_rejects = [ + json.loads(line) for line in Path(f"{output_path}.rejects.jsonl").read_text().splitlines() + ] + assert len(hf_rejects) == 1 + assert hf_rejects[0]["reasons"] == ["context_overlength"] + assert hf_rejects[0]["lengths"]["token_count_method"] == "tokenizer" + manifest = json.loads(manifest_path.read_text()) + assert manifest["parameters"]["length_tokenizer"] == tokenizer.name_or_path + assert manifest["validation"]["token_length_filter"]["token_count_method"] == "tokenizer" + + builder = DatasetBuilder("dummy", output_dir=str(tmp_path / "builder")) + for idx, (body, tac) in enumerate( + [(short_body, short_tac), (long_body, long_tac)], start=1 + ): + builder._store_function_pair( + FunctionPair( + function_name="short" if idx == 1 else "longer", + tac_representation=tac, + solidity_code=body, + function_signature="function short()" if idx == 1 else "function longer()", + visibility="public", + is_payable=False, + is_view=False, + contract_address=f"0x{idx:040x}", + ) + ) + builder_output = Path( + builder.export_dataset("jsonl", max_seq_length=budget, length_tokenizer=tokenizer) + ) + assert len(builder_output.read_text().splitlines()) == 1 + builder_rejects = [ + json.loads(line) + for line in Path(f"{builder_output}.rejects.jsonl").read_text().splitlines() + ] + assert len(builder_rejects) == 1 + assert builder_rejects[0]["reasons"] == ["context_overlength"] + builder_manifest = json.loads(Path(f"{builder_output}.manifest.json").read_text()) + assert builder_manifest["parameters"]["length_tokenizer"] == tokenizer.name_or_path + + +def test_export_cli_loads_exact_tokenizer_offline(tmp_path, monkeypatch): + import download_hf_contracts + from transformers import AutoTokenizer + + db_path = tmp_path / "contracts.db" + output_path = tmp_path / "dataset.jsonl" + download_hf_contracts.init_database(db_path) + body = "function short() public { uint256 x = 1; emit Done(x); }" + download_hf_contracts._store_pairs_batch( + db_path, [_make_pair(download_hf_contracts, 1, body)] + ) + loaded = [] + + def load_tokenizer(name, **kwargs): + loaded.append((name, kwargs)) + return _CharacterTokenizer() + + monkeypatch.setattr(AutoTokenizer, "from_pretrained", load_tokenizer) + monkeypatch.setattr( + sys, "argv", + [ + "download_hf_contracts.py", "--export-only", "--db", str(db_path), + "--output", str(output_path), "--length-tokenizer", "locally-cached-tokenizer", + "--max-seq-length", "128", + ], + ) + download_hf_contracts.main() + assert loaded == [("locally-cached-tokenizer", {"local_files_only": True})] + assert output_path.read_text() == "" + rejects = [ + json.loads(line) for line in Path(f"{output_path}.rejects.jsonl").read_text().splitlines() + ] + assert len(rejects) == 1 + assert "context_overlength" in rejects[0]["reasons"] + + def test_shared_primitives_produce_same_final_rows_across_export_paths(tmp_path): import download_hf_contracts from src import dataset_pipeline diff --git a/tests/test_dataset_pipeline.py b/tests/test_dataset_pipeline.py index 3335e1d..29f9db6 100644 --- a/tests/test_dataset_pipeline.py +++ b/tests/test_dataset_pipeline.py @@ -576,7 +576,7 @@ def fake_compile_source(*_args, **_kwargs): }, ) - def fake_match(solidity_functions, _bytecode_functions, _analyzer): + def fake_match(solidity_functions, _bytecode_functions, _analyzer, **_kwargs): return [ { "solidity_function": solidity_functions[0], @@ -896,6 +896,77 @@ def test_no_matches(self, builder): matches = builder._match_functions_by_selector(sol_funcs, bc_funcs, mock_analyzer) assert matches == [] + def test_conflicting_selector_targets_are_not_labeled(self, builder): + selector = "0x23b872dd" + erc20 = { + "name": "transferFrom", + "selector": selector, + "body": "function transferFrom(address from, address to, uint256 amount) public returns (bool) { return true; }", + } + erc721 = { + "name": "transferFrom", + "selector": selector, + "body": "function transferFrom(address from, address to, uint256 tokenId) public { ownerOf[tokenId] = to; }", + } + distinct = { + "name": "transfer", + "selector": "0xa9059cbb", + "body": "function transfer(address to, uint256 amount) public { emit Transfer(to, amount); }", + } + bytecode_funcs = { + "collision": SimpleNamespace(selector=selector, entry_block="block_0", basic_blocks=[]), + "transfer": SimpleNamespace( + selector=distinct["selector"], entry_block="block_1", basic_blocks=[] + ), + } + analyzer = MagicMock(basic_blocks={}) + ambiguous = set() + matches = builder._match_functions_by_selector( + [erc20, erc721, distinct], bytecode_funcs, analyzer, + ambiguous_selectors=ambiguous, + ) + assert ambiguous == {selector} + assert [match["solidity_function"] for match in matches] == [distinct] + assert builder._match_functions_by_selector( + [erc20], bytecode_funcs, analyzer + )[0]["solidity_function"] == erc20 + + def test_identical_source_targets_with_same_selector_remain_matchable(self, builder): + selector = "0x23b872dd" + source = { + "name": "transferFrom", + "selector": selector, + "body": "function transferFrom(address from, address to, uint256 value) public { emit Transfer(from, to, value); }", + } + bytecode = SimpleNamespace(selector=selector, entry_block="block_0", basic_blocks=[]) + ambiguous = set() + matches = builder._match_functions_by_selector( + [source, dict(source)], {"first": bytecode}, MagicMock(basic_blocks={}), + ambiguous_selectors=ambiguous, + ) + assert ambiguous == set() + assert len(matches) == 1 + assert matches[0]["solidity_function"] == source + + def test_duplicate_runtime_selector_with_distinct_entries_is_ambiguous(self, builder): + selector = "0x23b872dd" + source = { + "name": "transferFrom", + "selector": selector, + "body": "function transferFrom(address from, address to, uint256 amount) public { emit Transfer(from, to, amount); }", + } + bytecode_funcs = { + "first": SimpleNamespace(selector=selector, entry_block="block_0"), + "second": SimpleNamespace(selector=selector, entry_block="block_1"), + } + ambiguous = set() + matches = builder._match_functions_by_selector( + [source], bytecode_funcs, MagicMock(basic_blocks={}), + ambiguous_selectors=ambiguous, + ) + assert matches == [] + assert ambiguous == {selector} + class TestCollectFunctionBlocks: def test_traversal(self, builder): diff --git a/tests/test_decompile_cli.py b/tests/test_decompile_cli.py index b803439..f94003c 100644 --- a/tests/test_decompile_cli.py +++ b/tests/test_decompile_cli.py @@ -78,6 +78,7 @@ def test_cli_autodiscovers_newest_final_model(monkeypatch): def test_cli_function_limit_returns_structured_json(monkeypatch, capsys): decompile = load_decompile_module() + monkeypatch.setattr(decompile, "_resolve_model_path", lambda _model_path: ROOT) analyzer = SimpleNamespace(instructions=[1], basic_blocks={"b": 1}, functions={}) monkeypatch.setattr( @@ -96,6 +97,7 @@ def test_cli_function_limit_returns_structured_json(monkeypatch, capsys): def test_cli_timeout_returns_structured_json(monkeypatch, capsys): decompile = load_decompile_module() + monkeypatch.setattr(decompile, "_resolve_model_path", lambda _model_path: ROOT) def slow_analyze(bytecode): import time diff --git a/tests/test_eval_gate_suite.py b/tests/test_eval_gate_suite.py index 20f2338..16062af 100644 --- a/tests/test_eval_gate_suite.py +++ b/tests/test_eval_gate_suite.py @@ -1,3 +1,4 @@ +import hashlib import json from scripts.eval_gate_suite import evaluate_gate_suite, format_markdown_report @@ -7,6 +8,9 @@ def _write_eval(path, num_rows, f1, bytecode, semantic=0.8, valid=1.0): details = [ { "dataset_index": index, + "input_hash": hashlib.sha256(f"tac {index}".encode()).hexdigest(), + "output_hash": hashlib.sha256(f"sol {index}".encode()).hexdigest(), + "metadata": {"function_signature": f"f{index}()"}, "metrics": { "replication_f1": f1, "bytecode_semantic_score": bytecode, @@ -29,9 +33,18 @@ def _write_eval(path, num_rows, f1, bytecode, semantic=0.8, valid=1.0): "summary": { "num_evaluated": num_rows, "replication_f1_micro": f1, + "replication_behavior_only_f1_micro": f1, "bytecode_semantic_score_mean": bytecode, "semantic_similarity_mean": semantic, "solidity_valid_mean": valid, + "eval_batch_size": 1, + "eval_max_new_tokens": 512, + "eval_repetition_penalty": 1.05, + "include_selector_signature_metadata": True, + "selector_signature_prompt_policy": "bundled_only_v1", + "prompt_truncation_count": 0, + "eval_sampling_strategy": "all", + "eval_sample_indices": None, }, "details": details, } @@ -87,3 +100,43 @@ def test_eval_gate_suite_marks_any_small_pair_smoke_only(tmp_path): ) assert suite["decision"] == "smoke_only" + + +def test_eval_gate_suite_rejects_noncomparable_improvement(tmp_path): + base = tmp_path / "base.json" + candidate = tmp_path / "candidate.json" + _write_eval(base, 30, 0.5, 0.4) + _write_eval(candidate, 30, 0.6, 0.5) + payload = json.loads(candidate.read_text(encoding="utf-8")) + payload["details"][0]["input_hash"] = "0" * 64 + candidate.write_text(json.dumps(payload), encoding="utf-8") + + suite = evaluate_gate_suite( + [{"name": "broad", "baseline": base, "candidate": candidate, "min_rows": 30}] + ) + + assert suite["decision"] == "reject" + assert "incomparable evaluations" in suite["comparisons"][0]["decision_reason"] + + +def test_small_slice_improvement_cannot_alone_promote_candidate(tmp_path): + small_base = tmp_path / "small_base.json" + small_candidate = tmp_path / "small_candidate.json" + large_base = tmp_path / "large_base.json" + large_candidate = tmp_path / "large_candidate.json" + _write_eval(small_base, 17, 0.5, 0.4) + _write_eval(small_candidate, 17, 0.6, 0.5) + _write_eval(large_base, 64, 0.8, 0.7) + _write_eval(large_candidate, 64, 0.8, 0.7) + small = {"name": "state17", "baseline": small_base, "candidate": small_candidate, "min_rows": 1} + large = { + "name": "holdout64", + "baseline": large_base, + "candidate": large_candidate, + "min_rows": 30, + } + + assert evaluate_gate_suite([small])["decision"] == "smoke_only" + suite = evaluate_gate_suite([small, large]) + assert suite["decision"] == "no_change" + assert suite["comparisons"][0]["decision"] == "keep_candidate" diff --git a/tests/test_evaluation_quality_metrics.py b/tests/test_evaluation_quality_metrics.py index d114e3d..0a5120c 100644 --- a/tests/test_evaluation_quality_metrics.py +++ b/tests/test_evaluation_quality_metrics.py @@ -1,14 +1,18 @@ """Focused tests for evaluation quality metrics and reporting helpers.""" import difflib +import logging +import sys import pytest from src.training_pipeline import ( + SolidityValidityResult, SmartContractTrainingPipeline, compare_evaluation_to_baseline, compute_benchmark_suite_metrics, compute_metadata_segment_metrics, + evaluate_bytecode_semantics, extract_opcode_control_flow_slices, load_curated_evaluation_benchmarks, mean_confidence_interval, @@ -102,6 +106,292 @@ def fake_solc_validation(source_code, metadata): assert result.scaffold_valid is True +def test_bytecode_score_is_unchecked_without_reference_bytecode_or_opcode_evidence(): + code = "function foo() public { return; }" + scaffold = validate_generated_solidity(code, allow_compiler=False) + compiled_candidate = SolidityValidityResult( + valid=True, + method="compiler_ast", + scaffold_valid=True, + compiler_checked=True, + ast_valid=True, + bytecode_checked=True, + deployable=True, + compiled_runtime_bytecode="0x6000", + ) + + for validity in (scaffold, compiled_candidate): + result = evaluate_bytecode_semantics(code, code, {}, solidity_validity=validity) + assert result.checked is False + assert result.score == 0.0 + assert result.runtime_bytecode_checked is False + assert result.to_dict()["checked_kind"] == "reference_bytecode_or_opcode_evidence" + assert result.to_dict()["score_kind"] == "source_fact_overlap_proxy" + + +@pytest.fixture +def local_reference_contract(): + solcx = pytest.importorskip("solcx") + version = "0.8.20" + if version not in {str(installed) for installed in solcx.get_installed_solc_versions()}: + pytest.skip("solc 0.8.20 is not installed locally") + + source = ( + "pragma solidity ^0.8.20;\n" + "contract Decoy { function value() public pure returns (uint256) { return 99; } }\n" + "contract Counter { function value() public pure returns (uint256) { return 1; } }\n" + ) + + def compile_runtime(code, *, optimized=False): + output = solcx.compile_standard( + { + "language": "Solidity", + "sources": {"contract.sol": {"content": code}}, + "settings": { + "optimizer": {"enabled": optimized, "runs": 200}, + "outputSelection": {"*": {"*": ["evm.deployedBytecode.object"]}}, + }, + }, + solc_version=version, + ) + return output["contracts"]["contract.sol"]["Counter"]["evm"]["deployedBytecode"]["object"] + + runtime = compile_runtime(source) + metadata = { + "bytecode": "0x" + runtime, + "compiler_version": version, + "optimizer_enabled": False, + "optimizer_runs": 200, + "runtime_comparison": { + "contract_name": "Counter", + "compiler_version": version, + "optimizer_enabled": False, + "optimizer_runs": 200, + }, + } + return source, metadata, compile_runtime + + +def test_runtime_comparison_checks_exact_named_full_contract(local_reference_contract): + source, metadata, _ = local_reference_contract + equal = evaluate_bytecode_semantics(source, source, metadata) + + assert equal.runtime_bytecode_checked is True + assert equal.runtime_bytecode_match is True + assert equal.to_dict()["runtime_comparison_kind"] == "exact_full_contract_runtime_bytecode" + assert equal.to_dict()["score_kind"] == "source_fact_overlap_proxy" + assert equal.to_dict()["runtime_bytecode_skip_reason"] is None + assert ( + evaluate_bytecode_semantics( + source, source, {**metadata, "runtime_bytecode": metadata["bytecode"]} + ).runtime_bytecode_match + is True + ) + assert ( + evaluate_bytecode_semantics( + source, + source, + { + **metadata, + "bytecode": "", + "evm": {"deployedBytecode": {"object": metadata["bytecode"]}}, + }, + ).runtime_bytecode_match + is True + ) + + changed = source.replace("return 1;", "return 2;") + wrong = evaluate_bytecode_semantics( + source, changed, {**metadata, "candidate_runtime_bytecode": metadata["bytecode"]} + ) + assert wrong.runtime_bytecode_checked is True + assert wrong.runtime_bytecode_match is False + assert wrong.mismatch_buckets["runtime_bytecode_mismatch"] == ["compiled_runtime_differs"] + + +def test_runtime_comparison_rejects_unverified_reference_or_compiler_settings( + local_reference_contract, +): + source, metadata, compile_runtime = local_reference_contract + for reference, skip_reason in ( + ({**metadata, "bytecode": "0x6000"}, "reference_runtime_mismatch"), + ({**metadata, "runtime_bytecode": "0x6000"}, "reference_runtime_mismatch"), + ( + {**metadata, "bytecode": "0x" + compile_runtime(source, optimized=True)}, + "reference_runtime_mismatch", + ), + ( + {**metadata, "bytecode": "", "creation_bytecode": metadata["bytecode"]}, + "missing_or_invalid_reference_runtime", + ), + ( + {**metadata, "bytecode": "", "evm": {"bytecode": {"object": metadata["bytecode"]}}}, + "missing_or_invalid_reference_runtime", + ), + ({**metadata, "optimizer_enabled": True}, "conflicting_compiler_settings"), + ({**metadata, "compiler_version": "0.8.19"}, "conflicting_compiler_settings"), + ({**metadata, "runtime_comparison": True}, "invalid_compiler_settings"), + ( + { + **metadata, + "runtime_comparison": { + **metadata["runtime_comparison"], + "compiler_version": "9.9.9", + }, + "compiler_version": "9.9.9", + }, + "reference_solc_version_not_installed", + ), + ( + { + **metadata, + "runtime_comparison": { + **metadata["runtime_comparison"], + "contract_name": "Missing", + }, + }, + "reference_target_contract_missing", + ), + ( + { + **metadata, + "runtime_comparison": {**metadata["runtime_comparison"], "contract_name": "Decoy"}, + }, + "reference_runtime_mismatch", + ), + ): + result = evaluate_bytecode_semantics(source, source, reference) + assert result.runtime_bytecode_checked is False + assert result.runtime_bytecode_match is None + assert result.to_dict()["runtime_bytecode_skip_reason"] == skip_reason + + +def test_runtime_comparison_does_not_treat_compilable_fragments_as_full_contracts( + local_reference_contract, +): + _, metadata, _ = local_reference_contract + fragment = "function value() public pure returns (uint256) { return 1; }" + validity = validate_generated_solidity(fragment, metadata) + assert validity.ast_valid is True + assert validity.compiled_runtime_bytecode + + result = evaluate_bytecode_semantics( + fragment, + fragment, + {**metadata, "candidate_runtime_bytecode": metadata["bytecode"]}, + solidity_validity=validity, + ) + assert result.runtime_bytecode_checked is False + assert result.runtime_bytecode_match is None + assert result.to_dict()["runtime_bytecode_skip_reason"] == "reference_fragment" + + +@pytest.mark.parametrize( + "source", + [ + "contract Counter { uint256 public x; constructor() { x = 1; } }", + "contract Counter { uint256 public x = 1; }", + "contract Parent {}\n" + "contract Counter is Parent { function value() public pure returns (uint256) { return 1; } }", + ], +) +def test_runtime_comparison_excludes_deployment_or_inherited_context( + source, + local_reference_contract, +): + _, metadata, compile_runtime = local_reference_contract + metadata = {**metadata, "bytecode": "0x" + compile_runtime(source)} + result = evaluate_bytecode_semantics(source, source, metadata) + assert result.runtime_bytecode_checked is False + assert result.runtime_bytecode_match is None + assert result.to_dict()["runtime_bytecode_skip_reason"] in { + "reference_abstract_or_inherited_contract", + "reference_constructor_or_state_initializer", + } + + +def test_runtime_compiler_rejection_exposes_skip_reason_and_log( + local_reference_contract, + monkeypatch, + caplog, +): + import solcx + from solcx.exceptions import SolcError + + source, metadata, _ = local_reference_contract + validity = validate_generated_solidity(source, allow_compiler=False) + compiler = solcx.compile_standard + + def reject_candidate(*args, **kwargs): + if "return 2;" in args[0]["sources"]["contract.sol"]["content"]: + raise SolcError("Candidate compilation failed") + return compiler(*args, **kwargs) + + monkeypatch.setattr(solcx, "compile_standard", reject_candidate) + with caplog.at_level(logging.INFO, logger="src.training_pipeline"): + result = evaluate_bytecode_semantics( + source, + source.replace("return 1;", "return 2;"), + metadata, + solidity_validity=validity, + ) + assert result.runtime_bytecode_checked is False + assert result.runtime_bytecode_match is None + assert result.to_dict()["runtime_bytecode_skip_reason"] == "candidate_compiler_rejected_source" + assert "candidate_compiler_rejected_source" in caplog.text + + +def test_runtime_compiler_io_failure_is_unchecked_and_logged( + local_reference_contract, + monkeypatch, + caplog, +): + import solcx + + source, metadata, _ = local_reference_contract + validity = validate_generated_solidity(source, allow_compiler=False) + + def compiler_io_failure(*args, **kwargs): + raise OSError("solc binary inaccessible") + + monkeypatch.setattr(solcx, "compile_standard", compiler_io_failure) + with caplog.at_level(logging.WARNING, logger="src.training_pipeline"): + result = evaluate_bytecode_semantics(source, source, metadata, solidity_validity=validity) + assert result.runtime_bytecode_checked is False + assert result.to_dict()["runtime_bytecode_skip_reason"] == "reference_compiler_io_error" + assert "solc binary inaccessible" in caplog.text + + +def test_runtime_missing_local_solcx_is_unchecked_and_reported( + local_reference_contract, + monkeypatch, + caplog, +): + source, metadata, _ = local_reference_contract + validity = validate_generated_solidity(source, allow_compiler=False) + monkeypatch.setitem(sys.modules, "solcx", None) + + with caplog.at_level(logging.INFO, logger="src.training_pipeline"): + result = evaluate_bytecode_semantics(source, source, metadata, solidity_validity=validity) + assert result.runtime_bytecode_checked is False + assert result.to_dict()["runtime_bytecode_skip_reason"] == "reference_solcx_unavailable" + assert "reference_solcx_unavailable" in caplog.text + + +def test_runtime_unexpected_compiler_error_is_not_silenced(local_reference_contract, monkeypatch): + import solcx + + source, metadata, _ = local_reference_contract + validity = validate_generated_solidity(source, allow_compiler=False) + + def unexpected_error(*args, **kwargs): + raise RuntimeError("unexpected compiler bridge bug") + + monkeypatch.setattr(solcx, "compile_standard", unexpected_error) + with pytest.raises(RuntimeError, match="unexpected compiler bridge bug"): + evaluate_bytecode_semantics(source, source, metadata, solidity_validity=validity) + + def test_function_signature_match_compares_name_params_and_returns(): reference = """ function transfer(address to, uint256 amount) public returns (bool) { @@ -286,6 +576,36 @@ def test_training_pipeline_aggregate_stats_include_ci_segments_and_baseline(): assert stats["baseline_comparison"]["num_metrics_compared"] >= 2 +def test_runtime_aggregate_reports_checked_count_and_equality_among_checked_only(): + pipeline = SmartContractTrainingPipeline.__new__(SmartContractTrainingPipeline) + results = [ + {"metrics": {"bytecode_runtime_checked": True, "bytecode_runtime_match": True}}, + {"metrics": {"bytecode_runtime_checked": True, "bytecode_runtime_match": False}}, + { + "metrics": { + "bytecode_runtime_checked": False, + "bytecode_runtime_match": False, + "metadata": { + "bytecode_semantics": {"runtime_bytecode_skip_reason": "not_opted_in"} + }, + } + }, + ] + comparison = pipeline._compute_aggregate_statistics(results)["runtime_bytecode_comparison"] + assert comparison == { + "checked_n": 2, + "equal_n": 1, + "total_n": 3, + "equality_rate_checked": 0.5, + "skip_reasons": {"not_opted_in": 1}, + "scope": "exact_full_contract_runtime_bytecode_not_deployment_or_behavior", + } + no_checks = pipeline._compute_aggregate_statistics(results[-1:])["runtime_bytecode_comparison"] + assert no_checks["checked_n"] == 0 + assert no_checks["equality_rate_checked"] is None + assert no_checks["skip_reasons"] == {"not_opted_in": 1} + + def test_curated_evaluation_benchmarks_include_expected_facts_and_failures(): suites = load_curated_evaluation_benchmarks("test_data/evaluation") diff --git a/tests/test_evaluation_report.py b/tests/test_evaluation_report.py index 001de95..00fec0a 100644 --- a/tests/test_evaluation_report.py +++ b/tests/test_evaluation_report.py @@ -50,6 +50,14 @@ def test_format_latest_results_report_includes_quality_and_model_metadata(tmp_pa "bytecode_deployable_mean": 1.0, "bytecode_runtime_checked_mean": 1.0, "bytecode_runtime_match_mean": 0.0, + "aggregate_statistics": { + "runtime_bytecode_comparison": { + "checked_n": 1, + "equal_n": 0, + "total_n": 1, + "equality_rate_checked": 0.0, + } + }, "confidence_intervals": { "semantic_similarity_mean": { "mean": 0.9, @@ -143,9 +151,13 @@ def test_format_latest_results_report_includes_quality_and_model_metadata(tmp_pa assert "Examples evaluated: 1" in report assert "Semantic similarity mean: 0.9000" in report assert "Replication F1 micro: 0.7742" in report - assert "Solidity valid outputs: 100.00%" in report - assert "Bytecode semantic score mean: 0.8750" in report - assert "Runtime bytecode matches: 0.00%" in report + assert "Behavior-only source-fact F1 micro: n/a" in report + assert "Solidity best-effort valid (may be scaffold/context-limited): 100.00%" in report + assert "Source-fact overlap proxy mean: 0.8750" in report + assert "Runtime bytecode match rate (checked only): 0.00%" in report + assert "Exact full-contract runtime bytecode checked: 1 / 1" in report + assert "Exact runtime bytecode equal (checked only): 0 / 1" in report + assert "not constructor, deployment, or behavioral equivalence" in report assert "abi | 1.0000 | 0.5000 | 0.6667 | 1 | 0 | 1" in report assert "unsupported_calls | 2 | 66.67%" in report assert "Opcode and Control-Flow Coverage" in report @@ -156,6 +168,22 @@ def test_format_latest_results_report_includes_quality_and_model_metadata(tmp_pa assert "Metrics compared: 2" in report assert "edit_distance_mean | 0.1000 | 0.0800 | 0.0200 | 0.2500 | regressed" in report + summary["aggregate_statistics"]["runtime_bytecode_comparison"] = { + "checked_n": 0, + "equal_n": 0, + "total_n": 1, + "equality_rate_checked": None, + } + summary["bytecode_runtime_checked_mean"] = 0.0 + no_runtime_report = format_latest_results_report( + summary=summary, + model_path=str(model_dir), + test_dataset_path=str(dataset), + results_json_path="results/eval_test.json", + ) + assert "Exact full-contract runtime bytecode checked: 0 / 1" in no_runtime_report + assert "Runtime bytecode match rate (checked only): n/a (no comparisons)" in no_runtime_report + def test_format_latest_results_report_derives_hallucination_rates_from_details(tmp_path): model_dir = tmp_path / "model" @@ -419,7 +447,14 @@ def test_evaluation_diagnostics_accept_nested_replication_metrics_without_missin "precision": 0.82, "recall": 0.81, "f1": 0.815, - } + }, + "by_category_micro": { + "guard": { + "true_positives": 1, + "false_positives": 3, + "false_negatives": 2, + } + }, }, "solidity_valid_mean": 1.0, "bytecode_semantic_checked_mean": 1.0, @@ -427,9 +462,140 @@ def test_evaluation_diagnostics_accept_nested_replication_metrics_without_missin ) assert diagnostics["metrics_used"]["replication_f1_micro"] == 0.815 + assert abs(diagnostics["metrics_used"]["replication_behavior_only_f1_micro"] - 2 / 7) < 1e-8 assert not any("Replication metrics are missing" in caveat for caveat in diagnostics["caveats"]) +def test_reference_bytecode_coverage_does_not_claim_behavior_without_runtime_checks(tmp_path): + model_dir = tmp_path / "model" + model_dir.mkdir() + dataset = tmp_path / "test_dataset.jsonl" + dataset.write_text('{"input": "a", "output": "b"}\n') + summary = { + "num_evaluated": 1890, + "bytecode_semantic_checked_mean": 1.0, + "bytecode_semantic_score_mean": 0.5083, + "bytecode_runtime_checked_mean": 0.0, + "bytecode_runtime_match_mean": 0.0, + "solidity_compiler_checked_mean": 1.0, + "solidity_ast_valid_mean": 0.0243, + "solidity_valid_mean": 0.9619, + "bytecode_deployable_mean": 0.0243, + "evaluation_diagnostics": { + "overall_status": "needs_attention", + "issues": [ + { + "title": "Bytecode-grounded behavior does not match reference behavior often enough." + } + ], + "strengths": [], + "caveats": [], + "next_experiments": [], + "issue_counts_by_severity": {}, + "metrics_used": {}, + }, + } + + diagnostics = build_evaluation_diagnostics(summary) + assert any("source-fact overlap proxy" in caveat for caveat in diagnostics["caveats"]) + assert any( + "No generated/reference runtime comparisons" in caveat for caveat in diagnostics["caveats"] + ) + assert any("compiler-only checks" in caveat for caveat in diagnostics["caveats"]) + assert any( + "solidity_valid_mean includes scaffold" in caveat for caveat in diagnostics["caveats"] + ) + assert any(issue["id"] == "bytecode_runtime_coverage_gap" for issue in diagnostics["issues"]) + assert not any( + "Bytecode-grounded behavior" in issue["title"] for issue in diagnostics["issues"] + ) + + report = format_latest_results_report( + summary=summary, + model_path=str(model_dir), + test_dataset_path=str(dataset), + results_json_path="results/missing.json", + ) + + assert "Source-fact overlap proxy mean: 0.5083" in report + assert "Legacy bytecode semantic checked flag: 100.00%" in report + assert "Reference bytecode/opcode coverage: 100.00%" not in report + assert "Runtime bytecode match rate (checked only): n/a (no comparisons)" in report + assert "No generated/reference runtime comparisons" in report + assert ( + "Bytecode-grounded behavior does not match reference behavior often enough." not in report + ) + + +def test_runtime_match_diagnostics_use_only_checked_rows(tmp_path): + summary = { + "num_evaluated": 40, + "bytecode_semantic_checked_mean": 1.0, + "bytecode_semantic_score_mean": 0.9, + "bytecode_runtime_checked_mean": 0.1, + "bytecode_runtime_match_mean": 0.1, + } + + diagnostics = build_evaluation_diagnostics(summary) + assert not any(issue["id"] == "runtime_bytecode_mismatch" for issue in diagnostics["issues"]) + report = format_latest_results_report( + summary=summary, + model_path=str(tmp_path), + test_dataset_path=str(tmp_path / "missing.jsonl"), + results_json_path="results/missing.json", + ) + assert "Runtime bytecode match rate (checked only): 100.00%" in report + + +def test_behavior_only_f1_from_historical_category_counts_is_separate_from_aggregate(tmp_path): + counts = { + "abi": (9034, 397, 379), + "call": (683, 1619, 2413), + "control_flow": (89, 205, 225), + "event": (83, 161, 210), + "guard": (204, 1349, 1050), + "member_call": (238, 574, 898), + "modifier": (310, 425, 601), + "mutability": (815, 54, 50), + "return": (238, 818, 775), + "state_write": (197, 750, 917), + "visibility": (1120, 770, 770), + } + categories = { + category: { + "true_positives": tp, + "false_positives": fp, + "false_negatives": fn, + } + for category, (tp, fp, fn) in counts.items() + } + summary = { + "num_evaluated": 1890, + "replication_f1_micro": 0.6281, + "replication_by_category_micro": categories, + } + + diagnostics = build_evaluation_diagnostics(summary) + assert ( + abs(diagnostics["metrics_used"]["replication_behavior_only_f1_micro"] - 0.2245268343) < 1e-8 + ) + report = format_latest_results_report( + summary=summary, + model_path=str(tmp_path), + test_dataset_path=str(tmp_path / "missing.jsonl"), + results_json_path="results/missing.json", + ) + + assert "Replication F1 micro: 0.6281" in report + assert "Behavior-only source-fact F1 micro: 0.2245" in report + assert ( + "Behavior facts: call, control_flow, event, guard, member_call, return, state_write" + in report + ) + assert "excludes ABI, modifier, mutability, visibility; not bytecode equivalence" in report + assert summary["replication_f1_micro"] == 0.6281 + + def test_write_latest_results_report_creates_file(tmp_path): model_dir = tmp_path / "model" model_dir.mkdir() diff --git a/tests/test_gate_dataset.py b/tests/test_gate_dataset.py new file mode 100644 index 0000000..3b06647 --- /dev/null +++ b/tests/test_gate_dataset.py @@ -0,0 +1,529 @@ +import hashlib +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from scripts.gate_dataset import ( + body_identity, + canonicalize_body_hash, + exclude_eval_rows, + file_sha256, + load_jsonl, + row_keys, + sample_dataset, + verify_baseline_artifact, + verify_eval_artifact, + verify_model_gate_exclusion, +) + +ROOT = Path(__file__).resolve().parents[1] +GATE_VARS = ( + "BROAD_DATASET", + "CALLS_DATASET", + "STATE_DATASET", + "HOLDOUT64_DATASET", + "PURE_NEGATIVE_DATASET", + "LARGE192_DATASET", +) +BASELINE_VARS = ( + "BROAD_BASELINE", + "CALLS_BASELINE", + "STATE_BASELINE", + "HOLDOUT64_BASELINE", + "PURE_NEGATIVE_BASELINE", + "LARGE192_BASELINE", +) + + +def _row(name, *, output=None, **metadata): + return { + "input": f"tac {name}", + "output": output or f"function {name}() external {{ return; }}", + "metadata": metadata, + } + + +def _write_jsonl(path, rows): + path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + + +def test_gate_exclusion_uses_recomputed_body_with_missing_and_stale_hashes(): + gate = _row("shared", body_hash="declared-gate-hash") + duplicate = _row( + "different", + output="FUNCTION shared() external { /*comment*/ return; }", + ) + stale = _row("shared", body_hash="different-stale-hash") + variant = _row("variant", body_hash="declared-gate-hash") + clean = _row("clean") + + selected, count = exclude_eval_rows([duplicate, stale, variant, clean], [gate]) + + assert selected == [clean] + assert count == 3 + assert body_identity(duplicate) == body_identity(gate) + assert body_identity(variant) != body_identity(gate) + + +def test_gate_exclusion_keeps_colliding_metadata_namespaces_separate(): + gate = _row("gate", contract_address="a:b", selector="c", body_hash="same") + unrelated = _row("unrelated", contract_address="a", selector="b:c") + # A colon-delimited "address:selector" string would incorrectly match. + selected, count = exclude_eval_rows([unrelated], [gate]) + + assert selected == [unrelated] + assert count == 0 + assert row_keys(unrelated).isdisjoint(row_keys(gate)) + + +def test_gate_exclusion_matches_contract_and_exact_input_without_body_hash(): + gate = _row("gate", contract_address="0xabc") + same_address = _row("other", contract_address="0xAbC") + same_input = _row("gate", output="function other() external {}") + selected, count = exclude_eval_rows([same_address, same_input, _row("clean")], [gate]) + assert selected == [_row("clean")] + assert count == 2 + + +def test_canonicalized_rows_keep_missing_hash_duplicates_together_in_split(): + import train + + duplicates = [ + _row("first", output="function shared() external { return; }"), + _row( + "other", + output="FUNCTION shared() external { /* comment */ return; }", + body_hash="stale", + ), + ] + others = [_row(f"other{i}") for i in range(20)] + selected = [canonicalize_body_hash(row) for row in duplicates + others] + + groups = train._grouped_split(selected, 0.6, 0.2, seed=0) + locations = [ + next(index for index, group in enumerate(groups) if row in group) for row in selected[:2] + ] + assert locations[0] == locations[1] + + +def test_sample_cache_requires_source_gate_and_output_hashes(tmp_path): + source = tmp_path / "source.jsonl" + gate = tmp_path / "gate.jsonl" + output = tmp_path / "sample.jsonl" + _write_jsonl(source, [_row("gate"), _row("keep"), _row("extra")]) + _write_jsonl(gate, [_row("gate")]) + + manifest = sample_dataset(source, output, 2, 42, [gate]) + assert manifest["excluded_source_rows"] == 1 + assert sample_dataset(source, output, 2, 42, [gate]) == manifest + assert all(row["metadata"]["body_hash"] == body_identity(row) for row in load_jsonl(output)) + assert {row["input"] for row in load_jsonl(output)} == {"tac keep", "tac extra"} + + _write_jsonl(gate, [_row("keep")]) + with pytest.raises(ValueError, match="unverified or stale"): + sample_dataset(source, output, 2, 42, [gate]) + _write_jsonl(source, [_row("gate"), _row("keep"), _row("extra"), _row("another")]) + with pytest.raises(ValueError, match="unverified or stale"): + sample_dataset(source, output, 2, 42, [gate]) + + +def test_gate_sources_fail_closed_on_missing_target_or_empty_gate(tmp_path): + source = tmp_path / "source.jsonl" + gate = tmp_path / "gate.jsonl" + output = tmp_path / "sample.jsonl" + _write_jsonl(source, [{"input": "tac", "metadata": {"body_hash": "stale"}}]) + _write_jsonl(gate, [_row("gate")]) + with pytest.raises(ValueError, match="no usable output"): + sample_dataset(source, output, 1, 42, [gate]) + _write_jsonl(source, [_row("clean")]) + gate.write_text("", encoding="utf-8") + with pytest.raises(ValueError, match="has no rows"): + sample_dataset(source, output, 1, 42, [gate]) + + +def _write_model_lineage(tmp_path, splits): + model = tmp_path / "model" / "final_model" + model.mkdir(parents=True) + split_dir = tmp_path / "splits" + split_dir.mkdir() + paths = {} + for name, rows in splits.items(): + path = split_dir / f"{name}_dataset.jsonl" + _write_jsonl(path, rows) + paths[name] = path + (split_dir / "split_manifest.json").write_text( + json.dumps( + { + "manifest_kind": "dataset_split", + "leakage_validation": {"status": "passed"}, + "split_quality": {"status": "passed"}, + "outputs": { + name: {"path": str(path), "sha256": file_sha256(path)} + for name, path in paths.items() + }, + } + ), + encoding="utf-8", + ) + (model / "training_input_manifest.json").write_text( + json.dumps( + { + "manifest_kind": "training_inputs", + "status": "completed", + "datasets": { + "train": { + "artifact": { + "path": str(paths["train"]), + "sha256": file_sha256(paths["train"]), + } + } + }, + } + ), + encoding="utf-8", + ) + return model, paths + + +def test_model_gate_check_rejects_eval_overlap_in_any_split(tmp_path): + gate = tmp_path / "gate.jsonl" + _write_jsonl(gate, [_row("heldout", body_hash="expected")]) + for leaking_split in ("train", "val", "test"): + run_dir = tmp_path / leaking_split + run_dir.mkdir() + splits = {name: [_row(name)] for name in ("train", "val", "test")} + splits[leaking_split] = [ + _row( + "heldout", + output="FUNCTION heldout() external { /* same */ return; }", + body_hash="missing-or-stale", + ) + ] + model, _ = _write_model_lineage(run_dir, splits) + with pytest.raises(ValueError, match=f"{leaking_split} split overlaps fixed"): + verify_model_gate_exclusion(model, [gate]) + + +def test_model_gate_check_rejects_unverifiable_and_cross_split_lineage(tmp_path): + gate = tmp_path / "gate.jsonl" + _write_jsonl(gate, [_row("heldout")]) + model, paths = _write_model_lineage( + tmp_path, + { + "train": [_row("first")], + "val": [_row("second")], + "test": [_row("third")], + }, + ) + assert verify_model_gate_exclusion(model, [gate]) == paths["train"] + checked = subprocess.run( + [sys.executable, "-m", "scripts.gate_dataset", "verify-model", str(model), str(gate)], + cwd=ROOT, + text=True, + capture_output=True, + ) + assert checked.returncode == 0, checked.stderr + assert checked.stdout.strip() == str(paths["train"]) + + _write_jsonl(paths["val"], [_row("first", output="function other() external {}")]) + with pytest.raises(ValueError, match="changed"): + verify_model_gate_exclusion(model, [gate]) + split_manifest_path = paths["val"].parent / "split_manifest.json" + manifest = json.loads(split_manifest_path.read_text(encoding="utf-8")) + manifest["outputs"]["val"]["sha256"] = file_sha256(paths["val"]) + split_manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ValueError, match="overlaps train split"): + verify_model_gate_exclusion(model, [gate]) + + +def test_model_gate_check_verifies_train_time_eval_artifact(tmp_path): + gate = tmp_path / "gate.jsonl" + _write_jsonl(gate, [_row("heldout")]) + model, paths = _write_model_lineage( + tmp_path, + {"train": [_row("first")], "val": [_row("second")], "test": [_row("third")]}, + ) + manifest_path = model / "training_input_manifest.json" + manifest = json.loads(manifest_path.read_text()) + manifest["datasets"]["eval"] = {"artifact": {"path": str(gate), "sha256": file_sha256(gate)}} + manifest_path.write_text(json.dumps(manifest)) + + with pytest.raises(ValueError, match="eval dataset differs"): + verify_model_gate_exclusion(model, [gate]) + + manifest["datasets"]["eval"]["artifact"]["path"] = str(paths["val"]) + with pytest.raises(ValueError, match="eval artifact hash differs"): + manifest_path.write_text(json.dumps(manifest)) + verify_model_gate_exclusion(model, [gate]) + + manifest["datasets"]["eval"]["artifact"]["sha256"] = file_sha256(paths["val"]) + manifest_path.write_text(json.dumps(manifest)) + assert verify_model_gate_exclusion(model, [gate]) == paths["train"] + + +def test_eval_artifact_cannot_be_misattributed_to_a_different_model_or_gate(tmp_path): + eval_path = tmp_path / "eval.json" + model = tmp_path / "model" + dataset = tmp_path / "gate.jsonl" + eval_path.write_text( + json.dumps( + { + "summary": { + "model_path": str(model), + "test_dataset": str(dataset), + "eval_max_new_tokens": 512, + "eval_repetition_penalty": 1.05, + } + } + ), + encoding="utf-8", + ) + + verify_eval_artifact(eval_path, model, dataset, 512, 1.05) + with pytest.raises(ValueError, match="does not match requested"): + verify_eval_artifact(eval_path, tmp_path / "other-model", dataset, 512, 1.05) + with pytest.raises(ValueError, match="does not match requested"): + verify_eval_artifact(eval_path, model, tmp_path / "other-gate", 512, 1.05) + with pytest.raises(ValueError, match="does not match requested"): + verify_eval_artifact(eval_path, model, dataset, 1024, 1.05) + + +def test_baseline_preflight_rejects_old_prompt_policy_and_missing_behavior_metric(tmp_path): + gate = tmp_path / "gate.jsonl" + baseline = tmp_path / "baseline.json" + row = _row("gate") + _write_jsonl(gate, [row]) + detail = { + "dataset_index": 0, + "input_hash": hashlib.sha256(row["input"].encode()).hexdigest(), + "output_hash": hashlib.sha256(row["output"].encode()).hexdigest(), + "metadata": row["metadata"], + } + summary = { + "model_path": str(tmp_path / "model"), + "test_dataset": str(gate), + "eval_max_new_tokens": 512, + "eval_repetition_penalty": 1.05, + "num_evaluated": 1, + "selector_signature_prompt_policy": "bundled_only_v1", + "prompt_truncation_count": 0, + "replication_f1_micro": 0.5, + "replication_behavior_only_f1_micro": 0.4, + "bytecode_semantic_score_mean": 0.3, + "semantic_similarity_mean": 0.6, + "solidity_valid_mean": 1.0, + } + baseline.write_text(json.dumps({"summary": summary, "details": [detail]})) + verify_baseline_artifact(baseline, gate, 512, 1.05) + + summary["selector_signature_prompt_policy"] = "source_db" + baseline.write_text(json.dumps({"summary": summary, "details": [detail]})) + with pytest.raises(ValueError, match="bundled_only_v1"): + verify_baseline_artifact(baseline, gate, 512, 1.05) + + summary["selector_signature_prompt_policy"] = "bundled_only_v1" + del summary["replication_behavior_only_f1_micro"] + baseline.write_text(json.dumps({"summary": summary, "details": [detail]})) + with pytest.raises(ValueError, match="replication_behavior_only_f1_micro"): + verify_baseline_artifact(baseline, gate, 512, 1.05) + + summary["replication_behavior_only_f1_micro"] = 0.4 + summary["num_evaluated"] = 2 + baseline.write_text(json.dumps({"summary": summary, "details": [detail]})) + with pytest.raises(ValueError, match="every fixed-gate row"): + verify_baseline_artifact(baseline, gate, 512, 1.05) + + summary["num_evaluated"] = 1 + summary["prompt_truncation_count"] = 1 + baseline.write_text(json.dumps({"summary": summary, "details": [detail]})) + with pytest.raises(ValueError, match="prompt truncation"): + verify_baseline_artifact(baseline, gate, 512, 1.05) + + summary["prompt_truncation_count"] = 0 + baseline.write_text(json.dumps({"summary": summary, "details": [detail]})) + _write_jsonl(gate, [_row("changed")]) + with pytest.raises(ValueError, match="differs from current gate dataset"): + verify_baseline_artifact(baseline, gate, 512, 1.05) + + +def test_full_runner_rejects_missing_baseline_before_materializing_training_data(tmp_path): + source = tmp_path / "source.jsonl" + gate = tmp_path / "gate.jsonl" + _write_jsonl(source, [_row("train"), _row("other")]) + _write_jsonl(gate, [_row("gate")]) + data_dir = tmp_path / "training" + output_dir = tmp_path / "model" + env = { + **os.environ, + "SOURCE_DATASET": str(source), + "DATA_DIR": str(data_dir), + "OUTPUT_DIR": str(output_dir), + "RUN_GATES": "1", + "DRY_RUN": "0", + "BROAD_BASELINE": str(tmp_path / "missing-baseline.json"), + **{key: str(gate) for key in GATE_VARS}, + } + result = subprocess.run( + ["bash", str(ROOT / "run_train_qwen_qlora_full_body_balanced.sh")], + cwd=ROOT, + env=env, + text=True, + capture_output=True, + ) + + assert result.returncode != 0 + assert "Required broad30 baseline eval not found" in result.stderr + assert not data_dir.exists() + assert not output_dir.exists() + + +def test_standalone_gate_runner_rejects_model_without_provenance_before_eval(tmp_path): + gate = tmp_path / "gate.jsonl" + model = tmp_path / "model" + model.mkdir() + _write_jsonl(gate, [_row("gate")]) + env = { + **os.environ, + **{name: str(gate) for name in GATE_VARS + BASELINE_VARS}, + "MODEL_PATH": str(model), + "GATE_DIR": str(tmp_path / "gates"), + } + result = subprocess.run( + ["bash", str(ROOT / "run_eval_gate_suite_for_model.sh")], + cwd=ROOT, + env=env, + text=True, + capture_output=True, + ) + assert result.returncode != 0 + assert "Cannot verify eval-clean model lineage" in result.stderr + assert not (tmp_path / "gates").exists() + + +@pytest.mark.parametrize( + "runner,output_name", + [ + ("run_train_qwen_qlora_500.sh", "qwen_qlora_2_sample.jsonl"), + ("run_train_qwen_qlora_full_body_balanced.sh", "body_balanced_dataset.jsonl"), + ], +) +def test_runner_dry_run_excludes_fixed_gates_and_rejects_stale_cache(tmp_path, runner, output_name): + source = tmp_path / "source.jsonl" + gate = tmp_path / "gate.jsonl" + data_dir = tmp_path / "run" + _write_jsonl(source, [_row("gate"), _row("clean1"), _row("clean2")]) + _write_jsonl(gate, [_row("gate")]) + env = { + **os.environ, + "SOURCE_DATASET": str(source), + "DATA_DIR": str(data_dir), + "OUTPUT_DIR": str(tmp_path / "model"), + "SAMPLE_COUNT": "2", + "DRY_RUN": "1", + **{key: str(gate) for key in GATE_VARS}, + } + command = ["bash", str(ROOT / runner)] + + built = subprocess.run(command, cwd=ROOT, env=env, text=True, capture_output=True) + assert built.returncode == 0, built.stdout + built.stderr + rows = load_jsonl(data_dir / output_name) + assert len(rows) == 2 + assert {row["input"] for row in rows} == {"tac clean1", "tac clean2"} + + reused = subprocess.run(command, cwd=ROOT, env=env, text=True, capture_output=True) + assert reused.returncode == 0, reused.stdout + reused.stderr + _write_jsonl(gate, [_row("clean1")]) + stale = subprocess.run(command, cwd=ROOT, env=env, text=True, capture_output=True) + assert stale.returncode != 0 + assert "unverified or stale" in stale.stderr + + +def test_body_balanced_runner_caps_duplicate_targets_without_body_hash(tmp_path): + source = tmp_path / "source.jsonl" + gate = tmp_path / "gate.jsonl" + data_dir = tmp_path / "run" + duplicate = _row( + "one", + output="FUNCTION shared() external { /* same target */ return; }", + body_hash="stale", + ) + _write_jsonl( + source, + [ + _row("first", output="function shared() external { return; }"), + duplicate, + _row("distinct"), + _row("also_distinct"), + ], + ) + _write_jsonl(gate, [_row("heldout")]) + env = { + **os.environ, + "SOURCE_DATASET": str(source), + "DATA_DIR": str(data_dir), + "OUTPUT_DIR": str(tmp_path / "model"), + "DRY_RUN": "1", + **{key: str(gate) for key in GATE_VARS}, + } + + result = subprocess.run( + ["bash", str(ROOT / "run_train_qwen_qlora_full_body_balanced.sh")], + cwd=ROOT, + env=env, + text=True, + capture_output=True, + ) + + assert result.returncode == 0, result.stdout + result.stderr + selected = load_jsonl(data_dir / "body_balanced_dataset.jsonl") + assert len(selected) == 3 + assert len({body_identity(row) for row in selected}) == 3 + assert all(row["metadata"]["body_hash"] == body_identity(row) for row in selected) + manifest = json.loads((data_dir / "body_balanced_manifest.json").read_text()) + assert manifest["source_body_duplicate_rows"] == 1 + assert manifest["missing_body_hash_source_rows"] == 3 + + +def test_sample_runner_repairs_duplicate_body_hashes_before_grouped_split(tmp_path): + source = tmp_path / "source.jsonl" + gate = tmp_path / "gate.jsonl" + data_dir = tmp_path / "run" + _write_jsonl( + source, + [ + _row("first", output="function shared() external { return; }"), + _row( + "other", + output="FUNCTION shared() external { /* duplicate */ return; }", + body_hash="stale", + ), + _row("distinct"), + ], + ) + _write_jsonl(gate, [_row("heldout")]) + env = { + **os.environ, + "SOURCE_DATASET": str(source), + "DATA_DIR": str(data_dir), + "OUTPUT_DIR": str(tmp_path / "model"), + "SAMPLE_COUNT": "3", + "DRY_RUN": "1", + **{key: str(gate) for key in GATE_VARS}, + } + + result = subprocess.run( + ["bash", str(ROOT / "run_train_qwen_qlora_500.sh")], + cwd=ROOT, + env=env, + text=True, + capture_output=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + selected = load_jsonl(data_dir / "qwen_qlora_3_sample.jsonl") + assert len(selected) == 3 + assert len({row["metadata"]["body_hash"] for row in selected}) == 2 + assert all(row["metadata"]["body_hash"] == body_identity(row) for row in selected) diff --git a/tests/test_inference.py b/tests/test_inference.py new file mode 100644 index 0000000..8f1cc30 --- /dev/null +++ b/tests/test_inference.py @@ -0,0 +1,121 @@ +import pytest + +from src.bytecode_analyzer import BytecodeAnalyzer +from src.inference import run_bytecode_inference +from src.model_setup import format_prompt_metadata + + +TWO_FUNCTION_BYTECODE = "0x60003560e01c80631111111114601c57806322222222146020575b005b6001005b600200" + + +def test_bytecode_inference_uses_training_compatible_function_prompt_metadata(monkeypatch): + monkeypatch.setattr( + "src.inference.validate_solidity_output", + lambda source, metadata=None: { + "valid": True, + "method": "scaffold", + "compiler_checked": False, + }, + ) + + class RecordingDecompiler: + def __init__(self): + self.calls = [] + + def decompile_tac_to_solidity(self, tac, metadata=None, **kwargs): + self.calls.append((tac, metadata, format_prompt_metadata(metadata, tac_input=tac))) + return f"function recovered{len(self.calls)}() public {{}}" + + decompiler = RecordingDecompiler() + result = run_bytecode_inference(TWO_FUNCTION_BYTECODE, decompiler=decompiler) + + assert result["success"] is True + assert len(decompiler.calls) == 3 + assert result["analysis"]["num_instructions"] == 22 + assert result["analysis"]["num_functions"] == 3 + assert result["reconstruction"]["contract_facts"]["function_count"] == 3 + for tac, metadata, prompt_metadata in decompiler.calls: + training_metadata = {"selector": metadata["selector"]} if "selector" in metadata else {} + assert prompt_metadata == format_prompt_metadata(training_metadata, tac_input=tac) + assert "bytecode_instruction_count" not in metadata + assert "function_count" not in metadata + + +@pytest.mark.parametrize("legacy_whole_contract_fallback", [False, True]) +@pytest.mark.parametrize("tac_only", [False, True]) +def test_no_recoverable_functions_never_become_model_generated( + monkeypatch, legacy_whole_contract_fallback, tac_only +): + bytecode = "0x60003560e01c80631234567814602057" + monkeypatch.setattr( + "src.inference.validate_solidity_output", + lambda source, metadata=None: {"valid": True, "method": "scaffold"}, + ) + + class RecordingDecompiler: + def __init__(self): + self.calls = [] + + def decompile_tac_to_solidity(self, tac, **kwargs): + self.calls.append(tac) + return "function invented() public {}" + + def legacy_analysis(code): + analyzer = BytecodeAnalyzer(code) + assert analyzer.generate_per_function_tac() == {} + contract_tac = analyzer.generate_tac_representation() + return analyzer, {"contract": contract_tac}, contract_tac + + decompiler = RecordingDecompiler() + result = run_bytecode_inference( + bytecode, + decompiler=decompiler, + analyze_tac_fn=legacy_analysis if legacy_whole_contract_fallback else None, + tac_only=tac_only, + ) + + assert decompiler.calls == [] + assert result["success"] is False + assert result["decompilation_status"] == "analysis_failed" + assert "no recoverable function" in result["error"].lower() + assert result["tac"] + assert result["tac_per_function"] == {} + assert result["functions"] == {} + assert result["function_results"] == [] + assert result["analysis"]["num_functions"] == 0 + assert result["reconstruction"]["chunk_count"] == 0 + assert result["trace"]["status"] == "failed" + + +@pytest.mark.parametrize("tac_only", [False, True]) +def test_invalid_selector_with_valid_fallback_is_reported_partial(monkeypatch, tac_only): + monkeypatch.setattr( + "src.inference.validate_solidity_output", + lambda source, metadata=None: {"valid": True, "method": "scaffold"}, + ) + + class RecordingDecompiler: + def __init__(self): + self.calls = [] + + def decompile_tac_to_solidity(self, tac, **kwargs): + self.calls.append(tac) + return "fallback() external {}" + + decompiler = RecordingDecompiler() + result = run_bytecode_inference( + "0x60003560e01c806312345678146020575b00", + decompiler=decompiler, + tac_only=tac_only, + ) + + assert result["success"] is False + assert result["partial_success"] is True + assert result["decompilation_status"] == "partial_analysis" + assert result["analysis"]["rejected_dispatcher_targets"] == {"0x12345678": 0x20} + assert "function_0x12345678" in result["quality"]["unresolved_chunks"] + assert result["quality"]["severity"] == "error" + assert "0x12345678 -> 0x20" in result["error"] + assert set(result["tac_per_function"]) == {"fallback_function"} + assert len(decompiler.calls) == (0 if tac_only else 1) + assert result["trace"]["status"] == "partial" diff --git a/tests/test_model_training_issues.py b/tests/test_model_training_issues.py index e29002b..ce0bd31 100644 --- a/tests/test_model_training_issues.py +++ b/tests/test_model_training_issues.py @@ -4,6 +4,8 @@ from pathlib import Path from types import SimpleNamespace +import pytest + import train from src import model_setup from src.model_setup import ( @@ -240,6 +242,17 @@ def test_dataset_truncation_preserves_full_short_target_span(): assert supervised_labels == ["TARGET_A", "TARGET_B", "TARGET_C"] +def test_dataset_rejects_target_that_cannot_fit_with_context_and_eos(): + dataset = _dataset_for_item( + {"input": "tac", "output": "TARGET_A TARGET_B", "metadata": {}}, + max_length=3, + ) + dataset.tokenizer = NumericTokenizer() + + with pytest.raises(ValueError, match="target.*max_length"): + dataset[0] + + def test_numeric_tokenizer_targets_include_eos_for_stop_learning(): dataset = SmartContractDataset.__new__(SmartContractDataset) dataset.tokenizer = NumericTokenizer() @@ -704,6 +717,7 @@ def fake_run_data_preflight(dataset_paths, **kwargs): train_test_split=0.6, validation_split=0.2, ) + pipeline.config.model_config.include_selector_signature_metadata = False train_path, val_path, test_path = pipeline._split_dataset(str(dataset_path)) @@ -716,6 +730,7 @@ def fake_run_data_preflight(dataset_paths, **kwargs): "test": test_path, } assert captured["kwargs"]["allow_legacy_metadata_schema"] is False + assert captured["kwargs"]["include_selector_signature_metadata"] is False assert pipeline.last_preflight_report["status"] == "passed" @@ -814,6 +829,38 @@ def test_tokenized_dataset_cache_reuses_examples_and_max_length_invalidates(tmp_ assert changed_vocab_tokenizer.calls > 0 +def test_tokenized_cache_invalidates_legacy_selector_signatures(tmp_path): + dataset_path = tmp_path / "dataset.jsonl" + _write_jsonl( + dataset_path, + [{"input": "function function_0x5312ea8e:\n return", "output": "TARGET", "metadata": {}}], + ) + cache_dir = tmp_path / "token-cache" + cache_dir.mkdir() + tokenizer = CountingTokenizer() + dataset = SmartContractDataset(str(dataset_path), tokenizer, max_length=64) + legacy_metadata = dataset._cache_metadata(str(dataset_path)) + legacy_metadata["cache_version"] = 2 + cache_path, metadata_path = dataset._cache_paths(str(dataset_path), cache_dir, legacy_metadata) + dataset._write_tokenization_cache( + cache_path, + metadata_path, + legacy_metadata, + [{"input_ids": ["selector_signature=oracle()", "TARGET"], + "attention_mask": [1, 1], "labels": [-100, "TARGET"]}], + ) + + current = SmartContractDataset( + str(dataset_path), + tokenizer, + max_length=64, + tokenization_cache=TokenizationCacheConfig(enabled=True, cache_dir=str(cache_dir)), + ) + + assert tokenizer.calls > 0 + assert "selector_signature=oracle()" not in current[0]["input_ids"] + + def test_tokenized_dataset_cache_invalidates_prompt_flags_and_dataset_fingerprint(tmp_path): dataset_path = tmp_path / "dataset.jsonl" rows = [ @@ -910,6 +957,10 @@ def __init__(self, **kwargs): class FakeTrainer: def __init__(self, **kwargs): captured["eval_ids"] = [row["metadata"]["id"] for row in kwargs["eval_dataset"].data] + captured["prompt_flags"] = { + split: getattr(kwargs[f"{split}_dataset"], "include_selector_signature_metadata") + for split in ("train", "eval") + } self.state = SimpleNamespace(log_history=[{"loss": 0.5, "step": 1}]) def train(self, **_kwargs): @@ -934,7 +985,11 @@ def fake_setup_model(self, *args, **kwargs): monkeypatch.setattr(SmartContractModelTrainer, "setup_model", fake_setup_model) trainer = SmartContractModelTrainer( - ModelConfig(max_sequence_length=128, use_quantization=False), + ModelConfig( + max_sequence_length=128, + use_quantization=False, + include_selector_signature_metadata=False, + ), output_dir=str(tmp_path / "models"), ) model_path = trainer.train( @@ -961,6 +1016,7 @@ def fake_setup_model(self, *args, **kwargs): assert captured["eval_ids"] == expected_indices assert captured["eval_ids"] != [0, 1, 2, 3] + assert captured["prompt_flags"] == {"train": False, "eval": False} assert manifest["status"] == "completed" assert manifest["seed"] == 123 assert manifest["datasets"]["train"]["artifact"]["row_count"] == 6 diff --git a/tests/test_prompt_metadata.py b/tests/test_prompt_metadata.py index f8b319e..989b794 100644 --- a/tests/test_prompt_metadata.py +++ b/tests/test_prompt_metadata.py @@ -1,10 +1,16 @@ +import sqlite3 + +import pytest + from src.model_setup import ( ModelConfig, SmartContractDataset, SmartContractDecompiler, format_prompt_metadata, + resolve_selector_signature_for_prompt, sanitize_tac_for_prompt, ) +from src.selector_resolver import SelectorResolver class FakeTokenizer: @@ -77,6 +83,38 @@ def test_format_prompt_metadata_uses_only_bytecode_context(): assert_source_metadata_excluded(line) +def test_prompt_ignores_unshipped_heldout_signatures_in_local_registry(tmp_path, monkeypatch): + selector = "0x5312ea8e" + db_path = tmp_path / "contracts.db" + with sqlite3.connect(db_path) as conn: + conn.execute( + "CREATE TABLE selector_registry " + "(selector TEXT, signature TEXT, source TEXT, occurrences INTEGER)" + ) + conn.execute( + "INSERT INTO selector_registry VALUES (?, ?, ?, ?)", + (selector, "emergencyWithdraw(uint256)", "compiled", 10), + ) + + resolver = SelectorResolver( + use_remote=False, + db_path=db_path, + json_path=tmp_path / "absent-selectors.json", + ) + assert resolver.resolve(selector).best_match.source == "db" + monkeypatch.setattr("src.selector_resolver.get_resolver", lambda **_kwargs: resolver) + + assert resolve_selector_signature_for_prompt(selector) is None + assert resolve_selector_signature_for_prompt("0xa9059cbb") == "transfer(address,uint256)" + prompt_metadata = format_prompt_metadata( + {"selector": selector, "function_signature": "emergencyWithdraw(uint256)"}, + tac_input=f"function function_{selector}:\nblock_entry:\n return", + ) + assert f"selector={selector}" in prompt_metadata + assert "selector_signature=" not in prompt_metadata + assert "emergencyWithdraw" not in prompt_metadata + + def test_dataset_prompt_excludes_source_metadata_with_deprecated_flag_enabled(): tac = """// Compiler: solc 0.6.12 // Visibility: public @@ -155,6 +193,37 @@ def test_decompiler_prompt_excludes_source_metadata_with_deprecated_config_flag( assert "### Response:" in prompt +@pytest.mark.parametrize("selector_signature_enabled", [False, True]) +def test_training_and_inference_prompts_match_without_truncation(selector_signature_enabled): + metadata = { + "selector": "0xa9059cbb", + "function_name": "transfer", + "compiler_version": "0.8.20", + "bytecode_length": 256, + } + tac = """// Compiler: solc 0.8.20 +function transfer(address to, uint256 value): +block_entry: + v0 = CALLDATALOAD 0x04 +""" + dataset = SmartContractDataset.__new__(SmartContractDataset) + dataset.template_format = "alpaca" + dataset.include_bytecode_metadata = True + dataset.include_selector_signature_metadata = selector_signature_enabled + train_prefix, _, _ = dataset._format_prompt_parts(tac, "function transfer() {}", metadata) + + decompiler = SmartContractDecompiler.__new__(SmartContractDecompiler) + decompiler.tokenizer = FakeTokenizer() + decompiler.config = ModelConfig( + include_selector_signature_metadata=selector_signature_enabled + ) + inference_prompt = decompiler._build_prompt(tac, metadata, max_new_tokens=256) + + assert train_prefix == inference_prompt + assert ("selector_signature=transfer(address,uint256)" in train_prefix) is selector_signature_enabled + assert_source_metadata_excluded(train_prefix) + + def test_decompiler_inference_context_keeps_prompt_budget_above_training_sweep_length(): decompiler = SmartContractDecompiler.__new__(SmartContractDecompiler) decompiler.tokenizer = FakeTokenizer() diff --git a/tests/test_replication_metrics.py b/tests/test_replication_metrics.py index d6423d0..d4d4943 100644 --- a/tests/test_replication_metrics.py +++ b/tests/test_replication_metrics.py @@ -4,6 +4,7 @@ from src.replication_metrics import ( aggregate_replication_scores, + behavior_only_replication_micro, evaluate_replication, extract_solidity_facts, ) @@ -57,6 +58,53 @@ def test_extracts_nested_mapping_state_writes(self): "freemintaddresses[addresses[i]]", } + def test_nested_mapping_parameter_type_changes_reduce_abi_f1(self): + reference = ( + "function f(mapping(address => uint256) storage ledger) internal " + "{ ledger[msg.sender] = 1; }" + ) + candidate = ( + "function f(mapping(address => bool) storage ledger) internal " + "{ ledger[msg.sender] = true; }" + ) + + assert "param_type:0:mapping(address=>uint256)" in extract_solidity_facts(reference)["abi"] + assert "param_type:0:mapping(address=>bool)" in extract_solidity_facts(candidate)["abi"] + result = evaluate_replication(reference, candidate) + assert result.by_category["abi"].f1 < 1.0 + assert "modifier" not in result.by_category + + def test_nested_function_type_signature_keeps_outer_parameter_and_return(self): + code = ( + "function f(function(uint256) external returns (bool) callback) public " + "returns (uint256) { return callback(1) ? 1 : 0; }" + ) + facts = extract_solidity_facts(code) + + assert "param_count:1" in facts["abi"] + assert "param_type:0:function(uint256)external returns(bool)" in facts["abi"] + assert "return_type:0:uint256" in facts["abi"] + assert facts["visibility"] == {"public"} + + def test_integer_and_fixed_bytes_casts_are_not_calls(self): + code = ( + "function f(uint8 value) external pure returns (bytes32) { " + "_record(bytes32(uint256(value))); return bytes32(uint8(value)); }" + ) + + assert extract_solidity_facts(code)["call"] == {"_record"} + + def test_bodyless_declaration_does_not_borrow_next_function_body(self): + code = ( + "abstract contract C { function first() public virtual; " + "function second() public { _unrelatedCall(); } }" + ) + + facts = extract_solidity_facts(code) + + assert "function_name:first" in facts["abi"] + assert "call" not in facts + def test_parameter_renames_do_not_penalize_guard_matching(self): reference = """ function setOwner(address newOwner) external onlyOwner { @@ -157,6 +205,22 @@ def test_extra_generated_facts_are_bucketed_as_grounded_hallucinations(self): class TestAggregateReplicationScores: + def test_behavior_only_micro_excludes_abi_and_requires_behavior_facts(self): + categories = { + "abi": {"true_positives": 9000, "false_positives": 0, "false_negatives": 0}, + "modifier": {"true_positives": 100, "false_positives": 0, "false_negatives": 0}, + "call": {"true_positives": 0, "false_positives": 3, "false_negatives": 2}, + "state_write": {"true_positives": 1, "false_positives": 0, "false_negatives": 1}, + } + + assert behavior_only_replication_micro({"abi": categories["abi"]}) is None + behavior = behavior_only_replication_micro(categories) + + assert behavior["true_positives"] == 1 + assert behavior["false_positives"] == 3 + assert behavior["false_negatives"] == 3 + assert behavior["f1"] == 0.25 + def test_aggregates_mean_and_micro_scores(self): first = evaluate_replication( "function a() public { count = 1; }", @@ -190,6 +254,9 @@ def test_aggregates_mean_and_micro_scores(self): assert summary["micro"]["false_negatives"] == 1 assert summary["fact_error_totals"] == {"matched": 10, "extra": 0, "missing": 1} assert summary["by_category_micro"]["state_write"]["false_negatives"] == 1 + assert summary["behavior_only_micro"]["f1"] == 0.8 + assert summary["behavior_only_micro"]["true_positives"] == 2 + assert summary["micro"]["true_positives"] == 10 assert summary["category_gap_summary"][0]["category"] == "state_write" assert summary["category_gap_summary"][0]["primary_error"] == "recall" diff --git a/tests/test_train_cli_issues.py b/tests/test_train_cli_issues.py index c568d21..c2b2f7b 100644 --- a/tests/test_train_cli_issues.py +++ b/tests/test_train_cli_issues.py @@ -338,6 +338,65 @@ def test_jsonl_preflight_reports_schema_and_token_length_errors(tmp_path): assert report["error_counts"]["target_overlength"] == 1 +def test_jsonl_preflight_counts_training_eos_token(tmp_path): + class EosTokenizer(TinyTokenizer): + eos_token_id = 99 + + def __call__(self, text, **_kwargs): + return {"input_ids": list(range(len(str(text).split())))} + + row = {"input": "tac", "output": "target", "metadata": {"schema_version": 1}} + dataset_path = tmp_path / "eos.jsonl" + _write_jsonl(dataset_path, [row]) + prefix, target, suffix = train._preflight_prompt_parts(row, True, True, "alpaca") + max_seq_length = len(prefix.split()) + len(f"{target}{suffix}".split()) + + report = train.validate_jsonl_schema_and_lengths( + dataset_path, tokenizer=EosTokenizer(), max_seq_length=max_seq_length + ) + + assert report["status"] == "failed" + assert report["lengths"]["max_total_tokens"] == max_seq_length + 1 + assert report["error_counts"]["context_overlength"] == 1 + + +def test_dataset_only_preflight_honors_selector_metadata_flag(tmp_path, monkeypatch): + dataset_path = tmp_path / "source.jsonl" + _write_jsonl( + dataset_path, + [{"input": f"tac {i}", "output": f"sol {i}", "metadata": {}} for i in range(6)], + ) + captured = {} + + def fake_preflight(_paths, **kwargs): + captured.update(kwargs) + return {"status": "passed"} + + monkeypatch.setattr(train, "setup_logging", lambda *args, **kwargs: None) + monkeypatch.setattr(train, "run_data_preflight", fake_preflight) + monkeypatch.setattr( + sys, + "argv", + [ + "train.py", + "--skip-collection", + "--dataset", + str(dataset_path), + "--data-dir", + str(tmp_path / "data"), + "--dataset-only", + "--no-selector-signature-metadata", + "--no-auto-torchrun", + "--run-manifest", + str(tmp_path / "manifest.json"), + ], + ) + + train.main() + + assert captured["include_selector_signature_metadata"] is False + + def test_jsonl_preflight_rejects_malformed_versioned_metadata(tmp_path): dataset_path = tmp_path / "bad_metadata.jsonl" _write_jsonl( @@ -428,6 +487,27 @@ def evaluate_function(self, _expected, _actual, _metadata): return FakeMetrics() +def test_aggregate_summary_exposes_behavior_only_f1(monkeypatch): + from src.training_pipeline import SmartContractTrainingPipeline + + monkeypatch.setattr( + SmartContractTrainingPipeline, + "_compute_aggregate_statistics", + lambda self, results, baseline_summary=None: { + "replication_metrics": { + "micro": {"f1": 0.63}, + "behavior_only_micro": {"f1": 0.22}, + "by_category_micro": {"call": {"f1": 0.22}}, + } + }, + ) + summary = {} + train._merge_aggregate_statistics(summary, [{"metrics": {}}], None) + + assert summary["replication_f1_micro"] == 0.63 + assert summary["replication_behavior_only_f1_micro"] == 0.22 + + def _patch_evaluation_dependencies(monkeypatch, decompiler_cls): import src.evaluation_report as evaluation_report import src.model_setup as model_setup @@ -503,8 +583,11 @@ def decompile_tac_to_solidity( assert summary["eval_batch_size"] == 2 assert summary["eval_max_new_tokens"] == 77 assert summary["eval_repetition_penalty"] == 1.05 + assert summary["selector_signature_prompt_policy"] == "bundled_only_v1" assert summary["num_evaluated"] == 3 - assert Path(summary["results_path"]).exists() + assert json.loads(Path(summary["results_path"]).read_text())["summary"][ + "selector_signature_prompt_policy" + ] == "bundled_only_v1" def test_evaluate_model_batch_oom_falls_back_to_single_examples(tmp_path, monkeypatch): @@ -1143,6 +1226,50 @@ def fail_if_resplit(*_args, **_kwargs): assert train.split_dataset.last_status["reused"] is True +def test_split_dataset_groups_normalized_targets_even_with_stale_or_missing_body_hash(tmp_path): + rows = [ + {"input": "tac duplicate A", "output": "function f() public { return; }", + "metadata": {"body_hash": "stale-hash"}}, + {"input": "tac duplicate B", + "output": "function f() public { /* different comment */ return; }", + "metadata": {}}, + ] + rows.extend( + {"input": f"tac distinct {index}", + "output": f"function distinct{index}() public {{ return; }}", + "metadata": {}} + for index in range(7) + ) + dataset_path = tmp_path / "source.jsonl" + _write_jsonl(dataset_path, rows) + + split_paths = train.split_dataset( + str(dataset_path), str(tmp_path / "splits"), + train_ratio=0.6, val_ratio=0.2, seed=0, + ) + + duplicates = [ + split_name + for split_name, path in zip(("train", "val", "test"), split_paths) + for row in (json.loads(line) for line in Path(path).read_text().splitlines()) + if row["input"].startswith("tac duplicate") + ] + assert len(duplicates) == 2 + assert len(set(duplicates)) == 1 + manifest_path = tmp_path / "splits" / "split_manifest.json" + manifest = json.loads(manifest_path.read_text()) + assert manifest["leakage_validation"]["status"] == "passed" + + manifest["schema_version"] -= 1 + manifest_path.write_text(json.dumps(manifest)) + train.split_dataset( + str(dataset_path), str(tmp_path / "splits"), + train_ratio=0.6, val_ratio=0.2, seed=0, reuse_existing=True, + ) + assert train.split_dataset.last_status["reused"] is False + assert train.split_dataset.last_status["reason"] == "schema_version_mismatch" + + def test_split_dataset_creates_nested_output_directory(tmp_path): rows = [ { diff --git a/tests/test_web_app.py b/tests/test_web_app.py index 4875b9b..59e25ce 100644 --- a/tests/test_web_app.py +++ b/tests/test_web_app.py @@ -95,6 +95,7 @@ def test_ui_provides_in_memory_api_key_field_and_auth_headers(client): assert 'fetch("/api/gpu-stats", { headers: apiHeaders() })' in app_js assert 'headers: apiHeaders({ "Content-Type": "application/json" })' in app_js assert 'headers.Authorization = "Bearer " + apiKey' in app_js + assert "if (data.error)" in app_js def test_health_splits_liveness_readiness_and_redacts_public_details(client, monkeypatch): @@ -453,6 +454,57 @@ def resolve_function_names(self, fnames): assert disabled_result["lookup_config"]["benchmark_mode"] is True +def test_decompile_stream_rejects_unrecoverable_function_boundaries(client, monkeypatch): + monkeypatch.setattr(web_app, "decompiler", None) + monkeypatch.setattr(web_app, "tac_lookup", None) + monkeypatch.setattr(web_app, "_write_inference_trace", lambda trace: None) + response = client.post( + "/api/decompile", + json={"bytecode": "0x60003560e01c80631234567814602057"}, + environ_overrides={"REMOTE_ADDR": "127.0.0.1"}, + ) + + events = _parse_sse_events(response.get_data(as_text=True)) + assert not any(event == "result" and data.get("success") for event, data in events) + assert any( + event == "error" and "No recoverable function boundaries" in data["error"] + for event, data in events + ) + + +def test_decompile_stream_marks_rejected_selector_with_fallback_partial(client, monkeypatch): + class FakeDecompiler: + def decompile_tac_to_solidity(self, tac, **kwargs): + return "fallback() external {}" + + monkeypatch.setattr(web_app, "decompiler", FakeDecompiler()) + monkeypatch.setattr(web_app, "tac_lookup", None) + monkeypatch.setattr(web_app, "_write_inference_trace", lambda trace: None) + monkeypatch.setattr( + web_app, + "_validate_solidity_output", + lambda source, metadata=None: {"valid": True, "method": "scaffold"}, + ) + response = client.post( + "/api/decompile", + json={"bytecode": "0x60003560e01c806312345678146020575b00"}, + environ_overrides={"REMOTE_ADDR": "127.0.0.1"}, + ) + + results = [ + data for event, data in _parse_sse_events(response.get_data(as_text=True)) + if event == "result" + ] + assert len(results) == 1 + assert results[0]["success"] is False + assert results[0]["partial_success"] is True + assert results[0]["decompilation_status"] == "partial_analysis" + assert results[0]["analysis"]["rejected_dispatcher_targets"] == {"0x12345678": 0x20} + assert "function_0x12345678" in results[0]["quality"]["unresolved_chunks"] + assert results[0]["quality"]["severity"] == "error" + assert "0x12345678 -> 0x20" in results[0]["error"] + + def test_decompile_rejects_invalid_abi_json(client): response = client.post( "/api/decompile", diff --git a/train.py b/train.py index 448a97e..0af7d97 100644 --- a/train.py +++ b/train.py @@ -67,6 +67,8 @@ import yaml +from src.dataset_export_primitives import hash_normalized_body + DEFAULT_MODEL_NAME = "Qwen/Qwen2.5-Coder-7B-Instruct" DEFAULT_NUM_GPUS = 4 DEFAULT_BATCH_SIZE = 1 @@ -88,8 +90,8 @@ "validation_dataset.jsonl", "test_dataset.jsonl", } -SPLIT_CACHE_SCHEMA_VERSION = 2 -PREFLIGHT_CACHE_SCHEMA_VERSION = 2 +SPLIT_CACHE_SCHEMA_VERSION = 3 +PREFLIGHT_CACHE_SCHEMA_VERSION = 4 DEFAULT_MIN_SPLIT_TARGET_RATIO = 0.5 DEFAULT_MAX_COMPONENT_TARGET_RATIO = 1.0 DEFAULT_QUALITY_THRESHOLDS = { @@ -572,6 +574,9 @@ def _row_leakage_keys(item: dict) -> dict[str, set[str]]: "solidity_body_hash", ), ) + output = item.get("output") + if isinstance(output, str) and output.strip(): + _add_leakage_key(keys, "body_hash", hash_normalized_body(output)) _add_leakage_key( keys, @@ -1142,6 +1147,8 @@ def _split_manifest_matches( ) -> tuple[bool, str, dict[str, str]]: if manifest.get("manifest_kind") != "dataset_split": return False, "manifest_kind_mismatch", {} + if manifest.get("schema_version") != SPLIT_CACHE_SCHEMA_VERSION: + return False, "schema_version_mismatch", {} if manifest.get("input_sha256") != source_sha256: return False, "source_sha256_mismatch", {} manifest_params = manifest.get("parameters") @@ -1493,7 +1500,7 @@ def validate_jsonl_schema_and_lengths( max_errors: int = 50, ) -> dict: """Validate JSONL schema and token lengths before training/evaluation.""" - from src.model_setup import _tokenize_to_ids + from src.model_setup import _append_eos_if_supported, _tokenize_to_ids from src.dataset_pipeline import ( TRAINING_ROW_SCHEMA_VERSION, validate_training_metadata_schema, @@ -1641,7 +1648,11 @@ def validate_jsonl_schema_and_lengths( template_format=template_format, ) context_tokens = len(_tokenize_to_ids(tokenizer, prefix)) - target_tokens = len(_tokenize_to_ids(tokenizer, f"{target}{suffix}")) + target_tokens = len( + _append_eos_if_supported( + tokenizer, _tokenize_to_ids(tokenizer, f"{target}{suffix}") + ) + ) total_tokens = context_tokens + target_tokens except Exception as exc: _record_preflight_error( @@ -2549,6 +2560,7 @@ def _merge_aggregate_statistics( replication = aggregate.get("replication_metrics") if isinstance(replication, Mapping): micro = replication.get("micro", {}) + behavior_micro = replication.get("behavior_only_micro", {}) summary.update( { "replication_precision_mean": replication.get("precision_mean"), @@ -2562,6 +2574,9 @@ def _merge_aggregate_statistics( micro.get("recall") if isinstance(micro, Mapping) else None ), "replication_f1_micro": micro.get("f1") if isinstance(micro, Mapping) else None, + "replication_behavior_only_f1_micro": ( + behavior_micro.get("f1") if isinstance(behavior_micro, Mapping) else None + ), "replication_by_category_micro": replication.get("by_category_micro", {}), } ) @@ -2999,7 +3014,7 @@ def evaluate_model( extract_opcode_control_flow_slices, sample_evaluation_data, ) - from src.model_setup import SmartContractDecompiler + from src.model_setup import SELECTOR_SIGNATURE_PROMPT_POLICY, SmartContractDecompiler from src.evaluation_report import write_latest_results_report from src.replication_metrics import aggregate_replication_scores from dataclasses import asdict @@ -3554,6 +3569,7 @@ def _evaluate_single_with_logging(item: dict, item_number: int) -> None: "include_selector_signature_metadata": _decompiler_selector_signature_metadata( decompiler ), + "selector_signature_prompt_policy": SELECTOR_SIGNATURE_PROMPT_POLICY, "generation_config": generation_config, "model_path": model_path, "test_dataset": test_path, @@ -3576,6 +3592,7 @@ def _evaluate_single_with_logging(item: dict, item_number: int) -> None: summary["worst_samples"] = _worst_evaluation_samples(results) if replication_summary: replication_micro = replication_summary.get("micro", {}) + behavior_micro = replication_summary.get("behavior_only_micro", {}) summary.update( { "replication_precision_mean": replication_summary.get("precision_mean"), @@ -3585,6 +3602,7 @@ def _evaluate_single_with_logging(item: dict, item_number: int) -> None: "replication_precision_micro": replication_micro.get("precision"), "replication_recall_micro": replication_micro.get("recall"), "replication_f1_micro": replication_micro.get("f1"), + "replication_behavior_only_f1_micro": behavior_micro.get("f1"), "replication_by_category_micro": replication_summary.get( "by_category_micro", {} ), @@ -3608,6 +3626,7 @@ def _evaluate_single_with_logging(item: dict, item_number: int) -> None: "include_selector_signature_metadata": _decompiler_selector_signature_metadata( decompiler ), + "selector_signature_prompt_policy": SELECTOR_SIGNATURE_PROMPT_POLICY, "generation_config": generation_config, "model_path": model_path, "test_dataset": test_path, @@ -4596,6 +4615,7 @@ def main(): tokenizer_source=args.model_name, max_seq_length=args.max_seq_length, include_bytecode_metadata=not args.no_bytecode_metadata, + include_selector_signature_metadata=not args.no_selector_signature_metadata, skip=args.skip_data_preflight, allow_tokenizer_download=args.preflight_tokenizer_download, allow_whitespace_fallback=args.allow_whitespace_preflight_fallback, diff --git a/web/app.py b/web/app.py index 1d320f7..29cc447 100644 --- a/web/app.py +++ b/web/app.py @@ -2027,6 +2027,28 @@ def generate(): num_blocks = len(analyzer.basic_blocks) num_functions = len(analyzer.functions) func_names = list(func_tac_map.keys()) + rejected_targets = dict(getattr(analyzer, "rejected_dispatcher_targets", {}) or {}) + incomplete_reason = ( + "Some dispatcher selectors have invalid targets: " + + ", ".join( + f"{selector} -> 0x{target:x}" + for selector, target in sorted(rejected_targets.items()) + ) + if rejected_targets + else None + ) + trace["analysis"] = { + "num_instructions": num_instructions, + "num_basic_blocks": num_blocks, + "num_functions": num_functions, + "tac_generation_time_s": round(tac_time, 3), + "rejected_dispatcher_targets": rejected_targets, + } + if not func_names: + raise DecompileRequestError( + "No recoverable function boundaries in bytecode; refusing to infer " + "Solidity from whole-contract TAC." + ) if len(func_names) > MAX_DECOMPILE_FUNCTIONS: raise DecompileRequestError( ( @@ -2035,12 +2057,6 @@ def generate(): ) ) _check_timeout() - trace["analysis"] = { - "num_instructions": num_instructions, - "num_basic_blocks": num_blocks, - "num_functions": num_functions, - "tac_generation_time_s": round(tac_time, 3), - } # ---- Resolve function selectors ---- resolver = get_resolver(use_remote=ENABLE_REMOTE_SELECTOR_LOOKUP) @@ -2733,22 +2749,33 @@ def generate(): function_results, source_summary, reconstruction_plan, + rejected_dispatcher_targets=rejected_targets, ) failure_count = len(function_errors) validation_failed = not bool(validation.get("valid")) success = ( failure_count == 0 and not validation_failed + and incomplete_reason is None and not (decompiler is None and unresolved_fnames) ) - partial_success = (failure_count > 0 or validation_failed) and any( + partial_success = (failure_count > 0 or validation_failed or incomplete_reason is not None) and any( source != "error" for source in function_sources.values() ) + if incomplete_reason: + decompilation_status = "partial_analysis" + elif partial_success: + decompilation_status = "partial_error" + elif success: + decompilation_status = "model_generated" + else: + decompilation_status = "analysis_failed" analysis = { "num_instructions": num_instructions, "num_basic_blocks": num_blocks, "num_functions": num_functions, + "rejected_dispatcher_targets": rejected_targets, "tac_generation_time_s": round(tac_time, 3), "solidity_generation_time_s": round(gen_time, 3), "lookup_hits": lookup_hits, @@ -2778,7 +2805,7 @@ def generate(): trace["functions"].setdefault(item["name"], {}).update(item) trace_path = _finish_trace( "success" if success else "partial" if partial_success else "failed", - ( + incomplete_reason or ( None if success or partial_success else ( @@ -2795,6 +2822,8 @@ def generate(): "request_id": request_id, "success": success, "partial_success": partial_success, + "decompilation_status": decompilation_status, + "error": incomplete_reason, "tac": combined_tac, "tac_per_function": func_tac_map, "solidity": assembled, diff --git a/web/static/app.js b/web/static/app.js index 75485ee..2a5a3a6 100644 --- a/web/static/app.js +++ b/web/static/app.js @@ -1537,14 +1537,17 @@ solidityOutput.textContent = data.solidity || "(no Solidity output)"; // Model / partial-failure warning - if (data.model_error || data.partial_success || data.success === false) { + if (data.model_error || data.error || data.partial_success || data.success === false) { var warning = data.model_error || ""; + if (data.error) { + warning += (warning ? " " : "") + data.error; + } if (data.source_summary && data.source_summary.error) { warning += (warning ? " " : "") + data.source_summary.error + " function(s) failed; see Function Mapping for details."; - } else if (data.success === false) { + } else if (data.success === false && !data.error) { warning += (warning ? " " : "") + "Decompilation did not fully succeed."; } if (data.validation && !data.validation.valid) { From d2597ac03c2f85363608ffe9d3f13bbe124d5422 Mon Sep 17 00:00:00 2001 From: Alex Gorevski Date: Thu, 24 Sep 2026 14:22:50 -0700 Subject: [PATCH 2/4] Align training export with function-local inference TAC Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/runbook.md | 7 + learnings/learnings.md | 10 +- src/dataset_export_primitives.py | 9 + tests/test_data_generation_export_issues.py | 193 +++++++++++++------- 4 files changed, 147 insertions(+), 72 deletions(-) diff --git a/docs/runbook.md b/docs/runbook.md index 1bf0678..bca846b 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -152,6 +152,13 @@ real pairs fail fast unless `--allow-demo-fallback` is set. The preferred generator is `download_hf_contracts.py`. It reads verified Solidity contracts from Hugging Face `andstor/smart_contracts`, compiles each with one source-aligned `solc`/optimizer configuration, emits TAC with `BytecodeAnalyzer`, deduplicates and filters pairs, validates normalized-body duplicate caps, and exports JSONL plus lineage manifests. For production prompt design, treat compiler version and optimizer fields/comments in generated data as oracle-only and exclude or sanitize them before training. +Training export now uses the analyzer's caller-context per-function TAC, the +same path used at inference, rather than walking global CFG successor unions. +On one bounded 2,000-source audit with the actual Qwen tokenizer and an +8,192-token cap, this increased usable unique-body rows from 409 to 758 and +reduced overlength rejects from 618 to 276. The remaining long-input tail is +not solved; these counts are data coverage, **not** a model-quality improvement. + ```bash # Quick data-generation test uv run python download_hf_contracts.py --limit 20 diff --git a/learnings/learnings.md b/learnings/learnings.md index aef56ee..ff29838 100644 --- a/learnings/learnings.md +++ b/learnings/learnings.md @@ -12,6 +12,8 @@ This file is the persistent research log for TAC-to-Solidity training runs. It r **Bounded real-data audit (2026-09-24):** Regenerated 2,000 public verified-contract records locally with the exact cached Qwen tokenizer and source-aligned single-compiler policy. Of 1,027 unique-body candidate rows after other filters, 618 (60.2%) exceeded 8,192 true training tokens and only 409 exported. Rejected prompts had median 50,224 context tokens (max 204,976), mostly parameterized (475/618), despite the small 36-function benchmark's 36/36 source-map witness coverage. The 409-row dataset is an *audit artifact*, not a clean model run or acceptance result. Prioritize compact, behavior-preserving per-function TAC or hierarchical retrieval; do not silently truncate or train only the easy minority and claim broad decompilation quality. +**Export parity follow-up:** The old data exporter followed contract-wide CFG successor unions instead of the analyzer's stack-context-aware function traversal. In one 9-function real contract, different functions shared 97.9%-99.9% of their roughly 1,000 TAC blocks. Routing export through the same `generate_function_tac` used at inference shrank one `insureStake` row from 1,044 to 47 blocks. A repeat of the same bounded 2,000-source build with identical tokenizer/settings exported **758 rather than 409** unique-body rows (+85.3%) and quarantined **276 rather than 618** for overlength (-55.3%); 403 old accepted body hashes remained accepted and 355 newly entered. There are still 276/1,034 (26.7%) overlong unique-body rows (median rejected context 15,820.5 tokens). Neither structural improvement establishes trained-model quality; the remaining long-tail still needs work. + ## Iteration log | Iteration | Artifact(s) | Configuration | Key result | Learning | @@ -71,6 +73,8 @@ This file is the persistent research log for TAC-to-Solidity training runs. It r | 48. Pre-training fixed-gate baseline validation | `run_train_qwen_qlora_full_body_balanced.sh`, `scripts/gate_dataset.py`, `tests/test_gate_dataset.py` | Moved all six baseline requirements ahead of dataset materialization and GPU training when `RUN_GATES=1`; verify dataset path and every row's input/output hashes, decode settings, `bundled_only_v1` policy, nonmissing behavior-only F1, and measured zero prompt truncation. Diagnostic `RUN_GATES=0` and `DRY_RUN=1` remain available. | Missing/old/mismatched gate baselines now fail before GPU time is spent; no full model run or acceptance comparison was attempted because current model/source and results artifacts are absent. | Historical baselines cannot be quietly reused after prompt/metric changes. Regenerate clean, comparable baselines first; otherwise run diagnostic training explicitly without an acceptance claim. | | 49. Bounded public source-aligned regeneration with true Qwen length gate | Ignored local `data/audit_contracts.db`, `data/audit_hf_dataset.jsonl`, `data/audit_hf_dataset.rejects.jsonl`, manifests `data/audit_manifests/hf_{download,compile,export}_manifest.json`; cached tokenizer `data/audit_qwen_tokenizer/` | `download_hf_contracts.py --limit 2000 --workers 8 --max-body-dupes 1 --max-seq-length 8192 --length-tokenizer data/audit_qwen_tokenizer`; one source-aligned compiler job per prepared contract, using locally available solc versions where possible. No model was trained; generated data is ignored by Git. | Downloaded 2,000 source records, ran 1,957 compile jobs, generated 1,906 pairs (1,704 unique persisted), saw 1,027 unique normalized bodies, capped 677 duplicate-body rows, and **quarantined 618/1,027 unique bodies (60.2%) for actual Qwen context overlength**, leaving 409 exported rows. Rejects: context tokens min 8,144 / median 50,224 / p90 102,814 / max 204,976; target tokens median 61; 475/618 rejects had parameters. 370 compile failures and 2 ambiguous-alignment outcomes were recorded, not silently accepted. | Compiler-source-map witness coverage on 36 small functions does not imply 8k-trainable TAC on real contracts. Investigate long per-function CFG/shared-decoder duplication before scaling training or changing model capacity. | +| 50. Train/inference TAC parity and bounded regeneration | `src/dataset_export_primitives.py`, `tests/test_data_generation_export_issues.py`; ignored second-generation `data/audit_contracts_v2.db`, `data/audit_hf_dataset_v2.jsonl`, `data/audit_hf_dataset_v2.rejects.jsonl`, `data/audit_manifests_v2/hf_export_manifest.json` | The training exporter used `collect_blocks()` over global CFG successor unions, whereas inference used `BytecodeAnalyzer.generate_function_tac()` with caller-specific stack contexts. A real 9-function contract had 97.9%-99.9% overlapping blocks between large function TACs; its `insureStake` row was 1,044 global blocks vs 47 context-specific blocks. Updated real-analyzer export to reuse the inference generator, preserving a fallback for legacy test analyzers. Rebuilt the same first 2,000 public records with solc/source-aligned configs, `--max-body-dupes 1`, exact local Qwen tokenizer, and 8,192-token limit. | First export: 409 accepted / 618 context-overlength / 677 body-capped from 1,704 stored pairs. Revised export: **758 accepted / 276 overlength / 675 body-capped from 1,709 stored pairs**; 403 accepted body hashes overlap, 355 newly accepted. Overlength rate among unique-body candidates dropped 60.2% (618/1,027) -> 26.7% (276/1,034); new rejects had median context 15,820.5 tokens. All 758 revised rows passed real-token training preflight with max total 8,141. The 36-function source-map benchmark still reports 36/36 reachable witnesses. | Global CFG unions produced massive training/inference prompt mismatch and biased the 8k sample toward easy functions. The scoped export meaningfully improves usable coverage, but the two compile runs differ by five persisted pairs and no model was trained; more real, behavior-preserving TAC reduction and clean heldout evaluation remain necessary. | + ## Durable learnings 1. **The overfit sanity check is necessary and should run without train-time eval.** The earlier 10-row run stopped at epoch 9 because trainer eval enabled early stopping and selected an earlier checkpoint. Disabling train-time eval via `TRAIN_EVAL_STRATEGY=no` allowed the corrected 10-row run to reach 30 epochs and memorize exactly. @@ -128,6 +132,7 @@ This file is the persistent research log for TAC-to-Solidity training runs. It r 52. **Exact runtime equality is too strict to stand in for semantic equivalence.** In 30 full-contract controls, 30 identical candidates were byte-equal and 30 return/storage mutations byte-unequal, but a comment-only source change also produced byte-unequal runtime due to Solidity metadata despite an identical executable prefix. Preserve the exact-byte label and seek independent differential/semantic controls. 53. **Full-run gates must reject stale baselines before expensive training.** The full body-balanced runner now validates all six fixed-gate baseline JSONs for row identities, settings, prompt policy, behavior-only metric and zero truncation before training, rather than failing hours later. `RUN_GATES=0` is a diagnostic override, not an acceptance decision. 54. **Actual-token length is the immediate real-data bottleneck after CFG reachability repair.** In a bounded 2,000-source regeneration, 618/1,027 unique-body candidates (60.2%) exceeded the real Qwen 8,192-token prompt+target cap; rejected prompt median was 50,224 tokens despite target median only 61. Long samples can contain 2,417 TAC instruction lines at the median; 475/618 rejects are parameterized. Do not treat 409 surviving short rows as representative heldout decompilation. +55. **Training data must use the same caller-context per-function TAC as inference.** The old exporter copied global CFG closures into many functions (up to 99.9% common blocks in a real 9-function contract), creating six-digit-token prompts and discarding 60.2% of unique-body rows. Reusing `BytecodeAnalyzer.generate_function_tac` raised accepted rows 409 -> 758 on the bounded 2,000-source audit and cut overlength rows 618 -> 276 with actual Qwen tokenization. This is a data/input fidelity gain, not a measured model improvement. ## Hypotheses @@ -230,7 +235,7 @@ This file is the persistent research log for TAC-to-Solidity training runs. It r 35. **Extend the TAC witness benchmark to independently sourced contracts.** Current 36/36 structural coverage is six local families compiled together with solc 0.8.20, optimizer enabled and Shanghai. Success criterion: at least 30 distinct real, eval-clean contract identities per compiler/optimizer/behavior slice; report selector presence, source-map witness reachability, rendered TAC, bounded-traversal truncation, and sample failures separately. 36. **Add execution-level or metadata-normalized controls before promoting runtime identity to a model gate.** Keep 30/30 identical and 30/30 behavioral mutations correctly distinguished while recognizing the comment-only runtime identity false negative. Success criterion: executable-prefix or differential-call checks classify comment-only as same observable behavior without excusing altered returns or storage, with fully matched deployment state/compiler settings. 37. **Regenerate model/data artifacts before any claimed quality gain (partially addressed).** A 409-row exact-token audit export and Qwen tokenizer now exist only as ignored `data/audit_*` assets, but the pretrained 7B weights, accepted clean full dataset, model results, and six refreshed gate baselines are absent. Success criterion: first fix the 60.2% real-data overlength loss, then run a 1-5-row overfit sanity control plus 30+ nonoverlap and large192 paired heldout evals with behavior-only F1, hallucination, AST/bytecode validity, zero truncation and a runtime-checked subset. -38. **Measure and reduce true-token TAC blowup without erasing bytecode behavior.** On the ignored 2,000-source audit export, compare `function_pairs` TAC length to reachable CFG block counts, function count, shared decoder/helper block overlap and compiler source-map witnesses for both 409 kept and 618 quarantined unique bodies. Success criterion: materially lower overlength loss (aim <10% on the comparable short-target slice under 8k) while preserving selector and behavior-opcode witness coverage and maintaining zero prompt truncation; do not simply clip TAC or raise decode tokens. +38. **Partially addressed: reduce true-token TAC blowup without erasing bytecode behavior.** On the ignored 2,000-source audit export, caller-context training/inference TAC parity cut overlength 618 -> 276 and raised kept unique bodies 409 -> 758. Audit the remaining 276 long cases for shared helper cycles and real behavioral complexity; aim below 10% overlength on a comparable 8k short-target slice while preserving source-map witnesses and zero prompt truncation. Do not simply clip TAC or raise decode tokens. ## Artifact map @@ -301,6 +306,7 @@ This file is the persistent research log for TAC-to-Solidity training runs. It r - 36-case compiler-source-map benchmark: `scripts/benchmark_tac_fidelity.py`, `tests/test_benchmark_tac_fidelity.py` - 30-contract/121-control runtime identity benchmark: `scripts/benchmark_runtime_controls.py`, `tests/test_benchmark_runtime_controls.py` - 2,000-record bounded exact-token generation audit (local ignored assets): `data/audit_manifests/hf_download_manifest.json`, `data/audit_manifests/hf_compile_manifest.json`, `data/audit_manifests/hf_export_manifest.json`; 409 accepted `data/audit_hf_dataset.jsonl`, 618 length rejects `data/audit_hf_dataset.rejects.jsonl` +- Scoped per-function export rebuild (local ignored assets): `data/audit_manifests_v2/hf_export_manifest.json`; 758 accepted `data/audit_hf_dataset_v2.jsonl`, 276 length rejects `data/audit_hf_dataset_v2.rejects.jsonl` ## Path reflection (2026-09-24) @@ -311,3 +317,5 @@ The follow-up model-free checkpoint promotes **analyzer and evaluator control sl The 36-case benchmark **promotes parameter-decoder TAC fidelity** from a conjectured bottleneck to a measured 18/36 -> 36/36 structural gain on one compiler setting; this is the first code-level input-conditioning change to prioritize when models/data are restored. The runtime comment-only negative control **demotes exact byte identity as a stand-alone keep gate**: its false negative reflects metadata, not changed executable prefix. Continue clean model training only after baselines pass fail-fast provenance/truncation checks. No model-quality or world-leading claim is warranted without restored model weights, dataset, and independent heldout/differential evidence. The bounded 2,000-source regeneration **changes the immediate priority again**: actual-token length is now a measured 60.2% unique-body loss, not a speculation about prompt truncation on the short rows that survived earlier exports. Keep earlier zero-truncation findings scoped to *evaluated survivors*. First audit whether shared decoder/internal function blocks swell real per-function TAC and implement loss-aware compression or a longer-context training design before spending GPU time. The small cached tokenizer and 409 audit rows do not replace a representative accepted training set or establish model gains. + +The scoped export confirmed **global CFG closure duplication** as one concrete source of the length failure and improved usable bodies by 85.3% under identical 8k Qwen filtering. The remaining 26.7% overlength tail is still too large for a representative full run. Treat the earlier "scale before more input work" path as lower priority until those examples are characterized and clean heldout model scores are available; preserve paired metadata/prompt lineage when retraining because the training TAC distribution has changed. diff --git a/src/dataset_export_primitives.py b/src/dataset_export_primitives.py index 2c5518a..4018301 100644 --- a/src/dataset_export_primitives.py +++ b/src/dataset_export_primitives.py @@ -756,6 +756,15 @@ def extract_tac_for_function( *, logger: Optional[Any] = None, ) -> str: + generator = getattr(type(analyzer), "generate_function_tac", None) + if callable(generator): + tac = generator(analyzer, bytecode_function) + if not isinstance(tac, str) or not tac.strip(): + raise ValueError( + f"No recoverable TAC for function {getattr(bytecode_function, 'name', '')}" + ) + return tac + lines: List[str] = [] try: ensure_tac_integrated(analyzer) diff --git a/tests/test_data_generation_export_issues.py b/tests/test_data_generation_export_issues.py index ddc24de..66d2dd7 100644 --- a/tests/test_data_generation_export_issues.py +++ b/tests/test_data_generation_export_issues.py @@ -5,6 +5,44 @@ import pytest +from src.bytecode_analyzer import BytecodeAnalyzer +from src.dataset_export_primitives import collect_blocks, extract_tac_for_function + + +def test_real_analyzer_export_uses_inference_function_tac_not_global_cfg(): + bytecode = "0x60003560e01c80631111111114601c57806322222222146020575b005b6001005b600200" + analyzer = BytecodeAnalyzer(bytecode) + inference_tac = analyzer.generate_per_function_tac() + + for name, function in analyzer.functions.items(): + assert extract_tac_for_function(function, analyzer) == inference_tac[name] + assert "block_001c:" not in inference_tac["function_0x22222222"] + + +def test_compiler_shared_decoder_export_does_not_copy_sibling_function_blocks(): + # solc 0.8.20, optimizer enabled, metadata disabled: write(uint256) and read(uint256). + bytecode = ( + "6080604052348015600e575f80fd5b50600436106030575f3560e01c80632f048afa" + "146034578063ed2e5a97146045575b5f80fd5b6043603f3660046078565b5f55565b" + "005b605460503660046078565b6066565b60405190815260200160405180910390f35b5f" + "815f5460729190608e565b92915050565b5f602082840312156087575f80fd5b503591" + "9050565b80820180821115607257634e487b7160e01b5f52601160045260245ffd" + ) + analyzer = BytecodeAnalyzer(bytecode) + inference_tac = analyzer.generate_per_function_tac() + selectors = { + "function_0x2f048afa", + "function_0xed2e5a97", + } + + assert selectors.issubset(inference_tac) + for name in selectors: + function = analyzer.functions[name] + global_blocks = collect_blocks(function.entry_block, analyzer.basic_blocks) + local_blocks = analyzer._blocks_for_function(function) + assert len(local_blocks) < len(global_blocks) + assert extract_tac_for_function(function, analyzer) == inference_tac[name] + class _CharacterTokenizer: name_or_path = "character-test-tokenizer" @@ -231,8 +269,7 @@ def test_download_preserves_distinct_targets_and_verified_compiler_configs(tmp_p "language": ["Solidity"] * len(rows), "source_code": [source] * len(rows), "contract_address": [ - f"0x{idx:040x}" if idx <= 5 else "" - for idx in range(1, len(rows) + 1) + f"0x{idx:040x}" if idx <= 5 else "" for idx in range(1, len(rows) + 1) ], "contract_name": [row[0] for row in rows], "compiler_version": [row[1] for row in rows], @@ -253,9 +290,9 @@ def test_download_preserves_distinct_targets_and_verified_compiler_configs(tmp_p db_path = tmp_path / "contracts.db" manifest_path = tmp_path / "download.manifest.json" download_hf_contracts.init_database(db_path) - assert download_hf_contracts.download_contracts( - db_path=db_path, manifest_path=manifest_path - ) == 6 + assert ( + download_hf_contracts.download_contracts(db_path=db_path, manifest_path=manifest_path) == 6 + ) with sqlite3.connect(db_path) as conn: stored = conn.execute( "SELECT address, contract_name, compiler_version, optimization_enabled " @@ -270,17 +307,18 @@ def test_download_preserves_distinct_targets_and_verified_compiler_configs(tmp_p ("A", "v0.8.20", 0), ] assert len({row[0] for row in synthetic}) == 2 - assert {row[1:] for row in synthetic} == { - ("B", "v0.8.19", 1), ("B", "v0.8.19", 0) - } - assert download_hf_contracts.download_contracts( - db_path=db_path, manifest_path=manifest_path - ) == 0 + assert {row[1:] for row in synthetic} == {("B", "v0.8.19", 1), ("B", "v0.8.19", 0)} + assert ( + download_hf_contracts.download_contracts(db_path=db_path, manifest_path=manifest_path) == 0 + ) manifest = json.loads(manifest_path.read_text()) assert manifest["drop_counts"]["source_deduped"] == 7 assert manifest["parameters"]["dedup_key"] == [ - "source_hash", "contract_name", "compiler_version", - "optimization_enabled", "optimization_runs", + "source_hash", + "contract_name", + "compiler_version", + "optimization_enabled", + "optimization_runs", ] @@ -325,17 +363,19 @@ def test_hf_compile_quarantines_ambiguous_selector_alignment(monkeypatch): selector = "0x23b872dd" source_functions = [ { - "name": "transferFrom", "selector": selector, "contract_name": "ERC20", + "name": "transferFrom", + "selector": selector, + "contract_name": "ERC20", "body": "function transferFrom(address from, address to, uint256 amount) public returns (bool) { return true; }", }, { - "name": "transferFrom", "selector": selector, "contract_name": "ERC721", + "name": "transferFrom", + "selector": selector, + "contract_name": "ERC721", "body": "function transferFrom(address from, address to, uint256 tokenId) public { ownerOf[tokenId] = to; }", }, ] - bytecode_function = SimpleNamespace( - selector=selector, entry_block="block_0", basic_blocks=[] - ) + bytecode_function = SimpleNamespace(selector=selector, entry_block="block_0", basic_blocks=[]) class FakeAnalyzer: basic_blocks = {} @@ -352,7 +392,8 @@ def identify_functions(self): monkeypatch.setattr(download_hf_contracts, "install_solc_version", lambda _: True) monkeypatch.setattr(download_hf_contracts, "BytecodeAnalyzer", FakeAnalyzer) monkeypatch.setattr( - download_hf_contracts, "compile_source", + download_hf_contracts, + "compile_source", lambda *_args, **_kwargs: SimpleNamespace( success=True, contracts={ @@ -422,18 +463,21 @@ def identify_functions(self): builder = dataset_pipeline.DatasetBuilder("dummy", output_dir=str(tmp_path / "builder")) builder.etherscan.get_contract_source = lambda address: dataset_pipeline.ContractData( - address=address, source_code=source, bytecode="0x6000600055", - compiler_version="v0.8.20", optimization_enabled=True, optimization_runs=200, + address=address, + source_code=source, + bytecode="0x6000600055", + compiler_version="v0.8.20", + optimization_enabled=True, + optimization_runs=200, ) monkeypatch.setattr(dataset_pipeline, "install_solc_version", lambda _: True) monkeypatch.setattr(dataset_pipeline, "BytecodeAnalyzer", FakeAnalyzer) monkeypatch.setattr( - dataset_pipeline, "compile_source", + dataset_pipeline, + "compile_source", lambda *_args, **_kwargs: SimpleNamespace( success=True, - contracts={ - "Derived": SimpleNamespace(runtime_bytecode="6000600055", abi=[]) - }, + contracts={"Derived": SimpleNamespace(runtime_bytecode="6000600055", abi=[])}, ), ) outcome = builder._collect_compile_address( @@ -442,8 +486,7 @@ def identify_functions(self): assert outcome.pairs == [] assert outcome.status_update["status"] == "ambiguous_selector_alignment" assert any( - diag["status"] == "ambiguous_selector_alignment" - and "0x23b872dd" in diag["error"] + diag["status"] == "ambiguous_selector_alignment" and "0x23b872dd" in diag["error"] for diag in outcome.diagnostics ) @@ -531,9 +574,7 @@ class EosCharacterTokenizer(_CharacterTokenizer): record = {"input": "function example:\n return", "output": "function example() {}"} without_eos = export_length_report(record, 8192, _CharacterTokenizer()) - with_eos = export_length_report( - record, without_eos["total_tokens"], EosCharacterTokenizer() - ) + with_eos = export_length_report(record, without_eos["total_tokens"], EosCharacterTokenizer()) assert with_eos["target_tokens"] == without_eos["target_tokens"] + 1 assert with_eos["reasons"] == ["context_overlength"] @@ -549,15 +590,19 @@ def test_both_export_paths_quarantine_exact_tokenizer_overlength(tmp_path): long_body = "function longer() public { uint256 x = 2; emit Done(x); }" short_tac = "function same:\n block_1:\n temp = 1" long_tac = "function same:\n block_2:\n temp = " + "ADD 1 2 " * 45 - budget = export_length_report( - build_training_record(short_tac, short_body), 8192, tokenizer - )["total_tokens"] + 1 - assert not export_length_report( - build_training_record(long_tac, long_body), budget - )["reasons"] - assert "context_overlength" in export_length_report( - build_training_record(long_tac, long_body), budget, tokenizer - )["reasons"] + budget = ( + export_length_report(build_training_record(short_tac, short_body), 8192, tokenizer)[ + "total_tokens" + ] + + 1 + ) + assert not export_length_report(build_training_record(long_tac, long_body), budget)["reasons"] + assert ( + "context_overlength" + in export_length_report(build_training_record(long_tac, long_body), budget, tokenizer)[ + "reasons" + ] + ) db_path = tmp_path / "hf.db" output_path = tmp_path / "hf.jsonl" @@ -565,9 +610,7 @@ def test_both_export_paths_quarantine_exact_tokenizer_overlength(tmp_path): download_hf_contracts.init_database(db_path) pairs = [ {**_make_pair(download_hf_contracts, idx, body), "tac_representation": tac} - for idx, (body, tac) in enumerate( - [(short_body, short_tac), (long_body, long_tac)], start=1 - ) + for idx, (body, tac) in enumerate([(short_body, short_tac), (long_body, long_tac)], start=1) ] assert download_hf_contracts._store_pairs_batch(db_path, pairs) == 2 download_hf_contracts.export_training_data( @@ -589,9 +632,7 @@ def test_both_export_paths_quarantine_exact_tokenizer_overlength(tmp_path): assert manifest["validation"]["token_length_filter"]["token_count_method"] == "tokenizer" builder = DatasetBuilder("dummy", output_dir=str(tmp_path / "builder")) - for idx, (body, tac) in enumerate( - [(short_body, short_tac), (long_body, long_tac)], start=1 - ): + for idx, (body, tac) in enumerate([(short_body, short_tac), (long_body, long_tac)], start=1): builder._store_function_pair( FunctionPair( function_name="short" if idx == 1 else "longer", @@ -626,9 +667,7 @@ def test_export_cli_loads_exact_tokenizer_offline(tmp_path, monkeypatch): output_path = tmp_path / "dataset.jsonl" download_hf_contracts.init_database(db_path) body = "function short() public { uint256 x = 1; emit Done(x); }" - download_hf_contracts._store_pairs_batch( - db_path, [_make_pair(download_hf_contracts, 1, body)] - ) + download_hf_contracts._store_pairs_batch(db_path, [_make_pair(download_hf_contracts, 1, body)]) loaded = [] def load_tokenizer(name, **kwargs): @@ -637,11 +676,19 @@ def load_tokenizer(name, **kwargs): monkeypatch.setattr(AutoTokenizer, "from_pretrained", load_tokenizer) monkeypatch.setattr( - sys, "argv", + sys, + "argv", [ - "download_hf_contracts.py", "--export-only", "--db", str(db_path), - "--output", str(output_path), "--length-tokenizer", "locally-cached-tokenizer", - "--max-seq-length", "128", + "download_hf_contracts.py", + "--export-only", + "--db", + str(db_path), + "--output", + str(output_path), + "--length-tokenizer", + "locally-cached-tokenizer", + "--max-seq-length", + "128", ], ) download_hf_contracts.main() @@ -661,7 +708,10 @@ def test_shared_primitives_produce_same_final_rows_across_export_paths(tmp_path) from src.dataset_pipeline import DatasetBuilder, FunctionPair assert download_hf_contracts.hash_normalized_pair is dataset_pipeline.hash_normalized_pair - assert download_hf_contracts.sanitize_tac_prompt_input is dataset_pipeline.sanitize_tac_prompt_input + assert ( + download_hf_contracts.sanitize_tac_prompt_input + is dataset_pipeline.sanitize_tac_prompt_input + ) tac = ( "function transfer(address to):\n" @@ -675,10 +725,13 @@ def test_shared_primitives_produce_same_final_rows_across_export_paths(tmp_path) hf_db = tmp_path / "hf.db" hf_output = tmp_path / "hf.jsonl" download_hf_contracts.init_database(hf_db) - assert download_hf_contracts._store_pairs_batch( - hf_db, - [_make_pair(download_hf_contracts, 1, body) | {"tac_representation": tac}], - ) == 1 + assert ( + download_hf_contracts._store_pairs_batch( + hf_db, + [_make_pair(download_hf_contracts, 1, body) | {"tac_representation": tac}], + ) + == 1 + ) download_hf_contracts.export_training_data( str(hf_output), max_body_dupes=5, @@ -721,16 +774,10 @@ def test_export_training_data_deduplicates_after_final_sanitization(tmp_path): body = "function transfer(address to) public { uint256 x = 1; emit Done(to, x); }" tac_a = ( - "function transfer(address to):\n" - " // Selector: 0xa9059cbb\n" - " block:\n" - " temp = 1" + "function transfer(address to):\n" " // Selector: 0xa9059cbb\n" " block:\n" " temp = 1" ) tac_b = ( - "function selector_a9059cbb:\n" - " // Selector: 0xa9059cbb\n" - " block:\n" - " temp = 1" + "function selector_a9059cbb:\n" " // Selector: 0xa9059cbb\n" " block:\n" " temp = 1" ) with sqlite3.connect(db_path) as conn: for idx, tac in enumerate([tac_a, tac_b], start=1): @@ -767,7 +814,9 @@ def test_export_training_data_deduplicates_after_final_sanitization(tmp_path): ) rows = output_path.read_text().splitlines() - rejects = [json.loads(line) for line in Path(f"{output_path}.rejects.jsonl").read_text().splitlines()] + rejects = [ + json.loads(line) for line in Path(f"{output_path}.rejects.jsonl").read_text().splitlines() + ] manifest = json.loads(manifest_path.read_text()) assert len(rows) == 1 assert len(rejects) == 1 @@ -844,7 +893,9 @@ def test_export_training_data_quarantines_tac_errors_and_auxiliary_contracts(tmp ) exported = [json.loads(line) for line in output_path.read_text().splitlines()] - rejects = [json.loads(line) for line in Path(f"{output_path}.rejects.jsonl").read_text().splitlines()] + rejects = [ + json.loads(line) for line in Path(f"{output_path}.rejects.jsonl").read_text().splitlines() + ] manifest = json.loads(manifest_path.read_text()) assert [row["metadata"]["function_name"] for row in exported] == ["good"] assert {reason for row in rejects for reason in row["reasons"]} >= { @@ -1006,9 +1057,7 @@ def test_sanitized_training_record_adds_hashes_and_quality_signals(): assert "likely:" not in sanitized assert prompt_leakage_reject_reasons(sanitized) == [] assert prompt_leakage_reject_reasons(" return memory[temp_1:temp_2]") == [] - assert prompt_leakage_reject_reasons("// Returns: amount") == [ - "prompt_source_metadata_leak" - ] + assert prompt_leakage_reject_reasons("// Returns: amount") == ["prompt_source_metadata_leak"] record = build_training_record( tac, @@ -1054,7 +1103,9 @@ def test_export_training_data_quarantines_malformed_metadata_with_quality_signal ) rows = [json.loads(line) for line in output_path.read_text().splitlines()] - rejects = [json.loads(line) for line in Path(f"{output_path}.rejects.jsonl").read_text().splitlines()] + rejects = [ + json.loads(line) for line in Path(f"{output_path}.rejects.jsonl").read_text().splitlines() + ] manifest = json.loads(manifest_path.read_text()) assert len(rows) == 1 From 6f68c4cdd12121b0e911f89cdf161e26a7b6815a Mon Sep 17 00:00:00 2001 From: Alex Gorevski Date: Thu, 24 Sep 2026 14:44:35 -0700 Subject: [PATCH 3/4] Run compiler TAC benchmark with pinned solc in CI Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/data-quality.yml | 5 +++++ docs/runbook.md | 3 +++ tests/test_benchmark_tac_fidelity.py | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/.github/workflows/data-quality.yml b/.github/workflows/data-quality.yml index a7a297c..b5edce9 100644 --- a/.github/workflows/data-quality.yml +++ b/.github/workflows/data-quality.yml @@ -48,6 +48,11 @@ jobs: uv run --frozen --extra training --extra web --extra analysis --extra cpu \ python -c "import torch; assert torch.version.cuda is None, 'CI must use CPU-only PyTorch'" + - name: Install pinned compiler for TAC fidelity benchmark + run: | + uv run --frozen --extra training --extra web --extra analysis --extra cpu \ + python -c 'import solcx; solcx.install_solc("0.8.20")' + - name: Run CPU-only regression suite run: | uv run --frozen --extra training --extra web --extra analysis --extra cpu pytest diff --git a/docs/runbook.md b/docs/runbook.md index 269a655..ce288eb 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -719,6 +719,9 @@ python -m scripts.benchmark_runtime_controls The first benchmark compiles 36 distinct selector-bearing functions with local solc 0.8.20 (getter, storage write, guard, external call, event, and branch). +CI installs that pinned compiler before running its benchmark tests; a local +environment without it skips those tests rather than fetching a compiler +implicitly. It checks ABI selector recovery and source-mapped opcode reachability in each function's TAC: the measured coverage improved from 18/36 to 36/36 after the parameter-decoder CFG fix. Opcode coverage is **not** semantic correctness. diff --git a/tests/test_benchmark_tac_fidelity.py b/tests/test_benchmark_tac_fidelity.py index 5dcfbeb..94874b0 100644 --- a/tests/test_benchmark_tac_fidelity.py +++ b/tests/test_benchmark_tac_fidelity.py @@ -9,6 +9,10 @@ @pytest.fixture(scope="module") def report(): + if benchmark.SOLC_VERSION not in { + str(version) for version in benchmark.solcx.get_installed_solc_versions() + }: + pytest.skip(f"locally installed solc {benchmark.SOLC_VERSION} required") return benchmark.run_benchmark() From 29128c6e2159f79151afbfe37fc2e6f9f54bf873 Mon Sep 17 00:00:00 2001 From: Alex Gorevski Date: Thu, 24 Sep 2026 14:47:18 -0700 Subject: [PATCH 4/4] Keep CI gate smoke fail-closed on missing bytecode evidence Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/data-quality.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/data-quality.yml b/.github/workflows/data-quality.yml index b5edce9..096e339 100644 --- a/.github/workflows/data-quality.yml +++ b/.github/workflows/data-quality.yml @@ -85,7 +85,7 @@ jobs: "pct_below_0.4_edit_dist": 1.0, "replication_f1_mean": 0.90, "solidity_valid_mean": 1.0, - "solidity_compiler_checked_mean": 0.0, + "solidity_compiler_checked_mean": 1.0, "solidity_ast_valid_mean": 1.0, "baseline_comparison": {"num_regressions": 0, "num_metrics_compared": 1}, } @@ -104,6 +104,20 @@ jobs: "max_baseline_regressions": 0, } + missing_evidence = evaluate_quality_gate(summary, gate_config) + assert missing_evidence["status"] == "failed", missing_evidence + assert { + "bytecode_semantic_score_mean", + "bytecode_semantic_checked_mean", + "bytecode_deployable_mean", + }.issubset({item["metric"] for item in missing_evidence["failures"]}) + + # Synthetic fully checked statistics exercise the positive gate/report path. + summary.update({ + "bytecode_semantic_score_mean": 0.9, + "bytecode_semantic_checked_mean": 1.0, + "bytecode_deployable_mean": 1.0, + }) summary["quality_gate"] = evaluate_quality_gate(summary, gate_config) assert summary["quality_gate"]["status"] == "passed", summary["quality_gate"]