From e83b169abe147830f9082a29844eb87d49be66c1 Mon Sep 17 00:00:00 2001 From: Spyros Date: Tue, 29 Sep 2026 11:50:41 +0200 Subject: [PATCH 1/2] feat(oie): full-pipeline runner, docs, and experimental-flag notes Makefile/full-pipeline glue, OIE human docs, and experiment YAMLs. Experimental flags are documented as removable on merge if they are not useful to keep. --- .env.example | 15 +- .gitignore | 2 + Makefile | 24 +- application/tests/oie_full_pipeline_test.py | 131 +++ application/tests/web_main_test.py | 32 + docs/CONTRIBUTING.md | 2 + docs/gsoc_2026_module_c/module_d_contract.md | 1 + docs/gsoc_2026_module_c/runbook.md | 5 + docs/oie/README.md | 18 + docs/oie/evaluation-and-metrics.md | 70 ++ docs/oie/experimental-feature-flags.md | 56 ++ docs/oie/feature-flags.md | 81 ++ docs/oie/full-pipeline.md | 64 ++ docs/oie/how-it-works.md | 134 +++ docs/owasp-graph/apis/module-c-librarian.md | 2 +- .../owasp-graph/apis/schemas/review-item.json | 3 +- requirements-dev.txt | 7 + .../ASVS_EXTRACT_RERUN_20260923.md | 64 ++ .../experiments/EXPERIMENTAL_BRANCH.md | 10 + .../experiments/NEIGHBORHOOD_METRIC.md | 17 + scripts/oie_owasp_eval/experiments/WINNERS.md | 52 ++ .../oie_owasp_eval/experiments/baseline.yaml | 6 + .../experiments/context_enrich.yaml | 7 + .../experiments/cre_summary.yaml | 6 + .../experiments/dual_index.yaml | 7 + .../experiments/hub_leaf_cage.yaml | 7 + .../oie_owasp_eval/experiments/margin.yaml | 7 + scripts/oie_owasp_eval/experiments/rrf.yaml | 9 + .../experiments/stack_winners.yaml | 9 + .../oie_owasp_eval/full_pipeline_repos.yaml | 68 ++ scripts/oie_owasp_eval/run_full_pipeline.py | 844 ++++++++++++++++++ scripts/oie_owasp_eval/tarball_harvest.py | 253 ++++++ 32 files changed, 2007 insertions(+), 6 deletions(-) create mode 100644 application/tests/oie_full_pipeline_test.py create mode 100644 docs/oie/README.md create mode 100644 docs/oie/evaluation-and-metrics.md create mode 100644 docs/oie/experimental-feature-flags.md create mode 100644 docs/oie/feature-flags.md create mode 100644 docs/oie/full-pipeline.md create mode 100644 docs/oie/how-it-works.md create mode 100644 scripts/oie_owasp_eval/experiments/ASVS_EXTRACT_RERUN_20260923.md create mode 100644 scripts/oie_owasp_eval/experiments/EXPERIMENTAL_BRANCH.md create mode 100644 scripts/oie_owasp_eval/experiments/NEIGHBORHOOD_METRIC.md create mode 100644 scripts/oie_owasp_eval/experiments/WINNERS.md create mode 100644 scripts/oie_owasp_eval/experiments/baseline.yaml create mode 100644 scripts/oie_owasp_eval/experiments/context_enrich.yaml create mode 100644 scripts/oie_owasp_eval/experiments/cre_summary.yaml create mode 100644 scripts/oie_owasp_eval/experiments/dual_index.yaml create mode 100644 scripts/oie_owasp_eval/experiments/hub_leaf_cage.yaml create mode 100644 scripts/oie_owasp_eval/experiments/margin.yaml create mode 100644 scripts/oie_owasp_eval/experiments/rrf.yaml create mode 100644 scripts/oie_owasp_eval/experiments/stack_winners.yaml create mode 100644 scripts/oie_owasp_eval/full_pipeline_repos.yaml create mode 100644 scripts/oie_owasp_eval/run_full_pipeline.py create mode 100644 scripts/oie_owasp_eval/tarball_harvest.py diff --git a/.env.example b/.env.example index 17ad95d27..e5e607699 100644 --- a/.env.example +++ b/.env.example @@ -52,14 +52,25 @@ GOOGLE_CLIENT_SECRET=your-client-secret GOOGLE_SECRET_JSON=path/to/secret.json LOGIN_ALLOWED_DOMAINS=example.com -# GCP +# GCP / Gemini (optional — any LiteLLM provider works for Module B) GCP_NATIVE=false GEMINI_API_KEY=your-gemini-api-key VERTEX_CHAT_MODEL=gemini-2.5-flash VERTEX_EMBED_CONTENT_MODEL=gemini-embedding-001 -# Noise / Relevance Filter (Module B) +# Other chat providers (LiteLLM). Set the key that matches CRE_NOISE_FILTER_LLM_MODEL. +# OPENAI_API_KEY=sk-... +# ANTHROPIC_API_KEY=sk-ant-... +# GROQ_API_KEY=gsk_... +# MISTRAL_API_KEY=... +# OPENROUTER_API_KEY=... + +# Noise / Relevance Filter (Module B) — LiteLLM model id (provider/model) +# Examples: gemini/gemini-2.5-flash-lite | openai/gpt-4o-mini | +# anthropic/claude-haiku-4-5 | groq/llama-3.3-70b-versatile +# Any chat model LiteLLM supports; mid-size models usually work best +# (cheap + reliable for this classify gate — avoid tiny completion stubs). CRE_NOISE_FILTER_LLM_MODEL=gemini/gemini-2.5-flash-lite CRE_NOISE_FILTER_BATCH_SIZE=10 diff --git a/.gitignore b/.gitignore index 9efaa738d..14eea981f 100644 --- a/.gitignore +++ b/.gitignore @@ -79,6 +79,8 @@ standards_cache.sqlite !docs/gsoc_2026_module_b/module_c_contract.md !docs/gsoc_2026_module_b/module_b_runbook.md !docs/gsoc_2026_module_b/final_metrics.md +!docs/oie/*.md +!scripts/oie_owasp_eval/experiments/*.md ### Dev DBDumps *.sql diff --git a/Makefile b/Makefile index 17b8c0ea1..e78a0bf8b 100644 --- a/Makefile +++ b/Makefile @@ -141,8 +141,12 @@ install-python: make install-deps-python &&\ playwright install # Python embeddings/scraping (prompt_client); NOT frontend e2e — keep when migrating to Cypress -install-typescript: - yarn add webpack && cd application/frontend && yarn build +# Yarn install only. Do not `yarn add` or `yarn build` here: webpack is +# already in package.json, and a local build rewrites +# application/frontend/www/bundle.js (tracked so Heroku can serve the UI +# without a Node compile step). That was dirtying contributor trees after +# `make install`. Rebuild with `make frontend` when you change the UI. +install-typescript: install-deps-typescript install: install-typescript install-python migrate-upgrade @@ -185,6 +189,22 @@ oie-pipeline: [ -d "./venv" ] && . ./venv/bin/activate &&\ PYTHONPATH=. python scripts/run_oie_pipeline.py --cache_file "$(or $(CACHE_FILE),sqlite:///$(CURDIR)/standards_cache.sqlite)" $(OIE_ARGS) +# Deterministic OIE base metadata (Node then CRE document_metadata.oie). Fill-if-missing. +# Example: +# make oie-tag-base CACHE_FILE=postgresql://cre:password@127.0.0.1:5432/cre_prodclone +oie-tag-base: + [ -d "./venv" ] && . ./venv/bin/activate &&\ + PYTHONPATH=. python scripts/oie_ensure_base_metadata.py \ + --cache-file "$(or $(CACHE_FILE),$(or $(DEV_DATABASE_URL),postgresql://cre:password@127.0.0.1:5432/cre))" \ + $(OIE_TAG_ARGS) + +# Local OIE bootstrap: start cre-postgres, migrate, sync CRE hub (local|upstream). +# Examples: +# make setup-oie +# make setup-oie SETUP_OIE_ARGS='--source upstream --install-ml --calibrate' +setup-oie: + bash ./scripts/setup_oie.sh $(SETUP_OIE_ARGS) + oie-e2e-smoke: [ -d "./venv" ] && . ./venv/bin/activate &&\ PYTHONPATH=. python scripts/run_oie_e2e_smoke.py diff --git a/application/tests/oie_full_pipeline_test.py b/application/tests/oie_full_pipeline_test.py new file mode 100644 index 000000000..519317c5b --- /dev/null +++ b/application/tests/oie_full_pipeline_test.py @@ -0,0 +1,131 @@ +"""Unit tests for full-pipeline gold key guessing and CRE pick parity (no DB).""" + +from __future__ import annotations + +import importlib.util +import sys +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +_SCRIPT = ROOT / "scripts" / "oie_owasp_eval" / "run_full_pipeline.py" +_B2 = ROOT / "scripts" / "oie_owasp_eval" / "run_b2_pr_mappings.py" + + +def _load_fp(): + spec = importlib.util.spec_from_file_location("run_full_pipeline", _SCRIPT) + assert spec and spec.loader + mod = importlib.util.module_from_spec(spec) + sys.modules["run_full_pipeline"] = mod + spec.loader.exec_module(mod) + return mod + + +def _load_b2(): + spec = importlib.util.spec_from_file_location("run_b2_pr_mappings", _B2) + assert spec and spec.loader + mod = importlib.util.module_from_spec(spec) + sys.modules["run_b2_pr_mappings"] = mod + spec.loader.exec_module(mod) + return mod + + +_fp = _load_fp() +_b2 = _load_b2() + + +class FullPipelineKeyGuessTest(unittest.TestCase): + def test_asvs_section_from_path_and_text(self) -> None: + keys = _fp._guess_keys_for_decision( + path="5.0/en/0x10-V1-Encoding-and-Sanitization.md", + text="### V1.1.2\nVerify that...", + repo="OWASP/ASVS", + ) + self.assertIn("asvs::V1.1.2", keys) + + def test_asvs_prefers_explicit_single_section_id(self) -> None: + """Outer heading soup must not dilute a clean extractor Section-ID.""" + keys = _fp._guess_keys_for_decision( + path="5.0/en/0x10-V1-Encoding-and-Sanitization.md", + text=( + "Standard: ASVS\n" + "Version: 5.0\n" + "Source: 0x10-V1-Encoding-and-Sanitization.md\n" + "Section-ID: V1.1.2\n" + "Section: V1 Encoding\n\n" + "Source: V1.1.2\n" + "Section-ID: V1.1.2\n" + "Section: Verify that output encoding is applied.\n\n" + "Verify that output encoding is applied.\n" + ), + repo="OWASP/ASVS", + ) + self.assertEqual(keys, {"asvs::V1.1.2"}) + + def test_aisvs_from_filename(self) -> None: + keys = _fp._guess_keys_for_decision( + path="1.0/en/0x10-C01-Training-Data-Integrity-and-Traceability.md", + text="", + repo="OWASP/AISVS", + ) + self.assertIn("aisvs::AISVS1", keys) + + def test_cheatsheet_stem(self) -> None: + keys = _fp._guess_keys_for_decision( + path="cheatsheets/Authorization_Cheat_Sheet.md", + text="", + repo="OWASP/CheatSheetSeries", + ) + self.assertIn("cheatsheets::Authorization_Cheat_Sheet", keys) + + def test_cheatsheet_section_id_from_hyperlink(self) -> None: + sid = _fp._cheatsheet_section_id( + { + "section": "Authorization Cheat Sheet", + "hyperlink": ( + "https://cheatsheetseries.owasp.org/cheatsheets/" + "Authorization_Cheat_Sheet.html" + ), + "cre_ids": ["128-128"], + } + ) + self.assertEqual(sid, "Authorization_Cheat_Sheet") + + +class FullPipelineTop2ParityTest(unittest.TestCase): + def test_top2_matches_b2_rerank_union_vector(self) -> None: + uuid_to_ext = { + "u-rerank-1": "111-111", + "u-rerank-2": "222-222", + "u-rerank-3": "333-333", + "u-vec-1": "444-444", + "u-vec-2": "555-555", + "u-link": "666-666", + } + envelope = { + "suggested_links": [{"cre_id": "u-link", "confidence": 0.9}], + "retrieval": { + "reranked": [ + {"cre_id": "u-rerank-1", "score_rerank": 1.0}, + {"cre_id": "u-rerank-2", "score_rerank": 0.9}, + {"cre_id": "u-rerank-3", "score_rerank": 0.8}, + ], + "candidates": [ + {"cre_id": "u-vec-2", "score_vector": 0.5}, + {"cre_id": "u-vec-1", "score_vector": 0.9}, + {"cre_id": "u-rerank-1", "score_vector": 0.2}, + ], + }, + } + b2 = _b2.top2_cre_external_ids(envelope, uuid_to_ext) + fp = _fp._top2_cre_ids(envelope, uuid_to_ext) + self.assertEqual(fp, b2) + # Must not stop at suggested_links-only (legacy fullpipe bug). + self.assertIn("111-111", fp) + self.assertIn("222-222", fp) + self.assertIn("444-444", fp) + self.assertNotIn("666-666", fp) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/tests/web_main_test.py b/application/tests/web_main_test.py index 13b30f921..0c1d5f430 100644 --- a/application/tests/web_main_test.py +++ b/application/tests/web_main_test.py @@ -10,6 +10,7 @@ import json import unittest import tempfile +from pathlib import Path from types import SimpleNamespace from unittest.mock import patch, Mock @@ -187,6 +188,37 @@ def test_find_by_id_tracks_normalized_source(self, posthog_mock) -> None: f"id:{cre.id};nameNone;source:wstg-v4.2-draft-main", ) + def test_rest_cre_payload_omits_librarian_summary(self) -> None: + """Hidden librarian blurbs live in the cache, never the public CRE document.""" + blurb = "HIDDEN_LIBRARIAN_BLURB_MUST_NOT_APPEAR_IN_REST_OR_SPA" + cre = defs.CRE(id="481-710", description="", name="Access Control") + self.collection.add_cre(cre) + with tempfile.TemporaryDirectory() as tmp: + cache = Path(tmp) / "481-710.json" + cache.write_text( + json.dumps( + { + "cre_id": "cre-uuid-1", + "external_id": "481-710", + "name": "Access Control", + "summary": blurb, + } + ) + ) + with patch.dict( + os.environ, {"CRE_LIBRARIAN_CRE_SUMMARY_CACHE": tmp}, clear=False + ): + with self.app.test_client() as client: + response = client.get(f"/rest/v1/id/{cre.id}") + self.assertEqual(200, response.status_code) + payload = json.loads(response.data.decode()) + dumped = json.dumps(payload) + self.assertNotIn(blurb, dumped) + self.assertNotIn("embeddings_content", dumped) + self.assertNotIn("librarian_summary", dumped) + self.assertEqual(payload["data"].get("description", ""), "") + self.assertEqual(payload["data"]["name"], "Access Control") + @patch("application.web.web_main.posthog") def test_find_by_id_ignores_empty_source(self, posthog_mock) -> None: collection = self.collection diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 89cad9742..9e8cfdc61 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -226,3 +226,5 @@ Then in a separate terminal 
cd application/frontend
and finally start the frontend in a debug session: 
yarn start
+ +`make frontend` / `yarn build` regenerates `application/frontend/www/bundle.js` (and `index.html` / `bundle.js.LICENSE.txt`). Those files are committed so production Heroku can serve the UI without a Node buildpack. Do not include them in a PR unless you intentionally changed the frontend. After `make install` they should not change; if they do, discard the diffs. diff --git a/docs/gsoc_2026_module_c/module_d_contract.md b/docs/gsoc_2026_module_c/module_d_contract.md index 5c53549e9..ce2209c92 100644 --- a/docs/gsoc_2026_module_c/module_d_contract.md +++ b/docs/gsoc_2026_module_c/module_d_contract.md @@ -111,6 +111,7 @@ Review rows carry one, in this precedence order: | `reason_code` | meaning | |---|---| | `NO_CANDIDATES` | retrieval returned nothing to link to | +| `CRE_GAP` | prior cage found no existing CRE; `suggested_links` has a `Proposed new CRE` | | `ADVERSARIAL_FLAG` | the safety guard flagged the content | | `UPDATE_AMBIGUOUS` | it restates an existing link, ambiguously | | `BELOW_THRESHOLD` | the calibrated confidence did not clear τ | diff --git a/docs/gsoc_2026_module_c/runbook.md b/docs/gsoc_2026_module_c/runbook.md index b666eb494..44d095d02 100644 --- a/docs/gsoc_2026_module_c/runbook.md +++ b/docs/gsoc_2026_module_c/runbook.md @@ -2,6 +2,11 @@ How to run Module C, what each knob does, and what to do when a run goes wrong. +Human-oriented OIE overview (pipeline, metrics, promoted flags): +[`docs/oie/`](../oie/README.md). Experimental flags that did not improve the +exact Links bar are in-tree as dead code (default off): +[`docs/oie/experimental-feature-flags.md`](../oie/experimental-feature-flags.md). + The package overview lives in [`application/utils/librarian/README.md`](../../application/utils/librarian/README.md). The table C reads from is specified in diff --git a/docs/oie/README.md b/docs/oie/README.md new file mode 100644 index 000000000..f9abefa69 --- /dev/null +++ b/docs/oie/README.md @@ -0,0 +1,18 @@ +# OpenCRE Integrated Ecosystem (OIE) — documentation + +This directory describes how the OWASP Integrated Ecosystem maps standards into +Common Requirements Enumeration (CRE) nodes, how Module C (the Librarian) +decides automatic links, and how the B2 evaluation harness measures accuracy. + +| Document | Contents | +|----------|----------| +| [How it works](how-it-works.md) | Pipeline stages A→B→C, retrieval and decision theory | +| [Evaluation and metrics](evaluation-and-metrics.md) | Exact Links bar vs CRE neighborhood soft bar | +| [Feature flags](feature-flags.md) | Promoted `CRE_LIBRARIAN_*` settings and how to reproduce | +| [Full pipeline](full-pipeline.md) | Repo URL harvest (ASVS, AISVS, Cheat Sheets) + AIX gold arm | +| [Experimental feature flags](experimental-feature-flags.md) | Flags tried and not promoted — shipped as dead code (default off) | + +Package-level API notes remain in +[`application/utils/librarian/README.md`](../../application/utils/librarian/README.md). +Operator steps for live drains are in +[`docs/gsoc_2026_module_c/runbook.md`](../gsoc_2026_module_c/runbook.md). diff --git a/docs/oie/evaluation-and-metrics.md b/docs/oie/evaluation-and-metrics.md new file mode 100644 index 000000000..f27c9c121 --- /dev/null +++ b/docs/oie/evaluation-and-metrics.md @@ -0,0 +1,70 @@ +# Evaluation and metrics + +OIE mapping quality is measured on fixed **gold** fixtures: each standard section +lists acceptable CRE identifiers. Two bars are reported. They answer different +questions and must not be mixed when claiming a ship number. + +## Exact Links bar + +A section is a **hit** if the intersection of predicted CRE ids and gold CRE ids +is non-empty. + +- Use this bar for: deciding whether to write Links, PR gate headlines, regression + against canonical fixtures under `application/tests/fixtures/owasp_mappings/`. +- Do **not** expand gold with Related or Contains neighbors. Expanding gold + trains the system to accept near-misses as if they were correct Links. + +On the promoted stack (`CRE_SUMMARY=1`, `MARGIN_GAMMA=0.85`) against AI-topic +LLM Top10 gold (2026-09-22 offline rescore of cached predictions): + +**33 / 62 (53.2%)** exact. + +## CRE neighborhood soft bar + +A section is a **soft hit** if any predicted CRE is an exact gold id **or** lies +one hop from a gold id on **Related ∪ Contains** in `cre_links`. PartOf is the +reverse Contains edge and is not stored separately. + +- Use this bar for: gap-analysis / RAG usefulness (“did we land in the right part + of the CRE graph?”). +- Scorer: `scripts/oie_owasp_eval/score_b2_hop_distance.py` (see also + `hop_distance.py` and `experiments/NEIGHBORHOOD_METRIC.md`). + +Same predictions as above: + +**52 / 62 (83.9%)** neighborhood (distance ≤ 1). + +## Gold sets used in B2 + +| Fixture | Role | +|---------|------| +| Classical mapping fixtures (Top10, API, K8s, …) | Gate-style exact scoring | +| LLM Top10 / AI-topic CREs | Canonical LLM gold shared with AI Exchange hub CREs | +| `owasp_asvs_5_0_provisional` | Provisional ASVS 5.0 answer key (v4→v5 remap); 190 sections | + +ASVS 5 was historically excluded from default B2 denominators when chapter-sized +bodies caused Module C out-of-memory failures. Section-scoped sources under +`scripts/oie_owasp_eval/fixtures/b2_sources/owasp_asvs_5_0_provisional/` are the +intended input for ASVS-only runs (`run_b2_pr_mappings.py` with other fixtures +excluded, or `run_experiment.py --include-asvs5`). + +## How to reproduce a soft score + +```bash +# After a B2 report with non-empty predicted ids exists: +PYTHONPATH=. python scripts/oie_owasp_eval/score_b2_hop_distance.py \ + --reports-glob 'asvs5_stack_winners.b2_report.json' \ + --force +``` + +Reports land under `tmp/oie_owasp_eval/experiments/hop_analysis/`. + +## Exhaustive switch grid (context) + +A 512-combo factorial over librarian flags (CPU cross-encoder, shortlist judge +off) established that **CRE summaries on** dominate exact accuracy. Full tables +live in `scripts/oie_owasp_eval/experiments/WINNERS.md` and (locally) +`tmp/oie_owasp_eval/experiments/grid/`. Promoted flags are summarized in +[Feature flags](feature-flags.md). Approaches that did not help the exact bar +are documented in [experimental-feature-flags.md](experimental-feature-flags.md) +(dead code; default off). diff --git a/docs/oie/experimental-feature-flags.md b/docs/oie/experimental-feature-flags.md new file mode 100644 index 000000000..11eb725d9 --- /dev/null +++ b/docs/oie/experimental-feature-flags.md @@ -0,0 +1,56 @@ +# Experimental feature flags + +These librarian levers and experiment YAMLs were evaluated on the OIE B2 harness +and **not** promoted for the exact Links bar. They remain in-tree as **dead code** +(env default off) so a later merge can keep the implementations without enabling +them. The promoted stack is only: + +> **Merge note:** These experimental flags and dead-code paths will be **removed on merge** if they are not useful to keep (no production callers, no planned follow-up). Prefer deleting unused levers over shipping permanent dead code. + +- `CRE_LIBRARIAN_CRE_SUMMARY=1` +- `CRE_LIBRARIAN_MARGIN_GAMMA=0.85` + +See `docs/oie/` on the evaluation branch for the human-facing guide. + +## What was tried and did not win (exact Links) + +Evidence layers: (A) 512-combo grid on prior canonical gold, +(B) one-factor Step 2/3 matrices, (C) hop soft scores. Prefer citing which layer. + +| Flag / stack | Exact evidence | Soft / notes | Verdict | +|--------------|----------------|--------------|---------| +| `CRE_SUMMARY=0` | Grid mean 0.354 vs 0.389 with summary; max 0.377 vs 0.436 | — | Clear loser | +| `DUAL_INDEX=1` | Step2 32/62 vs summary 36/62; with summary, grid mean lower | Can raise neighborhood d1 while hurting exact | Not for Links bar | +| `USE_RRF=1` (needs dual) | Step2 34/62 < summary alone; RRF mean lower with dual | [Cormack et al. RRF](https://dl.acm.org/doi/10.1145/1571941.1572114) | Does not beat summary | +| `HUB_LEAF_CAGE=1` | Step3 28/62 (45.2%); grid hub=1 mean 0.359 vs 0.383 | High-risk allowlist | Hurts | +| `CONTEXT_ENRICH=1` alone | 22/62 (35.5%); grid ~flat | Prefix Standard/Section labels | No help as one-factor | +| `FOCUS_QUERY=1` | Grid mean 0.368 vs 0.375 off | First CE on stripped focus text | Neutral / slight drag | +| `CRE_TEXT_ENRICH=1` | Tied at top 0/1; mean slightly lower when on | Append linked standard prose to C.2 | Neutral | +| Hybrid nameheavy (β=3 / γ=0.15) | Tied max with Lawrence; mean lower | Title-heavy mix | Prefer Lawrence defaults | +| `MARGIN_GAMMA=0.85` vs off | Identical max/mean when SUMMARY=1 on grid | Still promoted for multi-link precision | Neutral on exact alone | +| `SHORTLIST_JUDGE=1` | Not in grid | Gemini hangs | Untested | +| Trained CE | Deferred | — | Untested | + +Worst grid cells (~30.6% exact): SUMMARY=0 + hub_leaf=1 + nameheavy + cre_text_enrich. + +## Experiment YAMLs kept here + +| File | Flag under test | +|------|-----------------| +| `scripts/oie_owasp_eval/experiments/dual_index.yaml` | `CRE_LIBRARIAN_DUAL_INDEX` | +| `scripts/oie_owasp_eval/experiments/rrf.yaml` | `CRE_LIBRARIAN_USE_RRF` (+ dual) | +| `scripts/oie_owasp_eval/experiments/hub_leaf_cage.yaml` | `CRE_LIBRARIAN_HUB_LEAF_CAGE` | +| `scripts/oie_owasp_eval/experiments/context_enrich.yaml` | `CRE_LIBRARIAN_CONTEXT_ENRICH` | + +Implementation modules live in `application/utils/librarian/` (flag-gated, default +off). Experiment YAMLs under `scripts/oie_owasp_eval/experiments/` re-run the +non-winning stacks for research only — do not write live Links from them. + +## Theory pointers (why these existed) + +- Dual index (header→names, body→summaries): multi-representation retrieval; length-based routing failed on several Top10 sections. +- RRF: fuse uncalibrated pools by rank ([SIGIR 2009](https://dl.acm.org/doi/10.1145/1571941.1572114)). +- Hub→leaf cage: restrict to Contains children of top hub hits — precise when hubs are right, brittle when hubs are wrong. +- Context enrich: weakly supervised query rewriting via labels; insufficient alone on this corpus. + +## Do not write live Links from these matrices diff --git a/docs/oie/feature-flags.md b/docs/oie/feature-flags.md new file mode 100644 index 000000000..e0cdca0be --- /dev/null +++ b/docs/oie/feature-flags.md @@ -0,0 +1,81 @@ +# Feature flags (promoted) + +Environment variables for Module C live under the `CRE_LIBRARIAN_*` prefix +(loader: `application/utils/librarian/config_loader.py`). This page lists what +**shipped evaluation** uses. Flags that were tried and did not improve the exact Links bar are documented in +[experimental-feature-flags.md](experimental-feature-flags.md) (in-tree dead code, +default off). + +## Promoted stack + +| Variable | Value | Why | +|----------|-------|-----| +| `CRE_LIBRARIAN_CRE_SUMMARY` | `1` | Strongest exact-bar lever in the 512-combo grid and one-factor matrix. | +| `CRE_LIBRARIAN_MARGIN_GAMMA` | `0.85` | Relative cutoff after rerank; ties summary alone on exact, keeps multi-CRE sections without dumping the full shortlist. | + +YAML: `scripts/oie_owasp_eval/experiments/stack_winners.yaml`. + +### Fixed defaults used in the winning grid cells + +These were held constant or selected among tied winners: + +| Variable | Value | Notes | +|----------|-------|-------| +| `CRE_LIBRARIAN_PRIOR_CAGE` | on | Prior-caged C.1 | +| `CRE_LIBRARIAN_PREF_INJECT` | on | Inject preferred CRE ids | +| `CRE_LIBRARIAN_PREFER_AUDIT_IDS` | on | Prefer those ids in order | +| `CRE_LIBRARIAN_HYBRID_BETA` | `0` | “Lawrence” mix | +| `CRE_LIBRARIAN_HYBRID_GAMMA` | `0.70` | Cross-encoder weight in the hybrid mix (≠ margin γ) | +| `CRE_LIBRARIAN_RETRIEVER_BACKEND` | `pgvector` | Live B2 | +| `CRE_LIBRARIAN_DEVICE` | `cpu` | Eval matrix | +| `CRE_LIBRARIAN_SHORTLIST_JUDGE` | `0` | Gemini judge hangs blocked judge-ON factorial | +| `CRE_LIBRARIAN_DUAL_INDEX` | off | See [experimental-feature-flags.md](experimental-feature-flags.md) | +| `CRE_LIBRARIAN_USE_RRF` | off | See [experimental-feature-flags.md](experimental-feature-flags.md) | +| `CRE_LIBRARIAN_HUB_LEAF_CAGE` | off | See [experimental-feature-flags.md](experimental-feature-flags.md) | +| `CRE_LIBRARIAN_CONTEXT_ENRICH` | off | See [experimental-feature-flags.md](experimental-feature-flags.md) | +| `CRE_LIBRARIAN_FOCUS_QUERY` | off | Neutral / slight drag on grid mean | +| `CRE_LIBRARIAN_CRE_TEXT_ENRICH` | off | Tied at top; mean slightly lower when on | + +## Headline scores (promoted stack) + +| Gold / method | Exact | Neighborhood (Related ∪ Contains, 1 hop) | +|---------------|------:|------------------------------------------:| +| AI-topic LLM gold, offline rescore of SUMMARY+margin preds | 33/62 (53.2%) | 52/62 (83.9%) | +| Earlier grid best cell (prior canonical gold) | 27/62 (43.55%) | — | + +Neighborhood definition: +[Evaluation and metrics](evaluation-and-metrics.md). + +## Reproduce + +```bash +export CRE_LIBRARIAN_CRE_SUMMARY=1 +export CRE_LIBRARIAN_MARGIN_GAMMA=0.85 +export CRE_LIBRARIAN_RETRIEVER_BACKEND=pgvector +export CRE_LIBRARIAN_SHORTLIST_JUDGE=0 +export NO_LOAD_GRAPH_DB=1 + +PYTHONPATH=. python scripts/oie_owasp_eval/run_experiment.py \ + --config scripts/oie_owasp_eval/experiments/stack_winners.yaml \ + --keep-all-knowledge +``` + +ASVS 5 only (exclude other B2 families): + +```bash +PYTHONPATH=. python scripts/oie_owasp_eval/run_b2_pr_mappings.py \ + --keep-all-knowledge \ + --exclude-fixtures \ + owasp_top10_2025 owasp_api_top10_2023 owasp_llm_top10_2025 \ + owasp_aisvs_1_0 owasp_kubernetes_top10_2025 owasp_kubernetes_top10_2022 \ + --out tmp/oie_owasp_eval/experiments/asvs5_stack_winners.b2_report.json +``` + +Then score neighborhood with `score_b2_hop_distance.py`. + +## Open items + +- Trained cross-encoder: deferred; not in the grid. +- Shortlist judge ON: blocked by provider hangs during factorial runs. +- Live Module C re-run on every gold revision still required when fixtures change; + offline rescore reuses cached predictions. diff --git a/docs/oie/full-pipeline.md b/docs/oie/full-pipeline.md new file mode 100644 index 000000000..79f230a83 --- /dev/null +++ b/docs/oie/full-pipeline.md @@ -0,0 +1,64 @@ +# Full pipeline (repo URL → A → B → C) + +End-to-end evaluation for the four agreed families: + +| Family | Module A source | Gold | +|--------|-----------------|------| +| ASVS | GitHub `OWASP/ASVS` (`5.0/en`) | `owasp_asvs_5_0_provisional` | +| AISVS | GitHub `OWASP/AISVS` (`1.0/en`) | `owasp_aisvs_1_0` | +| Cheat Sheets | GitHub `OWASP/CheatSheetSeries` | `owasp_cheatsheets_supplement` | +| AI Exchange (AIX) | *Not* GitHub — hub / site CSV | `owasp_llm_top10_2025` (CRE proxy) | +| NIST 800-53 v5 | OSCAL prose (`b2_sources/nist_800_53_v5`) | `nist_800_53_v5` (hub Links) | +| Top 10 2025 | Cached hyperlink text | `owasp_top10_2025` | +| API Top 10 2023 | Cached hyperlink text | `owasp_api_top10_2023` | + +Build NIST gold + sources:: + +```bash +PYTHONPATH=. python scripts/oie_owasp_eval/build_nist_800_53_v5_eval.py --sample 48 +# full catalog: omit --sample +``` + +Example expanded arms:: + +```bash +PYTHONPATH=. python scripts/oie_owasp_eval/run_full_pipeline.py \ + --repos nist,top10,api --keep-all-knowledge --neighborhood +``` + +## Command + +```bash +export CRE_LIBRARIAN_CRE_SUMMARY=1 +export CRE_LIBRARIAN_MARGIN_GAMMA=0.85 +export DEV_DATABASE_URL=postgresql://cre:password@127.0.0.1:5432/cre + +PYTHONPATH=. python scripts/oie_owasp_eval/run_full_pipeline.py \ + --repos asvs,aisvs,cheatsheets,aix \ + --keep-all-knowledge \ + --neighborhood +``` + +Or: `make oie-full-pipeline`. + +Harvest uses GitHub **tarballs** (no `.git`). Promoted librarian flags default on. +Reports land under `tmp/oie_owasp_eval/experiments/full_pipeline_*.b2_report.json` +and `tmp/oie_owasp_eval/full_pipeline/summary.json`. + +Production Module A list (`application/utils/harvester/repos.yaml`) includes +ASVS 4+5, AISVS, and CheatSheetSeries for live `make oie-pipeline` runs. + +## Optional requirement extract + +ASVS/AISVS gold is requirement-grain (`V1.1.2`), not chapter. Heading/docling +chunks alone under-harvest exact match. Optional extractor: + +| `chunking.requirement_extract` | Behavior | +|--------------------------------|----------| +| `off` (default) | Normal chunk + A.2 merge | +| `auto` | Extract only when `requirements_needed(text)` (tables / dense catalogs) | +| `on` | Always attempt extract; fall back to normal chunking if none found | + +When extract yields segments, each chunk is prefixed with `Section-ID: V…` +(B2-style) and A.2 merge is skipped. Cheat sheets stay `off` / narrative. +YAML: ASVS + AISVS use `auto` in `repos.yaml` and `full_pipeline_repos.yaml`. diff --git a/docs/oie/how-it-works.md b/docs/oie/how-it-works.md new file mode 100644 index 000000000..8504d3245 --- /dev/null +++ b/docs/oie/how-it-works.md @@ -0,0 +1,134 @@ +# How OIE mapping works + +OpenCRE’s Integrated Ecosystem (OIE) turns published OWASP (and related) +standards into candidate links against the CRE graph. The system is a staged +pipeline: harvest text, drop obvious noise, retrieve candidate CREs, rerank +them, then either auto-link or send the chunk to human review. + +Nothing in this article invents CRE edges for gold evaluation. Canonical gold +is a fixed answer key of CRE identifiers. Soft scoring may credit a prediction +that lands in the CRE’s graph neighborhood; that is a separate metric (see +[Evaluation and metrics](evaluation-and-metrics.md)). + +## Pipeline overview + +```text +Module A (harvester) + → harvest_input +Module B (noise filter) + → knowledge_queue +Module C (librarian) + → LinkProposal or ReviewItem → Module D (human-in-the-loop) +``` + +| Module | Role | +|--------|------| +| **A** | Fetch repository revisions (often via GitHub tarballs or per-section URLs) and emit section-sized chunks. | +| **B** | Label chunks `KNOWLEDGE`, `UNCERTAIN`, or `NOISE`. Recall-first: uncertain text still reaches C. | +| **C** | Retrieve and decide. Never silently invents a link: every outcome carries a retrieval audit. | +| **D** | Human review for `review_required` envelopes (handoff table: `decision_queue`). | + +Module C is hermetic by default: database access is confined to a factory +boundary so unit tests need no Postgres, API key, or downloaded model. Live runs +use pgvector CRE embeddings and a cross-encoder reranker. + +## Librarian stages (Module C) + +| Stage | Purpose | +|-------|---------| +| **C.0** | Validate a `knowledge_queue` row into an internal `Section` without rewriting body text. | +| **C.0.5** | Deterministic path: an explicit CRE id citation links without ML. | +| **C.0.4 / prior** | Optional problem-class / family prior; may cage the retrieval pool. | +| **C.1** | Dense retrieval over CRE embeddings → shortlist (typical *k* = 20). | +| **C.2** | Cross-encoder rerank of section↔CRE pairs (typical top-*k* = 5). | +| **C.3** | Temperature scaling toward calibrated probabilities (ECE gate in live eval). | +| **C.4** | Threshold τ and emit `LinkProposal` or `ReviewItem`. | + +Supporting ideas that affect quality but are not separate numbered stages: + +- **CRE summaries** — short librarian-only blurbs used as ranking text; they are + not written to public `CRE.description` or Postgres embeddings. +- **Margin cutoff** — after rerank, keep the top score and any later score that + clears a fraction γ of that top score (relative thresholding). +- **Prior cage / preferred-id inject** — graph and audit preferences that bias + which CREs appear on the shortlist (defaults chosen for production-like eval). + +## Theory behind the main design choices + +### Dense retrieval (C.1) + +Candidate CREs are found by embedding similarity in a vector index (pgvector in +live eval). This follows the dual-encoder / dense-passage retrieval pattern: +encode queries and documents into a shared space, then retrieve nearest +neighbors ([Karpukhin et al., Dense Passage Retrieval](https://arxiv.org/abs/2004.04906); +survey context in [Dense retrieval](https://en.wikipedia.org/wiki/Vector_database) +and [Semantic search](https://en.wikipedia.org/wiki/Semantic_search)). + +OpenCRE stores one (or more) embedding rows per CRE. Hygiene of the embedded +string (titles, linked standard prose) matters more than exotic fusion when the +corpus is a few hundred CREs. + +### Cross-encoder reranking (C.2) + +A bi-encoder is fast but coarse. A cross-encoder scores the full +(section, CRE) pair and reorders the shortlist. The default model family is the +MS MARCO MiniLM cross-encoder line commonly used for passage reranking +([Nogueira & Cho on passage reranking](https://arxiv.org/abs/1901.04085); +[Sentence-Transformers cross-encoders](https://www.sbert.net/examples/applications/cross-encoder/README.html)). + +Hybrid mixes of vector score, title overlap, and cross-encoder score appear in +learning-to-rank practice ([Learning to rank](https://en.wikipedia.org/wiki/Learning_to_rank)). +The promoted “Lawrence” mix uses β = 0 (no title boost) and γ = 0.70 on the +minmax-scaled cross-encoder term. That γ is **not** the margin γ below. + +### Relative margin cutoff + +When several CREs score close to the winner, emitting only rank-1 under-serves +multi-link sections; emitting the whole shortlist floods false positives. A +relative margin keeps rank 1 and every later item *i* with +`score_i ≥ γ · score_1` (promoted γ = 0.85). Relative thresholds of this form +are standard in retrieval and detection when absolute scores are poorly +calibrated ([Thresholding (decision theory)](https://en.wikipedia.org/wiki/Thresholding_(image_processing)) +as analogy; ranking cutoffs in LTR systems). + +### CRE summaries as ranking text + +Cross-encoders need informative CRE-side text. Raw CRE names are short; public +descriptions may be missing or uneven. Cached LLM summaries built from name, +description, and linked standard prose improve C.2 without mutating the public +graph. Summaries stay off the REST/UI CRE page by design. + +### Graph neighborhood (soft metric only) + +The CRE graph uses typed edges such as **Related** and **Contains**. In OpenCRE +storage, PartOf is the reverse of Contains, not a separate edge type. Soft +evaluation credits a prediction if it matches gold **or** sits one hop away on +Related ∪ Contains. That measures “useful for gap analysis / RAG,” not +permission to expand gold when writing Links. Background on concept graphs: +[Knowledge graph](https://en.wikipedia.org/wiki/Knowledge_graph), +[Ontology (information science)](https://en.wikipedia.org/wiki/Ontology_(information_science)). + +### Reciprocal Rank Fusion (experiment only) + +When two retrieval pools are not score-calibrated, [Reciprocal Rank Fusion +(Cormack et al., SIGIR 2009)](https://dl.acm.org/doi/10.1145/1571941.1572114) +combines ranks with `1 / (k + rank)`. Dual-index + RRF was evaluated and **not** +promoted for the exact Links bar; see the +[experimental feature flags](experimental-feature-flags.md) +page (dead code; default off). + +## What “a link” means in production + +An automatic link is a `LinkProposal` that cleared τ after calibrated scoring +and (eventually) a graph writer that refuses to commit behind an unevaluated +safety guard. Evaluation harnesses used for PR #1088 **do not write live Links**; +they score predicted CRE ids against gold fixtures. + +## Related code + +| Area | Path | +|------|------| +| Librarian package | `application/utils/librarian/` | +| Config / flags | `application/utils/librarian/config_loader.py` | +| B2 harness | `scripts/oie_owasp_eval/` | +| Promoted experiment YAML | `scripts/oie_owasp_eval/experiments/stack_winners.yaml` | diff --git a/docs/owasp-graph/apis/module-c-librarian.md b/docs/owasp-graph/apis/module-c-librarian.md index 812724e59..dc7a82717 100644 --- a/docs/owasp-graph/apis/module-c-librarian.md +++ b/docs/owasp-graph/apis/module-c-librarian.md @@ -90,7 +90,7 @@ Mapping to existing `cre_defs.Document` + `register_standard` is an implementati | `pipeline_run_id` | string | yes | | | `created_at` | string (date-time) | yes | | | `status` | `review_required` | yes | | -| `reason_code` | string | yes | `BELOW_THRESHOLD`, `NO_CANDIDATES`, `ADVERSARIAL_FLAG`, `UPDATE_AMBIGUOUS` | +| `reason_code` | string | yes | `BELOW_THRESHOLD`, `NO_CANDIDATES`, `ADVERSARIAL_FLAG`, `UPDATE_AMBIGUOUS`, `CRE_GAP` | | `knowledge` | `KnowledgeSnapshot` | yes | | | `retrieval` | `RetrievalAudit` | yes | Maintainer needs top candidates | | `suggested_links` | `ProposedLink[]` | no | Best guess even if below threshold | diff --git a/docs/owasp-graph/apis/schemas/review-item.json b/docs/owasp-graph/apis/schemas/review-item.json index f5a1de19c..4c83355cf 100644 --- a/docs/owasp-graph/apis/schemas/review-item.json +++ b/docs/owasp-graph/apis/schemas/review-item.json @@ -31,7 +31,8 @@ "BELOW_THRESHOLD", "NO_CANDIDATES", "ADVERSARIAL_FLAG", - "UPDATE_AMBIGUOUS" + "UPDATE_AMBIGUOUS", + "CRE_GAP" ] }, "knowledge": { "$ref": "link-proposal.json#/$defs/knowledge_snapshot" }, diff --git a/requirements-dev.txt b/requirements-dev.txt index 7bf1f2bdb..74e95a139 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,6 +6,13 @@ sentence-transformers>=5.0.0,<6.0.0 setuptools +# OIE ingest + orchestration (worker/dev only — not Heroku web slug) +docling>=2.0.0 +llama-index-core>=0.14.0 +llama-index-readers-docling>=0.4.0 +llama-index-node-parser-docling>=0.4.0 +langgraph>=0.2.0 + # importer / embeddings scrape tooling (not needed for prod chat) playwright nltk diff --git a/scripts/oie_owasp_eval/experiments/ASVS_EXTRACT_RERUN_20260923.md b/scripts/oie_owasp_eval/experiments/ASVS_EXTRACT_RERUN_20260923.md new file mode 100644 index 000000000..d0b9fa7af --- /dev/null +++ b/scripts/oie_owasp_eval/experiments/ASVS_EXTRACT_RERUN_20260923.md @@ -0,0 +1,64 @@ +# ASVS full-pipeline + requirement_extract=auto (2026-09-23) + +## Setup +- Worktree / branch: `OpenCRE-wt-oie-1088-harness` / `wip/oie-pipeline-eval-harness` @ `d6d4e5cb` +- Baseline tag lineage: `oie-eval-asvs-neighborhood-v1` @ `01f83175` +- Run id: `fullpipe-asvs-extract-20260923T181734Z` +- Stack: SUMMARY+MARGIN (`CRE_LIBRARIAN_CRE_SUMMARY=1`, `CRE_LIBRARIAN_MARGIN_GAMMA=0.85`) +- Extract: `requirement_extract=auto` (wired in `run_full_pipeline.py` ChunkingConfig) +- Queues: cleared all local `harvest_input` / `knowledge_queue` / `decision_queue` before replay (4712 leftovers) +- Gold: canonical provisional ASVS 5.0; soft metric = Related ∪ Contains 1-hop only (`score_b2_hop_distance.py`) + +## Harvest effect +- This run: **461** ASVS harvest chunks (extract fired per-chapter, e.g. V6 Auth → 47 segments) +- Prior fullpipe-all ASVS arm: **275** chunks logged — consistent with silent `content_hash` skip when queues were dirty + +## Scores (ASVS only, n=190) + +| Metric | Prior fullpipe ASVS slice | This re-run | Δ | +|--------|--------------------------:|------------:|--:| +| Exact | 3/190 (1.6%) | 11/190 (5.8%) | +4.2 pp | +| Neighborhood d1 | 70/190 (36.8%) | 82/190 (43.2%) | +6.3 pp | + +Buckets this run: `{'one_hop_contains': 64, 'exact': 11, 'multi_hop': 108, 'one_hop_related': 7}` + +## Reference (different path — B2 fixture arm, not tarball) +- `asvs5_stack_winners` SUMMARY+MARGIN: exact **45/190 (23.7%)**, d1 **71.6%** +- Gap to fixture exact: **-17.89 pp** + +## Takeaway +- Clearing KQ + extract=auto raised exact **3→11** and d1 **70→82** on the full-pipeline GitHub path. +- Absolute accuracy remains far below the B2 fixture arm; Module C still mostly `review_required` (460/461) with only 1 hard `linked`. +- Next accuracy work should chase why tarball+extract underperforms the fixture arm (alignment of section_id / pred CRE selection), not gold fattening. + +Artifacts: `tmp/oie_owasp_eval/experiments/full_pipeline_github.b2_report.json`, hop under `hop_analysis/full_pipeline_github.hop.json`, prior snapshot in `experiments/baselines/`. + +## Scorer parity + extractor hygiene (same day, code) + +### Offline scorecard (same envelopes, no LLM) +| Gate | Exact | +|------|------:| +| Legacy fullpipe suggested_links top2 + chapter union | 11/190 (5.8%) | +| B2 rerank∪vector top2 + per-chunk (parity target) | 20/190 (10.5%) | + +### After code: rescored same envelopes +- Exact **20/190 (10.5%)**, d1 **83/190 (43.7%)** +- Confirms scorer under-count was ~half the gap vs B2 fixture (45/190) + +### Extractor / builder fixes (needs fresh Module C re-run) +- Table segments = prose only (no pipes, no next-heading bleed) +- Record builder keeps a single `Section-ID: Vn.n.n` when extract prefix present +- Fullpipe `_top2_cre_ids` matches B2; attribution is per-chunk + +Next: clear KQ and re-run `--repos asvs` to measure extract impact on ranks. + +## Clean extract re-run (`fullpipe-asvs-extract-clean-20260923T190122Z`) + +Harvest: 421/461 chunks with a single non-comma Section-ID; V1.1.2 body is prose-only (no table pipes). + +| Metric | Dirty+legacy score | Dirty+parity score | **Clean extract+parity** | B2 fixture | +|--------|-------------------:|-------------------:|-------------------------:|-----------:| +| Exact | 5.8% | 10.5% | **65.3%** (124/190) | 23.7% | +| d1 | 43.2% | 43.7% | **76.3%** (145/190) | 71.6% | + +Gold in rerank top-2: **0/196** mentions (was ~0 on dirty run). Rank dist when found: `{3: 33, 4: 24, 5: 19, 6: 17, 7: 11}`. diff --git a/scripts/oie_owasp_eval/experiments/EXPERIMENTAL_BRANCH.md b/scripts/oie_owasp_eval/experiments/EXPERIMENTAL_BRANCH.md new file mode 100644 index 000000000..5bf9302ba --- /dev/null +++ b/scripts/oie_owasp_eval/experiments/EXPERIMENTAL_BRANCH.md @@ -0,0 +1,10 @@ +# Experimental feature flags (dead code on this PR) + +Non-promoted B2 experiment configs. Defaults stay **off**; do not enable in +production. Research write-up: `docs/oie/experimental-feature-flags.md`. + +Promoted stack: `stack_winners.yaml` (`CRE_SUMMARY=1`, `MARGIN_GAMMA=0.85`). + +Historical research branch (optional): `experimental-feature-flags`. + +**Merge note:** Remove these experiment YAMLs/flags on merge if they are not useful to keep. diff --git a/scripts/oie_owasp_eval/experiments/NEIGHBORHOOD_METRIC.md b/scripts/oie_owasp_eval/experiments/NEIGHBORHOOD_METRIC.md new file mode 100644 index 000000000..db72b8e9f --- /dev/null +++ b/scripts/oie_owasp_eval/experiments/NEIGHBORHOOD_METRIC.md @@ -0,0 +1,17 @@ +# Soft production metric: CRE neighborhood (Related ∪ Contains) + +**Exact Links bar** stays canonical gold ID intersection (ship / write Links). + +**Neighborhood soft bar** (gap-analysis / RAG usefulness): a section hits if any +predicted CRE is exact **or** 1-hop Related **or** 1-hop Contains (PartOf is the +reverse Contains edge in `cre_links` — not stored separately). + +On SUMMARY+margin preds vs AI-topic LLM gold (2026-09-22): + +| Metric | Score | +|--------|------:| +| Exact | 33/62 (53.2%) | +| Neighborhood (d1) | 52/62 (83.9%) | + +Scorer: `scripts/oie_owasp_eval/score_b2_hop_distance.py` + `hop_distance.py`. +Do **not** put Related-expand back into canonical gold. diff --git a/scripts/oie_owasp_eval/experiments/WINNERS.md b/scripts/oie_owasp_eval/experiments/WINNERS.md new file mode 100644 index 000000000..5364697f0 --- /dev/null +++ b/scripts/oie_owasp_eval/experiments/WINNERS.md @@ -0,0 +1,52 @@ +# OIE experiment winners (exhaustive switch grid) + +**Date:** 2026-09-22 +**Worktree:** `OpenCRE-wt-oie-1088-harness` +**Shared pipeline knowledge:** `orch-exp-baseline-20260921` (Module C re-run per combo; CE on CPU; shortlist judge off — Gemini hangs blocked judge-ON factorial) +**Gold:** canonical (`local_gold: false`); Agentic stub excluded +**Grid:** 512 / 512 combos (see `tmp/oie_owasp_eval/experiments/grid/`) + +## Best combo + +- **id:** `g_1_0_0_0_0.85_0_0_lawrence_0` +- **score:** 27/62 (0.4355) +- **axes:** `{"cre_summary": "1", "dual_index": "0", "use_rrf": "0", "context_enrich": "0", "margin_gamma": "0.85", "hub_leaf_cage": "0", "focus_query": "0", "hybrid": "lawrence", "cre_text_enrich": "0"}` +- **env flags:** `{"CRE_LIBRARIAN_CRE_SUMMARY": "1", "CRE_LIBRARIAN_DUAL_INDEX": "0", "CRE_LIBRARIAN_USE_RRF": "0", "CRE_LIBRARIAN_CONTEXT_ENRICH": "0", "CRE_LIBRARIAN_MARGIN_GAMMA": "0.85", "CRE_LIBRARIAN_HUB_LEAF_CAGE": "0", "CRE_LIBRARIAN_FOCUS_QUERY": "0", "CRE_LIBRARIAN_HYBRID_BETA": "0", "CRE_LIBRARIAN_HYBRID_GAMMA": "0.70", "CRE_LIBRARIAN_CRE_TEXT_ENRICH": "0"}` + +## Top 5 + +| rank | id | hits/scorable | accuracy | notes | +|---:|---|---:|---:|---| +| 1 | `g_1_0_0_0_0.85_0_0_lawrence_0` | 27/62 | 0.4355 | — | +| 2 | `g_1_0_0_0_0.85_0_0_lawrence_1` | 27/62 | 0.4355 | — | +| 3 | `g_1_0_0_0_0.85_0_0_nameheavy_0` | 27/62 | 0.4355 | — | +| 4 | `g_1_0_0_0_0.85_0_0_nameheavy_1` | 27/62 | 0.4355 | — | +| 5 | `g_1_0_0_0_0.85_0_1_lawrence_0` | 27/62 | 0.4355 | — | + +## Promoted stack_winners.yaml flags + +``` +flags: + CRE_LIBRARIAN_CRE_SUMMARY: '1' + CRE_LIBRARIAN_MARGIN_GAMMA: '0.85' +``` + +## No-ops / caveats + +- `USE_RRF=1` without `DUAL_INDEX=1` is a no-op (copied from rrf=0 sibling; still in master.csv). +- ASVS5 harvest still excluded (chapter HTML OOM); not in denominator. +- Trained CE deferred — not in this grid. +- `SHORTLIST_JUDGE=0` fixed (Gemini judge hangs blocked judge-ON factorial). + +## Parallel Module C (isolated DBs) + +4 workers × `cre_grid_w0`…`cre_grid_w3` (~500MB TEMPLATE clones of `cre`). + +```bash +WORKERS=4 scripts/oie_owasp_eval/clone_grid_worker_dbs.sh +python -u scripts/oie_owasp_eval/run_exhaustive_switch_grid_parallel.py --workers 4 --clone +# re-attach aggregator if it exits while workers keep running: +python -u scripts/oie_owasp_eval/run_exhaustive_switch_grid_parallel.py --monitor-only +``` + +Do not write live Links from this matrix. diff --git a/scripts/oie_owasp_eval/experiments/baseline.yaml b/scripts/oie_owasp_eval/experiments/baseline.yaml new file mode 100644 index 000000000..6216e4520 --- /dev/null +++ b/scripts/oie_owasp_eval/experiments/baseline.yaml @@ -0,0 +1,6 @@ +experiment_id: baseline +description: > + Exp0 — all new retrieval/precision flags off. PR hybrid defaults remain + (FOCUS_QUERY off, CE hybrid γ=0.70). Canonical gold only. +flags: {} +include_agentic: false diff --git a/scripts/oie_owasp_eval/experiments/context_enrich.yaml b/scripts/oie_owasp_eval/experiments/context_enrich.yaml new file mode 100644 index 000000000..838392bf5 --- /dev/null +++ b/scripts/oie_owasp_eval/experiments/context_enrich.yaml @@ -0,0 +1,7 @@ +experiment_id: context_enrich +description: > + Step 2 — prefix Standard/Section when missing. Flag not yet wired in + config_loader; runner still sets env for when it lands. +flags: + CRE_LIBRARIAN_CONTEXT_ENRICH: "1" +include_agentic: false diff --git a/scripts/oie_owasp_eval/experiments/cre_summary.yaml b/scripts/oie_owasp_eval/experiments/cre_summary.yaml new file mode 100644 index 000000000..809d1b51a --- /dev/null +++ b/scripts/oie_owasp_eval/experiments/cre_summary.yaml @@ -0,0 +1,6 @@ +experiment_id: cre_summary +description: > + Step 2 one-factor — CRE_LIBRARIAN_CRE_SUMMARY alone (search-only blurbs). +flags: + CRE_LIBRARIAN_CRE_SUMMARY: "1" +include_agentic: false diff --git a/scripts/oie_owasp_eval/experiments/dual_index.yaml b/scripts/oie_owasp_eval/experiments/dual_index.yaml new file mode 100644 index 000000000..b3c05305a --- /dev/null +++ b/scripts/oie_owasp_eval/experiments/dual_index.yaml @@ -0,0 +1,7 @@ +experiment_id: dual_index +description: > + Step 2 — dual-index (title vs body pools). Requires CRE summaries per PR wiring. +flags: + CRE_LIBRARIAN_CRE_SUMMARY: "1" + CRE_LIBRARIAN_DUAL_INDEX: "1" +include_agentic: false diff --git a/scripts/oie_owasp_eval/experiments/hub_leaf_cage.yaml b/scripts/oie_owasp_eval/experiments/hub_leaf_cage.yaml new file mode 100644 index 000000000..47351bacb --- /dev/null +++ b/scripts/oie_owasp_eval/experiments/hub_leaf_cage.yaml @@ -0,0 +1,7 @@ +experiment_id: hub_leaf_cage +description: > + Step 3 — hub→leaf cage (high risk; ablate last). Flag not yet wired in + config_loader. +flags: + CRE_LIBRARIAN_HUB_LEAF_CAGE: "1" +include_agentic: false diff --git a/scripts/oie_owasp_eval/experiments/margin.yaml b/scripts/oie_owasp_eval/experiments/margin.yaml new file mode 100644 index 000000000..a718a320b --- /dev/null +++ b/scripts/oie_owasp_eval/experiments/margin.yaml @@ -0,0 +1,7 @@ +experiment_id: margin +description: > + Step 3 — margin cutoff (NOT HYBRID_GAMMA). Report B2 hit and decision-FP when + FP hook exists. Flag not yet wired in config_loader. +flags: + CRE_LIBRARIAN_MARGIN_GAMMA: "0.85" +include_agentic: false diff --git a/scripts/oie_owasp_eval/experiments/rrf.yaml b/scripts/oie_owasp_eval/experiments/rrf.yaml new file mode 100644 index 000000000..36946d6b5 --- /dev/null +++ b/scripts/oie_owasp_eval/experiments/rrf.yaml @@ -0,0 +1,9 @@ +experiment_id: rrf +description: > + Step 2 — RRF merge on top of dual-index (k=60). Only meaningful with dual_index. + Flag not yet wired in config_loader; runner still sets env for when it lands. +flags: + CRE_LIBRARIAN_CRE_SUMMARY: "1" + CRE_LIBRARIAN_DUAL_INDEX: "1" + CRE_LIBRARIAN_USE_RRF: "1" +include_agentic: false diff --git a/scripts/oie_owasp_eval/experiments/stack_winners.yaml b/scripts/oie_owasp_eval/experiments/stack_winners.yaml new file mode 100644 index 000000000..f2c5b01ac --- /dev/null +++ b/scripts/oie_owasp_eval/experiments/stack_winners.yaml @@ -0,0 +1,9 @@ +experiment_id: stack_winners +description: > + Soft neighborhood winner stack: CRE_SUMMARY=1 + MARGIN_GAMMA=0.85. + Exact ~53% on AI-topic LLM gold; neighborhood (Related∪Contains 1-hop) ~84%. + Trained CE deferred. No live Links writes. +flags: + CRE_LIBRARIAN_CRE_SUMMARY: "1" + CRE_LIBRARIAN_MARGIN_GAMMA: "0.85" +include_agentic: false diff --git a/scripts/oie_owasp_eval/full_pipeline_repos.yaml b/scripts/oie_owasp_eval/full_pipeline_repos.yaml new file mode 100644 index 000000000..5ea4a77c3 --- /dev/null +++ b/scripts/oie_owasp_eval/full_pipeline_repos.yaml @@ -0,0 +1,68 @@ +# Full-pipeline GitHub harvest targets (eval / production-shaped). +# Used by scripts/oie_owasp_eval/run_full_pipeline.py +# +# AI Exchange (AIX) is not a GitHub Module A target — see run_full_pipeline.py +# (--aix uses LLM Top10 / hub CRE gold as the AIX scoring arm). + +repositories: + - id: owasp-asvs + type: github + enabled: true + owner: OWASP + repo: ASVS + branch: master + paths: + include: + - "5.0/en/**/*.md" + exclude: + - "**/archive/**" + chunking: + strategy: docling + max_tokens: 1200 + overlap_tokens: 100 + merge_profile: requirements + requirement_extract: auto + polling: + mode: incremental + interval_minutes: 60 + + - id: owasp-aisvs + type: github + enabled: true + owner: OWASP + repo: AISVS + branch: main + paths: + include: + - "1.0/en/**/*.md" + exclude: + - "**/archive/**" + chunking: + strategy: docling + max_tokens: 1200 + overlap_tokens: 100 + merge_profile: requirements + requirement_extract: auto + polling: + mode: incremental + interval_minutes: 60 + + - id: owasp-cheatsheets + type: github + enabled: true + owner: OWASP + repo: CheatSheetSeries + branch: master + paths: + include: + - "cheatsheets/**/*.md" + exclude: + - "**/archive/**" + chunking: + strategy: docling + max_tokens: 1000 + overlap_tokens: 100 + merge_profile: narrative + polling: + mode: incremental + interval_minutes: 120 diff --git a/scripts/oie_owasp_eval/run_full_pipeline.py b/scripts/oie_owasp_eval/run_full_pipeline.py new file mode 100644 index 000000000..4b3bdefdf --- /dev/null +++ b/scripts/oie_owasp_eval/run_full_pipeline.py @@ -0,0 +1,844 @@ +#!/usr/bin/env python3 +"""Full OIE pipeline from GitHub repo tarballs → B → C → gold score. + +Repos (default):: + + OWASP/ASVS (5.0/en) + OWASP/AISVS (1.0/en) + OWASP/CheatSheetSeries + AIX (not GitHub — scores LLM Top10 / hub CRE gold via B2) + +Also supported as B2/source arms (cached ``b2_sources``):: + + nist NIST SP 800-53 Rev. 5 (hub Links + OSCAL prose) + top10 OWASP Top 10 2025 + api OWASP API Top 10 2023 + +Module A uses GitHub tarballs (no ``.git``) for ASVS/AISVS/CheatSheets. +Modules B and C run through ``run_oie_pipeline`` with the promoted librarian +stack (``CRE_SUMMARY=1``, ``MARGIN_GAMMA=0.85``) unless overridden. + +Examples:: + + PYTHONPATH=. python scripts/oie_owasp_eval/run_full_pipeline.py \\ + --repos asvs,aisvs,cheatsheets,aix \\ + --keep-all-knowledge --neighborhood + + PYTHONPATH=. python scripts/oie_owasp_eval/run_full_pipeline.py \\ + --repos nist,top10,api --keep-all-knowledge --neighborhood +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +import tarfile +import urllib.request +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath +from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple + +ROOT = Path(__file__).resolve().parents[2] +ART = ROOT / "tmp" / "oie_owasp_eval" / "full_pipeline" +EXP = ROOT / "tmp" / "oie_owasp_eval" / "experiments" +TARBALL_DIR = ART / "tarballs" +FIXTURES = ROOT / "application" / "tests" / "fixtures" / "owasp_mappings" + +# Logical name → (github repo, branch, include globs, gold fixture stem or None) +GITHUB_TARGETS: Dict[str, Tuple[str, str, List[str], Optional[str]]] = { + "asvs": ( + "ASVS", + "master", + ["5.0/en/**/*.md"], + "owasp_asvs_5_0_provisional", + ), + "aisvs": ( + "AISVS", + "main", + ["1.0/en/**/*.md"], + "owasp_aisvs_1_0", + ), + "cheatsheets": ( + "CheatSheetSeries", + "master", + ["cheatsheets/**/*.md"], + "owasp_cheatsheets_supplement", + ), +} + +# AIX has no GitHub Module A source in this repo; B2 LLM gold is the CRE proxy. +AIX_GOLD = "owasp_llm_top10_2025" + +# Logical name → B2 fixture stem(s) run via run_b2_pr_mappings --only-fixtures +B2_ARMS: Dict[str, List[str]] = { + "aix": ["owasp_llm_top10_2025"], + "nist": ["nist_800_53_v5"], + "top10": ["owasp_top10_2025"], + "api": ["owasp_api_top10_2023"], +} + +EXCLUDE = [ + "**/archive/**", + "**/.github/**", + "**/node_modules/**", + "**/LICENSE*", + "**/CHANGELOG*", + "**/CONTRIBUTING*", +] + +ASVS_SID_RE = re.compile(r"\b(V\d+\.\d+(?:\.\d+)?)\b", re.IGNORECASE) +# ASVS 5.0 chapter tables often omit the leading "V" (e.g. **1.1.2**). +ASVS_BARE_REQ_RE = re.compile(r"(? None: + env_path = ROOT / ".env" + if not env_path.is_file(): + return + for line in env_path.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, _, v = line.partition("=") + os.environ.setdefault(k.strip(), v.strip().strip('"').strip("'")) + + +def _match(path: str, patterns: Sequence[str]) -> bool: + posix = PurePosixPath(path) + expanded: List[str] = [] + for pat in patterns: + expanded.append(pat) + if "/**/" in pat: + expanded.append(pat.replace("/**/", "/")) + if pat.startswith("**/"): + expanded.append(pat[3:]) + for pat in expanded: + try: + if posix.match(pat): + return True + except ValueError: + pass + import fnmatch + + if fnmatch.fnmatch(path, pat): + return True + return False + + +def _download_tarball(repo: str, branch: str, dest: Path) -> str: + dest.parent.mkdir(parents=True, exist_ok=True) + if dest.is_file() and dest.stat().st_size > 1000: + try: + with tarfile.open(dest, "r:gz") as tf: + tf.getmembers()[:1] + return branch + except Exception: + dest.unlink(missing_ok=True) + for try_branch in (branch, "main" if branch == "master" else "master"): + url = f"https://codeload.github.com/OWASP/{repo}/tar.gz/refs/heads/{try_branch}" + print(f"GET {url}", flush=True) + try: + urllib.request.urlretrieve(url, dest) + with tarfile.open(dest, "r:gz") as tf: + tf.getmembers()[:1] + return try_branch + except Exception as exc: # noqa: BLE001 + print(f" fail {try_branch}: {exc}", flush=True) + dest.unlink(missing_ok=True) + raise RuntimeError(f"could not download tarball for {repo}") + + +def _iter_md(tgz: Path, includes: Sequence[str]) -> List[Tuple[str, str]]: + out: List[Tuple[str, str]] = [] + with tarfile.open(tgz, "r:gz") as tf: + for member in tf.getmembers(): + if not member.isfile(): + continue + parts = Path(member.name).parts + if len(parts) < 2: + continue + rel = str(Path(*parts[1:])) + if not rel.lower().endswith(".md"): + continue + if _match(rel, EXCLUDE): + continue + if not _match(rel, includes): + continue + extracted = tf.extractfile(member) + if extracted is None: + continue + raw = extracted.read() + try: + text = raw.decode("utf-8") + except UnicodeDecodeError: + text = raw.decode("utf-8", errors="replace") + if len(text.strip()) < 40: + continue + out.append((rel, text)) + return out + + +def _apply_promoted_flags() -> None: + os.environ.setdefault("CRE_LIBRARIAN_CRE_SUMMARY", "1") + os.environ.setdefault("CRE_LIBRARIAN_MARGIN_GAMMA", "0.85") + os.environ.setdefault("CRE_LIBRARIAN_RETRIEVER_BACKEND", "pgvector") + os.environ.setdefault("CRE_LIBRARIAN_SHORTLIST_JUDGE", "0") + os.environ.setdefault("CRE_LIBRARIAN_DEVICE", "cpu") + os.environ.setdefault("NO_LOAD_GRAPH_DB", "1") + + +def _make_tarball_harvester( + targets: Dict[str, Tuple[str, str, List[str], Optional[str]]], +) -> Any: + """Return a run_harvester-compatible callable that pulls GitHub tarballs.""" + + def run_harvester( + session: Any, + rid: str, + *, + dry_run: bool = False, + sync_repos: bool = True, + ) -> Any: + from application.utils.harvester.chunk_pipeline import DocumentChunkPipeline + from application.utils.harvester.harvest_writer import write_harvest_input + from application.utils.harvester.heading_extractor import HeadingExtractor + from application.utils.harvester.models import Document, Locator, SourceInfo + from application.utils.harvester.pipeline import RunSummary + from application.utils.harvester.schemas import ChunkingConfig + + summary = RunSummary(run_id=rid, dry_run=dry_run, repositories=0) + if dry_run: + return summary + + chunking = ChunkingConfig( + strategy="markdown_heading", + max_tokens=1200, + overlap_tokens=100, + merge_profile="requirements", + requirement_extract="auto", + ) + pipeline = DocumentChunkPipeline(chunking=chunking) + headings = HeadingExtractor() + committed_at = datetime.now(timezone.utc) + sha = "fullpipe0000000000000000000000000000001" + TARBALL_DIR.mkdir(parents=True, exist_ok=True) + + for logical, (repo, branch, includes, _gold) in targets.items(): + summary.repositories += 1 + try: + tgz = TARBALL_DIR / f"{repo}.tgz" + _download_tarball(repo, branch, tgz) + files = _iter_md(tgz, includes) + records = [] + for rel, text in files: + artifact_id = f"art:OWASP/{repo}:{rel}" + doc = Document( + schema_version="0.2.0", + artifact_id=artifact_id, + pipeline_run_id=rid, + text=text, + heading_structure=headings.extract(text), + source=SourceInfo( + type="github", + repository=f"OWASP/{repo}", + commit_sha=sha, + committed_at=committed_at, + ), + locator=Locator(kind="repo_path", id=rel, path=rel), + ) + records.extend(pipeline.chunk(doc)) + written = write_harvest_input(session, rid, records) + summary.chunks_written += written + print( + f" [{logical}] OWASP/{repo}: files={len(files)} chunks={written}", + flush=True, + ) + except Exception as exc: # noqa: BLE001 + summary.errors += 1 + print(f" [{logical}] ERROR {exc}", flush=True) + + if summary.errors and summary.chunks_written == 0: + summary.status = "degraded" + elif summary.errors: + summary.status = "degraded" + return summary + + return run_harvester + + +def _keep_all_noise_filter(session: Any, pipeline_run_id: str, **_kwargs: Any) -> Any: + """Eval-only: promote every harvest chunk to KNOWLEDGE (score Module C).""" + from application.database.db import HarvestInput + from application.utils.noise_filter.hashing import compute_content_hash + from application.utils.noise_filter.pipeline import RunSummary as BSummary + from application.utils.noise_filter.queue_writer import write_verdicts + from application.utils.noise_filter.schemas import ChangeRecord, ClassifyResult + + summary = BSummary(run_id=pipeline_run_id) + rows = ( + session.query(HarvestInput) + .filter_by(pipeline_run_id=pipeline_run_id, status="pending") + .all() + ) + summary.read = len(rows) + triples = [] + for row in rows: + try: + record = ChangeRecord.model_validate(row.payload) + except Exception: # noqa: BLE001 + summary.parse_errors += 1 + continue + verdict = ClassifyResult( + label="KNOWLEDGE", + confidence=1.0, + reasoning="full-pipeline-keep-all-knowledge", + ) + triples.append((record, verdict, compute_content_hash(record.text))) + row.status = "processed" + summary.kept_knowledge += 1 + stats = write_verdicts(session, triples) + summary.inserted = stats.inserted + summary.deduped = stats.deduped + session.commit() + summary.status = "ok" + return summary + + +def _load_gold(stem: str) -> List[Dict[str, Any]]: + path = FIXTURES / f"{stem}.json" + if not path.is_file(): + raise FileNotFoundError(path) + data = json.loads(path.read_text()) + if not isinstance(data, list): + raise ValueError(f"gold {stem} must be a list") + return data + + +def _cheatsheet_section_id(row: Dict[str, Any]) -> str: + sid = str(row.get("section_id") or "").strip() + if sid: + return sid + href = str(row.get("hyperlink") or "") + name = Path(href).stem or str(row.get("section") or "sheet") + return name.replace(" ", "_") + + +def _top2_cre_ids(envelope: Dict[str, Any], uuid_to_ext: Dict[str, str]) -> List[str]: + """Union of rerank top-2 and vector top-2 (up to 4 ids). + + Keep in sync with ``run_b2_pr_mappings.top2_cre_external_ids`` — full-pipeline + GitHub scoring must use the same gate as the B2 fixture arm. + """ + ordered: List[str] = [] + + def add(cre_id: Any) -> None: + if cre_id is None: + return + s = str(cre_id).strip() + if not s: + return + ext = uuid_to_ext.get(s, s) + if ext not in ordered: + ordered.append(ext) + + retrieval = envelope.get("retrieval") or {} + reranked = list(retrieval.get("reranked") or []) + candidates = list(retrieval.get("candidates") or []) + + for cand in reranked[:2]: + if isinstance(cand, dict): + add(cand.get("cre_id")) + + by_vec = sorted( + (c for c in candidates if isinstance(c, dict)), + key=lambda c: float(c.get("score_vector") or 0.0), + reverse=True, + ) + for cand in by_vec[:2]: + add(cand.get("cre_id")) + + if ordered: + return ordered + + for field in ("links", "suggested_links"): + for link in envelope.get(field) or []: + if isinstance(link, dict): + add(link.get("cre_id")) + if len(ordered) >= 2: + return ordered[:2] + return ordered + + +def _guess_keys_for_decision( + *, + path: str, + text: str, + repo: str, +) -> Set[str]: + """Map a decision to gold keys ``{resource}::{section_id}``.""" + keys: Set[str] = set() + blob = f"{path}\n{text}" + repo_l = repo.lower() + path_l = path.lower() + # Infer family from path when source.repo is missing/odd. + is_asvs = "asvs" in repo_l or "/asvs" in path_l or "asvs" in path_l + is_aisvs = "aisvs" in repo_l or "aisvs" in path_l + is_cs = "cheatsheet" in repo_l or "cheatsheet" in path_l + + if is_asvs and not is_aisvs: + # Prefer an explicit single requirement Section-ID (extractor / B2 shape). + primary: Set[str] = set() + for m in re.finditer(r"Section-ID:\s*([^\n]+)", blob, flags=re.IGNORECASE): + parts = [p.strip() for p in re.split(r"[,;]+", m.group(1)) if p.strip()] + three: List[str] = [] + for part in parts: + if ASVS_SID_RE.fullmatch(part) and part.count(".") == 2: + three.append(f"asvs::{part.upper()}") + elif ASVS_BARE_REQ_RE.fullmatch(part): + three.append(f"asvs::V{part}") + if len(parts) == 1 and three: + primary.add(three[0]) + if primary: + keys |= primary + else: + for m in ASVS_SID_RE.finditer(blob): + sid = m.group(1).upper() + # Prefer requirement grain (V1.1.2); keep V1.1 for rare gold rows + keys.add(f"asvs::{sid}") + for m in ASVS_BARE_REQ_RE.finditer(blob): + keys.add(f"asvs::V{m.group(1)}") + for m in re.finditer(r"Section-ID:\s*([^\n]+)", blob, flags=re.IGNORECASE): + for part in re.split(r"[,;\s]+", m.group(1)): + part = part.strip() + if ASVS_SID_RE.fullmatch(part): + keys.add(f"asvs::{part.upper()}") + elif ASVS_BARE_REQ_RE.fullmatch(part): + keys.add(f"asvs::V{part}") + if is_aisvs: + for m in AISVS_SID_RE.finditer(blob): + keys.add(f"aisvs::{m.group(1).upper()}") + m = re.search(r"C(\d+)", path) + if m: + keys.add(f"aisvs::AISVS{int(m.group(1))}") + if is_cs: + stem = Path(path).stem + keys.add(f"cheatsheets::{stem}") + return keys + + +def score_github_decisions( + run_id: str, + cache: str, + github_targets: Dict[str, Tuple[str, str, List[str], Optional[str]]], +) -> Dict[str, Any]: + from application import sqla + from application.cmd.cre_main import db_connect + from application.database.db import CRE, DecisionQueueItem + + db_connect(cache) + uuid_to_ext = { + row.id: (row.external_id or row.id) + for row in sqla.session.query(CRE.id, CRE.external_id).all() + } + decisions = ( + sqla.session.query(DecisionQueueItem).filter_by(pipeline_run_id=run_id).all() + ) + + # gold key → cre set + gold_map: Dict[str, Set[str]] = {} + gold_meta: Dict[str, Dict[str, Any]] = {} + for logical, (_repo, _br, _inc, stem) in github_targets.items(): + if not stem: + continue + for row in _load_gold(stem): + if logical == "cheatsheets": + sid = _cheatsheet_section_id(row) + else: + sid = str(row.get("section_id") or "").strip() + if not sid: + continue + key = f"{logical}::{sid.upper() if logical != 'cheatsheets' else sid}" + gold_map[key] = {c for c in (row.get("cre_ids") or []) if c} + gold_meta[key] = {"logical": logical, "section_id": sid, "row": row} + + # Per-chunk attribution: each decision's preds only land on section ids + # found in that chunk (no chapter-file union that smears CRE soup). + pred: Dict[str, Set[str]] = {} + for row in decisions: + env = row.envelope if isinstance(row.envelope, dict) else {} + knowledge = env.get("knowledge") or {} + loc = knowledge.get("locator") or {} + path = str(loc.get("path") or loc.get("id") or "") + text = str(knowledge.get("text") or knowledge.get("body") or "") + src = knowledge.get("source") or {} + repo = str(src.get("repository") or src.get("repo") or src.get("name") or "") + if not repo: + repo = str(env.get("artifact_id") or path) + suggested = set(_top2_cre_ids(env, uuid_to_ext)) + if not suggested and not path: + continue + keys = _guess_keys_for_decision(path=path, text=text, repo=repo) + asvs_three = { + k + for k in keys + if k.startswith("asvs::") and k.split("::", 1)[1].count(".") == 2 + } + if asvs_three: + keys = (keys - {k for k in keys if k.startswith("asvs::")}) | asvs_three + # Normalize cheatsheet keys against gold + norm_keys: Set[str] = set() + for key in keys: + if key.startswith("cheatsheets::"): + stem = key.split("::", 1)[1] + matched = None + for gk in gold_map: + if ( + gk.startswith("cheatsheets::") + and gk.split("::", 1)[1].casefold() == stem.casefold() + ): + matched = gk + break + if matched: + norm_keys.add(matched) + else: + norm_keys.add(key) + if not suggested: + for key in norm_keys: + pred.setdefault(key, set()) + continue + for key in norm_keys: + pred.setdefault(key, set()).update(suggested) + + details: List[Dict[str, Any]] = [] + by_resource: Dict[str, Dict[str, int]] = {} + hits = scorable = unaligned = 0 + for key, gold in gold_map.items(): + logical = gold_meta[key]["logical"] + bucket = by_resource.setdefault( + logical, {"scorable": 0, "hits": 0, "unaligned": 0} + ) + suggested = pred.get(key) or set() + scorable += 1 + bucket["scorable"] += 1 + hit = bool(suggested & gold) + aligned = key in pred + if not aligned: + unaligned += 1 + bucket["unaligned"] += 1 + if hit: + hits += 1 + bucket["hits"] += 1 + details.append( + { + "resource": logical, + "section_id": gold_meta[key]["section_id"], + "aligned": aligned, + "gold": sorted(gold), + "predicted": sorted(suggested), + "hit": hit, + } + ) + + for bucket in by_resource.values(): + s = bucket["scorable"] + bucket["accuracy"] = (bucket["hits"] / s) if s else 0.0 + + return { + "run_id": run_id, + "mode": "github_aligned", + "decisions_total": len(decisions), + "scorable": scorable, + "unaligned": unaligned, + "hits": hits, + "accuracy": (hits / scorable) if scorable else 0.0, + "by_resource": by_resource, + "details": details, + } + + +def _write_b2_shaped_report(report: Dict[str, Any], out: Path) -> None: + """Write a report score_b2_hop_distance can consume (details + predicted).""" + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(report, indent=2) + "\n") + + +def _run_b2_arm( + *, + run_id: str, + cache: str, + keep_all: bool, + fixtures: Sequence[str], + arm_name: str, + out_name: str, +) -> Dict[str, Any]: + """Score one or more B2 fixture families (cached sources under b2_sources/).""" + import subprocess + + from application import sqla + from application.cmd.cre_main import db_connect + from application.database.db import ( + DecisionQueueItem, + HarvestInput, + KnowledgeQueueItem, + ) + + stems = [str(s).strip() for s in fixtures if str(s).strip()] + if not stems: + raise ValueError(f"B2 arm {arm_name!r} has no fixtures") + + db_connect(cache) + # content_hash is unique — leftover KQ rows from prior arms block replay. + for stem in stems: + for pattern in (f"%{stem}%", f"%{arm_name}%"): + sqla.session.query(KnowledgeQueueItem).filter( + KnowledgeQueueItem.artifact_id.like(pattern) + ).delete(synchronize_session=False) + sqla.session.query(KnowledgeQueueItem).filter( + KnowledgeQueueItem.chunk_id.like(pattern) + ).delete(synchronize_session=False) + sqla.session.query(HarvestInput).filter( + HarvestInput.pipeline_run_id.like(f"%{arm_name}%") + ).delete(synchronize_session=False) + sqla.session.query(KnowledgeQueueItem).filter( + KnowledgeQueueItem.pipeline_run_id.like(f"%{arm_name}%") + ).delete(synchronize_session=False) + sqla.session.query(HarvestInput).filter( + HarvestInput.pipeline_run_id.like(f"%{arm_name}%") + ).delete(synchronize_session=False) + sqla.session.query(DecisionQueueItem).filter( + DecisionQueueItem.pipeline_run_id.like(f"%{arm_name}%") + ).delete(synchronize_session=False) + sqla.session.commit() + + arm_run = f"{run_id}-{arm_name}" + out = EXP / out_name + EXP.mkdir(parents=True, exist_ok=True) + cmd = [ + str(ROOT / "venv" / "bin" / "python"), + str(ROOT / "scripts" / "oie_owasp_eval" / "run_b2_pr_mappings.py"), + "--run-id", + arm_run, + "--out", + str(out), + "--only-fixtures", + *stems, + ] + if keep_all: + cmd.append("--keep-all-knowledge") + env = os.environ.copy() + env["DEV_DATABASE_URL"] = cache + env["DATABASE_URL"] = cache + print(" ".join(cmd), flush=True) + ret = subprocess.call(cmd, cwd=str(ROOT), env=env) + if not out.is_file(): + raise RuntimeError(f"B2 arm {arm_name} produced no report (exit={ret})") + report = json.loads(out.read_text()) + report["arm"] = arm_name + report["fixtures"] = stems + report["b2_exit_code"] = ret + return report + + +def _run_aix_b2_arm(*, run_id: str, cache: str, keep_all: bool) -> Dict[str, Any]: + """Score AIX via existing LLM Top10 B2 gold (hub CRE proxy).""" + return _run_b2_arm( + run_id=run_id, + cache=cache, + keep_all=keep_all, + fixtures=B2_ARMS["aix"], + arm_name="aix-b2", + out_name="full_pipeline_aix_llm.b2_report.json", + ) + + +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--repos", + default="asvs,aisvs,cheatsheets,aix", + help=( + "Comma list: asvs,aisvs,cheatsheets,aix,nist,top10,api " + "(nist/top10/api are B2 source arms)" + ), + ) + parser.add_argument( + "--cache_file", + default=os.environ.get("DEV_DATABASE_URL") + or "postgresql://cre:password@127.0.0.1:5432/cre", + ) + parser.add_argument("--run-id", default="") + parser.add_argument( + "--keep-all-knowledge", + action="store_true", + help="Skip Module B noise filter; forward all harvest chunks as KNOWLEDGE", + ) + parser.add_argument( + "--skip-score", + action="store_true", + help="Run A→B→C only; do not align to gold", + ) + parser.add_argument( + "--neighborhood", + action="store_true", + help="After exact score, run score_b2_hop_distance on reports", + ) + parser.add_argument( + "--out-dir", + type=Path, + default=ART, + help="Report directory (default tmp/oie_owasp_eval/full_pipeline)", + ) + args = parser.parse_args(list(argv) if argv is not None else None) + + _load_dotenv() + _apply_promoted_flags() + os.environ.setdefault("FLASK_CONFIG", "development") + sys.path.insert(0, str(ROOT)) + + wanted = [x.strip().lower() for x in args.repos.split(",") if x.strip()] + known = set(GITHUB_TARGETS) | set(B2_ARMS) + unknown = [x for x in wanted if x not in known] + if unknown: + print(f"unknown --repos entries: {unknown}", file=sys.stderr) + return 2 + + github = {k: GITHUB_TARGETS[k] for k in wanted if k in GITHUB_TARGETS} + b2_wanted = [k for k in wanted if k in B2_ARMS] + run_id = args.run_id or ( + "fullpipe-" + datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + ) + out_dir: Path = args.out_dir + out_dir.mkdir(parents=True, exist_ok=True) + + from application import sqla + from application.cmd.cre_main import db_connect + from application.database.db import ( + DecisionQueueItem, + HarvestInput, + KnowledgeQueueItem, + ) + from application.utils.oie_orchestrator import run_oie_pipeline + + cache = args.cache_file + orch_result = None + github_report: Optional[Dict[str, Any]] = None + b2_reports: Dict[str, Dict[str, Any]] = {} + + if github: + print(f"=== Module A (tarball) + B + C run_id={run_id} ===", flush=True) + db_connect(cache) + for model in (HarvestInput, KnowledgeQueueItem, DecisionQueueItem): + deleted = ( + sqla.session.query(model) + .filter_by(pipeline_run_id=run_id) + .delete(synchronize_session=False) + ) + print(f"cleared {model.__tablename__} for {run_id}: {deleted}", flush=True) + sqla.session.commit() + + harvester = _make_tarball_harvester(github) + orch_result = run_oie_pipeline( + cache_file=cache, + pipeline_run_id=run_id, + sync_repos=False, + run_harvester_fn=harvester, + run_noise_filter_fn=( + _keep_all_noise_filter if args.keep_all_knowledge else None + ), + use_langgraph=True, + ) + orch_path = out_dir / "orchestrator_result.json" + orch_path.write_text(orch_result.to_json() + "\n") + print(f"wrote {orch_path}", flush=True) + + if not args.skip_score: + print("=== Score GitHub arms vs gold ===", flush=True) + github_report = score_github_decisions(run_id, cache, github) + gh_out = EXP / "full_pipeline_github.b2_report.json" + EXP.mkdir(parents=True, exist_ok=True) + _write_b2_shaped_report(github_report, gh_out) + # Mirror under ART for humans browsing full_pipeline/ + _write_b2_shaped_report( + github_report, out_dir / "github_arms.b2_report.json" + ) + print( + json.dumps( + {k: github_report[k] for k in github_report if k != "details"}, + indent=2, + ), + flush=True, + ) + print(f"wrote {gh_out}", flush=True) + + if b2_wanted and not args.skip_score: + for arm in b2_wanted: + stems = B2_ARMS[arm] + print(f"=== B2 arm {arm} ({', '.join(stems)}) ===", flush=True) + out_name = f"full_pipeline_{arm}.b2_report.json" + report = _run_b2_arm( + run_id=run_id, + cache=cache, + keep_all=args.keep_all_knowledge, + fixtures=stems, + arm_name=arm, + out_name=out_name, + ) + b2_reports[arm] = report + _write_b2_shaped_report(report, out_dir / out_name) + print( + json.dumps( + {k: report[k] for k in report if k != "details"}, + indent=2, + ), + flush=True, + ) + + summary: Dict[str, Any] = { + "run_id": run_id, + "repos": wanted, + "github_accuracy": (github_report or {}).get("accuracy"), + "github_hits": (github_report or {}).get("hits"), + "github_scorable": (github_report or {}).get("scorable"), + "orchestrator_ok": ( + None if orch_result is None else orch_result.to_dict().get("ok") + ), + "b2_arms": { + name: { + "accuracy": rep.get("accuracy"), + "hits": rep.get("hits"), + "scorable": rep.get("scorable"), + "fixtures": rep.get("fixtures"), + } + for name, rep in b2_reports.items() + }, + } + # Back-compat keys for AIX-only consumers. + if "aix" in b2_reports: + summary["aix_accuracy"] = b2_reports["aix"].get("accuracy") + summary["aix_hits"] = b2_reports["aix"].get("hits") + summary["aix_scorable"] = b2_reports["aix"].get("scorable") + (out_dir / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") + print(json.dumps(summary, indent=2), flush=True) + + if args.neighborhood and not args.skip_score: + print("=== Neighborhood hop score ===", flush=True) + import subprocess + + hop_cmd = [ + sys.executable, + str(ROOT / "scripts" / "oie_owasp_eval" / "score_b2_hop_distance.py"), + "--reports-glob", + "full_pipeline_*.b2_report.json", + "--force", + ] + subprocess.check_call(hop_cmd, cwd=str(ROOT), env=os.environ.copy()) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/oie_owasp_eval/tarball_harvest.py b/scripts/oie_owasp_eval/tarball_harvest.py new file mode 100644 index 000000000..a2bf8e4aa --- /dev/null +++ b/scripts/oie_owasp_eval/tarball_harvest.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python3 +"""Wave1 harvest without git: download GitHub tarballs and emit harvest_input. + +Used when the agent sandbox forbids writing ``.git/`` (hooks/config). Produces +real ChangeRecord-shaped rows so Module B dual-classify can run. +""" + +from __future__ import annotations + +import fnmatch +import json +import os +import sys +import tarfile +import urllib.request +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath +from typing import Dict, List, Tuple + +ROOT = Path(__file__).resolve().parents[2] +ART = ROOT / "tmp" / "oie_owasp_eval" + +WAVE1: List[Tuple[str, str, List[str]]] = [ + ("ASVS", "master", ["4.0/en/**/*.md", "5.0/en/**/*.md"]), + ("CheatSheetSeries", "master", ["cheatsheets/**/*.md"]), + ("wstg", "master", ["document/**/*.md"]), + ("SAMM", "master", ["**/*.md"]), + ("Top10", "master", ["**/*.md"]), + ("API-Security", "master", ["**/*.md"]), + ("mastg", "master", ["**/*.md"]), + ("masvs", "master", ["**/*.md"]), + ("AISVS", "main", ["**/*.md"]), + ( + "www-project-top-10-for-large-language-model-applications", + "main", + ["**/*.md"], + ), + ("www-project-proactive-controls", "main", ["**/*.md"]), + ("www-project-ai-testing-guide", "main", ["**/*.md"]), + ("www-project-ai-security-and-privacy-guide", "main", ["**/*.md"]), + ("www-project-devsecops-guideline", "main", ["**/*.md"]), + ("secure-coding-practices-quick-reference-guide", "master", ["**/*.md"]), + ("DevSecOpsGuideline", "main", ["**/*.md"]), + ("MCP-Security-Testing-Guide", "main", ["**/*.md"]), + ("Software-Component-Verification-Standard", "main", ["**/*.md"]), + ("IoT-Security-Verification-Standard-ISVS", "master", ["**/*.md"]), + ("cornucopia", "master", ["**/*.md"]), +] + +EXCLUDE = [ + "**/archive/**", + "**/.github/**", + "**/node_modules/**", + "**/LICENSE*", + "**/CHANGELOG*", + "**/CONTRIBUTING*", +] + + +def _match(path: str, patterns: List[str]) -> bool: + """Glob match with ``**`` support; also try the non-recursive collapse.""" + posix = PurePosixPath(path) + expanded: List[str] = [] + for pat in patterns: + expanded.append(pat) + if "/**/" in pat: + expanded.append(pat.replace("/**/", "/")) + if pat.startswith("**/"): + expanded.append(pat[3:]) + for pat in expanded: + try: + if posix.match(pat): + return True + except ValueError: + pass + if fnmatch.fnmatch(path, pat): + return True + return False + + +def _download_tarball(repo: str, branch: str, dest: Path) -> str: + """Download tarball; return the branch that worked.""" + dest.parent.mkdir(parents=True, exist_ok=True) + # Re-download if truncated/corrupt (small or unreadable) + if dest.is_file() and dest.stat().st_size > 1000: + try: + with tarfile.open(dest, "r:gz") as tf: + tf.getmembers()[:1] + return branch + except Exception: + dest.unlink(missing_ok=True) + for try_branch in (branch, "main" if branch == "master" else "master"): + url = f"https://codeload.github.com/OWASP/{repo}/tar.gz/refs/heads/{try_branch}" + print(f"GET {url}", flush=True) + try: + urllib.request.urlretrieve(url, dest) + with tarfile.open(dest, "r:gz") as tf: + tf.getmembers()[:1] + return try_branch + except Exception as exc: # noqa: BLE001 + print(f" fail {try_branch}: {exc}", flush=True) + dest.unlink(missing_ok=True) + raise RuntimeError(f"could not download tarball for {repo}") + + +def _iter_md(tgz: Path, includes: List[str]) -> List[Tuple[str, str]]: + out: List[Tuple[str, str]] = [] + with tarfile.open(tgz, "r:gz") as tf: + for member in tf.getmembers(): + if not member.isfile(): + continue + parts = Path(member.name).parts + if len(parts) < 2: + continue + rel = str(Path(*parts[1:])) + if not rel.lower().endswith(".md"): + continue + if _match(rel, EXCLUDE): + continue + if not _match(rel, includes): + continue + extracted = tf.extractfile(member) + if extracted is None: + continue + raw = extracted.read() + try: + text = raw.decode("utf-8") + except UnicodeDecodeError: + text = raw.decode("utf-8", errors="replace") + if len(text.strip()) < 40: + continue + out.append((rel, text)) + return out + + +def main() -> int: + os.environ.setdefault("FLASK_CONFIG", "development") + os.environ.setdefault("NO_LOAD_GRAPH_DB", "1") + sys.path.insert(0, str(ROOT)) + + state = json.loads((ART / "state.json").read_text()) + run_id = state["run_id"] + "-w1" + db_path = ART / "eval.sqlite" + if db_path.exists(): + db_path.unlink() + db_url = f"sqlite:///{db_path}" + + from application import sqla + from application.cmd.cre_main import db_connect + from application.utils.harvester.chunk_pipeline import DocumentChunkPipeline + from application.utils.harvester.harvest_writer import write_harvest_input + from application.utils.harvester.heading_extractor import HeadingExtractor + from application.utils.harvester.models import Document, Locator, SourceInfo + from application.utils.harvester.schemas import ChunkingConfig + + db_connect(db_url) + sqla.create_all() + + chunking = ChunkingConfig( + strategy="markdown_heading", max_tokens=1200, overlap_tokens=100 + ) + pipeline = DocumentChunkPipeline(chunking=chunking) + headings = HeadingExtractor() + committed_at = datetime.now(timezone.utc) + sha = "tarball00000000000000000000000000000001" + + by_repo: Dict[str, int] = {} + errors: Dict[str, str] = {} + tarball_dir = ART / "tarballs" + tarball_dir.mkdir(parents=True, exist_ok=True) + + for repo, branch, includes in WAVE1: + try: + tgz = tarball_dir / f"{repo}.tgz" + _download_tarball(repo, branch, tgz) + files = _iter_md(tgz, includes) + repo_records = [] + for rel, text in files: + artifact_id = f"art:OWASP/{repo}:{rel}" + doc = Document( + schema_version="0.2.0", + artifact_id=artifact_id, + pipeline_run_id=run_id, + text=text, + heading_structure=headings.extract(text), + source=SourceInfo( + type="github", + repository=f"OWASP/{repo}", + commit_sha=sha, + committed_at=committed_at, + ), + locator=Locator(kind="repo_path", id=rel, path=rel), + ) + repo_records.extend(pipeline.chunk(doc)) + written = write_harvest_input(sqla.session, run_id, repo_records) + by_repo[f"OWASP/{repo}"] = written + print(f"{repo}: files={len(files)} chunks={written}", flush=True) + except Exception as exc: # noqa: BLE001 + errors[repo] = str(exc) + print(f"{repo}: ERROR {exc}", flush=True) + + snap = { + "run_id": run_id, + "mode": "tarball", + "total_rows": sum(by_repo.values()), + "by_repo": dict(sorted(by_repo.items(), key=lambda kv: -kv[1])), + "errors": errors, + } + (ART / "harvest_wave1_snapshot.json").write_text(json.dumps(snap, indent=2) + "\n") + (ART / "harvest_wave1.json").write_text( + json.dumps( + { + "exit_code": 0 if by_repo else 1, + "summary": { + "run_id": run_id, + "repositories": len(WAVE1), + "chunks_written": snap["total_rows"], + "errors": len(errors), + "status": "ok" if snap["total_rows"] else "degraded", + "mode": "tarball", + }, + }, + indent=2, + ) + + "\n" + ) + + now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + state["wave1_run_id"] = run_id + state["db_url"] = db_url + state["phase"] = "classify_dual" if snap["total_rows"] else "harvest_wave1" + state["status"] = "running" if snap["total_rows"] else "error" + state["last_error"] = None if snap["total_rows"] else json.dumps(errors) + state["phases"]["harvest_wave1"] = { + "status": "ok" if snap["total_rows"] else "error", + "detail": f"tarball chunks={snap['total_rows']} errors={len(errors)}", + "at": now, + } + state["updated_at"] = now + (ART / "state.json").write_text(json.dumps(state, indent=2) + "\n") + (ART / "STATUS.md").write_text( + f"# OIE OWASP eval status\n\n" + f"- phase: **{state['phase']}**\n" + f"- status: **{state['status']}**\n" + f"- harvest chunks: **{snap['total_rows']}**\n" + f"- updated: {now}\n" + ) + print(json.dumps(snap, indent=2)) + return 0 if snap["total_rows"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) From 0b797a503d01dcb826bdfed12da2c5517887aad7 Mon Sep 17 00:00:00 2001 From: Spyros Date: Thu, 1 Oct 2026 00:53:45 +0200 Subject: [PATCH 2/2] feat(librarian): promote d1-winner combo defaults (judge+cap8+top_k=3) Six-lever ablations selected SHORTLIST_JUDGE=1, UMBRELLA_PROMOTE_CAP=8, and GitHub/B2 score top_k=3 (KEEP_HUB/HUB_FIRST off). Wire those as pipeline defaults with leaf-drilldown gating and the d1 lever experiment harness. --- .../tests/librarian/config_loader_test.py | 55 ++ .../tests/librarian/leaf_drilldown_test.py | 337 ++++++++++ application/tests/librarian/pipeline_test.py | 175 ++++++ .../tests/librarian/shortlist_judge_test.py | 60 +- application/utils/librarian/config_loader.py | 118 +++- application/utils/librarian/factory.py | 28 + application/utils/librarian/leaf_drilldown.py | 235 +++++++ application/utils/librarian/pipeline.py | 189 +++++- application/utils/librarian/queue_runner.py | 24 + .../utils/librarian/shortlist_judge.py | 75 ++- .../utils/librarian/umbrella_promote.py | 17 +- docs/oie/experimental-feature-flags.md | 67 +- docs/oie/feature-flags.md | 74 ++- docs/oie/full-pipeline.md | 2 +- scripts/oie_owasp_eval/run_b2_pr_mappings.py | 29 +- .../run_d1_lever_experiments.py | 578 ++++++++++++++++++ .../oie_owasp_eval/run_d1_levers_detached.sh | 29 + scripts/oie_owasp_eval/run_full_pipeline.py | 127 +++- .../oie_owasp_eval/run_fullpipe_detached.sh | 29 + 19 files changed, 2117 insertions(+), 131 deletions(-) create mode 100644 application/tests/librarian/leaf_drilldown_test.py create mode 100644 application/utils/librarian/leaf_drilldown.py create mode 100644 scripts/oie_owasp_eval/run_d1_lever_experiments.py create mode 100755 scripts/oie_owasp_eval/run_d1_levers_detached.sh create mode 100755 scripts/oie_owasp_eval/run_fullpipe_detached.sh diff --git a/application/tests/librarian/config_loader_test.py b/application/tests/librarian/config_loader_test.py index 336ff4276..917d3b0bc 100644 --- a/application/tests/librarian/config_loader_test.py +++ b/application/tests/librarian/config_loader_test.py @@ -42,6 +42,16 @@ def test_defaults_when_env_unset(self): self.assertAlmostEqual(cfg.hybrid_gamma, 0.70) self.assertAlmostEqual(cfg.hybrid_beta, HYBRID_BETA) self.assertAlmostEqual(cfg.hybrid_gamma, HYBRID_GAMMA) + self.assertTrue(cfg.leaf_drilldown) + self.assertEqual(cfg.leaf_drilldown_resources, ()) + self.assertEqual(cfg.leaf_drilldown_force_resources, ()) + self.assertEqual(cfg.leaf_drilldown_min_children, 3) + self.assertEqual(cfg.leaf_drilldown_min_sections, 20) + self.assertFalse(cfg.leaf_drilldown_keep_hub) + self.assertFalse(cfg.leaf_drilldown_hub_first) + self.assertEqual(cfg.umbrella_promote_cap, 8) + self.assertEqual(cfg.shortlist_judge_max_picks, 3) + self.assertIsNone(cfg.margin_gamma) def test_config_is_frozen(self): with mock.patch.dict(os.environ, {}, clear=True): @@ -114,6 +124,51 @@ def test_link_threshold_above_one_raises(self): with self.assertRaises(ValueError): load_config() + def test_empty_env_string_means_all_resources(self): + with mock.patch.dict( + os.environ, + {"CRE_LIBRARIAN_LEAF_DRILLDOWN_RESOURCES": ""}, + clear=True, + ): + cfg = load_config() + self.assertEqual(cfg.leaf_drilldown_resources, ()) + + def test_leaf_drilldown_gate_env_overrides(self): + with mock.patch.dict( + os.environ, + { + "CRE_LIBRARIAN_LEAF_DRILLDOWN_RESOURCES": "asvs", + "CRE_LIBRARIAN_LEAF_DRILLDOWN_FORCE_RESOURCES": "api", + "CRE_LIBRARIAN_LEAF_DRILLDOWN_MIN_CHILDREN": "5", + "CRE_LIBRARIAN_LEAF_DRILLDOWN_MIN_SECTIONS": "50", + "CRE_LIBRARIAN_LEAF_DRILLDOWN_KEEP_HUB": "1", + "CRE_LIBRARIAN_LEAF_DRILLDOWN_HUB_FIRST": "true", + "CRE_LIBRARIAN_UMBRELLA_PROMOTE_CAP": "8", + "CRE_LIBRARIAN_SHORTLIST_JUDGE_MAX_PICKS": "3", + "CRE_LIBRARIAN_MARGIN_GAMMA": "0.85", + }, + clear=True, + ): + cfg = load_config() + self.assertEqual(cfg.leaf_drilldown_resources, ("asvs",)) + self.assertEqual(cfg.leaf_drilldown_force_resources, ("api",)) + self.assertEqual(cfg.leaf_drilldown_min_children, 5) + self.assertEqual(cfg.leaf_drilldown_min_sections, 50) + self.assertTrue(cfg.leaf_drilldown_keep_hub) + self.assertTrue(cfg.leaf_drilldown_hub_first) + self.assertEqual(cfg.umbrella_promote_cap, 8) + self.assertEqual(cfg.shortlist_judge_max_picks, 3) + self.assertAlmostEqual(cfg.margin_gamma or 0.0, 0.85) + + def test_leaf_drilldown_resources_star_means_all(self): + with mock.patch.dict( + os.environ, + {"CRE_LIBRARIAN_LEAF_DRILLDOWN_RESOURCES": "*"}, + clear=True, + ): + cfg = load_config() + self.assertEqual(cfg.leaf_drilldown_resources, ()) + def test_non_positive_temperature_raises(self): """T divides the logits, so zero or negative is undefined, not merely a bad setting — the same guard TemperatureScaler applies.""" diff --git a/application/tests/librarian/leaf_drilldown_test.py b/application/tests/librarian/leaf_drilldown_test.py new file mode 100644 index 000000000..29500ca09 --- /dev/null +++ b/application/tests/librarian/leaf_drilldown_test.py @@ -0,0 +1,337 @@ +"""Leaf drill-down: promote Contains children over hub umbrellas when they score better.""" + +from __future__ import annotations + +import unittest +from typing import List, Sequence, Tuple + +from application.utils.librarian.leaf_drilldown import ( + apply_leaf_drilldown, + count_resource_families, + resource_allows_leaf_drilldown, + resource_family_key, + resource_is_fine_grained, +) +from application.utils.librarian.umbrella_promote import ParentIndex + + +def _score_by_doc_token( + pairs: Sequence[Tuple[str, str]], +) -> List[float]: + """Hermetic scorer: higher when query token appears in doc text.""" + out: List[float] = [] + for query, doc in pairs: + q = (query or "").lower() + d = (doc or "").lower() + hits = sum(1 for tok in q.split() if len(tok) >= 4 and tok in d) + out.append(float(hits) + (0.1 if "leaf" in d else 0.0)) + return out + + +class LeafDrilldownTest(unittest.TestCase): + def test_promotes_better_child_ahead_of_hub(self) -> None: + parents = ParentIndex( + child_to_parents={ + "leaf-encode": ("hub-appsec",), + "leaf-other": ("hub-appsec",), + }, + parent_names={"hub-appsec": "Technical application security controls"}, + parent_to_children={ + "hub-appsec": ("leaf-encode", "leaf-other"), + }, + ) + cre_texts = { + "hub-appsec": "broad application security controls umbrella", + "leaf-encode": "leaf encode output near the consuming interpreter", + "leaf-other": "leaf password storage hashing", + } + out = apply_leaf_drilldown( + ["hub-appsec", "noise-cre"], + "Section: Encode output near interpreter context", + parent_index=parents, + cre_texts=cre_texts, + score_fn=_score_by_doc_token, + ) + self.assertEqual(out[0], "leaf-encode") + self.assertIn("hub-appsec", out) + self.assertLess(out.index("leaf-encode"), out.index("hub-appsec")) + + def test_attaches_best_child_even_when_hub_wins_ce(self) -> None: + """Specificity: hub may still lead, but best leaf must appear next.""" + parents = ParentIndex( + child_to_parents={"leaf-a": ("hub",)}, + parent_to_children={"hub": ("leaf-a",)}, + ) + cre_texts = { + "hub": "authentication session management leaf umbrella keywords", + "leaf-a": "unrelated cryptography", + } + out = apply_leaf_drilldown( + ["hub", "noise"], + "authentication session management", + parent_index=parents, + cre_texts=cre_texts, + score_fn=_score_by_doc_token, + ) + self.assertEqual(out[0], "hub") + self.assertEqual(out[1], "leaf-a") + + def test_require_beat_hub_keeps_list_when_child_loses(self) -> None: + parents = ParentIndex( + child_to_parents={"leaf-a": ("hub",)}, + parent_to_children={"hub": ("leaf-a",)}, + ) + cre_texts = { + "hub": "authentication session management leaf umbrella keywords", + "leaf-a": "unrelated cryptography", + } + out = apply_leaf_drilldown( + ["hub"], + "authentication session management", + parent_index=parents, + cre_texts=cre_texts, + score_fn=_score_by_doc_token, + require_beat_hub=True, + ) + self.assertEqual(out, ["hub"]) + + def test_noop_without_parent_index_or_children(self) -> None: + ids = ["a", "b"] + self.assertEqual( + apply_leaf_drilldown( + ids, + "q", + parent_index=None, + cre_texts={}, + score_fn=_score_by_doc_token, + ), + ids, + ) + empty = ParentIndex(child_to_parents={}, parent_to_children={}) + self.assertEqual( + apply_leaf_drilldown( + ids, + "q", + parent_index=empty, + cre_texts={"a": "x", "b": "y"}, + score_fn=_score_by_doc_token, + ), + ids, + ) + + def test_skips_children_without_text(self) -> None: + parents = ParentIndex( + child_to_parents={"leaf-ok": ("hub",), "leaf-missing": ("hub",)}, + parent_to_children={"hub": ("leaf-ok", "leaf-missing")}, + ) + cre_texts = { + "hub": "hub text", + "leaf-ok": "leaf encode interpreter", + } + out = apply_leaf_drilldown( + ["hub"], + "encode interpreter", + parent_index=parents, + cre_texts=cre_texts, + score_fn=_score_by_doc_token, + ) + self.assertEqual(out[0], "leaf-ok") + self.assertNotIn("leaf-missing", out) + + def test_skips_hub_below_min_children(self) -> None: + parents = ParentIndex( + child_to_parents={ + "leaf-a": ("skinny",), + "leaf-b": ("fat",), + "leaf-c": ("fat",), + "leaf-d": ("fat",), + }, + parent_to_children={ + "skinny": ("leaf-a",), + "fat": ("leaf-b", "leaf-c", "leaf-d"), + }, + ) + cre_texts = { + "skinny": "hub skinny", + "fat": "hub fat", + "leaf-a": "leaf encode interpreter", + "leaf-b": "leaf encode interpreter", + "leaf-c": "other", + "leaf-d": "other", + } + out = apply_leaf_drilldown( + ["skinny", "fat"], + "encode interpreter", + parent_index=parents, + cre_texts=cre_texts, + score_fn=_score_by_doc_token, + min_children=3, + ) + self.assertEqual(out[0], "skinny") + self.assertNotIn("leaf-a", out) + self.assertEqual(out[1], "leaf-b") + self.assertLess(out.index("leaf-b"), out.index("fat")) + + def test_hub_first_keeps_hub_ahead_of_beating_leaf(self) -> None: + """E2: even when leaf beats hub on CE, hub stays #1 and leaf is #2.""" + parents = ParentIndex( + child_to_parents={ + "leaf-encode": ("hub-appsec",), + "leaf-other": ("hub-appsec",), + }, + parent_to_children={ + "hub-appsec": ("leaf-encode", "leaf-other"), + }, + ) + cre_texts = { + "hub-appsec": "broad application security controls umbrella", + "leaf-encode": "leaf encode output near the consuming interpreter", + "leaf-other": "leaf password storage hashing", + } + out = apply_leaf_drilldown( + ["hub-appsec", "noise-cre"], + "Section: Encode output near interpreter context", + parent_index=parents, + cre_texts=cre_texts, + score_fn=_score_by_doc_token, + hub_first=True, + ) + self.assertEqual(out[0], "hub-appsec") + self.assertEqual(out[1], "leaf-encode") + + def test_keep_hub_in_top2_when_leaf_displaces_hub(self) -> None: + """E1: after leaf leads, drilled hub must remain inside scored top-2.""" + parents = ParentIndex( + child_to_parents={ + "leaf-encode": ("hub-appsec",), + "leaf-other": ("hub-appsec",), + }, + parent_to_children={ + "hub-appsec": ("leaf-encode", "leaf-other"), + }, + ) + cre_texts = { + "hub-appsec": "broad application security controls umbrella", + "leaf-encode": "leaf encode output near the consuming interpreter", + "leaf-other": "leaf password storage hashing", + } + # Noise already occupies slot #2 before drill; without keep_hub the hub + # can fall to #3 after leaf insert-ahead. + out = apply_leaf_drilldown( + ["hub-appsec", "noise-cre", "other-noise"], + "Section: Encode output near interpreter context", + parent_index=parents, + cre_texts=cre_texts, + score_fn=_score_by_doc_token, + keep_hub_in_top2=True, + ) + self.assertEqual(out[0], "leaf-encode") + self.assertIn("hub-appsec", out[:2]) + + +class ResourceGrainHelpersTest(unittest.TestCase): + def test_family_key_b2_and_github(self) -> None: + self.assertEqual( + resource_family_key("art:B2/owasp_top10_2025:A01"), + "art:B2/owasp_top10_2025", + ) + self.assertEqual( + resource_family_key("art:OWASP/ASVS:5.0/en/x.md"), + "art:OWASP/ASVS", + ) + + def test_count_and_fine_grained(self) -> None: + counts = count_resource_families( + [ + "art:B2/owasp_top10_2025:A01", + "art:B2/owasp_top10_2025:A02", + "art:OWASP/ASVS:a.md", + ] + + [f"art:OWASP/ASVS:{i}.md" for i in range(25)] + ) + self.assertEqual(counts["art:B2/owasp_top10_2025"], 2) + self.assertGreaterEqual(counts["art:OWASP/ASVS"], 20) + self.assertFalse(resource_is_fine_grained(2, 20)) + self.assertTrue(resource_is_fine_grained(26, 20)) + self.assertTrue(resource_is_fine_grained(1, 1)) + + +class ResourceAllowsLeafDrilldownTest(unittest.TestCase): + def test_empty_or_star_allows_all(self) -> None: + self.assertTrue( + resource_allows_leaf_drilldown( + artifact_id="art:B2/owasp_top10_2025:A01", + text="Standard: OWASP Top 10", + allowed=(), + ) + ) + self.assertTrue( + resource_allows_leaf_drilldown( + artifact_id="art:B2/owasp_top10_2025:A01", + text="", + allowed=("*",), + ) + ) + + def test_asvs_artifact_allowed(self) -> None: + self.assertTrue( + resource_allows_leaf_drilldown( + artifact_id="art:OWASP/ASVS:5.0/en/0x10.md", + text="Standard: OWASP ASVS\nSection: V1.2.3", + allowed=("asvs", "aisvs"), + ) + ) + + def test_aisvs_not_matched_by_asvs_token_alone(self) -> None: + self.assertFalse( + resource_allows_leaf_drilldown( + artifact_id="art:OWASP/AISVS:1.0/en/x.md", + text="Standard: OWASP AISVS", + allowed=("asvs",), + ) + ) + self.assertTrue( + resource_allows_leaf_drilldown( + artifact_id="art:OWASP/AISVS:1.0/en/x.md", + text="Standard: OWASP AISVS", + allowed=("asvs", "aisvs"), + ) + ) + + def test_coarse_top10_denied(self) -> None: + self.assertFalse( + resource_allows_leaf_drilldown( + artifact_id="art:B2/owasp_top10_2025:A01", + text="Standard: OWASP Top 10 2025", + allowed=("asvs", "aisvs"), + ) + ) + + def test_standard_header_alone_can_allow(self) -> None: + self.assertTrue( + resource_allows_leaf_drilldown( + artifact_id="art:unknown/source:1", + text="Standard: OWASP Application Security Verification Standard (ASVS)\n", + allowed=("asvs",), + ) + ) + + def test_api_token_matches_b2_api_fixture(self) -> None: + self.assertTrue( + resource_allows_leaf_drilldown( + artifact_id="art:B2/owasp_api_top10_2023:API4", + text="Standard: OWASP API Security Top 10", + allowed=("api",), + ) + ) + self.assertFalse( + resource_allows_leaf_drilldown( + artifact_id="art:B2/owasp_top10_2025:A01", + text="Standard: OWASP Top 10 2025", + allowed=("api",), + ) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/tests/librarian/pipeline_test.py b/application/tests/librarian/pipeline_test.py index a147531cc..a22c52632 100644 --- a/application/tests/librarian/pipeline_test.py +++ b/application/tests/librarian/pipeline_test.py @@ -460,6 +460,181 @@ def llm(system: str, user: str) -> str: self.assertNotIn("shortlist-judge", result.envelopes[0].retrieval.retriever) +class LeafDrilldownGatePipelineTest(unittest.TestCase): + """Resource deny must not rewrite audit.reranked (scored preds).""" + + def test_denied_resource_keeps_pre_drill_reranked(self) -> None: + from application.utils.librarian.umbrella_promote import ParentIndex + + hub = "hub-uuid" + leaf = "leaf-uuid" + parents = ParentIndex( + child_to_parents={leaf: (hub,)}, + parent_to_children={hub: (leaf, "c2", "c3")}, + ) + reranked = [ + CreCandidate(cre_id=hub, score_rerank=2.0), + CreCandidate(cre_id="noise", score_rerank=1.0), + ] + + class _DrillReranker(_Reranker): + _cre_texts = { + hub: "hub authentication umbrella", + leaf: "leaf encode interpreter", + "c2": "other", + "c3": "other", + } + + @staticmethod + def _score_fn(pairs): + return [0.1] + [9.0] + [0.0] * (len(pairs) - 2) + + row = _row( + text="Standard: OWASP Top 10 2025\nSection: A01", + row_id="top10", + ) + # Force Top10-shaped artifact (deny under asvs,aisvs allowlist). + row = row.model_copy( + update={ + "artifact_id": "art:B2/owasp_top10_2025:A01", + "chunk_id": "chk:art:B2/owasp_top10_2025:A01:0", + } + ) + result = LibrarianPipeline( + _Source([row]), + _Retriever(), + _DrillReranker(reranked), + _Scaler(0.5), + threshold=0.8, + pipeline_run_id=RUN, + parent_index=parents, + leaf_drilldown=True, + leaf_drilldown_resources=(), + leaf_drilldown_min_children=3, + leaf_drilldown_min_sections=20, + ).run(at=AT) + env = result.envelopes[0] + ids = [c.cre_id for c in env.retrieval.reranked] + self.assertEqual(ids[:2], [hub, "noise"]) + self.assertNotIn(leaf, ids) + + def test_force_resources_bypasses_min_sections_for_api(self) -> None: + """Coarse API (n≈10) can still leaf-drill when force list matches.""" + from application.utils.librarian.umbrella_promote import ParentIndex + + hub = "hub-uuid" + leaf = "leaf-uuid" + parents = ParentIndex( + child_to_parents={leaf: (hub,)}, + parent_to_children={hub: (leaf, "c2", "c3")}, + ) + reranked = [ + CreCandidate(cre_id=hub, score_rerank=2.0), + CreCandidate(cre_id="noise", score_rerank=1.0), + ] + + class _DrillReranker(_Reranker): + _cre_texts = { + hub: "hub authentication umbrella", + leaf: "leaf encode interpreter", + "c2": "other", + "c3": "other", + } + + @staticmethod + def _score_fn(pairs): + return [0.1, 9.0, 0.0, 0.0][: len(pairs)] + + row = _row( + text="Standard: OWASP API Security Top 10\nSection: API4", + row_id="api4", + ) + row = row.model_copy( + update={ + "artifact_id": "art:B2/owasp_api_top10_2023:API4", + "chunk_id": "chk:art:B2/owasp_api_top10_2023:API4:0", + } + ) + result = LibrarianPipeline( + _Source([row]), + _Retriever(), + _DrillReranker(reranked), + _Scaler(0.5), + threshold=0.8, + pipeline_run_id=RUN, + parent_index=parents, + leaf_drilldown=True, + leaf_drilldown_resources=(), + leaf_drilldown_force_resources=("api",), + leaf_drilldown_min_children=3, + leaf_drilldown_min_sections=20, + ).run(at=AT) + env = result.envelopes[0] + ids = [c.cre_id for c in env.retrieval.reranked] + self.assertEqual(ids[0], leaf) + self.assertIn(hub, ids) + + def test_allowed_asvs_attaches_leaf_into_reranked(self) -> None: + from application.utils.librarian.umbrella_promote import ParentIndex + + hub = "hub-uuid" + leaf = "leaf-uuid" + parents = ParentIndex( + child_to_parents={leaf: (hub,)}, + parent_to_children={hub: (leaf, "c2", "c3")}, + ) + reranked = [ + CreCandidate(cre_id=hub, score_rerank=2.0), + CreCandidate(cre_id="noise", score_rerank=1.0), + ] + + class _DrillReranker(_Reranker): + _cre_texts = { + hub: "hub authentication umbrella", + leaf: "leaf encode interpreter", + "c2": "other", + "c3": "other", + } + + @staticmethod + def _score_fn(pairs): + # Child beats hub. + return [0.1, 9.0, 0.0, 0.0][: len(pairs)] + + # Enough ASVS siblings in-batch to pass min_sections=20. + rows = [] + for i in range(20): + r = _row( + text="Standard: OWASP ASVS\nSection: V1.2.3 Encode output", + row_id=f"asvs{i}", + ) + rows.append( + r.model_copy( + update={ + "artifact_id": f"art:OWASP/ASVS:5.0/en/x{i}.md", + "chunk_id": f"chk:art:OWASP/ASVS:5.0/en/x{i}.md:0", + } + ) + ) + result = LibrarianPipeline( + _Source(rows), + _Retriever(), + _DrillReranker(reranked), + _Scaler(0.5), + threshold=0.8, + pipeline_run_id=RUN, + parent_index=parents, + leaf_drilldown=True, + leaf_drilldown_resources=(), + leaf_drilldown_min_children=3, + leaf_drilldown_min_sections=20, + ).run(at=AT) + env = result.envelopes[0] + ids = [c.cre_id for c in env.retrieval.reranked] + self.assertEqual(ids[0], leaf) + self.assertIn(hub, ids) + + def failing_component_stub(kind): """A stub whose single method always raises, for the given seam.""" diff --git a/application/tests/librarian/shortlist_judge_test.py b/application/tests/librarian/shortlist_judge_test.py index cc884c222..7bb6066e4 100644 --- a/application/tests/librarian/shortlist_judge_test.py +++ b/application/tests/librarian/shortlist_judge_test.py @@ -9,6 +9,7 @@ from application.utils.librarian.schemas import CreCandidate, RetrievalAudit from application.utils.librarian.shortlist_judge import ( ShortlistJudgeCache, + cache_key, candidates_from_audit, judge_shortlist, parse_judge_json, @@ -41,9 +42,16 @@ def test_wrapper_and_fences(self) -> None: ["111-111", "222-222"], ) - def test_caps_at_two(self) -> None: - allow = ["a", "b", "c"] - self.assertEqual(parse_judge_json('["a","b","c"]', allow), ["a", "b"]) + def test_caps_at_default_three(self) -> None: + allow = ["a", "b", "c", "d"] + self.assertEqual(parse_judge_json('["a","b","c","d"]', allow), ["a", "b", "c"]) + + def test_caps_at_max_picks(self) -> None: + allow = ["a", "b", "c", "d"] + self.assertEqual( + parse_judge_json('["a","b","c","d"]', allow, max_picks=2), + ["a", "b"], + ) class JudgeShortlistTest(unittest.TestCase): @@ -62,6 +70,23 @@ def llm(system: str, user: str) -> str: ) self.assertEqual(out, ["cre-a"]) + def test_max_picks_three(self) -> None: + def llm(system: str, user: str) -> str: + self.assertIn("1–3", user) + return '["cre-a", "cre-b", "cre-c"]' + + out = judge_shortlist( + "Section: X", + [ + {"cre_id": "cre-a", "name": "A"}, + {"cre_id": "cre-b", "name": "B"}, + {"cre_id": "cre-c", "name": "C"}, + ], + llm_fn=llm, + max_picks=3, + ) + self.assertEqual(out, ["cre-a", "cre-b", "cre-c"]) + def test_llm_error_fail_open(self) -> None: def llm(system: str, user: str) -> str: raise RuntimeError("network down") @@ -96,6 +121,35 @@ def llm(system: str, user: str) -> str: self.assertEqual(second, ["cre-a"]) self.assertEqual(calls["n"], 1) + def test_cache_key_includes_max_picks(self) -> None: + ids = ["a", "b"] + self.assertNotEqual( + cache_key("q", ids, max_picks=2), + cache_key("q", ids, max_picks=3), + ) + + def test_max_picks_change_bypasses_stale_cache(self) -> None: + calls = {"n": 0} + + def llm(system: str, user: str) -> str: + calls["n"] += 1 + return '["cre-a", "cre-b", "cre-c"]' + + with tempfile.TemporaryDirectory() as tmp: + cache = ShortlistJudgeCache(disk_dir=Path(tmp)) + cands = [ + {"cre_id": "cre-a", "name": "A"}, + {"cre_id": "cre-b", "name": "B"}, + {"cre_id": "cre-c", "name": "C"}, + ] + two = judge_shortlist("focus", cands, llm_fn=llm, cache=cache, max_picks=2) + three = judge_shortlist( + "focus", cands, llm_fn=llm, cache=cache, max_picks=3 + ) + self.assertEqual(two, ["cre-a", "cre-b"]) + self.assertEqual(three, ["cre-a", "cre-b", "cre-c"]) + self.assertEqual(calls["n"], 2) + class CandidatesFromAuditTest(unittest.TestCase): def test_top_n_with_names(self) -> None: diff --git a/application/utils/librarian/config_loader.py b/application/utils/librarian/config_loader.py index 7769742a5..2c62b94b1 100644 --- a/application/utils/librarian/config_loader.py +++ b/application/utils/librarian/config_loader.py @@ -56,18 +56,6 @@ def _env_optional_margin_gamma(name: str) -> Optional[float]: return float(stripped) -def _env_bool(name: str, default: bool = False) -> bool: - raw = os.getenv(name) - if raw is None: - return default - return raw.strip().lower() in ("1", "true", "yes", "on") - - -def _env_csv(name: str) -> tuple[str, ...]: - raw = os.getenv(name, "") - return tuple(part.strip() for part in raw.split(",") if part.strip()) - - @dataclass(frozen=True) class LibrarianConfig: crossencoder_model: str @@ -92,6 +80,50 @@ class LibrarianConfig: prefer_audit_ids: bool = True hybrid_beta: float = 0.0 hybrid_gamma: float = 0.70 + #: After C.2, promote Contains children over hub umbrellas when they score better. + leaf_drilldown: bool = True + #: Optional name allowlist (empty = no name filter). Length gate is primary. + leaf_drilldown_resources: tuple[str, ...] = () + #: Bypass ``min_sections`` for these resources (empty = none; set ``api`` to force). + leaf_drilldown_force_resources: tuple[str, ...] = () + #: Skip Contains hubs with fewer than this many children. + leaf_drilldown_min_children: int = 3 + #: Skip drill when resource family has fewer chunks than this (≤1 disables). + leaf_drilldown_min_sections: int = 20 + #: E1: keep drilled hubs inside scored top-2. + leaf_drilldown_keep_hub: bool = False + #: E2: never put leaf ahead of hub (leaf always specificity #2). + leaf_drilldown_hub_first: bool = False + #: E5: max umbrella parents promoted into preferred shortlist. + umbrella_promote_cap: int = 8 + #: Grounded judge: max CRE ids (3 aligns with score top_k=3). + shortlist_judge_max_picks: int = 3 + #: Relative CE shortlist cutoff after C.2 (None = off). + margin_gamma: Optional[float] = None + + +def _env_leaf_drilldown_resources() -> tuple[str, ...]: + """Unset / ``*`` / empty → no name filter. Otherwise comma list.""" + raw = os.getenv("CRE_LIBRARIAN_LEAF_DRILLDOWN_RESOURCES") + if raw is None: + return () + stripped = raw.strip() + if not stripped or stripped == "*": + return () + return tuple(part.strip().lower() for part in stripped.split(",") if part.strip()) + + +def _env_leaf_drilldown_force_resources() -> tuple[str, ...]: + """Comma list; empty/unset → no length-gate bypass. + + Cold-start (``FORCE=api``) raised GitHub d1 but dropped API exact 60%→50%; + keep off by default — set ``CRE_LIBRARIAN_LEAF_DRILLDOWN_FORCE_RESOURCES=api`` + only for explicit experiments. + """ + raw = os.getenv("CRE_LIBRARIAN_LEAF_DRILLDOWN_FORCE_RESOURCES") + if raw is None or not raw.strip(): + return () + return tuple(part.strip().lower() for part in raw.split(",") if part.strip()) def load_config() -> LibrarianConfig: @@ -126,6 +158,24 @@ def load_config() -> LibrarianConfig: prefer_audit_ids = _env_bool("CRE_LIBRARIAN_PREFER_AUDIT_IDS", True) hybrid_beta = float(os.getenv("CRE_LIBRARIAN_HYBRID_BETA", "0")) hybrid_gamma = float(os.getenv("CRE_LIBRARIAN_HYBRID_GAMMA", "0.70")) + leaf_drilldown = _env_bool("CRE_LIBRARIAN_LEAF_DRILLDOWN", True) + leaf_drilldown_resources = _env_leaf_drilldown_resources() + leaf_drilldown_force_resources = _env_leaf_drilldown_force_resources() + leaf_drilldown_min_children = int( + os.getenv("CRE_LIBRARIAN_LEAF_DRILLDOWN_MIN_CHILDREN", "3") + ) + leaf_drilldown_min_sections = int( + os.getenv("CRE_LIBRARIAN_LEAF_DRILLDOWN_MIN_SECTIONS", "20") + ) + leaf_drilldown_keep_hub = _env_bool("CRE_LIBRARIAN_LEAF_DRILLDOWN_KEEP_HUB", False) + leaf_drilldown_hub_first = _env_bool( + "CRE_LIBRARIAN_LEAF_DRILLDOWN_HUB_FIRST", False + ) + umbrella_promote_cap = int(os.getenv("CRE_LIBRARIAN_UMBRELLA_PROMOTE_CAP", "8")) + shortlist_judge_max_picks = int( + os.getenv("CRE_LIBRARIAN_SHORTLIST_JUDGE_MAX_PICKS", "3") + ) + margin_gamma = _env_optional_margin_gamma("CRE_LIBRARIAN_MARGIN_GAMMA") if retriever_backend not in _RETRIEVER_BACKENDS: raise ValueError( @@ -180,6 +230,40 @@ def load_config() -> LibrarianConfig: raise ValueError( f"CRE_LIBRARIAN_HYBRID_GAMMA must be finite and >= 0, got {hybrid_gamma}" ) + if leaf_drilldown_min_children < 1: + raise ValueError( + "CRE_LIBRARIAN_LEAF_DRILLDOWN_MIN_CHILDREN must be >= 1, " + f"got {leaf_drilldown_min_children}" + ) + if leaf_drilldown_min_sections < 0: + raise ValueError( + "CRE_LIBRARIAN_LEAF_DRILLDOWN_MIN_SECTIONS must be >= 0, " + f"got {leaf_drilldown_min_sections}" + ) + if umbrella_promote_cap < 1: + raise ValueError( + "CRE_LIBRARIAN_UMBRELLA_PROMOTE_CAP must be >= 1, " + f"got {umbrella_promote_cap}" + ) + if shortlist_judge_max_picks < 1 or shortlist_judge_max_picks > 5: + raise ValueError( + "CRE_LIBRARIAN_SHORTLIST_JUDGE_MAX_PICKS must be in [1, 5], " + f"got {shortlist_judge_max_picks}" + ) + if margin_gamma is not None and ( + not math.isfinite(margin_gamma) or margin_gamma <= 0 + ): + raise ValueError( + "CRE_LIBRARIAN_MARGIN_GAMMA must be finite and > 0 when set, " + f"got {margin_gamma}" + ) + + # Mirror promote cap into the process env so ParentIndex._promote_cap sees it + # without plumbing the index through every call site. + os.environ["CRE_LIBRARIAN_UMBRELLA_PROMOTE_CAP"] = str(umbrella_promote_cap) + os.environ["CRE_LIBRARIAN_SHORTLIST_JUDGE_MAX_PICKS"] = str( + shortlist_judge_max_picks + ) return LibrarianConfig( crossencoder_model=crossencoder_model, @@ -204,4 +288,14 @@ def load_config() -> LibrarianConfig: prefer_audit_ids=prefer_audit_ids, hybrid_beta=hybrid_beta, hybrid_gamma=hybrid_gamma, + leaf_drilldown=leaf_drilldown, + leaf_drilldown_resources=leaf_drilldown_resources, + leaf_drilldown_force_resources=leaf_drilldown_force_resources, + leaf_drilldown_min_children=leaf_drilldown_min_children, + leaf_drilldown_min_sections=leaf_drilldown_min_sections, + leaf_drilldown_keep_hub=leaf_drilldown_keep_hub, + leaf_drilldown_hub_first=leaf_drilldown_hub_first, + umbrella_promote_cap=umbrella_promote_cap, + shortlist_judge_max_picks=shortlist_judge_max_picks, + margin_gamma=margin_gamma, ) diff --git a/application/utils/librarian/factory.py b/application/utils/librarian/factory.py index bb6b55d9e..caa9e868f 100644 --- a/application/utils/librarian/factory.py +++ b/application/utils/librarian/factory.py @@ -54,6 +54,23 @@ class LibrarianComponents: shortlist_llm_fn: Optional[Callable[[str, str], str]] = None #: C.4 safety guard (None → pipeline keeps NullSafetyGuard). safety_guard: Optional[Any] = None +#: Contains parent→children index for leaf drill-down (None = skip). + parent_index: Optional[Any] = None + #: When True, promote better Contains children over hub umbrellas after C.2. + leaf_drilldown: bool = True + #: Optional name allowlist (empty = no name filter). + leaf_drilldown_resources: tuple[str, ...] = () + #: Bypass min_sections for these resources (empty = none). + leaf_drilldown_force_resources: tuple[str, ...] = () + #: Skip Contains hubs with fewer than this many children. + leaf_drilldown_min_children: int = 3 + #: Skip drill when resource family has fewer chunks than this (≤1 disables). + leaf_drilldown_min_sections: int = 20 + leaf_drilldown_keep_hub: bool = False + leaf_drilldown_hub_first: bool = False + umbrella_promote_cap: int = 8 + shortlist_judge_max_picks: int = 3 + margin_gamma: Optional[float] = None>>>>>>> 7208db90 (feat(librarian): promote d1-winner combo defaults (judge+cap8+top_k=3)) def build_scaler(config: Optional[LibrarianConfig] = None) -> Scaler: @@ -327,6 +344,17 @@ def build_components( cre_prior=cre_prior, shortlist_llm_fn=shortlist_llm, safety_guard=safety_guard, +parent_index=parent_index, + leaf_drilldown=config.leaf_drilldown, + leaf_drilldown_resources=config.leaf_drilldown_resources, + leaf_drilldown_force_resources=config.leaf_drilldown_force_resources, + leaf_drilldown_min_children=config.leaf_drilldown_min_children, + leaf_drilldown_min_sections=config.leaf_drilldown_min_sections, + leaf_drilldown_keep_hub=config.leaf_drilldown_keep_hub, + leaf_drilldown_hub_first=config.leaf_drilldown_hub_first, + umbrella_promote_cap=config.umbrella_promote_cap, + shortlist_judge_max_picks=config.shortlist_judge_max_picks, + margin_gamma=config.margin_gamma,>>>>>>> 7208db90 (feat(librarian): promote d1-winner combo defaults (judge+cap8+top_k=3)) ) diff --git a/application/utils/librarian/leaf_drilldown.py b/application/utils/librarian/leaf_drilldown.py new file mode 100644 index 000000000..f0b638800 --- /dev/null +++ b/application/utils/librarian/leaf_drilldown.py @@ -0,0 +1,235 @@ +"""Post-rerank leaf drill-down for Module C. + +When a top-ranked CRE is a Contains parent (umbrella / hub), score its children +against the chunk text and promote the best leaf into the suggestion list: + +* If the leaf scores higher than the hub → leaf leads, hub stays next. +* Else (hub still wins CE) → still insert the best leaf immediately after the + hub so specificity is available for exact gold without dropping the umbrella. + +Gated by ``CRE_LIBRARIAN_LEAF_DRILLDOWN`` (default on), plus resource grain +(section-count / optional allowlist) and hub fan-out (``min_children``) so +coarse standards do not regress. +""" + +from __future__ import annotations + +import re +from collections import Counter +from typing import Callable, List, Mapping, Optional, Sequence, Tuple + +from application.utils.librarian.umbrella_promote import ParentIndex + +ScoreFn = Callable[[Sequence[Tuple[str, str]]], Sequence[float]] + +_DEFAULT_TOP_HUBS = 3 +_DEFAULT_MAX_CHILDREN = 32 +_DEFAULT_MIN_CHILDREN = 1 + +# ``asvs`` must not match ``aisvs`` (lookbehind rejects the leading ``i``). +_ASVS_TOKEN_RE = re.compile(r"(? str: + """Stable resource family for grain census. + + Examples:: + + art:B2/owasp_top10_2025:A01 → art:B2/owasp_top10_2025 + art:OWASP/ASVS:5.0/en/x.md → art:OWASP/ASVS + """ + art = (artifact_id or "").strip() + if not art: + return "" + if art.startswith("art:B2/"): + rest = art[len("art:B2/") :] + stem = rest.split(":", 1)[0] + return f"art:B2/{stem}" if stem else art + if art.startswith("art:"): + parts = art.split(":") + if len(parts) >= 2 and parts[1]: + return f"{parts[0]}:{parts[1]}" + return art + + +def count_resource_families( + artifact_ids: Sequence[str], +) -> Mapping[str, int]: + """Count chunks/sections per ``resource_family_key``.""" + counts: Counter[str] = Counter() + for art in artifact_ids: + key = resource_family_key(art) + if key: + counts[key] += 1 + return dict(counts) + + +def resource_is_fine_grained(section_count: int, min_sections: int) -> bool: + """True when length/grain gate allows drill. + + ``min_sections <= 1`` disables the length gate (always fine-grained). + """ + if int(min_sections) <= 1: + return True + return int(section_count) >= int(min_sections) + + +def resource_allows_leaf_drilldown( + *, + artifact_id: str = "", + text: str = "", + allowed: Sequence[str] = (), +) -> bool: + """Return True when this section's resource may run leaf drill-down. + + Empty ``allowed`` or a lone ``*`` token means no name filter (all OK). + Tokens match case-insensitively against ``artifact_id`` and any + ``Standard:`` header line in ``text``. The ``asvs`` token does not match + ``aisvs``. + """ + tokens = tuple(t.strip().lower() for t in allowed if t and str(t).strip()) + if not tokens or tokens == ("*",) or "*" in tokens: + return True + + haystack_parts = [artifact_id or ""] + for m in _STANDARD_LINE_RE.finditer(text or ""): + haystack_parts.append(m.group(1)) + haystack = "\n".join(haystack_parts) + + for token in tokens: + if token == "*": + return True + if token == "asvs": + if _ASVS_TOKEN_RE.search(haystack): + return True + continue + # Word-ish boundary so short tokens do not false-positive inside longer ids. + pat = re.compile( + rf"(? List[str]: + """Return ``cre_ids`` with Contains-children promoted for specificity. + + By default (``require_beat_hub=False``) the best child is always attached + next to a hub top-hit — even when the hub's CE score is higher — because + umbrella-tolerant product metrics still see the hub, while exact leaf gold + needs the child present. Set ``require_beat_hub=True`` to only promote when + the child strictly outscores the hub. + + ``hub_first=True`` always keeps the hub ahead of the attached leaf (leaf + as specificity #2). ``keep_hub_in_top2=True`` ensures each drilled hub + remains among the first two ids after all attaches (product-d1 safety). + + ``min_children`` skips Contains hubs with fewer than K children (fan-out + gate). Soft-fail: missing texts, empty children, or scoring errors leave + that hub unchanged. + """ + if not cre_ids or parent_index is None: + return list(cre_ids) + + parent_to_children = getattr(parent_index, "parent_to_children", None) or {} + if not parent_to_children: + return list(cre_ids) + + need = max(1, int(min_children)) + out: List[str] = list(cre_ids) + drilled_hubs: List[str] = [] + hubs_seen = 0 + for hub in list(cre_ids): + if hubs_seen >= top_hubs: + break + children = tuple(parent_to_children.get(hub) or ()) + if len(children) < need: + continue + hubs_seen += 1 + if hub not in cre_texts: + continue + + scored_ids: List[str] = [hub] + for child in children: + if child in cre_texts and child not in scored_ids: + scored_ids.append(child) + if len(scored_ids) > max_children: + break + if len(scored_ids) == 1: + continue + + pairs = [(query, cre_texts[cid]) for cid in scored_ids] + try: + scores = [float(s) for s in score_fn(pairs)] + except Exception: + continue + if len(scores) != len(scored_ids): + continue + + # Best among children only (index 0 is the hub). + child_idxs = list(range(1, len(scored_ids))) + best_i = max(child_idxs, key=lambda i: scores[i]) + best_id = scored_ids[best_i] + hub_score = scores[0] + child_beats = scores[best_i] > hub_score + if require_beat_hub and not child_beats: + continue + + out = [cid for cid in out if cid != best_id] + try: + hub_pos = out.index(hub) + except ValueError: + out.insert(0, best_id) + drilled_hubs.append(hub) + continue + if child_beats and not hub_first: + # Leaf leads; keep hub immediately after. + out.insert(hub_pos, best_id) + else: + # Hub preferred by CE, or hub_first — attach leaf as specificity #2. + out.insert(hub_pos + 1, best_id) + drilled_hubs.append(hub) + + if keep_hub_in_top2 and drilled_hubs: + # Ensure each drilled hub remains in the scored top-2 window. + head = out[:2] + for hub in drilled_hubs: + if hub in head: + continue + # Prefer: [hub, leaf...] or [leaf, hub, ...] if leaf already leads. + if hub in out: + out = [cid for cid in out if cid != hub] + if out and out[0] != hub: + # Keep current #1; place hub at #2. + out.insert(1, hub) + else: + out.insert(0, hub) + head = out[:2] + + return out + + +__all__ = [ + "apply_leaf_drilldown", + "count_resource_families", + "resource_allows_leaf_drilldown", + "resource_family_key", + "resource_is_fine_grained", + "ScoreFn", +] diff --git a/application/utils/librarian/pipeline.py b/application/utils/librarian/pipeline.py index fa0404179..21a49fb91 100644 --- a/application/utils/librarian/pipeline.py +++ b/application/utils/librarian/pipeline.py @@ -223,6 +223,16 @@ def __init__( use_focus_query: bool = False, pref_inject: bool = True, prefer_audit: bool = True, + parent_index: Optional[Any] = None, + leaf_drilldown: bool = False, + leaf_drilldown_resources: Sequence[str] = (), + leaf_drilldown_force_resources: Sequence[str] = (), + leaf_drilldown_min_children: int = 3, + leaf_drilldown_min_sections: int = 20, + leaf_drilldown_keep_hub: bool = False, + leaf_drilldown_hub_first: bool = False, + shortlist_judge_max_picks: int = 3, + margin_gamma: Optional[float] = None, ) -> None: self._source = source self._retriever = retriever @@ -242,6 +252,21 @@ def __init__( self._use_focus_query = use_focus_query self._pref_inject = pref_inject self._prefer_audit = prefer_audit + self._parent_index = parent_index + self._leaf_drilldown = leaf_drilldown + self._leaf_drilldown_resources = tuple(leaf_drilldown_resources or ()) + self._leaf_drilldown_force_resources = tuple( + leaf_drilldown_force_resources or () + ) + self._leaf_drilldown_min_children = max(1, int(leaf_drilldown_min_children)) + self._leaf_drilldown_min_sections = max(0, int(leaf_drilldown_min_sections)) + self._leaf_drilldown_keep_hub = bool(leaf_drilldown_keep_hub) + self._leaf_drilldown_hub_first = bool(leaf_drilldown_hub_first) + self._shortlist_judge_max_picks = max(1, min(5, int(shortlist_judge_max_picks))) + self._margin_gamma: Optional[float] = ( + float(margin_gamma) if margin_gamma is not None else None + ) + self._resource_family_counts: Mapping[str, int] = {} # Emit-time grounding: prefer an injected registry; otherwise build from # membership (+ cre_id_map for canonicalisation). None membership keeps # hermetic stubs passthrough until they opt in. @@ -254,6 +279,105 @@ def __init__( else: self._cre_registry = CreRegistry.disabled() + def _apply_leaf_drilldown( + self, + cre_ids: Sequence[str], + query: str, + *, + artifact_id: str = "", + section_text: str = "", + ) -> list: + """Promote Contains children over hubs when the child scores higher.""" + if not self._leaf_drilldown or not self._parent_index or not cre_ids: + return list(cre_ids) + from application.utils.librarian.leaf_drilldown import ( + apply_leaf_drilldown, + resource_allows_leaf_drilldown, + resource_family_key, + resource_is_fine_grained, + ) + + family = resource_family_key(artifact_id) + family_count = int(self._resource_family_counts.get(family, 0)) + force_length = bool(self._leaf_drilldown_force_resources) and ( + resource_allows_leaf_drilldown( + artifact_id=artifact_id, + text=section_text, + allowed=self._leaf_drilldown_force_resources, + ) + ) + if not force_length and not resource_is_fine_grained( + family_count, self._leaf_drilldown_min_sections + ): + return list(cre_ids) + + if not resource_allows_leaf_drilldown( + artifact_id=artifact_id, + text=section_text, + allowed=self._leaf_drilldown_resources, + ): + return list(cre_ids) + + reranker = self._reranker + cre_texts = getattr(reranker, "_cre_texts", None) + score_fn = getattr(reranker, "_score_fn", None) + if not cre_texts or score_fn is None: + return list(cre_ids) + try: + return apply_leaf_drilldown( + cre_ids, + query, + parent_index=self._parent_index, + cre_texts=cre_texts, + score_fn=score_fn, + min_children=self._leaf_drilldown_min_children, + hub_first=self._leaf_drilldown_hub_first, + keep_hub_in_top2=self._leaf_drilldown_keep_hub, + ) + except Exception: + logger.warning( + "leaf drill-down failed; keeping pre-drill ranking", + exc_info=True, + ) + return list(cre_ids) + + def _sync_audit_after_drilldown( + self, audit: RetrievalAudit, cre_ids: Sequence[str] + ) -> RetrievalAudit: + """Mirror drilled ``cre_ids`` into ``audit.reranked``. + + Eval scorers (B2 / full-pipeline) read ``retrieval.reranked``, not only + ``suggested_links``. Without this sync, leaf drill-down changes the + decide() shortlist but scoring still sees the pre-drill hubs. + """ + if not cre_ids: + return audit + from application.utils.librarian.schemas import CreCandidate + + by_id: dict = {} + for cand in list(audit.reranked or []) + list(audit.candidates or []): + cid = getattr(cand, "cre_id", None) + if cid and cid not in by_id: + by_id[cid] = cand + new_reranked = [] + seen = set() + for cid in cre_ids: + if not cid or cid in seen: + continue + seen.add(cid) + existing = by_id.get(cid) + if existing is not None: + new_reranked.append(existing) + else: + # Drilled leaf may be outside the C.1 shortlist. + new_reranked.append(CreCandidate(cre_id=cid, score_vector=0.0)) + for cand in audit.reranked or []: + cid = getattr(cand, "cre_id", None) + if cid and cid not in seen: + seen.add(cid) + new_reranked.append(cand) + return audit.model_copy(update={"reranked": new_reranked}) + def _map_cre_ids(self, external_ids: Sequence[str]) -> tuple: return tuple(self._cre_id_map.get(cid, cid) for cid in external_ids) @@ -271,7 +395,25 @@ def run(self, *, at: datetime) -> RunResult: source_labels: Dict[str, str] = {} linked = review = skipped = errored = total = 0 safety_unevaluated = 0 - for item in self._source.items(): + # Materialize once so we can census resource families for the grain gate + # without consuming a one-shot DB cursor twice. + batch_items = list(self._source.items()) + if self._leaf_drilldown: + from application.utils.librarian.leaf_drilldown import ( + count_resource_families, + ) + + arts: List[str] = [] + for raw in batch_items: + if isinstance(raw, dict): + arts.append(str(raw.get("artifact_id") or "")) + else: + arts.append(str(getattr(raw, "artifact_id", "") or "")) + self._resource_family_counts = count_resource_families(arts) + else: + self._resource_family_counts = {} + + for item in batch_items: total += 1 row_id = _row_id(item) label = _source_label(item) @@ -387,7 +529,7 @@ def run(self, *, at: datetime) -> RunResult: } ) - # Lever C: grounded shortlist judge (Gemini) — pick top-2 + # Lever C: grounded shortlist judge — pick up to max_picks # from the retrieval allowlist only; fail open on errors. focus = ( focus_query_text(section.text) @@ -405,6 +547,7 @@ def run(self, *, at: datetime) -> RunResult: if judge_enabled(): judge_query = focus or section.text + judge_cap = self._shortlist_judge_max_picks judged = judge_shortlist( judge_query, candidates_from_audit(retrieved), @@ -412,10 +555,14 @@ def run(self, *, at: datetime) -> RunResult: cache=ShortlistJudgeCache( disk_dir=default_cache_dir() ), + max_picks=judge_cap, ) if judged: preferred = prefer_ids_first( - judged, preferred, limit=12, lead=2 + judged, + preferred, + limit=12, + lead=judge_cap, ) retrieved = retrieved.model_copy( update={ @@ -427,18 +574,32 @@ def run(self, *, at: datetime) -> RunResult: def _decide_from_audit(query_text: str): ranked_audit = self._reranker.rerank(query_text, retrieved) + # Relative margin cutoff after C.2 (promoted γ=0.85). + if self._margin_gamma is not None: + from application.utils.librarian.margin_cutoff import ( + apply_margin_gamma, + ) + + filtered = apply_margin_gamma( + ranked_audit.reranked or [], + gamma=self._margin_gamma, + ) + ranked_audit = ranked_audit.model_copy( + update={"reranked": filtered} + ) # Confidence stays on CE logits (fitted T); prefer may - # inject into reranked so focus/CE top-2 sees winners. + # inject into reranked so focus/CE lead window sees winners. ce_logits = [ float(c.score_rerank) for c in ranked_audit.reranked if c.score_rerank is not None ] + lead_n = self._shortlist_judge_max_picks if self._prefer_audit: ranked_audit = prefer_audit_ids( ranked_audit, preferred, - lead=2, + lead=lead_n, inject_missing=True, ) ranked_ids = vector_rerank_union_ids(ranked_audit) @@ -471,6 +632,24 @@ def _decide_from_audit(query_text: str): section.text ) +<<<<<<< HEAD +======= + query_for_drill = focus or section.text + pre_drill = list(cre_ids) + cre_ids = self._apply_leaf_drilldown( + cre_ids, + query_for_drill, + artifact_id=section.artifact_id, + section_text=section.text, + ) + # Only rewrite audit.reranked when drill mutated the + # shortlist — otherwise coarse arms (deny path) keep + # pre-drill scored preds bit-identical. + if list(cre_ids) != pre_drill: + audit = self._sync_audit_after_drilldown(audit, cre_ids) + + verdict = self._safety_guard.evaluate(section) +>>>>>>> 7208db90 (feat(librarian): promote d1-winner combo defaults (judge+cap8+top_k=3)) result = decide( confidence, cre_ids, diff --git a/application/utils/librarian/queue_runner.py b/application/utils/librarian/queue_runner.py index 7db3f26c4..697bb8813 100644 --- a/application/utils/librarian/queue_runner.py +++ b/application/utils/librarian/queue_runner.py @@ -209,6 +209,30 @@ def run_librarian_queue( pref_inject=config.pref_inject, prefer_audit=config.prefer_audit_ids, safety_guard=getattr(components, "safety_guard", None), +parent_index=getattr(components, "parent_index", None), + leaf_drilldown=bool(getattr(components, "leaf_drilldown", False)), + leaf_drilldown_resources=tuple( + getattr(components, "leaf_drilldown_resources", ()) or () + ), + leaf_drilldown_force_resources=tuple( + getattr(components, "leaf_drilldown_force_resources", ()) or () + ), + leaf_drilldown_min_children=int( + getattr(components, "leaf_drilldown_min_children", 3) or 3 + ), + leaf_drilldown_min_sections=int( + getattr(components, "leaf_drilldown_min_sections", 20) or 20 + ), + leaf_drilldown_keep_hub=bool( + getattr(components, "leaf_drilldown_keep_hub", False) + ), + leaf_drilldown_hub_first=bool( + getattr(components, "leaf_drilldown_hub_first", False) + ), + shortlist_judge_max_picks=int( + getattr(components, "shortlist_judge_max_picks", 3) or 3 + ), + margin_gamma=getattr(components, "margin_gamma", None),>>>>>>> 7208db90 (feat(librarian): promote d1-winner combo defaults (judge+cap8+top_k=3)) ) # A locked run must not leave its claim outstanding. `FOR UPDATE SKIP LOCKED` diff --git a/application/utils/librarian/shortlist_judge.py b/application/utils/librarian/shortlist_judge.py index 894c97c05..033d93d11 100644 --- a/application/utils/librarian/shortlist_judge.py +++ b/application/utils/librarian/shortlist_judge.py @@ -1,8 +1,9 @@ -"""Grounded shortlist judge: LLM picks top-1..2 CRE ids from a provided list only. +"""Grounded shortlist judge: LLM picks top-1..N CRE ids from a provided list only. Module C lever C — not a freeform linker. The model may only return ids that appear in the retrieval shortlist; invented ids are dropped. Failures (bad JSON, network, empty allowlist) return ``[]`` so the pipeline keeps its existing rank. +Default N is 3 (score top_k); override via ``max_picks`` / env. """ from __future__ import annotations @@ -22,13 +23,19 @@ DEFAULT_TOP_N = 15 DEFAULT_CACHE_DIR = "tmp/oie_shortlist_judge_cache" -_SYSTEM = ( - "You are a grounded CRE shortlist judge for OpenCRE. " - "Given a standards section focus text and a fixed list of candidate CREs, " - "pick the 1 or 2 best matching CRE ids. " - "Return ONLY a JSON array of cre_id strings drawn from the provided list. " - "Do not invent ids. Do not return names, objects, or commentary." -) + +def _system_prompt(max_picks: int) -> str: + n = max(1, int(max_picks)) + return ( + "You are a grounded CRE shortlist judge for OpenCRE. " + "Given a standards section focus text and a fixed list of candidate CREs, " + f"pick the 1–{n} best matching CRE ids. " + "Return ONLY a JSON array of cre_id strings drawn from the provided list. " + "Do not invent ids. Do not return names, objects, or commentary." + ) + + +_SYSTEM = _system_prompt(3) def _strip_fences(raw: str) -> str: @@ -39,11 +46,14 @@ def _strip_fences(raw: str) -> str: return text.strip() -def parse_judge_json(raw: str, allowlist: Sequence[str]) -> List[str]: - """Parse LLM output to 1–2 cre_ids present in ``allowlist`` (order preserved).""" +def parse_judge_json( + raw: str, allowlist: Sequence[str], *, max_picks: int = 3 +) -> List[str]: + """Parse LLM output to 1–N cre_ids present in ``allowlist`` (order preserved).""" allowed = {str(a) for a in allowlist if a} if not allowed: return [] + cap = max(1, int(max_picks)) try: data = json.loads(_strip_fences(raw)) except (TypeError, ValueError, json.JSONDecodeError): @@ -69,13 +79,16 @@ def parse_judge_json(raw: str, allowlist: Sequence[str]) -> List[str]: continue seen.add(cid) out.append(cid) - if len(out) >= 2: + if len(out) >= cap: break return out def build_judge_prompt( - query_text: str, candidates: Sequence[Mapping[str, Any]] + query_text: str, + candidates: Sequence[Mapping[str, Any]], + *, + max_picks: int = 3, ) -> tuple: """Return ``(system, user)`` for the grounded judge call.""" rows = [] @@ -85,19 +98,26 @@ def build_judge_prompt( continue name = str(c.get("name") or c.get("cre_name") or "").strip() rows.append({"cre_id": cid, "name": name}) + n = max(1, int(max_picks)) user = ( "Section focus:\n" f"{(query_text or '').strip()}\n\n" - "Candidates (choose 1–2 cre_id values from this list only):\n" + f"Candidates (choose 1–{n} cre_id values from this list only):\n" f"{json.dumps(rows, ensure_ascii=False)}\n\n" 'Respond with a JSON array, e.g. ["123-456", "789-012"].' ) - return _SYSTEM, user + return _system_prompt(n), user -def cache_key(query_text: str, candidate_ids: Sequence[str]) -> str: +def cache_key( + query_text: str, candidate_ids: Sequence[str], *, max_picks: int = 3 +) -> str: blob = json.dumps( - {"q": (query_text or "").strip(), "ids": list(candidate_ids)}, + { + "q": (query_text or "").strip(), + "ids": list(candidate_ids), + "max_picks": int(max_picks), + }, sort_keys=True, ensure_ascii=False, ) @@ -176,6 +196,16 @@ def candidates_from_audit( return out +def judge_max_picks() -> int: + """``CRE_LIBRARIAN_SHORTLIST_JUDGE_MAX_PICKS`` (default 3; capped to [1, 5]).""" + raw = os.environ.get("CRE_LIBRARIAN_SHORTLIST_JUDGE_MAX_PICKS", "3") + try: + n = int(raw) + except (TypeError, ValueError): + return 3 + return max(1, min(5, n)) + + def judge_shortlist( query_text: str, candidates: Sequence[Mapping[str, Any]], @@ -183,11 +213,13 @@ def judge_shortlist( *, cache: Optional[ShortlistJudgeCache] = None, top_n: int = DEFAULT_TOP_N, + max_picks: Optional[int] = None, ) -> List[str]: - """Ask the LLM for 1–2 cre_ids from ``candidates``; validate against allowlist. + """Ask the LLM for 1–N cre_ids from ``candidates``; validate against allowlist. Returns ``[]`` on missing input, LLM errors, or invalid JSON (fail-open). """ + cap = judge_max_picks() if max_picks is None else max(1, min(5, int(max_picks))) rows = [ { "cre_id": str(c.get("cre_id") or "").strip(), @@ -200,12 +232,12 @@ def judge_shortlist( return [] allowlist = [r["cre_id"] for r in rows] - key = cache_key(query_text, allowlist) + key = cache_key(query_text, allowlist, max_picks=cap) if cache is not None: hit = cache.get(key) if hit is not None: # Re-validate against current allowlist (cache may be stale). - return [cid for cid in hit if cid in set(allowlist)][:2] + return [cid for cid in hit if cid in set(allowlist)][:cap] fn = llm_fn if fn is None: @@ -215,10 +247,10 @@ def judge_shortlist( logger.warning("shortlist judge: cannot build default LLM", exc_info=True) return [] - system, user = build_judge_prompt(query_text, rows) + system, user = build_judge_prompt(query_text, rows, max_picks=cap) try: raw = fn(system, user) - chosen = parse_judge_json(raw, allowlist) + chosen = parse_judge_json(raw, allowlist, max_picks=cap) except Exception: # noqa: BLE001 logger.warning( "shortlist judge LLM failed; keeping existing rank", exc_info=True @@ -251,6 +283,7 @@ def judge_enabled() -> bool: "default_cache_dir", "default_litellm_fn", "judge_enabled", + "judge_max_picks", "judge_shortlist", "parse_judge_json", ] diff --git a/application/utils/librarian/umbrella_promote.py b/application/utils/librarian/umbrella_promote.py index 97d59fede..3bff4b16d 100644 --- a/application/utils/librarian/umbrella_promote.py +++ b/application/utils/librarian/umbrella_promote.py @@ -21,7 +21,20 @@ re.I, ) -_PROMOTE_CAP = 4 +_PROMOTE_CAP = 8 + + +def _promote_cap() -> int: + """``CRE_LIBRARIAN_UMBRELLA_PROMOTE_CAP`` (default 8; d1-winner E5).""" + import os + + raw = os.getenv("CRE_LIBRARIAN_UMBRELLA_PROMOTE_CAP") + if raw is None or not str(raw).strip(): + return _PROMOTE_CAP + try: + return max(1, int(raw)) + except ValueError: + return _PROMOTE_CAP def _norm_name(text: str) -> str: @@ -129,7 +142,7 @@ def promote_for_shortlist( pname = self.parent_names.get(parent, "") if _tokens_overlap(_significant_tokens(pname), title_toks): out.append(parent) - if len(out) >= _PROMOTE_CAP: + if len(out) >= _promote_cap(): break return out diff --git a/docs/oie/experimental-feature-flags.md b/docs/oie/experimental-feature-flags.md index 11eb725d9..7b6ed0229 100644 --- a/docs/oie/experimental-feature-flags.md +++ b/docs/oie/experimental-feature-flags.md @@ -1,34 +1,40 @@ # Experimental feature flags -These librarian levers and experiment YAMLs were evaluated on the OIE B2 harness -and **not** promoted for the exact Links bar. They remain in-tree as **dead code** -(env default off) so a later merge can keep the implementations without enabling -them. The promoted stack is only: +These librarian levers were evaluated on the OIE B2 / full-pipeline harness and +are **not** part of the shipped d1-winner defaults (or are default-off knobs for +follow-up). Implementations may remain in-tree behind env gates. -> **Merge note:** These experimental flags and dead-code paths will be **removed on merge** if they are not useful to keep (no production callers, no planned follow-up). Prefer deleting unused levers over shipping permanent dead code. +> **Merge note:** Unused experiment YAMLs / dead levers may be **removed on merge** +> if they have no production callers and no planned follow-up. -- `CRE_LIBRARIAN_CRE_SUMMARY=1` -- `CRE_LIBRARIAN_MARGIN_GAMMA=0.85` +## Promoted (see feature-flags.md) + +Current ship path is documented in [feature-flags.md](feature-flags.md): -See `docs/oie/` on the evaluation branch for the human-facing guide. +- `CRE_LIBRARIAN_CRE_SUMMARY=1` +- `CRE_LIBRARIAN_MARGIN_GAMMA=0.85` (wired in Module C) +- `CRE_LIBRARIAN_DUAL_INDEX=1` (code default on) +- `CRE_LIBRARIAN_SHORTLIST_JUDGE=1` + `MAX_PICKS=3` (full-pipeline) +- `CRE_LIBRARIAN_UMBRELLA_PROMOTE_CAP=8` +- `CRE_LIBRARIAN_LEAF_DRILLDOWN=1` (E1/E2/FORCE off) -## What was tried and did not win (exact Links) +## What was tried and did not win (or is opt-in only) Evidence layers: (A) 512-combo grid on prior canonical gold, -(B) one-factor Step 2/3 matrices, (C) hop soft scores. Prefer citing which layer. +(B) one-factor Step 2/3 matrices, (C) hop soft scores / cold-start. | Flag / stack | Exact evidence | Soft / notes | Verdict | |--------------|----------------|--------------|---------| -| `CRE_SUMMARY=0` | Grid mean 0.354 vs 0.389 with summary; max 0.377 vs 0.436 | — | Clear loser | -| `DUAL_INDEX=1` | Step2 32/62 vs summary 36/62; with summary, grid mean lower | Can raise neighborhood d1 while hurting exact | Not for Links bar | -| `USE_RRF=1` (needs dual) | Step2 34/62 < summary alone; RRF mean lower with dual | [Cormack et al. RRF](https://dl.acm.org/doi/10.1145/1571941.1572114) | Does not beat summary | -| `HUB_LEAF_CAGE=1` | Step3 28/62 (45.2%); grid hub=1 mean 0.359 vs 0.383 | High-risk allowlist | Hurts | -| `CONTEXT_ENRICH=1` alone | 22/62 (35.5%); grid ~flat | Prefix Standard/Section labels | No help as one-factor | -| `FOCUS_QUERY=1` | Grid mean 0.368 vs 0.375 off | First CE on stripped focus text | Neutral / slight drag | -| `CRE_TEXT_ENRICH=1` | Tied at top 0/1; mean slightly lower when on | Append linked standard prose to C.2 | Neutral | -| Hybrid nameheavy (β=3 / γ=0.15) | Tied max with Lawrence; mean lower | Title-heavy mix | Prefer Lawrence defaults | -| `MARGIN_GAMMA=0.85` vs off | Identical max/mean when SUMMARY=1 on grid | Still promoted for multi-link precision | Neutral on exact alone | -| `SHORTLIST_JUDGE=1` | Not in grid | Gemini hangs | Untested | +| `CRE_SUMMARY=0` | Grid mean 0.354 vs 0.389 with summary | — | Clear loser | +| `DUAL_INDEX=0` | Older Links-bar matrices preferred off | Product/d1 path ships dual **on** | Historical exact note only | +| `USE_RRF=1` (needs dual) | Step2 34/62 < summary alone | [Cormack et al. RRF](https://dl.acm.org/doi/10.1145/1571941.1572114) | Off | +| `HUB_LEAF_CAGE=1` | Step3 28/62 (45.2%) | High-risk allowlist | Off | +| `CONTEXT_ENRICH=1` alone | 22/62 (35.5%) | Prefix labels | Off | +| `FOCUS_QUERY=1` | Grid mean 0.368 vs 0.375 off | First CE on stripped focus | Off | +| `CRE_TEXT_ENRICH=1` | Tied at top; mean slightly lower | Append linked standard prose | Off | +| Hybrid nameheavy (β=3 / γ=0.15) | Tied max; mean lower | Prefer Lawrence defaults | Off | +| `LEAF_DRILLDOWN_FORCE_RESOURCES=api` | GitHub d1↑; API exact 60→50 | Cold-start post-60 | **Do not ship** | +| `LEAF_DRILLDOWN_KEEP_HUB` / `HUB_FIRST` | E1/E2 | Not in d1 winner | Off | | Trained CE | Deferred | — | Untested | Worst grid cells (~30.6% exact): SUMMARY=0 + hub_leaf=1 + nameheavy + cre_text_enrich. @@ -37,20 +43,27 @@ Worst grid cells (~30.6% exact): SUMMARY=0 + hub_leaf=1 + nameheavy + cre_text_e | File | Flag under test | |------|-----------------| -| `scripts/oie_owasp_eval/experiments/dual_index.yaml` | `CRE_LIBRARIAN_DUAL_INDEX` | +| `scripts/oie_owasp_eval/experiments/dual_index.yaml` | Historical dual-off matrix | | `scripts/oie_owasp_eval/experiments/rrf.yaml` | `CRE_LIBRARIAN_USE_RRF` (+ dual) | | `scripts/oie_owasp_eval/experiments/hub_leaf_cage.yaml` | `CRE_LIBRARIAN_HUB_LEAF_CAGE` | | `scripts/oie_owasp_eval/experiments/context_enrich.yaml` | `CRE_LIBRARIAN_CONTEXT_ENRICH` | -Implementation modules live in `application/utils/librarian/` (flag-gated, default -off). Experiment YAMLs under `scripts/oie_owasp_eval/experiments/` re-run the -non-winning stacks for research only — do not write live Links from them. +Implementation modules live in `application/utils/librarian/` (flag-gated). +Experiment YAMLs under `scripts/oie_owasp_eval/experiments/` re-run research +stacks only — do not write live Links from them. + +## Gold / overlay note + +Canonical LLM Top 10 mapping gold lives at +`application/tests/fixtures/owasp_mappings/owasp_llm_top10_2025.json` (hub AI +CREs, #1124). The copy under `scripts/oie_owasp_eval/fixtures/b2_gold/` is a +**non-scoring overlay** for rebuild helpers — do not treat it as the answer key. ## Theory pointers (why these existed) -- Dual index (header→names, body→summaries): multi-representation retrieval; length-based routing failed on several Top10 sections. +- Dual index (header→names, body→summaries): multi-representation retrieval. - RRF: fuse uncalibrated pools by rank ([SIGIR 2009](https://dl.acm.org/doi/10.1145/1571941.1572114)). -- Hub→leaf cage: restrict to Contains children of top hub hits — precise when hubs are right, brittle when hubs are wrong. -- Context enrich: weakly supervised query rewriting via labels; insufficient alone on this corpus. +- Hub→leaf cage: restrict to Contains children of top hub hits — brittle when hubs are wrong. +- Context enrich: weakly supervised query rewriting via labels; insufficient alone. ## Do not write live Links from these matrices diff --git a/docs/oie/feature-flags.md b/docs/oie/feature-flags.md index e0cdca0be..cae569db7 100644 --- a/docs/oie/feature-flags.md +++ b/docs/oie/feature-flags.md @@ -2,22 +2,28 @@ Environment variables for Module C live under the `CRE_LIBRARIAN_*` prefix (loader: `application/utils/librarian/config_loader.py`). This page lists what -**shipped evaluation** uses. Flags that were tried and did not improve the exact Links bar are documented in -[experimental-feature-flags.md](experimental-feature-flags.md) (in-tree dead code, -default off). +**shipped evaluation** / `run_full_pipeline.py` uses. Flags that were tried and +did not improve the product metrics are documented in +[experimental-feature-flags.md](experimental-feature-flags.md). -## Promoted stack +## Promoted stack (d1-winner + post-60) | Variable | Value | Why | |----------|-------|-----| -| `CRE_LIBRARIAN_CRE_SUMMARY` | `1` | Strongest exact-bar lever in the 512-combo grid and one-factor matrix. | -| `CRE_LIBRARIAN_MARGIN_GAMMA` | `0.85` | Relative cutoff after rerank; ties summary alone on exact, keeps multi-CRE sections without dumping the full shortlist. | +| `CRE_LIBRARIAN_CRE_SUMMARY` | `1` | Strongest exact-bar lever in the 512-combo grid. | +| `CRE_LIBRARIAN_MARGIN_GAMMA` | `0.85` | Relative cutoff after C.2; wired in `LibrarianPipeline`. | +| `CRE_LIBRARIAN_DUAL_INDEX` | `1` (code default) | Dual header/body index; on by default in `load_config`. | +| `CRE_LIBRARIAN_SHORTLIST_JUDGE` | `1` | Grounded judge (full-pipeline `setdefault`). | +| `CRE_LIBRARIAN_SHORTLIST_JUDGE_MAX_PICKS` | `3` | Aligns judge lead with score `top_k=3`. | +| `CRE_LIBRARIAN_UMBRELLA_PROMOTE_CAP` | `8` | E5 promote cap (d1 winner). | +| `CRE_LIBRARIAN_LEAF_DRILLDOWN` | `1` | Hub→leaf after C.2; grain gate `MIN_SECTIONS=20`. | +| `CRE_LIBRARIAN_LEAF_DRILLDOWN_KEEP_HUB` | `0` | E1 off. | +| `CRE_LIBRARIAN_LEAF_DRILLDOWN_HUB_FIRST` | `0` | E2 off. | +| `CRE_LIBRARIAN_LEAF_DRILLDOWN_FORCE_RESOURCES` | unset | `api` raised GitHub d1 but dropped API exact 60→50; leave off. | -YAML: `scripts/oie_owasp_eval/experiments/stack_winners.yaml`. +YAML (historical grid cell): `scripts/oie_owasp_eval/experiments/stack_winners.yaml`. -### Fixed defaults used in the winning grid cells - -These were held constant or selected among tied winners: +### Fixed defaults used with the promoted stack | Variable | Value | Notes | |----------|-------|-------| @@ -25,23 +31,30 @@ These were held constant or selected among tied winners: | `CRE_LIBRARIAN_PREF_INJECT` | on | Inject preferred CRE ids | | `CRE_LIBRARIAN_PREFER_AUDIT_IDS` | on | Prefer those ids in order | | `CRE_LIBRARIAN_HYBRID_BETA` | `0` | “Lawrence” mix | -| `CRE_LIBRARIAN_HYBRID_GAMMA` | `0.70` | Cross-encoder weight in the hybrid mix (≠ margin γ) | -| `CRE_LIBRARIAN_RETRIEVER_BACKEND` | `pgvector` | Live B2 | +| `CRE_LIBRARIAN_HYBRID_GAMMA` | `0.70` | Cross-encoder weight (≠ margin γ) | +| `CRE_LIBRARIAN_RETRIEVER_BACKEND` | `pgvector` | Live B2 / full-pipeline | | `CRE_LIBRARIAN_DEVICE` | `cpu` | Eval matrix | -| `CRE_LIBRARIAN_SHORTLIST_JUDGE` | `0` | Gemini judge hangs blocked judge-ON factorial | -| `CRE_LIBRARIAN_DUAL_INDEX` | off | See [experimental-feature-flags.md](experimental-feature-flags.md) | | `CRE_LIBRARIAN_USE_RRF` | off | See [experimental-feature-flags.md](experimental-feature-flags.md) | -| `CRE_LIBRARIAN_HUB_LEAF_CAGE` | off | See [experimental-feature-flags.md](experimental-feature-flags.md) | -| `CRE_LIBRARIAN_CONTEXT_ENRICH` | off | See [experimental-feature-flags.md](experimental-feature-flags.md) | +| `CRE_LIBRARIAN_HUB_LEAF_CAGE` | off | See experimental notes | +| `CRE_LIBRARIAN_CONTEXT_ENRICH` | off | See experimental notes | | `CRE_LIBRARIAN_FOCUS_QUERY` | off | Neutral / slight drag on grid mean | | `CRE_LIBRARIAN_CRE_TEXT_ENRICH` | off | Tied at top; mean slightly lower when on | -## Headline scores (promoted stack) +## Headline scores (cold-start post-60, hub LLM gold) + +Cold-start run `20261002T103821Z` (`tmp/oie_owasp_eval/full_pipeline_cold_start_post60/`) +used FORCE=`api` for that experiment. **Shipped default leaves FORCE unset** so API +exact stays on the ~60% lane; GitHub d1 is the product bar we optimize. + +| Arm | Exact | Notes | +|-----|------:|-------| +| GitHub (ASVS/AISVS/…) | 32.2% (65/202) | d1 ≈ 59.9% on that run | +| OWASP Top 10 | ~90% | B2 arm | +| OWASP API Top 10 | 50% with FORCE=`api`; ~60% with FORCE off | Do not ship FORCE | +| OWASP LLM Top 10 (hub AI gold, #1124) | 50% | Hub-aligned gold rewrite | -| Gold / method | Exact | Neighborhood (Related ∪ Contains, 1 hop) | -|---------------|------:|------------------------------------------:| -| AI-topic LLM gold, offline rescore of SUMMARY+margin preds | 33/62 (53.2%) | 52/62 (83.9%) | -| Earlier grid best cell (prior canonical gold) | 27/62 (43.55%) | — | +Older offline rescore on prior gold (SUMMARY+margin only): 33/62 exact / 52/62 +neighborhood — superseded for AIX by the hub gold + d1-winner stack above. Neighborhood definition: [Evaluation and metrics](evaluation-and-metrics.md). @@ -49,15 +62,23 @@ Neighborhood definition: ## Reproduce ```bash +# Matches scripts/oie_owasp_eval/run_full_pipeline.py _apply_promoted_flags export CRE_LIBRARIAN_CRE_SUMMARY=1 export CRE_LIBRARIAN_MARGIN_GAMMA=0.85 export CRE_LIBRARIAN_RETRIEVER_BACKEND=pgvector -export CRE_LIBRARIAN_SHORTLIST_JUDGE=0 +export CRE_LIBRARIAN_SHORTLIST_JUDGE=1 +export CRE_LIBRARIAN_SHORTLIST_JUDGE_MAX_PICKS=3 +export CRE_LIBRARIAN_UMBRELLA_PROMOTE_CAP=8 +export CRE_LIBRARIAN_LEAF_DRILLDOWN=1 +export CRE_LIBRARIAN_LEAF_DRILLDOWN_MIN_SECTIONS=20 +export CRE_LIBRARIAN_LEAF_DRILLDOWN_KEEP_HUB=0 +export CRE_LIBRARIAN_LEAF_DRILLDOWN_HUB_FIRST=0 +# leave CRE_LIBRARIAN_LEAF_DRILLDOWN_FORCE_RESOURCES unset export NO_LOAD_GRAPH_DB=1 -PYTHONPATH=. python scripts/oie_owasp_eval/run_experiment.py \ - --config scripts/oie_owasp_eval/experiments/stack_winners.yaml \ - --keep-all-knowledge +PYTHONPATH=. python scripts/oie_owasp_eval/run_full_pipeline.py \ + --cache-file "$CACHE" \ + --out tmp/oie_owasp_eval/full_pipeline_verify ``` ASVS 5 only (exclude other B2 families): @@ -76,6 +97,5 @@ Then score neighborhood with `score_b2_hop_distance.py`. ## Open items - Trained cross-encoder: deferred; not in the grid. -- Shortlist judge ON: blocked by provider hangs during factorial runs. - Live Module C re-run on every gold revision still required when fixtures change; - offline rescore reuses cached predictions. + see [evaluation-and-metrics.md](evaluation-and-metrics.md). diff --git a/docs/oie/full-pipeline.md b/docs/oie/full-pipeline.md index 79f230a83..0829e05b9 100644 --- a/docs/oie/full-pipeline.md +++ b/docs/oie/full-pipeline.md @@ -7,7 +7,7 @@ End-to-end evaluation for the four agreed families: | ASVS | GitHub `OWASP/ASVS` (`5.0/en`) | `owasp_asvs_5_0_provisional` | | AISVS | GitHub `OWASP/AISVS` (`1.0/en`) | `owasp_aisvs_1_0` | | Cheat Sheets | GitHub `OWASP/CheatSheetSeries` | `owasp_cheatsheets_supplement` | -| AI Exchange (AIX) | *Not* GitHub — hub / site CSV | `owasp_llm_top10_2025` (CRE proxy) | +| AI Exchange (AIX) | *Not* GitHub — hub / site CSV | `owasp_llm_top10_2025` (hub AI CREs; #1124) | | NIST 800-53 v5 | OSCAL prose (`b2_sources/nist_800_53_v5`) | `nist_800_53_v5` (hub Links) | | Top 10 2025 | Cached hyperlink text | `owasp_top10_2025` | | API Top 10 2023 | Cached hyperlink text | `owasp_api_top10_2023` | diff --git a/scripts/oie_owasp_eval/run_b2_pr_mappings.py b/scripts/oie_owasp_eval/run_b2_pr_mappings.py index b71e3dc8f..5c4683308 100644 --- a/scripts/oie_owasp_eval/run_b2_pr_mappings.py +++ b/scripts/oie_owasp_eval/run_b2_pr_mappings.py @@ -317,13 +317,17 @@ def load_gold(harness: Dict[str, Any]) -> List[Dict[str, Any]]: return load_owasp_mapping_fixture(str(harness["fixture_name"])) -def top2_cre_external_ids( - envelope: Dict[str, Any], uuid_to_ext: Dict[str, str] +def top_k_cre_external_ids( + envelope: Dict[str, Any], + uuid_to_ext: Dict[str, str], + *, + top_k: int = 3, ) -> List[str]: - """Union of rerank top-2 and vector top-2 (up to 4 ids). + """Union of rerank top-k and vector top-k (default 3 = d1-winner E6 gate). Hit if ≥1 of this union is in gold — CE cannot erase a strong cosine hit. """ + k = max(1, int(top_k)) ordered: List[str] = [] def add(cre_id: Optional[str]) -> None: @@ -337,7 +341,7 @@ def add(cre_id: Optional[str]) -> None: reranked = list(retrieval.get("reranked") or []) candidates = list(retrieval.get("candidates") or []) - for cand in reranked[:2]: + for cand in reranked[:k]: if isinstance(cand, dict): add(cand.get("cre_id")) @@ -346,7 +350,7 @@ def add(cre_id: Optional[str]) -> None: key=lambda c: float(c.get("score_vector") or 0.0), reverse=True, ) - for cand in by_vec[:2]: + for cand in by_vec[:k]: add(cand.get("cre_id")) if ordered: @@ -356,11 +360,18 @@ def add(cre_id: Optional[str]) -> None: for link in envelope.get(field) or []: if isinstance(link, dict): add(link.get("cre_id")) - if len(ordered) >= 2: - return ordered[:2] + if len(ordered) >= k: + return ordered[:k] return ordered +def top2_cre_external_ids( + envelope: Dict[str, Any], uuid_to_ext: Dict[str, str] +) -> List[str]: + """Backward-compatible alias for ``top_k_cre_external_ids(..., top_k=2)``.""" + return top_k_cre_external_ids(envelope, uuid_to_ext, top_k=2) + + def refresh_source_headers( harnesses: Sequence[Dict[str, Any]], ) -> Dict[str, Any]: @@ -609,8 +620,8 @@ def score_run(run_id: str, cache: str) -> Dict[str, Any]: resource = parts[1] sid = Path(parts[-1]).stem key = f"{resource}::{sid}" - top2 = top2_cre_external_ids(env, uuid_to_ext) - pred.setdefault(key, set()).update(t for t in top2 if t) + topk = top_k_cre_external_ids(env, uuid_to_ext) + pred.setdefault(key, set()).update(t for t in topk if t) meta[key] = {"resource": resource, "section_id": sid, "path": path} details: List[Dict[str, Any]] = [] diff --git a/scripts/oie_owasp_eval/run_d1_lever_experiments.py b/scripts/oie_owasp_eval/run_d1_lever_experiments.py new file mode 100644 index 000000000..45328d876 --- /dev/null +++ b/scripts/oie_owasp_eval/run_d1_lever_experiments.py @@ -0,0 +1,578 @@ +#!/usr/bin/env python3 +"""Six-lever product-d1 ablations vs MIN_SECTIONS=20 baseline. + +Arms (each alone vs baseline, then one combo of positive levers):: + + E1 CRE_LIBRARIAN_LEAF_DRILLDOWN_KEEP_HUB=1 + E2 CRE_LIBRARIAN_LEAF_DRILLDOWN_HUB_FIRST=1 + E3 B2 summary-cache fork (tmp/oie_cre_summaries_b2) + E4 CRE_LIBRARIAN_SHORTLIST_JUDGE=1 + E5 CRE_LIBRARIAN_UMBRELLA_PROMOTE_CAP=8 + E6 Offline score top_k=3 (rerank[:3] ∪ vector[:3]) — no Module C re-run + +Writes ``tmp/oie_owasp_eval/experiments/d1_levers/summary.json`` and prints +the winning combo env. Pick winner by highest product d1, then exact, with +hard floor exact ≥ 17%. + +Usage:: + + PYTHONPATH=. python scripts/oie_owasp_eval/run_d1_lever_experiments.py \\ + --cache_file postgresql://cre:password@127.0.0.1:5432/cre_prodclone + + # Offline E6 only (needs decision envelopes for --baseline-run-id): + PYTHONPATH=. python scripts/oie_owasp_eval/run_d1_lever_experiments.py \\ + --only E6 --baseline-run-id fullpipe-minsec-20260930T121607Z +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT_DIR = Path(__file__).resolve().parent +ART = ROOT / "tmp" / "oie_owasp_eval" +OUT_DIR = ART / "experiments" / "d1_levers" +GITHUB_CACHE = ROOT / "tmp" / "oie_cre_summaries" +B2_CACHE = ROOT / "tmp" / "oie_cre_summaries_b2" +MINSEC_REPORT = ART / "full_pipeline_min_sections" / "github_arms.b2_report.json" +MINSEC_METRICS = ART / "experiments" / "product_metric_min_sections.json" + +EXACT_FLOOR = 0.17 +BASELINE_D1 = 0.4603960396039604 +BASELINE_EXACT = 0.18316831683168316 + +BASELINE_ENV: Dict[str, str] = { + "CRE_LIBRARIAN_CRE_SUMMARY": "1", + "CRE_LIBRARIAN_MARGIN_GAMMA": "0.85", + "CRE_LIBRARIAN_RETRIEVER_BACKEND": "pgvector", + "CRE_LIBRARIAN_SHORTLIST_JUDGE": "0", + "CRE_LIBRARIAN_DEVICE": "cpu", + "CRE_LIBRARIAN_LEAF_DRILLDOWN": "1", + "CRE_LIBRARIAN_LEAF_DRILLDOWN_MIN_SECTIONS": "20", + "CRE_LIBRARIAN_LEAF_DRILLDOWN_MIN_CHILDREN": "3", + "CRE_LIBRARIAN_LEAF_DRILLDOWN_KEEP_HUB": "0", + "CRE_LIBRARIAN_LEAF_DRILLDOWN_HUB_FIRST": "0", + "CRE_LIBRARIAN_UMBRELLA_PROMOTE_CAP": "4", + "NO_LOAD_GRAPH_DB": "1", + "CRE_LIBRARIAN_CRE_SUMMARY_CACHE": str(GITHUB_CACHE), +} + +# Lever id → env overrides (relative to baseline). E3/E6 handled specially. +LEVER_ENV: Dict[str, Dict[str, str]] = { + "E1": {"CRE_LIBRARIAN_LEAF_DRILLDOWN_KEEP_HUB": "1"}, + "E2": {"CRE_LIBRARIAN_LEAF_DRILLDOWN_HUB_FIRST": "1"}, + "E4": {"CRE_LIBRARIAN_SHORTLIST_JUDGE": "1"}, + "E5": {"CRE_LIBRARIAN_UMBRELLA_PROMOTE_CAP": "8"}, +} + + +def _load_dotenv() -> None: + for env_path in (ROOT / ".env", ROOT / "tmp" / "oie.env"): + if not env_path.is_file(): + continue + for line in env_path.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, _, v = line.partition("=") + if env_path.name == "oie.env" and ( + k.startswith("CRE_LIBRARIAN_") or k.startswith("DEV_DATABASE") + ): + os.environ[k.strip()] = v.strip() + else: + os.environ.setdefault(k.strip(), v.strip().strip('"').strip("'")) + + +def _py() -> str: + venv = ROOT / "venv" / "bin" / "python" + return str(venv) if venv.is_file() else sys.executable + + +def _merge_env(overrides: Mapping[str, str]) -> Dict[str, str]: + env = os.environ.copy() + for k, v in BASELINE_ENV.items(): + env[k] = v + for k, v in overrides.items(): + env[k] = v + env["PYTHONPATH"] = str(ROOT) + env.setdefault("FLASK_CONFIG", "development") + return env + + +def _hop_metrics(report_path: Path) -> Dict[str, float]: + """Exact + product d1 from a b2-shaped report via hop_distance.""" + sys.path.insert(0, str(ROOT)) + sys.path.insert(0, str(SCRIPT_DIR)) + from hop_distance import load_adjacency, score_details # noqa: E402 + from score_b2_hop_distance import ensure_adjacency # noqa: E402 + + adj_path = ART / "experiments" / "hop_analysis" / "cre_adjacency.json.gz" + if adj_path.is_file(): + adj = load_adjacency(adj_path) + else: + adj = ensure_adjacency(force=False, db_url=os.environ.get("DEV_DATABASE_URL")) + report = json.loads(report_path.read_text()) + scored = score_details(report.get("details") or [], adj) + rates = scored.get("rates") or {} + return { + "exact": float(rates.get("exact") or 0.0), + "d1": float(rates.get("distance1_hit") or 0.0), + "scorable": float(scored.get("scorable") or 0), + "b2_accuracy": float(report.get("accuracy") or 0.0), + "b2_hits": float(report.get("hits") or 0), + } + + +def _baseline_from_disk() -> Dict[str, Any]: + if MINSEC_METRICS.is_file(): + data = json.loads(MINSEC_METRICS.read_text()) + gh = (data.get("arms") or {}).get("github") or {} + return { + "id": "baseline", + "exact": float(gh.get("exact") or BASELINE_EXACT), + "d1": float(gh.get("d1") or BASELINE_D1), + "scorable": float(gh.get("scorable") or 202), + "source": str(MINSEC_METRICS.relative_to(ROOT)), + } + return { + "id": "baseline", + "exact": BASELINE_EXACT, + "d1": BASELINE_D1, + "scorable": 202.0, + "source": "hardcoded", + } + + +def _run_github_arm( + *, + arm_id: str, + cache: str, + overrides: Mapping[str, str], + out_dir: Path, + top_k: int = 2, +) -> Dict[str, Any]: + """Run Module A→B→C for ASVS+AISVS and score (+ hop d1).""" + run_id = f"d1-{arm_id.lower()}-" + datetime.now(timezone.utc).strftime( + "%Y%m%dT%H%M%SZ" + ) + arm_out = out_dir / arm_id.lower() + arm_out.mkdir(parents=True, exist_ok=True) + env = _merge_env(overrides) + env["DEV_DATABASE_URL"] = cache + env["DATABASE_URL"] = cache + env["CRE_LIBRARIAN_CRE_SUMMARY_CACHE"] = str(GITHUB_CACHE) + GITHUB_CACHE.mkdir(parents=True, exist_ok=True) + + cmd = [ + _py(), + str(SCRIPT_DIR / "run_full_pipeline.py"), + "--repos", + "asvs,aisvs", + "--cache_file", + cache, + "--run-id", + run_id, + "--keep-all-knowledge", + "--out-dir", + str(arm_out), + ] + print(f"=== {arm_id} GitHub C run_id={run_id} ===", flush=True) + print(" ".join(cmd), flush=True) + ret = subprocess.call(cmd, cwd=str(ROOT), env=env, start_new_session=True) + report_path = arm_out / "github_arms.b2_report.json" + if not report_path.is_file(): + return { + "id": arm_id, + "run_id": run_id, + "error": f"no github report (exit={ret})", + "exact": None, + "d1": None, + } + + # Re-score when ≠ pipeline default (d1-winner gate is top_k=3). + if top_k != 3: + sys.path.insert(0, str(ROOT)) + sys.path.insert(0, str(SCRIPT_DIR)) + import run_full_pipeline as rfp # noqa: E402 + + github = {k: rfp.GITHUB_TARGETS[k] for k in ("asvs", "aisvs")} + report = rfp.score_github_decisions(run_id, cache, github, top_k=top_k) + alt = arm_out / f"github_arms.topk{top_k}.b2_report.json" + rfp._write_b2_shaped_report(report, alt) + report_path = alt + + metrics = _hop_metrics(report_path) + return { + "id": arm_id, + "run_id": run_id, + "exit_code": ret, + "report": str(report_path.relative_to(ROOT)), + "top_k": top_k, + "env_overrides": dict(overrides), + **metrics, + } + + +def _run_e6_offline( + *, + cache: str, + baseline_run_id: str, + out_dir: Path, +) -> Dict[str, Any]: + """Rescore existing decisions with top_k=3 (no Module C).""" + sys.path.insert(0, str(ROOT)) + sys.path.insert(0, str(SCRIPT_DIR)) + from application import sqla + from application.cmd.cre_main import db_connect + from application.database.db import DecisionQueueItem + import run_full_pipeline as rfp # noqa: E402 + + db_connect(cache) + n = ( + sqla.session.query(DecisionQueueItem) + .filter_by(pipeline_run_id=baseline_run_id) + .count() + ) + if n <= 0: + return { + "id": "E6", + "error": f"no decisions for run_id={baseline_run_id!r}; need refresh C", + "exact": None, + "d1": None, + "needs_refresh": True, + } + + github = {k: rfp.GITHUB_TARGETS[k] for k in ("asvs", "aisvs")} + report = rfp.score_github_decisions(baseline_run_id, cache, github, top_k=3) + arm_out = out_dir / "e6" + arm_out.mkdir(parents=True, exist_ok=True) + report_path = arm_out / "github_arms.topk3.b2_report.json" + rfp._write_b2_shaped_report(report, report_path) + metrics = _hop_metrics(report_path) + return { + "id": "E6", + "run_id": baseline_run_id, + "report": str(report_path.relative_to(ROOT)), + "top_k": 3, + "offline": True, + **metrics, + } + + +def _run_e3_b2_fork( + *, + cache: str, + out_dir: Path, + github_run_id: str, +) -> Dict[str, Any]: + """After a GitHub pass, fork summary cache and score Top10+API B2.""" + sys.path.insert(0, str(ROOT)) + sys.path.insert(0, str(SCRIPT_DIR)) + import run_full_pipeline as rfp # noqa: E402 + + arm_out = out_dir / "e3" + arm_out.mkdir(parents=True, exist_ok=True) + # Fresh fork from GitHub cache after GitHub pass warmed it. + if B2_CACHE.exists(): + shutil.rmtree(B2_CACHE) + rfp.ensure_b2_summary_cache( + github_cache=GITHUB_CACHE, b2_cache=B2_CACHE, force=True + ) + + results: Dict[str, Any] = { + "id": "E3", + "b2_arms": {}, + "github_run_id": github_run_id, + } + env_before = os.environ.get("CRE_LIBRARIAN_CRE_SUMMARY_CACHE") + try: + os.environ["CRE_LIBRARIAN_CRE_SUMMARY_CACHE"] = str(B2_CACHE) + os.environ["DEV_DATABASE_URL"] = cache + os.environ["DATABASE_URL"] = cache + for arm, stems, out_name in ( + ("top10", ["owasp_top10_2025"], "full_pipeline_top10.b2_report.json"), + ("api", ["owasp_api_top10_2023"], "full_pipeline_api.b2_report.json"), + ): + try: + report = rfp._run_b2_arm( + run_id=github_run_id, + cache=cache, + keep_all=True, + fixtures=stems, + arm_name=f"e3-{arm}", + out_name=out_name, + ) + except Exception as exc: # noqa: BLE001 + results["b2_arms"][arm] = {"error": str(exc)} + continue + path = arm_out / out_name + path.write_text(json.dumps(report, indent=2) + "\n") + try: + metrics = _hop_metrics(path) + except Exception as exc: # noqa: BLE001 + metrics = {"error": str(exc), "b2_accuracy": report.get("accuracy")} + results["b2_arms"][arm] = { + "accuracy": report.get("accuracy"), + "hits": report.get("hits"), + "scorable": report.get("scorable"), + **{k: metrics.get(k) for k in ("exact", "d1") if k in metrics}, + } + finally: + rfp.restore_github_summary_cache(github_cache=GITHUB_CACHE) + if env_before is not None: + os.environ["CRE_LIBRARIAN_CRE_SUMMARY_CACHE"] = env_before + + # E3 is a B2 isolation lever; GitHub exact/d1 unchanged vs its parent run. + results["exact"] = None + results["d1"] = None + results["note"] = "B2 cache fork; GitHub metrics unchanged vs parent arm" + return results + + +def _positive(arm: Mapping[str, Any], baseline: Mapping[str, Any]) -> bool: + exact = arm.get("exact") + d1 = arm.get("d1") + if exact is None or d1 is None: + return False + if float(exact) < EXACT_FLOOR: + return False + return float(d1) > float(baseline.get("d1") or 0.0) + + +def _pick_winner( + arms: Sequence[Mapping[str, Any]], baseline: Mapping[str, Any] +) -> Dict[str, Any]: + candidates: List[Mapping[str, Any]] = [] + for arm in arms: + if arm.get("id") in ("baseline", "E3"): + continue + exact = arm.get("exact") + d1 = arm.get("d1") + if exact is None or d1 is None: + continue + if float(exact) < EXACT_FLOOR: + continue + candidates.append(arm) + # Always consider baseline itself. + candidates.append(baseline) + ranked = sorted( + candidates, + key=lambda a: ( + float(a.get("d1") or 0.0), + float(a.get("exact") or 0.0), + ), + reverse=True, + ) + return dict(ranked[0]) + + +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--cache_file", + default=os.environ.get("DEV_DATABASE_URL") + or "postgresql://cre:password@127.0.0.1:5432/cre_prodclone", + ) + parser.add_argument( + "--only", + default="", + help="Comma list of arms to run (E1,E2,E3,E4,E5,E6,combo). Empty=all.", + ) + parser.add_argument( + "--baseline-run-id", + default="", + help="Decision-queue run_id for offline E6 (default: from minsec summary).", + ) + parser.add_argument( + "--skip-combo", + action="store_true", + help="Do not run the combined positive-lever GitHub pass.", + ) + parser.add_argument( + "--out-dir", + type=Path, + default=OUT_DIR, + ) + args = parser.parse_args(list(argv) if argv is not None else None) + + _load_dotenv() + os.environ.setdefault("FLASK_CONFIG", "development") + os.environ.setdefault("NO_LOAD_GRAPH_DB", "1") + sys.path.insert(0, str(ROOT)) + + out_dir: Path = args.out_dir + out_dir.mkdir(parents=True, exist_ok=True) + cache = args.cache_file + os.environ["DEV_DATABASE_URL"] = cache + os.environ["DATABASE_URL"] = cache + + only = {x.strip().upper() for x in args.only.split(",") if x.strip()} + want = lambda x: (not only) or (x.upper() in only) # noqa: E731 + + baseline = _baseline_from_disk() + arms: List[Dict[str, Any]] = [baseline] + github_parent_run_id = "" + + # --- E6 offline first (instant when decisions present) --- + if want("E6"): + baseline_run_id = args.baseline_run_id.strip() + if not baseline_run_id and MINSEC_REPORT.is_file(): + baseline_run_id = str( + json.loads(MINSEC_REPORT.read_text()).get("run_id") or "" + ) + if not baseline_run_id: + baseline_run_id = "fullpipe-minsec-20260930T121607Z" + e6 = _run_e6_offline( + cache=cache, baseline_run_id=baseline_run_id, out_dir=out_dir + ) + if e6.get("needs_refresh"): + # One baseline GitHub refresh → score k=2 confirm + k=3 as E6. + print("E6: refreshing baseline GitHub C for envelopes…", flush=True) + refresh = _run_github_arm( + arm_id="baseline_refresh", + cache=cache, + overrides={}, + out_dir=out_dir, + top_k=2, + ) + arms.append(refresh) + github_parent_run_id = str(refresh.get("run_id") or "") + e6 = _run_e6_offline( + cache=cache, + baseline_run_id=github_parent_run_id, + out_dir=out_dir, + ) + arms.append(e6) + print( + f"E6 exact={e6.get('exact')} d1={e6.get('d1')} err={e6.get('error')}", + flush=True, + ) + + # --- Sequential GitHub levers --- + for lid in ("E1", "E2", "E4", "E5"): + if not want(lid): + continue + result = _run_github_arm( + arm_id=lid, + cache=cache, + overrides=LEVER_ENV[lid], + out_dir=out_dir, + ) + arms.append(result) + github_parent_run_id = str(result.get("run_id") or github_parent_run_id) + print( + f"{lid} exact={result.get('exact')} d1={result.get('d1')} " + f"err={result.get('error')}", + flush=True, + ) + + # --- E3 B2 cache fork (needs a GitHub parent run) --- + if want("E3"): + parent = github_parent_run_id + if not parent: + # Run a cheap parent if nothing else produced one. + parent_arm = _run_github_arm( + arm_id="e3_parent", + cache=cache, + overrides={}, + out_dir=out_dir, + ) + arms.append(parent_arm) + parent = str(parent_arm.get("run_id") or "") + e3 = _run_e3_b2_fork(cache=cache, out_dir=out_dir, github_run_id=parent) + arms.append(e3) + print(f"E3 b2_arms={json.dumps(e3.get('b2_arms'))}", flush=True) + + # --- Combo of positive GitHub levers --- + positive_ids = [ + a["id"] for a in arms if a.get("id") in LEVER_ENV and _positive(a, baseline) + ] + # E6 positive? include score top_k=3 in combo via env note only (scorer flag). + e6_arm = next((a for a in arms if a.get("id") == "E6"), None) + combo_top_k = 3 if e6_arm and _positive(e6_arm, baseline) else 2 + + combo_env: Dict[str, str] = {} + for pid in positive_ids: + combo_env.update(LEVER_ENV[pid]) + + if want("COMBO") and not args.skip_combo and (combo_env or combo_top_k != 2): + combo = _run_github_arm( + arm_id="combo", + cache=cache, + overrides=combo_env, + out_dir=out_dir, + top_k=combo_top_k, + ) + combo["combo_env"] = combo_env + combo["positive_levers"] = positive_ids + arms.append(combo) + print( + f"COMBO exact={combo.get('exact')} d1={combo.get('d1')} " + f"env={combo_env} top_k={combo_top_k}", + flush=True, + ) + else: + combo = { + "id": "combo", + "skipped": True, + "combo_env": combo_env, + "positive_levers": positive_ids, + "top_k": combo_top_k, + } + arms.append(combo) + + winner = _pick_winner( + [a for a in arms if a.get("id") not in ("E3",) or a.get("exact") is not None], + baseline, + ) + # Prefer combo when it beats baseline under the floor. + combo_arm = next((a for a in arms if a.get("id") == "combo"), None) + if ( + combo_arm + and combo_arm.get("exact") is not None + and float(combo_arm["exact"]) >= EXACT_FLOOR + and float(combo_arm.get("d1") or 0) >= float(winner.get("d1") or 0) + ): + winner = dict(combo_arm) + + summary = { + "created_at": datetime.now(timezone.utc).isoformat(), + "exact_floor": EXACT_FLOOR, + "baseline": baseline, + "arms": arms, + "positive_levers": positive_ids, + "combo_env": combo_env, + "combo_top_k": combo_top_k, + "winner": { + "id": winner.get("id"), + "exact": winner.get("exact"), + "d1": winner.get("d1"), + "env": winner.get("env_overrides") + or winner.get("combo_env") + or (combo_env if winner.get("id") == "combo" else {}), + "top_k": winner.get("top_k") + or (combo_top_k if winner.get("id") == "combo" else 2), + }, + } + summary_path = out_dir / "summary.json" + summary_path.write_text(json.dumps(summary, indent=2) + "\n") + print(json.dumps(summary, indent=2), flush=True) + print(f"wrote {summary_path}", flush=True) + wenv = summary["winner"].get("env") or {} + print( + "WINNER_ENV:", " ".join(f"{k}={v}" for k, v in sorted(wenv.items())), flush=True + ) + print(f"WINNER_TOP_K={summary['winner'].get('top_k')}", flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/oie_owasp_eval/run_d1_levers_detached.sh b/scripts/oie_owasp_eval/run_d1_levers_detached.sh new file mode 100755 index 000000000..40e66c718 --- /dev/null +++ b/scripts/oie_owasp_eval/run_d1_levers_detached.sh @@ -0,0 +1,29 @@ +#!/bin/zsh +# Detached six-lever d1 experiment runner (survives terminal close). +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +cd "$ROOT" +set -a +# shellcheck disable=SC1091 +[[ -f .env ]] && source .env +set +a +export PYTHONPATH=. FLASK_CONFIG=development NO_LOAD_GRAPH_DB=1 +RUN_TS="$(date -u +%Y%m%dT%H%M%SZ)" +mkdir -p logs tmp/oie_owasp_eval/experiments/d1_levers +LOG="logs/d1_levers_${RUN_TS}.log" +echo "$RUN_TS" > logs/d1_levers_latest.runid +echo "$LOG" > logs/d1_levers_latest.logpath +echo "START $RUN_TS $(date -u)" | tee "$LOG" +# setsid + nohup so nested Module C subprocesses are not SIGHUP'd +if command -v setsid >/dev/null 2>&1; then + WRAP=(setsid) +else + WRAP=() +fi +${WRAP[@]} nohup ./venv/bin/python -u scripts/oie_owasp_eval/run_d1_lever_experiments.py \ + --cache_file 'postgresql://cre:password@127.0.0.1:5432/cre_prodclone' \ + "$@" \ + >>"$LOG" 2>&1 & +echo $! | tee logs/d1_levers_latest.pid +disown || true +echo "LOG=$LOG pid=$(cat logs/d1_levers_latest.pid)" diff --git a/scripts/oie_owasp_eval/run_full_pipeline.py b/scripts/oie_owasp_eval/run_full_pipeline.py index 4b3bdefdf..f5a27d295 100644 --- a/scripts/oie_owasp_eval/run_full_pipeline.py +++ b/scripts/oie_owasp_eval/run_full_pipeline.py @@ -186,9 +186,62 @@ def _apply_promoted_flags() -> None: os.environ.setdefault("CRE_LIBRARIAN_CRE_SUMMARY", "1") os.environ.setdefault("CRE_LIBRARIAN_MARGIN_GAMMA", "0.85") os.environ.setdefault("CRE_LIBRARIAN_RETRIEVER_BACKEND", "pgvector") - os.environ.setdefault("CRE_LIBRARIAN_SHORTLIST_JUDGE", "0") + # d1 winner combo: E4 shortlist judge + E5 umbrella cap=8; E1/E2 off. + os.environ.setdefault("CRE_LIBRARIAN_SHORTLIST_JUDGE", "1") + # Post-60: judge fills score top_k=3 window (FORCE_RESOURCES left off below). + os.environ.setdefault("CRE_LIBRARIAN_SHORTLIST_JUDGE_MAX_PICKS", "3") os.environ.setdefault("CRE_LIBRARIAN_DEVICE", "cpu") + os.environ.setdefault("CRE_LIBRARIAN_LEAF_DRILLDOWN", "1") + # Grain gate: drill only when resource family has ≥20 chunks (ASVS/AISVS). + os.environ.setdefault("CRE_LIBRARIAN_LEAF_DRILLDOWN_MIN_SECTIONS", "20") + os.environ.setdefault("CRE_LIBRARIAN_LEAF_DRILLDOWN_MIN_CHILDREN", "3") + os.environ.setdefault("CRE_LIBRARIAN_LEAF_DRILLDOWN_KEEP_HUB", "0") + os.environ.setdefault("CRE_LIBRARIAN_LEAF_DRILLDOWN_HUB_FIRST", "0") + # FORCE_RESOURCES=api dropped API exact 60→50 on cold-start; leave unset/off. + os.environ.setdefault("CRE_LIBRARIAN_UMBRELLA_PROMOTE_CAP", "8") os.environ.setdefault("NO_LOAD_GRAPH_DB", "1") + # Shared CRE summary cache (GitHub). B2 may fork via ensure_b2_summary_cache. + os.environ.setdefault( + "CRE_LIBRARIAN_CRE_SUMMARY_CACHE", + str(ROOT / "tmp" / "oie_cre_summaries"), + ) + + +def ensure_b2_summary_cache( + *, + github_cache: Optional[Path] = None, + b2_cache: Optional[Path] = None, + force: bool = False, +) -> Path: + """E3: copy-on-write GitHub summary cache → B2-only cache directory. + + Returns the B2 cache path and sets ``CRE_LIBRARIAN_CRE_SUMMARY_CACHE`` to it. + """ + import shutil + + src = Path(github_cache or (ROOT / "tmp" / "oie_cre_summaries")) + dst = Path(b2_cache or (ROOT / "tmp" / "oie_cre_summaries_b2")) + if force and dst.exists(): + shutil.rmtree(dst) + if not dst.exists(): + dst.parent.mkdir(parents=True, exist_ok=True) + if src.is_dir(): + shutil.copytree(src, dst) + else: + dst.mkdir(parents=True, exist_ok=True) + os.environ["CRE_LIBRARIAN_CRE_SUMMARY_CACHE"] = str(dst) + return dst + + +def restore_github_summary_cache( + *, + github_cache: Optional[Path] = None, +) -> Path: + """Point env back at the shared GitHub summary cache.""" + path = Path(github_cache or (ROOT / "tmp" / "oie_cre_summaries")) + path.mkdir(parents=True, exist_ok=True) + os.environ["CRE_LIBRARIAN_CRE_SUMMARY_CACHE"] = str(path) + return path def _make_tarball_harvester( @@ -327,12 +380,19 @@ def _cheatsheet_section_id(row: Dict[str, Any]) -> str: return name.replace(" ", "_") -def _top2_cre_ids(envelope: Dict[str, Any], uuid_to_ext: Dict[str, str]) -> List[str]: - """Union of rerank top-2 and vector top-2 (up to 4 ids). +def _top_k_cre_ids( + envelope: Dict[str, Any], + uuid_to_ext: Dict[str, str], + *, + top_k: int = 3, +) -> List[str]: + """Union of rerank top-k and vector top-k. - Keep in sync with ``run_b2_pr_mappings.top2_cre_external_ids`` — full-pipeline - GitHub scoring must use the same gate as the B2 fixture arm. + Keep in sync with ``run_b2_pr_mappings.top_k_cre_external_ids`` — full-pipeline + GitHub scoring must use the same gate as the B2 fixture arm. Default ``top_k=3`` + is the d1-winner (E6) production gate. """ + k = max(1, int(top_k)) ordered: List[str] = [] def add(cre_id: Any) -> None: @@ -349,7 +409,7 @@ def add(cre_id: Any) -> None: reranked = list(retrieval.get("reranked") or []) candidates = list(retrieval.get("candidates") or []) - for cand in reranked[:2]: + for cand in reranked[:k]: if isinstance(cand, dict): add(cand.get("cre_id")) @@ -358,7 +418,7 @@ def add(cre_id: Any) -> None: key=lambda c: float(c.get("score_vector") or 0.0), reverse=True, ) - for cand in by_vec[:2]: + for cand in by_vec[:k]: add(cand.get("cre_id")) if ordered: @@ -368,11 +428,16 @@ def add(cre_id: Any) -> None: for link in envelope.get(field) or []: if isinstance(link, dict): add(link.get("cre_id")) - if len(ordered) >= 2: - return ordered[:2] + if len(ordered) >= k: + return ordered[:k] return ordered +def _top2_cre_ids(envelope: Dict[str, Any], uuid_to_ext: Dict[str, str]) -> List[str]: + """Backward-compatible alias for ``_top_k_cre_ids(..., top_k=2)``.""" + return _top_k_cre_ids(envelope, uuid_to_ext, top_k=2) + + def _guess_keys_for_decision( *, path: str, @@ -434,6 +499,8 @@ def score_github_decisions( run_id: str, cache: str, github_targets: Dict[str, Tuple[str, str, List[str], Optional[str]]], + *, + top_k: int = 3, ) -> Dict[str, Any]: from application import sqla from application.cmd.cre_main import db_connect @@ -478,7 +545,7 @@ def score_github_decisions( repo = str(src.get("repository") or src.get("repo") or src.get("name") or "") if not repo: repo = str(env.get("artifact_id") or path) - suggested = set(_top2_cre_ids(env, uuid_to_ext)) + suggested = set(_top_k_cre_ids(env, uuid_to_ext, top_k=top_k)) if not suggested and not path: continue keys = _guess_keys_for_decision(path=path, text=text, repo=repo) @@ -550,6 +617,7 @@ def score_github_decisions( return { "run_id": run_id, "mode": "github_aligned", + "top_k": int(top_k), "decisions_total": len(decisions), "scorable": scorable, "unaligned": unaligned, @@ -731,13 +799,12 @@ def main(argv: Optional[Sequence[str]] = None) -> int: if github: print(f"=== Module A (tarball) + B + C run_id={run_id} ===", flush=True) db_connect(cache) + # knowledge_queue.content_hash is globally UNIQUE. Clearing only this + # run_id leaves prior-run rows in place, Module B then inserts=0 + # (all ON CONFLICT deduped), and Module C reads nothing → 0% accuracy. for model in (HarvestInput, KnowledgeQueueItem, DecisionQueueItem): - deleted = ( - sqla.session.query(model) - .filter_by(pipeline_run_id=run_id) - .delete(synchronize_session=False) - ) - print(f"cleared {model.__tablename__} for {run_id}: {deleted}", flush=True) + deleted = sqla.session.query(model).delete(synchronize_session=False) + print(f"cleared ALL {model.__tablename__}: {deleted}", flush=True) sqla.session.commit() harvester = _make_tarball_harvester(github) @@ -779,14 +846,26 @@ def main(argv: Optional[Sequence[str]] = None) -> int: stems = B2_ARMS[arm] print(f"=== B2 arm {arm} ({', '.join(stems)}) ===", flush=True) out_name = f"full_pipeline_{arm}.b2_report.json" - report = _run_b2_arm( - run_id=run_id, - cache=cache, - keep_all=args.keep_all_knowledge, - fixtures=stems, - arm_name=arm, - out_name=out_name, - ) + try: + report = _run_b2_arm( + run_id=run_id, + cache=cache, + keep_all=args.keep_all_knowledge, + fixtures=stems, + arm_name=arm, + out_name=out_name, + ) + except Exception as exc: + # NIST (and similar) may lack a registered B2 harness — skip, + # do not abort Top10/API after a successful GitHub E2E. + print(f"B2 arm {arm} failed (continuing): {exc}", flush=True) + b2_reports[arm] = { + "arm": arm, + "fixtures": stems, + "error": str(exc), + "accuracy": None, + } + continue b2_reports[arm] = report _write_b2_shaped_report(report, out_dir / out_name) print( diff --git a/scripts/oie_owasp_eval/run_fullpipe_detached.sh b/scripts/oie_owasp_eval/run_fullpipe_detached.sh new file mode 100755 index 000000000..db3f8c155 --- /dev/null +++ b/scripts/oie_owasp_eval/run_fullpipe_detached.sh @@ -0,0 +1,29 @@ +#!/bin/zsh +# Durable local full-pipeline runner (ASVS+AISVS GitHub + B2 arms). +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +cd "$ROOT" +set -a +# shellcheck disable=SC1091 +source .env +set +a +export PYTHONPATH=. FLASK_CONFIG=development +RUN_ID="fullpipe-$(date -u +%Y%m%dT%H%M%SZ)" +mkdir -p logs tmp/oie_owasp_eval/full_pipeline +echo "$RUN_ID" > logs/full_pipeline_latest.runid +LOG="logs/full_pipeline_${RUN_ID}.log" +echo "START $RUN_ID $(date -u)" | tee "$LOG" +./venv/bin/python -u scripts/oie_owasp_eval/run_full_pipeline.py \ + --repos asvs,aisvs,aix,nist,top10,api \ + --cache_file 'postgresql://cre:password@127.0.0.1:5432/cre_prodclone' \ + --run-id "$RUN_ID" \ + --keep-all-knowledge \ + --neighborhood \ + --out-dir tmp/oie_owasp_eval/full_pipeline \ + >>"$LOG" 2>&1 +echo "EXIT:$?" | tee -a "$LOG" +if [[ -f tmp/oie_owasp_eval/full_pipeline/summary.json ]]; then + echo '==== summary.json ====' | tee -a "$LOG" + cat tmp/oie_owasp_eval/full_pipeline/summary.json | tee -a "$LOG" +fi +echo FULL_PIPELINE_DONE | tee -a "$LOG"