From 583e98723799a4897d7ffe54496ff2636fc2c68d Mon Sep 17 00:00:00 2001 From: Spyros Date: Tue, 29 Sep 2026 11:50:20 +0200 Subject: [PATCH 1/2] feat(oie): B2 eval harness and CRE neighborhood scorer Harness runners, hop-distance scoring, and gold loaders without bulky source fixtures (those land in follow-up stacked PRs). --- .../tests/librarian/aix_llm_link_seed_test.py | 38 + application/tests/oie_b2_gold_loader_test.py | 73 + application/tests/oie_b2_hop_distance_test.py | 126 ++ .../oie_orchestrator/langgraph_pipeline.py | 273 ++++ .../utils/oie_orchestrator/pipeline.py | 30 +- .../oie_owasp_eval/clone_grid_worker_dbs.sh | 70 + .../b2_gold/owasp_agentic_ai_stub.json | 32 + scripts/oie_owasp_eval/hop_distance.py | 356 +++++ .../rebuild_b2_gold_from_hub.py | 183 +++ .../rebuild_b2_llm_gold_from_hub.py | 213 +++ scripts/oie_owasp_eval/run_b2_pr_mappings.py | 811 +++++++++++ .../run_exhaustive_switch_grid.py | 1250 +++++++++++++++++ .../run_exhaustive_switch_grid_parallel.py | 435 ++++++ scripts/oie_owasp_eval/run_experiment.py | 718 ++++++++++ scripts/oie_owasp_eval/run_module_c.py | 348 +++++ .../run_official_orchestrator_local.py | 437 ++++++ scripts/oie_owasp_eval/run_phase.py | 829 +++++++++++ .../run_qwen_outside_sandbox.sh | 26 + .../oie_owasp_eval/score_b1_opencre_api.py | 432 ++++++ .../oie_owasp_eval/score_b2_hop_distance.py | 302 ++++ .../seed_hub_links_from_b2_gold.py | 181 +++ scripts/run_oie_e2e_smoke.py | 1 + scripts/setup_oie.sh | 274 ++++ 23 files changed, 7435 insertions(+), 3 deletions(-) create mode 100644 application/tests/oie_b2_gold_loader_test.py create mode 100644 application/tests/oie_b2_hop_distance_test.py create mode 100644 application/utils/oie_orchestrator/langgraph_pipeline.py create mode 100755 scripts/oie_owasp_eval/clone_grid_worker_dbs.sh create mode 100644 scripts/oie_owasp_eval/fixtures/b2_gold/owasp_agentic_ai_stub.json create mode 100644 scripts/oie_owasp_eval/hop_distance.py create mode 100644 scripts/oie_owasp_eval/rebuild_b2_gold_from_hub.py create mode 100644 scripts/oie_owasp_eval/rebuild_b2_llm_gold_from_hub.py create mode 100644 scripts/oie_owasp_eval/run_b2_pr_mappings.py create mode 100644 scripts/oie_owasp_eval/run_exhaustive_switch_grid.py create mode 100755 scripts/oie_owasp_eval/run_exhaustive_switch_grid_parallel.py create mode 100644 scripts/oie_owasp_eval/run_experiment.py create mode 100644 scripts/oie_owasp_eval/run_module_c.py create mode 100644 scripts/oie_owasp_eval/run_official_orchestrator_local.py create mode 100644 scripts/oie_owasp_eval/run_phase.py create mode 100755 scripts/oie_owasp_eval/run_qwen_outside_sandbox.sh create mode 100644 scripts/oie_owasp_eval/score_b1_opencre_api.py create mode 100644 scripts/oie_owasp_eval/score_b2_hop_distance.py create mode 100644 scripts/oie_owasp_eval/seed_hub_links_from_b2_gold.py create mode 100755 scripts/setup_oie.sh diff --git a/application/tests/librarian/aix_llm_link_seed_test.py b/application/tests/librarian/aix_llm_link_seed_test.py index e1df1c646..0255d2f21 100644 --- a/application/tests/librarian/aix_llm_link_seed_test.py +++ b/application/tests/librarian/aix_llm_link_seed_test.py @@ -106,5 +106,43 @@ def retrieve(self, text, *, allowlist=None): self.assertEqual(audit.candidates[0].cre_id, "uuid-direct") +class GoldRebuildHelpersTest(unittest.TestCase): + def test_build_gold_rows_marks_gaps(self) -> None: + # Import from script path via runpy-style relative — keep pure helper copy. + import importlib.util + from pathlib import Path + + path = ( + Path(__file__).resolve().parents[3] + / "scripts" + / "oie_owasp_eval" + / "rebuild_b2_llm_gold_from_hub.py" + ) + spec = importlib.util.spec_from_file_location("rebuild_b2_llm_gold", path) + assert spec and spec.loader + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + + hub = { + "LLM01": ["686-110", "012-625"], + "LLM03": ["701-654"], + } + existing = { + "LLM01": { + "section": "Prompt Injection", + "hyperlink": "https://example.test/llm01", + } + } + rows, gaps = mod.build_gold_rows(hub, existing=existing) + self.assertEqual(len(rows), 10) + llm01 = next(r for r in rows if r["section_id"] == "LLM01") + self.assertEqual(llm01["cre_ids"], ["012-625", "686-110"]) + self.assertEqual(llm01["hyperlink"], "https://example.test/llm01") + self.assertIn("LLM06", gaps) + self.assertIn("LLM09", gaps) + empty = next(r for r in rows if r["section_id"] == "LLM06") + self.assertEqual(empty["cre_ids"], []) + + if __name__ == "__main__": unittest.main() diff --git a/application/tests/oie_b2_gold_loader_test.py b/application/tests/oie_b2_gold_loader_test.py new file mode 100644 index 000000000..002cf6fed --- /dev/null +++ b/application/tests/oie_b2_gold_loader_test.py @@ -0,0 +1,73 @@ +import importlib.util +import unittest +from pathlib import Path + +from application.utils import mapping_fixtures + +ROOT = Path(__file__).resolve().parents[2] + + +def _load_run_b2(): + path = ROOT / "scripts" / "oie_owasp_eval" / "run_b2_pr_mappings.py" + spec = importlib.util.spec_from_file_location("oie_run_b2_pr_mappings", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class TestOieB2GoldLoader(unittest.TestCase): + def test_shared_harnesses_load_from_mapping_fixtures(self) -> None: + mod = _load_run_b2() + shared = 0 + for harness in mod.HARNESSES: + gold = mod.load_gold(harness) + self.assertIsInstance(gold, list) + self.assertGreater(len(gold), 0) + if harness.get("local_gold"): + continue + shared += 1 + self.assertEqual( + gold, + mapping_fixtures.load_owasp_mapping_fixture(harness["fixture_name"]), + ) + self.assertGreaterEqual(shared, 6) + + +class TestOieB2SourceIdentityPrefix(unittest.TestCase): + def test_k8s_years_emit_standard_and_version(self) -> None: + mod = _load_run_b2() + h22 = next( + h + for h in mod.HARNESSES + if h.get("fixture_name") == "owasp_kubernetes_top10_2022" + ) + h25 = next( + h + for h in mod.HARNESSES + if h.get("fixture_name") == "owasp_kubernetes_top10_2025" + ) + self.assertEqual(mod.harness_year(h22), 2022) + self.assertEqual(mod.harness_year(h25), 2025) + row = {"section_id": "K06", "section": "Broken Authentication Mechanisms"} + prefix = mod.source_identity_prefix(row, h22) + self.assertIn("Standard: OWASP Kubernetes Top 10 2022", prefix) + self.assertIn("Version: 2022", prefix) + self.assertIn("Section-ID: K06", prefix) + # Version before narrative so year_from_text does not pick body years. + self.assertLess(prefix.index("Version:"), prefix.index("Section-ID:")) + + def test_strip_and_rewrite_keeps_body(self) -> None: + mod = _load_run_b2() + harness = { + "label": "OWASP Kubernetes Top 10 2025", + "fixture_name": "owasp_kubernetes_top10_2025", + } + row = {"section_id": "K01", "section": "Insecure Workload Configurations"} + old = "Source: K01\nSection: Insecure Workload Configurations\nSection-ID: K01\n\nBODY\n" + body = mod.strip_source_identity_prefix(old) + self.assertEqual(body.strip(), "BODY") + rewritten = mod.source_identity_prefix(row, harness) + body + self.assertTrue(rewritten.startswith("Standard:")) + self.assertIn("Version: 2025", rewritten) + self.assertIn("BODY", rewritten) diff --git a/application/tests/oie_b2_hop_distance_test.py b/application/tests/oie_b2_hop_distance_test.py new file mode 100644 index 000000000..f481dd141 --- /dev/null +++ b/application/tests/oie_b2_hop_distance_test.py @@ -0,0 +1,126 @@ +"""Unit tests for B2 CRE hop-distance classification (no DB).""" + +from __future__ import annotations + +import importlib.util +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] + + +def _load_hop(): + import sys + + path = ROOT / "scripts" / "oie_owasp_eval" / "hop_distance.py" + name = "oie_hop_distance" + if name in sys.modules: + return sys.modules[name] + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +class TestOieB2HopDistance(unittest.TestCase): + def setUp(self) -> None: + self.hop = _load_hop() + # Graph: + # parent --Contains--> child --Related--> cousin + # island (known, disconnected) + edges = [ + self.hop.TypedEdge("parent", "child", "CONTAINS_DOWN"), + self.hop.TypedEdge("child", "parent", "CONTAINS_UP"), + self.hop.TypedEdge("child", "cousin", "RELATED"), + self.hop.TypedEdge("cousin", "child", "RELATED"), + ] + self.adj = self.hop.CreAdjacency( + known_ids=frozenset({"parent", "child", "cousin", "island"}), + edges=edges, + ) + + def test_exact(self) -> None: + row = self.hop.classify_section( + gold=["child"], predicted=["child", "999-999"], adj=self.adj + ) + self.assertEqual(row["bucket"], "exact") + self.assertEqual(row["hop_count"], 0) + + def test_one_hop_related(self) -> None: + row = self.hop.classify_section( + gold=["child"], predicted=["cousin"], adj=self.adj + ) + self.assertEqual(row["bucket"], "one_hop_related") + self.assertEqual(row["hop_count"], 1) + self.assertEqual(row["path_penalty"], 2) + + def test_one_hop_contains_down(self) -> None: + row = self.hop.classify_section( + gold=["child"], predicted=["parent"], adj=self.adj + ) + self.assertEqual(row["bucket"], "one_hop_contains") + self.assertEqual(row["hop_count"], 1) + # parent -> child is CONTAINS_DOWN from parent? Wait: predicted=parent, gold=child + # path parent -> child is CONTAINS_DOWN, penalty 1 + self.assertEqual(row["path_penalty"], 1) + + def test_one_hop_contains_up(self) -> None: + row = self.hop.classify_section( + gold=["parent"], predicted=["child"], adj=self.adj + ) + self.assertEqual(row["bucket"], "one_hop_contains") + self.assertEqual(row["hop_count"], 1) + self.assertEqual(row["path_penalty"], 2) # CONTAINS_UP + + def test_multi_hop(self) -> None: + row = self.hop.classify_section( + gold=["parent"], predicted=["cousin"], adj=self.adj + ) + self.assertEqual(row["bucket"], "multi_hop") + self.assertEqual(row["hop_count"], 2) + + def test_hallucination_invented_only(self) -> None: + row = self.hop.classify_section( + gold=["child"], predicted=["zzz-001", "zzz-002"], adj=self.adj + ) + self.assertEqual(row["bucket"], "hallucination") + self.assertEqual(sorted(row["invented"]), ["zzz-001", "zzz-002"]) + + def test_irrelevant_known_disconnected(self) -> None: + row = self.hop.classify_section( + gold=["child"], predicted=["island"], adj=self.adj + ) + self.assertEqual(row["bucket"], "irrelevant") + + def test_mixed_known_beats_invented(self) -> None: + row = self.hop.classify_section( + gold=["child"], predicted=["zzz-001", "cousin"], adj=self.adj + ) + self.assertEqual(row["bucket"], "one_hop_related") + self.assertIn("zzz-001", row["invented"]) + + def test_aggregate_percentages(self) -> None: + details = [ + {"gold": ["child"], "predicted": ["child"], "aligned": True}, + {"gold": ["child"], "predicted": ["cousin"], "aligned": True}, + {"gold": ["child"], "predicted": ["parent"], "aligned": True}, + {"gold": ["parent"], "predicted": ["cousin"], "aligned": True}, + {"gold": ["child"], "predicted": ["zzz-9"], "aligned": True}, + {"gold": ["child"], "predicted": ["island"], "aligned": True}, + ] + summary = self.hop.score_details(details, self.adj) + self.assertEqual(summary["scorable"], 6) + self.assertEqual(summary["counts"]["exact"], 1) + self.assertEqual(summary["counts"]["one_hop_related"], 1) + self.assertEqual(summary["counts"]["one_hop_contains"], 1) + self.assertEqual(summary["counts"]["multi_hop"], 1) + self.assertEqual(summary["counts"]["hallucination"], 1) + self.assertEqual(summary["counts"]["irrelevant"], 1) + self.assertAlmostEqual(summary["rates"]["exact"], 1 / 6, places=4) + self.assertAlmostEqual(summary["rates"]["distance1_hit"], 3 / 6, places=4) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/utils/oie_orchestrator/langgraph_pipeline.py b/application/utils/oie_orchestrator/langgraph_pipeline.py new file mode 100644 index 000000000..0efabbbef --- /dev/null +++ b/application/utils/oie_orchestrator/langgraph_pipeline.py @@ -0,0 +1,273 @@ +"""LangGraph + LlamaIndex-backed OIE orchestrator (A → B → C). + +LangGraph owns stage sequencing. LlamaIndex/Docling live under Module A's +``strategy: docling`` chunk path. Existing module entrypoints and DB queue +contracts are unchanged — this module only rewires *how* stages are invoked. +""" + +from __future__ import annotations + +import logging +from datetime import datetime, timezone +from typing import Any, Callable, Dict, Optional, TypedDict + +from application.utils.oie_orchestrator.pipeline import ( + OrchestratorResult, + StageResult, + _connect, + _stage_status_from_summary, + _summary_dict, +) + +logger = logging.getLogger(__name__) + + +class OieState(TypedDict, total=False): + run_id: str + cache_file: str + dry_run: bool + sync_repos: bool + skip_a: bool + skip_b: bool + skip_c: bool + stop_on_error: bool + stage_a: Dict[str, Any] + stage_b: Dict[str, Any] + stage_c: Dict[str, Any] + halt: bool + + +def _stage_to_dict(stage: StageResult) -> Dict[str, Any]: + return { + "name": stage.name, + "status": stage.status, + "detail": stage.detail, + "summary": stage.summary, + } + + +def _run_a( + state: OieState, run_harvester_fn: Optional[Callable[..., Any]] +) -> Dict[str, Any]: + if state.get("halt"): + return {} + if state.get("skip_a"): + stage = StageResult( + name="module_a_harvester", + status="skipped", + detail="skip_a=True; harvester not invoked", + ) + return {"stage_a": _stage_to_dict(stage)} + + fn = run_harvester_fn + if fn is None: + from application.utils.harvester.pipeline import run_harvester + + fn = run_harvester + try: + session = _connect(state["cache_file"]) + summary = fn( + session, + state["run_id"], + dry_run=bool(state.get("dry_run")), + sync_repos=bool(state.get("sync_repos", True)), + ) + stage = StageResult( + name="module_a_harvester", + status=_stage_status_from_summary(summary), + detail=f"run_harvester completed for run_id={state['run_id']!r}", + summary=_summary_dict(summary), + ) + except Exception as exc: # noqa: BLE001 + logger.exception("Module A stage failed") + stage = StageResult( + name="module_a_harvester", + status="error", + detail=f"run_harvester failed: {exc}", + ) + halt = bool(state.get("stop_on_error", True)) and stage.status == "error" + return {"stage_a": _stage_to_dict(stage), "halt": halt} + + +def _run_b( + state: OieState, run_noise_filter_fn: Optional[Callable[..., Any]] +) -> Dict[str, Any]: + if state.get("halt"): + return { + "stage_b": _stage_to_dict( + StageResult( + name="module_b_noise_filter", + status="skipped", + detail="halted after earlier stage error", + ) + ) + } + if state.get("skip_b"): + return { + "stage_b": _stage_to_dict( + StageResult( + name="module_b_noise_filter", + status="skipped", + detail="skip_b=True; noise filter not invoked", + ) + ) + } + + fn = run_noise_filter_fn + if fn is None: + from application.utils.noise_filter.pipeline import run_noise_filter + + fn = run_noise_filter + try: + session = _connect(state["cache_file"]) + summary = fn(session, state["run_id"], dry_run=bool(state.get("dry_run"))) + stage = StageResult( + name="module_b_noise_filter", + status=_stage_status_from_summary(summary), + detail=f"run_noise_filter completed for run_id={state['run_id']!r}", + summary=_summary_dict(summary), + ) + except Exception as exc: # noqa: BLE001 + logger.exception("Module B stage failed") + stage = StageResult( + name="module_b_noise_filter", + status="error", + detail=f"run_noise_filter failed: {exc}", + ) + halt = bool(state.get("stop_on_error", True)) and stage.status == "error" + return {"stage_b": _stage_to_dict(stage), "halt": halt} + + +def _run_c( + state: OieState, run_librarian_queue_fn: Optional[Callable[..., Any]] +) -> Dict[str, Any]: + if state.get("halt"): + return { + "stage_c": _stage_to_dict( + StageResult( + name="module_c_librarian", + status="skipped", + detail="halted after earlier stage error", + ) + ) + } + if state.get("skip_c"): + return { + "stage_c": _stage_to_dict( + StageResult( + name="module_c_librarian", + status="skipped", + detail="skip_c=True; librarian not invoked", + ) + ) + } + + try: + if run_librarian_queue_fn is not None: + summary = run_librarian_queue_fn( + state["run_id"], dry_run=bool(state.get("dry_run")) + ) + else: + from application.cmd.cre_main import db_connect + from application.utils.librarian.config_loader import load_config + from application.utils.librarian.envelope_sink import ( + DbEnvelopeSink, + NullEnvelopeSink, + ) + from application.utils.librarian.factory import build_components + from application.utils.librarian.queue_runner import run_librarian_queue + + cfg = load_config() + database = db_connect(path=state["cache_file"]) + components = build_components(database, config=cfg) + sink = ( + NullEnvelopeSink() + if state.get("dry_run") + else DbEnvelopeSink(database.session, state["run_id"]) + ) + summary = run_librarian_queue( + database.session, + state["run_id"], + components, + cfg, + at=datetime.now(timezone.utc), + sink=sink, + dry_run=bool(state.get("dry_run")), + ) + stage = StageResult( + name="module_c_librarian", + status=_stage_status_from_summary(summary), + detail=f"run_librarian_queue completed for run_id={state['run_id']!r}", + summary=( + _summary_dict(summary) if not isinstance(summary, dict) else summary + ), + ) + except Exception as exc: # noqa: BLE001 + logger.exception("Module C stage failed") + stage = StageResult( + name="module_c_librarian", + status="error", + detail=f"run_librarian_queue failed: {exc}", + ) + return {"stage_c": _stage_to_dict(stage)} + + +def run_oie_pipeline_langgraph( + *, + cache_file: str, + pipeline_run_id: Optional[str] = None, + skip_a: bool = False, + skip_b: bool = False, + skip_c: bool = False, + dry_run: bool = False, + sync_repos: bool = True, + stop_on_error: bool = True, + run_harvester_fn: Optional[Callable[..., Any]] = None, + run_noise_filter_fn: Optional[Callable[..., Any]] = None, + run_librarian_queue_fn: Optional[Callable[..., Any]] = None, +) -> OrchestratorResult: + """Compile a LangGraph ``A → B → C`` and invoke it once.""" + from langgraph.graph import END, START, StateGraph + + run_id = (pipeline_run_id or "").strip() or ( + datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + ) + + graph = StateGraph(OieState) + graph.add_node("module_a", lambda s: _run_a(s, run_harvester_fn)) + graph.add_node("module_b", lambda s: _run_b(s, run_noise_filter_fn)) + graph.add_node("module_c", lambda s: _run_c(s, run_librarian_queue_fn)) + graph.add_edge(START, "module_a") + graph.add_edge("module_a", "module_b") + graph.add_edge("module_b", "module_c") + graph.add_edge("module_c", END) + app = graph.compile() + + final = app.invoke( + { + "run_id": run_id, + "cache_file": cache_file, + "dry_run": dry_run, + "sync_repos": sync_repos, + "skip_a": skip_a, + "skip_b": skip_b, + "skip_c": skip_c, + "stop_on_error": stop_on_error, + "halt": False, + } + ) + + result = OrchestratorResult(run_id=run_id, dry_run=dry_run) + for key in ("stage_a", "stage_b", "stage_c"): + raw = final.get(key) + if not raw: + continue + result.stages.append( + StageResult( + name=raw["name"], + status=raw["status"], + detail=raw["detail"], + summary=raw.get("summary"), + ) + ) + return result diff --git a/application/utils/oie_orchestrator/pipeline.py b/application/utils/oie_orchestrator/pipeline.py index f0813f270..53554fb38 100644 --- a/application/utils/oie_orchestrator/pipeline.py +++ b/application/utils/oie_orchestrator/pipeline.py @@ -248,14 +248,38 @@ def run_oie_pipeline( run_harvester_fn: Optional[Callable[..., Any]] = None, run_noise_filter_fn: Optional[Callable[..., Any]] = None, run_librarian_queue_fn: Optional[Callable[..., Any]] = None, + use_langgraph: bool = True, ) -> OrchestratorResult: """ Run A→B→C for one ``pipeline_run_id``. - Defaults run all stages for real (not dry-run). Inject callables in tests. - When ``stop_on_error`` is True (default), later stages are skipped after - an earlier stage returns ``error``. + Default path uses LangGraph (``langgraph_pipeline``). Set + ``use_langgraph=False`` for the legacy sequential stages (tests/smoke). """ + if use_langgraph: + try: + from application.utils.oie_orchestrator.langgraph_pipeline import ( + run_oie_pipeline_langgraph, + ) + + return run_oie_pipeline_langgraph( + cache_file=cache_file, + pipeline_run_id=pipeline_run_id, + skip_a=skip_a, + skip_b=skip_b, + skip_c=skip_c, + dry_run=dry_run, + sync_repos=sync_repos, + stop_on_error=stop_on_error, + run_harvester_fn=run_harvester_fn, + run_noise_filter_fn=run_noise_filter_fn, + run_librarian_queue_fn=run_librarian_queue_fn, + ) + except ImportError: + logger.warning( + "langgraph not installed; falling back to sequential orchestrator" + ) + run_id = (pipeline_run_id or "").strip() or ( datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") ) diff --git a/scripts/oie_owasp_eval/clone_grid_worker_dbs.sh b/scripts/oie_owasp_eval/clone_grid_worker_dbs.sh new file mode 100755 index 000000000..b952e6f1b --- /dev/null +++ b/scripts/oie_owasp_eval/clone_grid_worker_dbs.sh @@ -0,0 +1,70 @@ +#!/bin/zsh +# Clone local `cre` into isolated Module C switch-grid worker DBs. +# +# Default: cre_grid_w0 .. cre_grid_w3 via CREATE DATABASE … TEMPLATE. +# Source DB (~500MB with embeddings) must have zero other connections. +# +# Usage: +# scripts/oie_owasp_eval/clone_grid_worker_dbs.sh +# WORKERS=4 SOURCE_DB=cre scripts/oie_owasp_eval/clone_grid_worker_dbs.sh +# FORCE=1 … # drop existing cre_grid_w* first +set -euo pipefail + +SOURCE_DB="${SOURCE_DB:-cre}" +WORKERS="${WORKERS:-4}" +HOST="${PGHOST:-127.0.0.1}" +PORT="${PGPORT:-5432}" +USER="${PGUSER:-cre}" +export PGPASSWORD="${PGPASSWORD:-password}" +PREFIX="${DB_PREFIX:-cre_grid_w}" + +psql_admin() { + psql -h "$HOST" -p "$PORT" -U "$USER" -d postgres -v ON_ERROR_STOP=1 "$@" +} + +echo "=== clone_grid_worker_dbs source=$SOURCE_DB workers=$WORKERS ===" + +size=$(psql -h "$HOST" -p "$PORT" -U "$USER" -d "$SOURCE_DB" -tAc \ + "SELECT pg_size_pretty(pg_database_size(current_database()))") +echo "SOURCE_SIZE $SOURCE_DB = $size" + +# Terminate other sessions on source (required for TEMPLATE clone). +echo "TERMINATE backends on $SOURCE_DB (except this session)" +psql_admin -c " +SELECT pg_terminate_backend(pid) +FROM pg_stat_activity +WHERE datname = '$SOURCE_DB' + AND pid <> pg_backend_pid(); +" >/dev/null + +for i in $(seq 0 $((WORKERS - 1))); do + name="${PREFIX}${i}" + exists=$(psql_admin -tAc "SELECT 1 FROM pg_database WHERE datname='$name'" | tr -d '[:space:]') + if [[ "$exists" == "1" ]]; then + if [[ "${FORCE:-0}" == "1" ]]; then + echo "DROP $name (FORCE=1)" + psql_admin -c " + SELECT pg_terminate_backend(pid) + FROM pg_stat_activity + WHERE datname = '$name' AND pid <> pg_backend_pid(); + " >/dev/null || true + psql_admin -c "DROP DATABASE IF EXISTS ${name};" + else + echo "KEEP $name (already exists; set FORCE=1 to recreate)" + continue + fi + fi + echo "CREATE $name TEMPLATE $SOURCE_DB …" + t0=$(date +%s) + psql_admin -c "CREATE DATABASE ${name} TEMPLATE ${SOURCE_DB} OWNER ${USER};" + t1=$(date +%s) + wsize=$(psql -h "$HOST" -p "$PORT" -U "$USER" -d "$name" -tAc \ + "SELECT pg_size_pretty(pg_database_size(current_database()))") + kq=$(psql -h "$HOST" -p "$PORT" -U "$USER" -d "$name" -tAc \ + "SELECT count(*) FROM knowledge_queue WHERE pipeline_run_id='orch-exp-baseline-20260921'") + echo "OK $name size=$wsize kq_harvest=$kq elapsed=$((t1 - t0))s" +done + +echo "=== DONE worker DBs: ${PREFIX}0 .. ${PREFIX}$((WORKERS - 1)) ===" +echo "Each worker uses DEV_DATABASE_URL=postgresql://${USER}:***@${HOST}:${PORT}/${PREFIX}N" +echo "Same pipeline_run_id orch-exp-baseline-20260921 per DB (isolated decision_queue)." diff --git a/scripts/oie_owasp_eval/fixtures/b2_gold/owasp_agentic_ai_stub.json b/scripts/oie_owasp_eval/fixtures/b2_gold/owasp_agentic_ai_stub.json new file mode 100644 index 000000000..a1aea3d3c --- /dev/null +++ b/scripts/oie_owasp_eval/fixtures/b2_gold/owasp_agentic_ai_stub.json @@ -0,0 +1,32 @@ +[ + { + "section_id": "AAI01", + "section": "Goal Hijacking and Agent Drift", + "hyperlink": "", + "cre_ids": ["177-260", "724-770"] + }, + { + "section_id": "AAI02", + "section": "Tool Misuse and Unsafe Actuation", + "hyperlink": "", + "cre_ids": ["486-813", "117-371"] + }, + { + "section_id": "AAI03", + "section": "Multi-Agent Collusion and Privilege Escalation", + "hyperlink": "", + "cre_ids": ["128-128", "633-428"] + }, + { + "section_id": "AAI04", + "section": "Memory Poisoning Across Agent Sessions", + "hyperlink": "", + "cre_ids": ["340-375", "613-287"] + }, + { + "section_id": "AAI05", + "section": "Unsupervised Cross-System Workflows", + "hyperlink": "", + "cre_ids": ["132-146", "515-021"] + } +] diff --git a/scripts/oie_owasp_eval/hop_distance.py b/scripts/oie_owasp_eval/hop_distance.py new file mode 100644 index 000000000..1716c9860 --- /dev/null +++ b/scripts/oie_owasp_eval/hop_distance.py @@ -0,0 +1,356 @@ +"""CRE-graph hop distance for B2 predictions (gap-analysis style weights). + +Classifies each scorable section into an exclusive bucket: + exact | one_hop_related | one_hop_contains | multi_hop | hallucination | irrelevant + +Hallucination = every predicted CRE id is unknown (invented). +Irrelevant = at least one predicted CRE exists in the graph but no path to gold. +""" + +from __future__ import annotations + +import gzip +import json +from collections import defaultdict, deque +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Set, Tuple + +# Mirror application.utils.gap_analysis.PENALTIES for CRE↔CRE edges. +PENALTIES = { + "RELATED": 2, + "CONTAINS_UP": 2, + "CONTAINS_DOWN": 1, + "SAME": 0, +} + +BUCKETS = ( + "exact", + "one_hop_related", + "one_hop_contains", + "multi_hop", + "hallucination", + "irrelevant", +) + + +@dataclass(frozen=True) +class TypedEdge: + src: str + dst: str + relationship: str # RELATED | CONTAINS_UP | CONTAINS_DOWN + + @property + def penalty(self) -> int: + return int(PENALTIES[self.relationship]) + + +@dataclass +class CreAdjacency: + known_ids: frozenset + # src -> list of TypedEdge + _out: Dict[str, List[TypedEdge]] + + def __init__( + self, + known_ids: Iterable[str], + edges: Sequence[TypedEdge], + ) -> None: + self.known_ids = frozenset(known_ids) + out: Dict[str, List[TypedEdge]] = defaultdict(list) + for e in edges: + out[e.src].append(e) + self._out = dict(out) + + def neighbors(self, node: str) -> List[TypedEdge]: + return self._out.get(node) or [] + + def to_serializable(self) -> Dict[str, Any]: + return { + "known_ids": sorted(self.known_ids), + "edges": [ + {"src": e.src, "dst": e.dst, "relationship": e.relationship} + for src_edges in self._out.values() + for e in src_edges + ], + } + + @classmethod + def from_serializable(cls, payload: Mapping[str, Any]) -> "CreAdjacency": + edges = [ + TypedEdge( + str(e["src"]), + str(e["dst"]), + str(e["relationship"]), + ) + for e in (payload.get("edges") or []) + ] + return cls(known_ids=payload.get("known_ids") or [], edges=edges) + + +def save_adjacency(adj: CreAdjacency, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + raw = json.dumps(adj.to_serializable(), separators=(",", ":")).encode("utf-8") + if str(path).endswith(".gz"): + path.write_bytes(gzip.compress(raw)) + else: + path.write_bytes(raw) + + +def load_adjacency(path: Path) -> CreAdjacency: + data = path.read_bytes() + if str(path).endswith(".gz"): + data = gzip.decompress(data) + return CreAdjacency.from_serializable(json.loads(data.decode("utf-8"))) + + +def build_adjacency_from_db(session: Any) -> CreAdjacency: + """Load CRE external_ids + Related/Contains edges from SQLAlchemy session. + + InternalLinks convention (application.database.db.InternalLinks): + group = higher / parent CRE uuid + cre = lower / child CRE uuid + type = Contains | Related + """ + from application.database.db import CRE, InternalLinks + + id_to_ext: Dict[str, str] = {} + for row in session.query(CRE.id, CRE.external_id).all(): + ext = (row.external_id or "").strip() + if not ext: + continue + id_to_ext[str(row.id)] = ext + + edges: List[TypedEdge] = [] + seen: Set[Tuple[str, str, str]] = set() + + def add(src: str, dst: str, rel: str) -> None: + key = (src, dst, rel) + if key in seen: + return + seen.add(key) + edges.append(TypedEdge(src, dst, rel)) + + for link in session.query(InternalLinks).all(): + ltype = (link.type or "").strip() + higher = id_to_ext.get(str(link.group)) + lower = id_to_ext.get(str(link.cre)) + if not higher or not lower or higher == lower: + continue + if ltype == "Related": + add(higher, lower, "RELATED") + add(lower, higher, "RELATED") + elif ltype == "Contains": + # group Contains cre → DOWN from parent, UP from child + add(higher, lower, "CONTAINS_DOWN") + add(lower, higher, "CONTAINS_UP") + # PartOf forbidden by DB constraint; ignore other types + + return CreAdjacency(known_ids=id_to_ext.values(), edges=edges) + + +PathEdge = TypedEdge + + +def _best_path( + start: str, + goals: Set[str], + adj: CreAdjacency, +) -> Optional[Tuple[int, int, List[TypedEdge]]]: + """BFS shortest hop path; among equal hops prefer lower GA path_penalty. + + Returns (hop_count, path_penalty, edges) or None. + """ + if start in goals: + return (0, 0, []) + if start not in adj.known_ids: + return None + + # state: node -> (hop_count, path_penalty, edges) + best: Dict[str, Tuple[int, int, List[TypedEdge]]] = { + start: (0, 0, []), + } + q: deque[str] = deque([start]) + found: List[Tuple[int, int, List[TypedEdge]]] = [] + + while q: + node = q.popleft() + hops, pen, path = best[node] + if found and hops > found[0][0]: + # BFS layer past shortest goal hop — stop + continue + for edge in adj.neighbors(node): + nxt = edge.dst + nh, np_ = hops + 1, pen + edge.penalty + npath = path + [edge] + prev = best.get(nxt) + if prev is None or (nh, np_) < (prev[0], prev[1]): + best[nxt] = (nh, np_, npath) + q.append(nxt) + if nxt in goals: + cand = (nh, np_, npath) + if not found or cand[:2] < found[0][:2]: + found = [cand] + elif cand[:2] == found[0][:2]: + found.append(cand) + + if not found: + return None + found.sort(key=lambda t: (t[0], t[1])) + return found[0] + + +def classify_section( + *, + gold: Sequence[str], + predicted: Sequence[str], + adj: CreAdjacency, +) -> Dict[str, Any]: + g = {str(x).strip() for x in gold if str(x).strip()} + p = [str(x).strip() for x in predicted if str(x).strip()] + p_set = set(p) + + invented = sorted(x for x in p_set if x not in adj.known_ids) + known_preds = sorted(x for x in p_set if x in adj.known_ids) + + base: Dict[str, Any] = { + "gold": sorted(g), + "predicted": sorted(p_set), + "invented": invented, + "known_predicted": known_preds, + "hop_count": None, + "path_penalty": None, + "path_edges": [], + "best_pred": None, + "best_gold": None, + "bucket": "irrelevant", + } + + if not g: + base["bucket"] = "irrelevant" + return base + + if p_set & g: + base["bucket"] = "exact" + base["hop_count"] = 0 + base["path_penalty"] = 0 + hit = sorted(p_set & g)[0] + base["best_pred"] = hit + base["best_gold"] = hit + return base + + if not known_preds: + base["bucket"] = "hallucination" if p_set else "irrelevant" + return base + + best: Optional[Tuple[int, int, List[TypedEdge], str, str]] = None + for pred in known_preds: + path = _best_path(pred, g, adj) + if path is None: + continue + hops, pen, edges = path + # destination is last edge dst or pred if empty + dest = edges[-1].dst if edges else pred + cand = (hops, pen, edges, pred, dest) + if best is None or (hops, pen) < (best[0], best[1]): + best = cand + + if best is None: + base["bucket"] = "irrelevant" + return base + + hops, pen, edges, pred, dest = best + base["hop_count"] = hops + base["path_penalty"] = pen + base["best_pred"] = pred + base["best_gold"] = dest + base["path_edges"] = [ + { + "src": e.src, + "dst": e.dst, + "relationship": e.relationship, + "penalty": e.penalty, + } + for e in edges + ] + + if hops == 1: + rel = edges[0].relationship + if rel == "RELATED": + base["bucket"] = "one_hop_related" + else: + base["bucket"] = "one_hop_contains" + else: + base["bucket"] = "multi_hop" + return base + + +def score_details( + details: Sequence[Mapping[str, Any]], + adj: CreAdjacency, +) -> Dict[str, Any]: + rows: List[Dict[str, Any]] = [] + counts = {b: 0 for b in BUCKETS} + scorable = 0 + pred_total = 0 + invented_total = 0 + + for d in details: + if d.get("aligned") is False: + continue + gold = d.get("gold") or [] + if not gold: + continue + predicted = d.get("predicted") or [] + scorable += 1 + row = classify_section(gold=gold, predicted=predicted, adj=adj) + row["resource"] = d.get("resource") + row["section_id"] = d.get("section_id") + row["exact_hit"] = bool(d.get("hit")) + rows.append(row) + counts[row["bucket"]] += 1 + pred_total += len(row["predicted"]) + invented_total += len(row["invented"]) + + rates = {b: round((counts[b] / scorable) if scorable else 0.0, 4) for b in BUCKETS} + d1 = counts["exact"] + counts["one_hop_related"] + counts["one_hop_contains"] + ga_useful = d1 + counts["multi_hop"] + rates["distance1_hit"] = round((d1 / scorable) if scorable else 0.0, 4) + rates["ga_path_hit"] = round((ga_useful / scorable) if scorable else 0.0, 4) + # Hop-capped soft hits (CRE graph is often one component → unbounded path ≈ always true) + within2 = sum( + 1 + for r in rows + if r["bucket"] == "exact" + or ( + r.get("hop_count") is not None + and int(r["hop_count"]) <= 2 + and r["bucket"] != "hallucination" + and r["bucket"] != "irrelevant" + ) + ) + within3 = sum( + 1 + for r in rows + if r["bucket"] == "exact" + or ( + r.get("hop_count") is not None + and int(r["hop_count"]) <= 3 + and r["bucket"] != "hallucination" + and r["bucket"] != "irrelevant" + ) + ) + rates["within_2_hops"] = round((within2 / scorable) if scorable else 0.0, 4) + rates["within_3_hops"] = round((within3 / scorable) if scorable else 0.0, 4) + rates["invented_pred"] = round( + (invented_total / pred_total) if pred_total else 0.0, 4 + ) + + return { + "scorable": scorable, + "counts": counts, + "rates": rates, + "pred_ids_total": pred_total, + "invented_ids_total": invented_total, + "rows": rows, + } diff --git a/scripts/oie_owasp_eval/rebuild_b2_gold_from_hub.py b/scripts/oie_owasp_eval/rebuild_b2_gold_from_hub.py new file mode 100644 index 000000000..252a73c58 --- /dev/null +++ b/scripts/oie_owasp_eval/rebuild_b2_gold_from_hub.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +"""Rebuild experimental B2 gold JSON from hub Links. + +Canonical eval gold is ``application/tests/fixtures/owasp_mappings`` +(``mapping_fixtures``). This script writes an overlay under +``scripts/oie_owasp_eval/fixtures/b2_gold/`` for hub-derived experiments; +B2 scoring does not read those copies. + +Usage: + PYTHONPATH=. python scripts/oie_owasp_eval/rebuild_b2_gold_from_hub.py \\ + --catalog k8s_top10 --out scripts/oie_owasp_eval/fixtures/b2_gold/owasp_kubernetes_top10_2025.json +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple + +ROOT = Path(__file__).resolve().parents[2] + +# catalog_key → default section order (sid, title, href) when existing gold missing +_DEFAULTS: Dict[str, Tuple[Tuple[str, str, str], ...]] = { + "llm_top10": ( + ("LLM01", "Prompt Injection", ""), + ("LLM02", "Sensitive Information Disclosure", ""), + ("LLM03", "Supply Chain", ""), + ("LLM04", "Data and Model Poisoning", ""), + ("LLM05", "Improper Output Handling", ""), + ("LLM06", "Excessive Agency", ""), + ("LLM07", "System Prompt Leakage", ""), + ("LLM08", "Vector and Embedding Weaknesses", ""), + ("LLM09", "Misinformation", ""), + ("LLM10", "Unbounded Consumption", ""), + ), +} + + +def _load_dotenv() -> None: + for env_path in (ROOT / ".env", ROOT / "tmp" / "oie.env"): + if not env_path.is_file(): + continue + for line in env_path.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, _, v = line.partition("=") + if env_path.name == "oie.env" and ( + k.startswith("CRE_LIBRARIAN_") or k.startswith("DEV_DATABASE") + ): + os.environ[k.strip()] = v.strip() + else: + os.environ.setdefault(k.strip(), v.strip().strip('"').strip("'")) + + +def load_existing_gold(path: Path) -> Dict[str, Dict[str, Any]]: + if not path.is_file(): + return {} + data = json.loads(path.read_text()) + if not isinstance(data, list): + return {} + out: Dict[str, Dict[str, Any]] = {} + for row in data: + if not isinstance(row, dict): + continue + sid = str(row.get("section_id") or "").strip().upper().split(":")[0] + if sid: + out[sid] = row + return out + + +def build_gold_rows( + hub_by_section: Mapping[str, Sequence[str]], + *, + existing: Mapping[str, Mapping[str, Any]], + defaults: Sequence[Tuple[str, str, str]], +) -> Tuple[List[Dict[str, Any]], List[str]]: + rows: List[Dict[str, Any]] = [] + gaps: List[str] = [] + # Prefer existing gold section order; else defaults; else hub keys sorted. + order: List[Tuple[str, str, str]] = [] + if existing: + for sid, old in existing.items(): + order.append( + ( + sid, + str(old.get("section") or sid), + str(old.get("hyperlink") or ""), + ) + ) + elif defaults: + order = list(defaults) + else: + order = [(sid, sid, "") for sid in sorted(hub_by_section.keys())] + + seen: set[str] = set() + for sid, default_title, default_href in order: + key = sid.upper() + if key in seen: + continue + seen.add(key) + old = existing.get(key) or existing.get(sid) or {} + cre_ids = sorted({c for c in (hub_by_section.get(key) or []) if c}) + if not cre_ids: + gaps.append(key) + row: Dict[str, Any] = { + "section_id": key if not key.startswith("AISVS") else sid, + "section": str(old.get("section") or default_title), + "hyperlink": str(old.get("hyperlink") or default_href), + "cre_ids": cre_ids, + } + if old.get("fallback_section_ids"): + row["fallback_section_ids"] = old["fallback_section_ids"] + rows.append(row) + return rows, gaps + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--catalog", + required=True, + help="Catalog key: llm_top10, k8s_top10, owasp_top10, api_top10, aisvs, …", + ) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--cache-file", default="") + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + + os.environ.setdefault("FLASK_CONFIG", "development") + os.environ.setdefault("NO_LOAD_GRAPH_DB", "1") + _load_dotenv() + cache = ( + args.cache_file + or os.environ.get("DEV_DATABASE_URL") + or os.environ.get("CRE_CACHE_FILE") + or "" + ) + if not cache: + print("DEV_DATABASE_URL / --cache-file required", file=sys.stderr) + return 2 + + from application import sqla + from application.cmd.cre_main import db_connect + from application.utils.librarian.standard_link_seed import ( + hub_external_ids_by_section, + ) + + db_connect(cache) + hub = hub_external_ids_by_section(sqla.session, catalog_key=args.catalog) + existing = load_existing_gold(args.out) + rows, gaps = build_gold_rows( + hub, + existing=existing, + defaults=_DEFAULTS.get(args.catalog, ()), + ) + report = { + "catalog": args.catalog, + "sections": len(rows), + "scorable": sum(1 for r in rows if r["cre_ids"]), + "gaps": gaps, + "hub_sections": sorted(hub.keys()), + "out": str(args.out), + } + print(json.dumps(report, indent=2), flush=True) + for row in rows: + print( + f" {row['section_id']}: {row['cre_ids'] or '(gap — unscorable)'}", + flush=True, + ) + if args.dry_run: + return 0 + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(rows, indent=2) + "\n") + print(f"wrote {args.out}", flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/oie_owasp_eval/rebuild_b2_llm_gold_from_hub.py b/scripts/oie_owasp_eval/rebuild_b2_llm_gold_from_hub.py new file mode 100644 index 000000000..21720f5fa --- /dev/null +++ b/scripts/oie_owasp_eval/rebuild_b2_llm_gold_from_hub.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +"""Rebuild experimental B2 LLM Top 10 gold from hub ``OWASP Top10 for LLM`` Links. + +Canonical eval gold is ``owasp_llm_top10_2025`` in +``application/tests/fixtures/owasp_mappings``. This script writes an overlay +under ``scripts/oie_owasp_eval/fixtures/b2_gold/``; B2 scoring does not read it. + +Usage: + set -a && source tmp/oie.env && set +a + PYTHONPATH=. python scripts/oie_owasp_eval/rebuild_b2_llm_gold_from_hub.py +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple + +ROOT = Path(__file__).resolve().parents[2] +DEFAULT_GOLD = ( + ROOT + / "scripts" + / "oie_owasp_eval" + / "fixtures" + / "b2_gold" + / "owasp_llm_top10_2025.json" +) + +# Canonical LLM Top 10 2025 section order (titles/hyperlinks filled from prior gold). +_DEFAULT_SECTIONS: Tuple[Tuple[str, str, str], ...] = ( + ( + "LLM01", + "Prompt Injection", + "https://genai.owasp.org/llmrisk/llm01-prompt-injection/", + ), + ( + "LLM02", + "Sensitive Information Disclosure", + "https://genai.owasp.org/llmrisk/llm022025-sensitive-information-disclosure/", + ), + ( + "LLM03", + "Supply Chain", + "https://genai.owasp.org/llmrisk/llm032025-supply-chain/", + ), + ( + "LLM04", + "Data and Model Poisoning", + "https://genai.owasp.org/llmrisk/llm042025-data-and-model-poisoning/", + ), + ( + "LLM05", + "Improper Output Handling", + "https://genai.owasp.org/llmrisk/llm052025-improper-output-handling/", + ), + ( + "LLM06", + "Excessive Agency", + "https://genai.owasp.org/llmrisk/llm062025-excessive-agency/", + ), + ( + "LLM07", + "System Prompt Leakage", + "https://genai.owasp.org/llmrisk/llm072025-system-prompt-leakage/", + ), + ( + "LLM08", + "Vector and Embedding Weaknesses", + "https://genai.owasp.org/llmrisk/llm082025-vector-and-embedding-weaknesses/", + ), + ( + "LLM09", + "Misinformation", + "https://genai.owasp.org/llmrisk/llm092025-misinformation/", + ), + ( + "LLM10", + "Unbounded Consumption", + "https://genai.owasp.org/llmrisk/llm102025-unbounded-consumption/", + ), +) + + +def _load_dotenv() -> None: + for env_path in (ROOT / ".env", ROOT / "tmp" / "oie.env"): + if not env_path.is_file(): + continue + for line in env_path.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, _, v = line.partition("=") + if env_path.name == "oie.env" and ( + k.startswith("CRE_LIBRARIAN_") or k.startswith("DEV_DATABASE") + ): + os.environ[k.strip()] = v.strip() + else: + os.environ.setdefault(k.strip(), v.strip().strip('"').strip("'")) + + +def load_existing_gold(path: Path) -> Dict[str, Dict[str, Any]]: + if not path.is_file(): + return {} + data = json.loads(path.read_text()) + if not isinstance(data, list): + return {} + out: Dict[str, Dict[str, Any]] = {} + for row in data: + if not isinstance(row, dict): + continue + sid = str(row.get("section_id") or "").strip().upper().split(":")[0] + if sid: + out[sid] = row + return out + + +def build_gold_rows( + hub_by_section: Mapping[str, Sequence[str]], + *, + existing: Optional[Mapping[str, Mapping[str, Any]]] = None, +) -> Tuple[List[Dict[str, Any]], List[str]]: + """Assemble gold rows; return (rows, gap_section_ids with empty cre_ids).""" + prev = existing or {} + rows: List[Dict[str, Any]] = [] + gaps: List[str] = [] + for sid, default_title, default_href in _DEFAULT_SECTIONS: + old = prev.get(sid) or {} + cre_ids = sorted({c for c in (hub_by_section.get(sid) or []) if c}) + if not cre_ids: + gaps.append(sid) + rows.append( + { + "section_id": sid, + "section": str(old.get("section") or default_title), + "hyperlink": str(old.get("hyperlink") or default_href), + "cre_ids": cre_ids, + } + ) + return rows, gaps + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--out", + type=Path, + default=DEFAULT_GOLD, + help="Output gold JSON path (harness only)", + ) + parser.add_argument( + "--cache-file", + default="", + help="Postgres URL (default: DEV_DATABASE_URL / CRE_CACHE_FILE)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print rows/gaps without writing", + ) + args = parser.parse_args() + + os.environ.setdefault("FLASK_CONFIG", "development") + os.environ.setdefault("NO_LOAD_GRAPH_DB", "1") + _load_dotenv() + cache = ( + args.cache_file + or os.environ.get("DEV_DATABASE_URL") + or os.environ.get("CRE_CACHE_FILE") + or "" + ) + if not cache: + print("DEV_DATABASE_URL / --cache-file required", file=sys.stderr) + return 2 + + from application.cmd.cre_main import db_connect + from application import sqla + from application.utils.librarian.aix_llm_link_seed import ( + hub_llm_external_ids_by_section, + ) + + db_connect(cache) + hub = hub_llm_external_ids_by_section(sqla.session) + existing = load_existing_gold(args.out) + rows, gaps = build_gold_rows(hub, existing=existing) + + report = { + "sections": len(rows), + "scorable": sum(1 for r in rows if r["cre_ids"]), + "gaps": gaps, + "hub_sections": sorted(hub.keys()), + "out": str(args.out), + } + print(json.dumps(report, indent=2), flush=True) + for row in rows: + print( + f" {row['section_id']}: {row['cre_ids'] or '(gap — unscorable)'}", + flush=True, + ) + + if args.dry_run: + return 0 + + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(rows, indent=2) + "\n") + print(f"wrote {args.out}", flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/oie_owasp_eval/run_b2_pr_mappings.py b/scripts/oie_owasp_eval/run_b2_pr_mappings.py new file mode 100644 index 000000000..9291ed12c --- /dev/null +++ b/scripts/oie_owasp_eval/run_b2_pr_mappings.py @@ -0,0 +1,811 @@ +#!/usr/bin/env python3 +"""B2: score Module C against OWASP mapping-fixture gold (harness only). + +Canonical gold is ``application/tests/fixtures/owasp_mappings`` via +``application.utils.mapping_fixtures``. Orphan umbrella CREs with no +Contains children include their ``Related`` neighbors in ``cre_ids`` +(``gold_remap: related_expand_orphan``). The agentic-AI stub (no hub Links) +stays under ``scripts/oie_owasp_eval/fixtures/b2_gold/``. Source material is +each row's ``hyperlink``, cached under +``scripts/oie_owasp_eval/fixtures/b2_sources/``. Alignment is +**section grain** (one gold row = one section_id). Hit if ≥1 of the union of +top-2 suggested CRE external_ids across that section's chunks is in the gold +``cre_ids``. + +Usage: + PYTHONPATH=. python scripts/oie_owasp_eval/run_b2_pr_mappings.py + PYTHONPATH=. python scripts/oie_owasp_eval/run_b2_pr_mappings.py --score-only --run-id +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +import urllib.error +import urllib.request +from datetime import datetime, timezone +from html.parser import HTMLParser +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Set, Tuple + +ROOT = Path(__file__).resolve().parents[2] +ART = ROOT / "tmp" / "oie_owasp_eval" +FIXTURES = Path(__file__).resolve().parent / "fixtures" +GOLD_DIR = FIXTURES / "b2_gold" +SOURCES_DIR = FIXTURES / "b2_sources" + +# Canonical gold lives under application/tests/fixtures/owasp_mappings +# (Related-expanded orphan umbrellas are part of those fixtures). Agentic stub +# stays harness-only under scripts/oie_owasp_eval/fixtures/b2_gold/. +HARNESSES: List[Dict[str, Any]] = [ + { + "fixture_name": "owasp_top10_2025", + "gold_file": "owasp_top10_2025.json", + "label": "OWASP Top 10 2025", + "pr": 960, + }, + { + "fixture_name": "owasp_api_top10_2023", + "gold_file": "owasp_api_top10_2023.json", + "label": "OWASP API Top 10 2023", + "pr": 960, + }, + { + "fixture_name": "owasp_llm_top10_2025", + "gold_file": "owasp_llm_top10_2025.json", + "label": "OWASP LLM Top 10 2025", + "pr": 960, + }, + { + "fixture_name": "owasp_aisvs_1_0", + "gold_file": "owasp_aisvs_1_0.json", + "label": "OWASP AISVS 1.0", + "pr": 960, + }, + { + "fixture_name": "owasp_kubernetes_top10_2025", + "gold_file": "owasp_kubernetes_top10_2025.json", + "label": "OWASP Kubernetes Top 10 2025", + "pr": 953, + }, + { + "fixture_name": "owasp_kubernetes_top10_2022", + "gold_file": "owasp_kubernetes_top10_2022.json", + # Cached hyperlink text still lives under the pre-merge PR #927 folder. + "source_dir": "owasp_kubernetes_top10_2022_pr927", + "label": "OWASP Kubernetes Top 10 2022", + "pr": 953, + }, + { + # Harness-only: brand-new family with no hub Nodes/Links → expect weak accuracy. + "pr": 0, + "local_gold": True, + "gold_file": "owasp_agentic_ai_stub.json", + "label": "OWASP Agentic AI (stub, no hub Links)", + "local_sources": True, + }, +] + + +class _HTMLTextExtractor(HTMLParser): + def __init__(self) -> None: + super().__init__() + self._chunks: List[str] = [] + self._skip = 0 + + def handle_starttag(self, tag: str, attrs: list) -> None: + if tag in {"script", "style", "noscript", "svg"}: + self._skip += 1 + + def handle_endtag(self, tag: str) -> None: + if tag in {"script", "style", "noscript", "svg"} and self._skip: + self._skip -= 1 + + def handle_data(self, data: str) -> None: + if self._skip: + return + text = data.strip() + if text: + self._chunks.append(text) + + def text(self) -> str: + return "\n".join(self._chunks) + + +def _load_dotenv() -> None: + for env_path in (ROOT / ".env", ROOT / "tmp" / "oie.env"): + if not env_path.is_file(): + continue + for line in env_path.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, _, v = line.partition("=") + if env_path.name == "oie.env" and ( + k.startswith("CRE_LIBRARIAN_") or k.startswith("DEV_DATABASE") + ): + os.environ[k.strip()] = v.strip() + else: + os.environ.setdefault(k.strip(), v.strip().strip('"').strip("'")) + + +def normalize_source_url(url: str) -> str: + """Prefer raw markdown for GitHub blob/tree links.""" + u = (url or "").strip() + if not u: + return u + m = re.match( + r"https?://github\.com/([^/]+)/([^/]+)/(?:blob|tree)/([^/]+)/(.+)$", + u, + ) + if m: + owner, repo, ref, path = m.groups() + return f"https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{path}" + return u + + +def fetch_url(url: str, *, timeout: int = 45) -> Tuple[str, str]: + """Return (content_type_hint, text). Raises on hard failure.""" + url = normalize_source_url(url) + req = urllib.request.Request( + url, + headers={"User-Agent": "OpenCRE-OIE-B2/1.0", "Accept": "text/*,*/*"}, + ) + with urllib.request.urlopen(req, timeout=timeout) as resp: + raw = resp.read() + ctype = (resp.headers.get("Content-Type") or "").lower() + text = raw.decode("utf-8", errors="replace") + if ( + "html" in ctype + or text.lstrip().lower().startswith(" 200: + return "html_text", extracted + return "text", text + + +# PR #927 hyperlinks use outdated path slugs; GitHub still has the markdown. +_K8S_2022_RAW = { + "K01": "K01-insecure-workload-configurations.md", + "K02": "K02-supply-chain-vulnerabilities.md", + "K03": "K03-overly-permissive-rbac.md", + "K04": "K04-policy-enforcement.md", + "K05": "K05-inadequate-logging.md", + "K06": "K06-broken-authentication.md", + "K07": "K07-network-segmentation.md", + "K08": "K08-secrets-management.md", + "K09": "K09-misconfigured-cluster-components.md", + "K10": "K10-vulnerable-components.md", +} + + +def source_url_candidates(row: Dict[str, Any], harness: Dict[str, Any]) -> List[str]: + href = str(row.get("hyperlink") or "").strip() + sid = str(row.get("section_id") or "").strip() + out: List[str] = [] + if href: + out.append(href) + if not href.endswith(".html"): + out.append(href.rstrip("/") + ".html") + if ( + harness.get("gold_file", "").startswith("owasp_kubernetes_top10_2022") + and sid in _K8S_2022_RAW + ): + out.append( + "https://raw.githubusercontent.com/OWASP/www-project-kubernetes-top-ten/" + f"main/2022/en/src/{_K8S_2022_RAW[sid]}" + ) + # dedupe preserve order + seen = set() + uniq = [] + for u in out: + if u not in seen: + seen.add(u) + uniq.append(u) + return uniq + + +def fetch_row_source(row: Dict[str, Any], harness: Dict[str, Any]) -> Tuple[str, str]: + last_exc: Optional[BaseException] = None + for url in source_url_candidates(row, harness): + try: + return fetch_url(url) + except Exception as exc: # noqa: BLE001 + last_exc = exc + assert last_exc is not None + raise last_exc + + +def gold_filename(harness: Dict[str, Any]) -> str: + if harness.get("gold_file"): + return str(harness["gold_file"]) + name = str(harness.get("fixture_name") or "") + return name if name.endswith(".json") else f"{name}.json" + + +def gold_stem(harness: Dict[str, Any]) -> str: + name = gold_filename(harness) + return name[:-5] if name.endswith(".json") else name + + +def source_stem(harness: Dict[str, Any]) -> str: + return str(harness.get("source_dir") or gold_stem(harness)) + + +_YEAR_TOKEN = re.compile(r"(20\d{2})") +_SOURCE_META_LINE = re.compile( + r"^(Standard|Version|Source|Section-ID|Section)\s*:", + re.I, +) + + +def harness_year(harness: Dict[str, Any]) -> Optional[int]: + """Edition year from harness label / fixture / source_dir names.""" + for blob in ( + harness.get("label"), + harness.get("gold_file"), + harness.get("fixture_name"), + harness.get("source_dir"), + ): + if not blob: + continue + m = _YEAR_TOKEN.search(str(blob)) + if m: + return int(m.group(1)) + return None + + +def source_identity_prefix(row: Dict[str, Any], harness: Dict[str, Any]) -> str: + """Metadata header so Module C can year-scope hub Links / edition remap. + + ``Standard:`` + ``Version:`` are required for ``year_from_text``; without + them bare ``K##`` seeds collapse across K8s 2022 and 2025 editions. + """ + sid = str(row.get("section_id") or "").strip() + section = str(row.get("section") or sid).strip() + label = str(harness.get("label") or harness.get("fixture_name") or "OWASP").strip() + year = harness_year(harness) + lines = [f"Standard: {label}"] + if year is not None: + lines.append(f"Version: {year}") + lines.extend( + [ + f"Source: {sid}", + f"Section: {section}", + f"Section-ID: {sid}", + "", + ] + ) + return "\n".join(lines) + + +def strip_source_identity_prefix(text: str) -> str: + """Drop leading Standard/Version/Source/Section metadata lines (+ blank).""" + lines = (text or "").splitlines() + i = 0 + while i < len(lines) and ( + not lines[i].strip() or _SOURCE_META_LINE.match(lines[i].strip()) + ): + i += 1 + return "\n".join(lines[i:]).lstrip("\n") + + +def write_source_with_prefix( + path: Path, row: Dict[str, Any], harness: Dict[str, Any], body: str +) -> None: + path.write_text(source_identity_prefix(row, harness) + body.lstrip("\n")) + + +def load_gold(harness: Dict[str, Any]) -> List[Dict[str, Any]]: + if harness.get("local_gold"): + path = GOLD_DIR / gold_filename(harness) + if not path.is_file(): + raise FileNotFoundError(f"missing local gold file {path}") + data = json.loads(path.read_text()) + if not isinstance(data, list): + raise ValueError(f"gold {path} must be a list of section mappings") + return data + from application.utils.mapping_fixtures import load_owasp_mapping_fixture + + return load_owasp_mapping_fixture(str(harness["fixture_name"])) + + +def top2_cre_external_ids( + envelope: Dict[str, Any], uuid_to_ext: Dict[str, str] +) -> List[str]: + """Union of rerank top-2 and vector top-2 (up to 4 ids). + + Hit if ≥1 of this union is in gold — CE cannot erase a strong cosine hit. + """ + ordered: List[str] = [] + + def add(cre_id: Optional[str]) -> None: + if not cre_id: + return + ext = uuid_to_ext.get(cre_id, cre_id) + if ext not in ordered: + ordered.append(ext) + + retrieval = envelope.get("retrieval") or {} + reranked = list(retrieval.get("reranked") or []) + candidates = list(retrieval.get("candidates") or []) + + for cand in reranked[:2]: + if isinstance(cand, dict): + add(cand.get("cre_id")) + + by_vec = sorted( + (c for c in candidates if isinstance(c, dict)), + key=lambda c: float(c.get("score_vector") or 0.0), + reverse=True, + ) + for cand in by_vec[:2]: + add(cand.get("cre_id")) + + if ordered: + return ordered + + for field in ("links", "suggested_links"): + for link in envelope.get(field) or []: + if isinstance(link, dict): + add(link.get("cre_id")) + if len(ordered) >= 2: + return ordered[:2] + return ordered + + +def refresh_source_headers( + harnesses: Sequence[Dict[str, Any]], +) -> Dict[str, Any]: + """Rewrite Standard/Version/Source/Section headers on existing source files.""" + report: Dict[str, Any] = {"refreshed": 0, "skipped": 0, "errors": []} + for harness in harnesses: + gold = load_gold(harness) + dest_dir = SOURCES_DIR / source_stem(harness) + for row in gold: + sid = str(row.get("section_id") or "").strip() + if not sid: + continue + out = dest_dir / f"{sid}.txt" + if not out.is_file() or out.stat().st_size < 80: + report["skipped"] += 1 + continue + try: + body = strip_source_identity_prefix(out.read_text()) + write_source_with_prefix(out, row, harness, body) + report["refreshed"] += 1 + except Exception as exc: # noqa: BLE001 + msg = f"{source_stem(harness)}:{sid}: {exc}" + report["errors"].append(msg) + return report + + +def ensure_sources( + harnesses: Sequence[Dict[str, Any]], *, refresh_headers: bool = False +) -> Dict[str, Any]: + """Download each gold row's hyperlink into b2_sources//.txt.""" + report: Dict[str, Any] = {"files": 0, "errors": [], "refreshed": 0} + SOURCES_DIR.mkdir(parents=True, exist_ok=True) + if refresh_headers: + hdr = refresh_source_headers(harnesses) + report["refreshed"] = hdr["refreshed"] + report["errors"].extend(hdr["errors"]) + for harness in harnesses: + gold = load_gold(harness) + dest_dir = SOURCES_DIR / source_stem(harness) + dest_dir.mkdir(parents=True, exist_ok=True) + for row in gold: + sid = str(row.get("section_id") or "").strip() + href = str(row.get("hyperlink") or "").strip() + if not sid: + continue + out = dest_dir / f"{sid}.txt" + if out.is_file() and out.stat().st_size > 200: + report["files"] += 1 + continue + if harness.get("local_sources") and not href: + report["files"] += 1 + continue + if not href: + continue + try: + kind, text = fetch_row_source(row, harness) + write_source_with_prefix(out, row, harness, text) + report["files"] += 1 + print( + f" fetched {gold_filename(harness)} {sid} ({kind}, {len(text)} chars)", + flush=True, + ) + except Exception as exc: # noqa: BLE001 + msg = f"{gold_filename(harness)}:{sid}: {exc}" + report["errors"].append(msg) + print(f" FAIL {msg}", flush=True) + return report + + +def run_pipeline( + run_id: str, cache: str, *, keep_all_knowledge: bool = False +) -> Dict[str, Any]: + from application import sqla + from application.cmd.cre_main import db_connect + from application.database.db import ( + DecisionQueueItem, + HarvestInput, + KnowledgeQueueItem, + ) + from application.utils.harvester.chunk_pipeline import DocumentChunkPipeline + from application.utils.harvester.harvest_writer import write_harvest_input + from application.utils.harvester.heading_extractor import HeadingExtractor + from application.utils.harvester.models import Document, Locator, SourceInfo + from application.utils.harvester.pipeline import RunSummary + from application.utils.harvester.schemas import ChunkingConfig + from application.utils.oie_orchestrator import run_oie_pipeline + + headings = HeadingExtractor() + committed_at = datetime.now(timezone.utc) + sha = "b2source0000000000000000000000000000001" + chunking = ChunkingConfig( + strategy="docling", + max_tokens=1000, + overlap_tokens=80, + merge_profile="narrative", + ) + pipeline = DocumentChunkPipeline(chunking=chunking) + + def run_harvester( + session: Any, rid: str, *, dry_run: bool = False, sync_repos: bool = True + ): + summary = RunSummary(run_id=rid, dry_run=dry_run, repositories=0) + all_records = [] + for harness in HARNESSES: + gold = load_gold(harness) + summary.repositories += 1 + src_dir = SOURCES_DIR / source_stem(harness) + for row in gold: + sid = str(row.get("section_id") or "").strip() + if not sid: + continue + if not (row.get("cre_ids") or []): + continue + path = src_dir / f"{sid}.txt" + if not path.is_file(): + continue + text = path.read_text() + if len(text.strip()) < 80: + continue + rel = f"b2/{source_stem(harness)}/{sid}.txt" + doc = Document( + schema_version="0.2.0", + artifact_id=f"art:B2/{source_stem(harness)}:{sid}", + pipeline_run_id=rid, + text=text, + heading_structure=headings.extract(text), + source=SourceInfo( + type="github", + repository=f"B2/{source_stem(harness)}", + commit_sha=sha, + committed_at=committed_at, + ), + locator=Locator(kind="repo_path", id=rel, path=rel), + ) + recs = pipeline.chunk(doc) + all_records.extend(recs) + summary.files_seen += 1 + summary.files_retained += 1 + print( + f" {harness['label']} {sid}: chunks={len(recs)}", + flush=True, + ) + summary.documents_emitted += sum( + 1 + for r in gold + if (r.get("cre_ids") or []) + and (src_dir / f"{r['section_id']}.txt").is_file() + ) + if dry_run: + summary.chunks_written = len(all_records) + summary.status = "ok" + return summary + written = write_harvest_input(session, rid, all_records) + summary.chunks_written = written + summary.status = "ok" if written else "degraded: no chunks" + return summary + + def keep_all_noise_filter(session: Any, pipeline_run_id: str, **_kwargs: Any): + """Harness-only: promote every harvest chunk to KNOWLEDGE (score Module C).""" + from application.utils.noise_filter.hashing import compute_content_hash + from application.utils.noise_filter.pipeline import RunSummary as BSummary + from application.utils.noise_filter.queue_writer import write_verdicts + from application.utils.noise_filter.schemas import ( + ChangeRecord, + ClassifyResult, + ) + + summary = BSummary(run_id=pipeline_run_id) + rows = ( + session.query(HarvestInput) + .filter_by(pipeline_run_id=pipeline_run_id, status="pending") + .all() + ) + summary.read = len(rows) + triples = [] + for row in rows: + try: + record = ChangeRecord.model_validate(row.payload) + except Exception: # noqa: BLE001 + summary.parse_errors += 1 + continue + verdict = ClassifyResult( + label="KNOWLEDGE", + confidence=1.0, + reasoning="b2-keep-all-knowledge", + ) + triples.append((record, verdict, compute_content_hash(record.text))) + row.status = "processed" + summary.kept_knowledge += 1 + stats = write_verdicts(session, triples) + summary.inserted = stats.inserted + summary.deduped = stats.deduped + session.commit() + summary.status = "ok" + return summary + + db_connect(cache) + for model in (HarvestInput, KnowledgeQueueItem, DecisionQueueItem): + deleted = sqla.session.query(model).delete(synchronize_session=False) + print(f"cleared {model.__tablename__}: {deleted}", flush=True) + sqla.session.commit() + + result = run_oie_pipeline( + cache_file=cache, + pipeline_run_id=run_id, + sync_repos=False, + run_harvester_fn=run_harvester, + run_noise_filter_fn=keep_all_noise_filter if keep_all_knowledge else None, + use_langgraph=True, + ) + return json.loads(result.to_json()) + + +def score_run(run_id: str, cache: str) -> Dict[str, Any]: + from application import sqla + from application.cmd.cre_main import db_connect + from application.database.db import CRE, DecisionQueueItem + + db_connect(cache) + uuid_to_ext = { + row.id: (row.external_id or row.id) + for row in sqla.session.query(CRE.id, CRE.external_id).all() + } + decisions = ( + sqla.session.query(DecisionQueueItem).filter_by(pipeline_run_id=run_id).all() + ) + + # section_key → predicted CRE set + pred: Dict[str, Set[str]] = {} + meta: Dict[str, Dict[str, Any]] = {} + for row in decisions: + env = row.envelope if isinstance(row.envelope, dict) else {} + knowledge = env.get("knowledge") or {} + loc = knowledge.get("locator") or {} + path = str(loc.get("path") or loc.get("id") or "") + # b2//
.txt + parts = Path(path).parts + if len(parts) < 3 or parts[0] != "b2": + # also try artifact_id art:B2/: + chunk_id = row.chunk_id or "" + m = re.search(r"art:B2/([^:]+):([^:]+)", chunk_id) + if not m: + continue + resource, sid = m.group(1), m.group(2) + else: + resource = parts[1] + sid = Path(parts[-1]).stem + key = f"{resource}::{sid}" + top2 = top2_cre_external_ids(env, uuid_to_ext) + pred.setdefault(key, set()).update(t for t in top2 if t) + meta[key] = {"resource": resource, "section_id": sid, "path": path} + + details: List[Dict[str, Any]] = [] + by_resource: Dict[str, Dict[str, int]] = {} + hits = scorable = unaligned = 0 + + for harness in HARNESSES: + resource = source_stem(harness) + gold_rows = load_gold(harness) + bucket = by_resource.setdefault( + harness["label"], + {"scorable": 0, "hits": 0, "unaligned": 0, "pr": harness["pr"]}, + ) + for row in gold_rows: + sid = str(row.get("section_id") or "").strip() + gold = {c for c in (row.get("cre_ids") or []) if c} + if not gold: + continue + key = f"{resource}::{sid}" + suggested = pred.get(key) or set() + if key not in pred: + unaligned += 1 + bucket["unaligned"] += 1 + details.append( + { + "resource": harness["label"], + "pr": harness["pr"], + "section_id": sid, + "aligned": False, + "gold": sorted(gold), + "predicted": [], + "hit": False, + } + ) + continue + scorable += 1 + bucket["scorable"] += 1 + hit = bool(suggested & gold) + if hit: + hits += 1 + bucket["hits"] += 1 + details.append( + { + "resource": harness["label"], + "pr": harness["pr"], + "section_id": sid, + "aligned": True, + "gold": sorted(gold), + "predicted": sorted(suggested), + "hit": hit, + } + ) + + rate = (hits / scorable) if scorable else 0.0 + for label, bucket in by_resource.items(): + s = bucket["scorable"] + bucket["accuracy"] = round((bucket["hits"] / s) if s else 0.0, 4) + + return { + "run_id": run_id, + "gate": "B2 PR JSON gold (section grain, ≥1 of union(rerank top-2 ∪ vector top-2) ∈ gold cre_ids)", + "harnesses": [ + { + "pr": h["pr"], + "label": h["label"], + "gold_file": gold_filename(h), + "fixture_name": h.get("fixture_name"), + } + for h in HARNESSES + ], + "decisions_total": len(decisions), + "scorable": scorable, + "unaligned": unaligned, + "hits": hits, + "accuracy": round(rate, 4), + "pass_gt_60": rate > 0.60, + "by_resource": by_resource, + "details": details, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--score-only", action="store_true") + parser.add_argument("--run-id", default="") + parser.add_argument( + "--keep-all-knowledge", + action="store_true", + help=( + "Bypass Gemini noise filter; enqueue every harvest chunk as " + "KNOWLEDGE so Module C is scored (HTML nav pages often false-NOISE)." + ), + ) + parser.add_argument( + "--refresh-source-headers", + action="store_true", + help=( + "Rewrite Standard/Version/Source/Section headers on cached " + "b2_sources without re-downloading bodies." + ), + ) + parser.add_argument( + "--headers-only", + action="store_true", + help="Only refresh source headers, then exit (no pipeline / score).", + ) + parser.add_argument( + "--only-fixtures", + nargs="*", + default=[], + metavar="STEM", + help=( + "If set, keep only these fixture stems. " + "Example: --only-fixtures owasp_aisvs_1_0" + ), + ) + parser.add_argument( + "--out", + type=Path, + default=ART / "b2_accuracy_report.json", + ) + args = parser.parse_args() + + os.environ.setdefault("FLASK_CONFIG", "development") + os.environ.setdefault("NO_LOAD_GRAPH_DB", "1") + os.environ.setdefault("CRE_LIBRARIAN_RETRIEVER_BACKEND", "pgvector") + _load_dotenv() + cache = os.environ.get( + "DEV_DATABASE_URL", "postgresql://cre:password@127.0.0.1:5432/cre" + ) + + only_fixtures = { + str(s).strip() for s in (args.only_fixtures or []) if str(s).strip() + } + if only_fixtures: + HARNESSES[:] = [ + h + for h in HARNESSES + if str(h.get("fixture_name") or "") in only_fixtures + or gold_stem(h) in only_fixtures + ] + if not HARNESSES: + print( + "no harnesses match --only-fixtures:", + sorted(only_fixtures), + file=sys.stderr, + ) + return 2 + + GOLD_DIR.mkdir(parents=True, exist_ok=True) + missing = [] + for harness in HARNESSES: + try: + load_gold(harness) + except FileNotFoundError: + missing.append(gold_filename(harness)) + if missing: + print( + "missing gold files — mapping fixtures or local stub not found:", + missing, + file=sys.stderr, + ) + return 2 + + if args.headers_only: + hdr = refresh_source_headers(HARNESSES) + print(json.dumps(hdr, indent=2), flush=True) + return 0 if not hdr["errors"] else 2 + + if not args.score_only: + print("fetching source pages…", flush=True) + src_report = ensure_sources( + HARNESSES, refresh_headers=args.refresh_source_headers + ) + print(json.dumps(src_report, indent=2), flush=True) + run_id = args.run_id or ( + "orch-b2-" + datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + ) + print(json.dumps({"run_id": run_id, "cache": cache}, indent=2), flush=True) + orch = run_pipeline(run_id, cache, keep_all_knowledge=args.keep_all_knowledge) + (ART / "b2_phase_status.json").write_text(json.dumps(orch, indent=2) + "\n") + print(json.dumps(orch, indent=2), flush=True) + else: + run_id = args.run_id + if not run_id: + print("--run-id required with --score-only", file=sys.stderr) + return 2 + + report = score_run(run_id, cache) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps({k: report[k] for k in report if k != "details"}, indent=2)) + print(f"wrote {args.out}", flush=True) + return 0 if report["scorable"] else 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/oie_owasp_eval/run_exhaustive_switch_grid.py b/scripts/oie_owasp_eval/run_exhaustive_switch_grid.py new file mode 100644 index 000000000..07b7e856a --- /dev/null +++ b/scripts/oie_owasp_eval/run_exhaustive_switch_grid.py @@ -0,0 +1,1250 @@ +#!/usr/bin/env python3 +"""Exhaustive B2 switch grid on shared harvest + Module C re-score. + +Canonical gold only (``local_gold: false``); Agentic excluded from headline. +Reuses ``orch-exp-baseline-20260921`` knowledge_queue; clears decisions and +re-runs Module C per combo. + +Writes: + tmp/oie_owasp_eval/experiments/grid/.json + tmp/oie_owasp_eval/experiments/grid/master.csv + tmp/oie_owasp_eval/experiments/grid/comparison.md + tmp/oie_owasp_eval/experiments/comparison.md (grid summary pointer + top) + +Fixed (not axes): PRIOR_CAGE / PREF_INJECT / PREFER_AUDIT_IDS = Lawrence on; +STANDARD_RETRIEVAL off; SHORTLIST_JUDGE off (deterministic, no Gemini judge); +CRE_LIBRARIAN_DEVICE=cpu; trained CE not in grid. +""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import itertools +import json +import os +import subprocess +import sys +import time +import traceback +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple +from urllib.parse import urlparse + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +EXP_ART = ROOT / "tmp" / "oie_owasp_eval" / "experiments" +GRID_DIR = EXP_ART / "grid" +LOCKS_DIR = GRID_DIR / ".locks" +COMPARISON_MD = EXP_ART / "comparison.md" +GRID_COMPARISON_MD = GRID_DIR / "comparison.md" +MASTER_CSV = GRID_DIR / "master.csv" +STATE_PATH = GRID_DIR / "_state.json" +ARTIFACTS_LOCKDIR = GRID_DIR / ".artifacts_rebuild.lockdir" +WINNERS_MD = ROOT / "scripts" / "oie_owasp_eval" / "experiments" / "WINNERS.md" +STACK_YAML = ROOT / "scripts" / "oie_owasp_eval" / "experiments" / "stack_winners.yaml" +EMBED_CACHE_DIR = ROOT / "tmp" / "oie_owasp_eval" / "embed_cache" + +RUN_ID = "orch-exp-baseline-20260921" +DEFAULT_DB_URL = "postgresql://cre:password@127.0.0.1:5432/cre" +_EMBED_CACHE_INSTALLED = False +_EMBED_CACHE_STATS = {"hit": 0, "miss": 0} +_DB_HANDLE: Any = None +_CE_WARMED = False + +# Axes inventory (eval-time boolean / discrete toggles). +# Order is stable and encoded into combo ids. +AXES: List[Dict[str, Any]] = [ + { + "key": "cre_summary", + "env": "CRE_LIBRARIAN_CRE_SUMMARY", + "levels": (("0", False), ("1", True)), + "note": "CRE search blurbs", + }, + { + "key": "dual_index", + "env": "CRE_LIBRARIAN_DUAL_INDEX", + "levels": (("0", False), ("1", True)), + "note": "title vs body pools; typically needs cre_summary", + }, + { + "key": "use_rrf", + "env": "CRE_LIBRARIAN_USE_RRF", + "levels": (("0", False), ("1", True)), + "note": "RRF merge; NO-OP unless dual_index=1", + }, + { + "key": "context_enrich", + "env": "CRE_LIBRARIAN_CONTEXT_ENRICH", + "levels": (("0", False), ("1", True)), + "note": "prefix Standard/Section when missing", + }, + { + "key": "margin_gamma", + "env": "CRE_LIBRARIAN_MARGIN_GAMMA", + "levels": (("off", None), ("0.85", "0.85")), + "note": "relative margin cutoff; off vs 0.85", + }, + { + "key": "hub_leaf_cage", + "env": "CRE_LIBRARIAN_HUB_LEAF_CAGE", + "levels": (("0", False), ("1", True)), + "note": "hub→leaf cage", + }, + { + "key": "focus_query", + "env": "CRE_LIBRARIAN_FOCUS_QUERY", + "levels": (("0", False), ("1", True)), + "note": "Lawrence FOCUS_QUERY on/off (default off)", + }, + { + "key": "hybrid", + "env": None, # multi-env + "levels": ( + ( + "lawrence", + { + "CRE_LIBRARIAN_HYBRID_BETA": "0", + "CRE_LIBRARIAN_HYBRID_GAMMA": "0.70", + }, + ), + ( + "nameheavy", + { + "CRE_LIBRARIAN_HYBRID_BETA": "3.0", + "CRE_LIBRARIAN_HYBRID_GAMMA": "0.15", + }, + ), + ), + "note": "CE-led β=0/γ=0.70 vs name-heavy β=3.0/γ=0.15", + }, + { + "key": "cre_text_enrich", + "env": "CRE_LIBRARIAN_CRE_TEXT_ENRICH", + "levels": (("0", False), ("1", True)), + "note": "CRE text enrich (already-wired boolean)", + }, +] + +FIXED_ENV: Dict[str, str] = { + "CRE_LIBRARIAN_PRIOR_CAGE": "1", + "CRE_LIBRARIAN_PREF_INJECT": "1", + "CRE_LIBRARIAN_PREFER_AUDIT_IDS": "1", + # Fixed OFF for the 512-combo grid: Gemini shortlist-judge hangs (0% CPU + # multi-minute stalls) made judge-ON factorial infeasible. Absolute scores + # are lower than Phase-3 one-factor (judge ON); relative ranking is under + # a consistent judge-off regime. Not a grid axis. + "CRE_LIBRARIAN_SHORTLIST_JUDGE": "0", + "CRE_LIBRARIAN_DEVICE": "cpu", + "CRE_LIBRARIAN_RETRIEVER_BACKEND": "pgvector", +} + +CLEAR_ENV: Tuple[str, ...] = ( + "CRE_LIBRARIAN_CRE_SUMMARY", + "CRE_LIBRARIAN_DUAL_INDEX", + "CRE_LIBRARIAN_USE_RRF", + "CRE_LIBRARIAN_CONTEXT_ENRICH", + "CRE_LIBRARIAN_MARGIN_GAMMA", + "CRE_LIBRARIAN_HUB_LEAF_CAGE", + "CRE_LIBRARIAN_FOCUS_QUERY", + "CRE_LIBRARIAN_HYBRID_BETA", + "CRE_LIBRARIAN_HYBRID_GAMMA", + "CRE_LIBRARIAN_CRE_TEXT_ENRICH", + "CRE_LIBRARIAN_STANDARD_RETRIEVAL", + "CRE_LIBRARIAN_CROSSENCODER_MODEL", +) + +AGENTIC_LABEL = "OWASP Agentic AI (stub, no hub Links)" + + +def total_combos() -> int: + n = 1 + for ax in AXES: + n *= len(ax["levels"]) + return n + + +def _level_tag(ax: Mapping[str, Any], level: Tuple[Any, Any]) -> str: + return str(level[0]) + + +def iter_combos() -> Iterable[Tuple[str, Dict[str, str], Dict[str, Any], List[str]]]: + """Yield (combo_id, env_flags, axis_vector, notes).""" + level_lists = [list(ax["levels"]) for ax in AXES] + for picks in itertools.product(*level_lists): + tags = [_level_tag(AXES[i], picks[i]) for i in range(len(AXES))] + # Compact id: g_ + combo_id = "g_" + "_".join(tags) + env: Dict[str, str] = {} + vector: Dict[str, Any] = {} + notes: List[str] = [] + for ax, pick in zip(AXES, picks): + tag, val = pick + vector[ax["key"]] = tag + if ax["key"] == "hybrid": + assert isinstance(val, dict) + env.update({k: str(v) for k, v in val.items()}) + elif ax["key"] == "margin_gamma": + if val is None: + # leave unset (= off) + pass + else: + env["CRE_LIBRARIAN_MARGIN_GAMMA"] = str(val) + else: + env[ax["env"]] = "1" if val else "0" + # No-op annotations + if vector.get("use_rrf") == "1" and vector.get("dual_index") == "0": + notes.append("noop: USE_RRF without DUAL_INDEX") + if vector.get("dual_index") == "1" and vector.get("cre_summary") == "0": + notes.append( + "dual_index without cre_summary (may be weak / empty summary pool)" + ) + yield combo_id, env, vector, notes + + +def parse_db_url(url: Optional[str] = None) -> Dict[str, str]: + """Parse DEV_DATABASE_URL into psql connection pieces.""" + raw = url or os.environ.get("DEV_DATABASE_URL") or DEFAULT_DB_URL + parsed = urlparse(raw) + db = (parsed.path or "/cre").lstrip("/") or "cre" + return { + "host": parsed.hostname or "127.0.0.1", + "port": str(parsed.port or 5432), + "user": parsed.username or "cre", + "password": parsed.password or os.environ.get("PGPASSWORD", "password"), + "dbname": db, + "url": raw, + } + + +def reset_queues() -> None: + """Clear decisions + unconsume KQ for RUN_ID on *this worker's* database.""" + conn = parse_db_url() + sql = ( + f"DELETE FROM decision_queue WHERE pipeline_run_id = '{RUN_ID}'; " + f"UPDATE knowledge_queue SET consumed_at = NULL " + f"WHERE pipeline_run_id = '{RUN_ID}';" + ) + subprocess.run( + [ + "psql", + "-h", + conn["host"], + "-p", + conn["port"], + "-U", + conn["user"], + "-d", + conn["dbname"], + "-v", + "ON_ERROR_STOP=1", + "-c", + sql, + ], + check=True, + env={**os.environ, "PGPASSWORD": conn["password"]}, + ) + + +def claim_combo(combo_id: str) -> bool: + """Exclusive claim via mkdir lockdir so parallel workers never double-write. + + Returns False if another worker holds the claim or a completed JSON exists. + """ + out = GRID_DIR / f"{combo_id}.json" + if out.is_file(): + try: + data = json.loads(out.read_text()) + if data.get("status") == "ok" and data.get("headline"): + return False + except (json.JSONDecodeError, OSError): + pass + LOCKS_DIR.mkdir(parents=True, exist_ok=True) + lock = LOCKS_DIR / combo_id + try: + lock.mkdir() + (lock / "pid").write_text(f"{os.getpid()}\n") + return True + except FileExistsError: + return False + + +def release_combo_claim(combo_id: str) -> None: + """Drop claim after a failed combo so another worker/resume can retry.""" + lock = LOCKS_DIR / combo_id + if not lock.is_dir(): + return + try: + for child in lock.iterdir(): + child.unlink(missing_ok=True) + lock.rmdir() + except OSError: + pass + + +def write_combo_json_atomic(combo_id: str, payload: Mapping[str, Any]) -> Path: + """Write g_.json via temp + rename (no partial JSON visible).""" + GRID_DIR.mkdir(parents=True, exist_ok=True) + out = GRID_DIR / f"{combo_id}.json" + tmp = GRID_DIR / f".{combo_id}.{os.getpid()}.tmp.json" + text = json.dumps(dict(payload), indent=2, default=str) + "\n" + tmp.write_text(text) + os.replace(tmp, out) + return out + + +def try_acquire_artifacts_lock(timeout_sec: float = 30.0) -> bool: + """mkdir flock for master.csv / comparison / winners rebuild.""" + deadline = time.time() + timeout_sec + ARTIFACTS_LOCKDIR.parent.mkdir(parents=True, exist_ok=True) + while time.time() < deadline: + try: + ARTIFACTS_LOCKDIR.mkdir() + (ARTIFACTS_LOCKDIR / "pid").write_text(f"{os.getpid()}\n") + return True + except FileExistsError: + # Stale lock reclaim if holder is dead. + try: + pid_s = (ARTIFACTS_LOCKDIR / "pid").read_text().strip() + pid = int(pid_s) if pid_s else 0 + except (OSError, ValueError): + pid = 0 + if pid and pid != os.getpid(): + try: + os.kill(pid, 0) + except OSError: + try: + for child in ARTIFACTS_LOCKDIR.iterdir(): + child.unlink(missing_ok=True) + ARTIFACTS_LOCKDIR.rmdir() + continue + except OSError: + pass + time.sleep(0.25) + return False + + +def release_artifacts_lock() -> None: + if not ARTIFACTS_LOCKDIR.is_dir(): + return + try: + for child in ARTIFACTS_LOCKDIR.iterdir(): + child.unlink(missing_ok=True) + ARTIFACTS_LOCKDIR.rmdir() + except OSError: + pass + + +def clear_and_apply(flags: Mapping[str, str]) -> None: + for k in CLEAR_ENV: + os.environ.pop(k, None) + for k, v in FIXED_ENV.items(): + os.environ[k] = v + for k, v in flags.items(): + os.environ[k] = str(v) + + +def install_query_embed_disk_cache() -> None: + """Cache PromptHandler.get_text_embeddings by sha256(text) across combos. + + Many grid axes (hybrid / margin / hub_leaf / dual / summary / rrf) do not + change the query string, so Module C re-embeds identical text hundreds of + times. Disk cache keeps Gemini calls off the hot path after the first miss. + Safe for this eval harness only (monkeypatch). + """ + global _EMBED_CACHE_INSTALLED + if _EMBED_CACHE_INSTALLED: + return + from application.prompt_client.prompt_client import PromptHandler + + EMBED_CACHE_DIR.mkdir(parents=True, exist_ok=True) + original = PromptHandler.get_text_embeddings + + def _cached(self: Any, text: Any) -> Any: + if not isinstance(text, str): + return original(self, text) + key = hashlib.sha256(text.encode("utf-8")).hexdigest() + path = EMBED_CACHE_DIR / f"{key}.json" + if path.is_file(): + try: + _EMBED_CACHE_STATS["hit"] += 1 + return json.loads(path.read_text()) + except (json.JSONDecodeError, OSError): + pass + _EMBED_CACHE_STATS["miss"] += 1 + vec = original(self, text) + try: + path.write_text(json.dumps(list(vec))) + except OSError: + pass + return vec + + PromptHandler.get_text_embeddings = _cached # type: ignore[method-assign] + _EMBED_CACHE_INSTALLED = True + print(f"EMBED_CACHE dir={EMBED_CACHE_DIR}", flush=True) + + +def get_database() -> Any: + """Reuse one DB handle across combos (same process, shared harvest).""" + global _DB_HANDLE + if _DB_HANDLE is None: + from application.cmd.cre_main import db_connect + + _DB_HANDLE = db_connect(path=os.environ["DEV_DATABASE_URL"]) + return _DB_HANDLE + + +def warm_cross_encoder() -> None: + """Load CE once before the first Module C combo (kept warm via process cache).""" + global _CE_WARMED + if _CE_WARMED: + return + from application.utils.librarian.cross_encoder import ( + DEFAULT_CROSSENCODER_MODEL, + build_cross_encoder_score_fn, + ) + + model = ( + os.environ.get("CRE_LIBRARIAN_CROSSENCODER_MODEL") or DEFAULT_CROSSENCODER_MODEL + ) + build_cross_encoder_score_fn(model) + _CE_WARMED = True + print( + f"CE_WARM model={model} device={os.environ.get('CRE_LIBRARIAN_DEVICE', '')}", + flush=True, + ) + + +def run_module_c(flags: Mapping[str, str]) -> Dict[str, Any]: + clear_and_apply(flags) + install_query_embed_disk_cache() + warm_cross_encoder() + from application.utils.librarian.config_loader import load_config + from application.utils.librarian.envelope_sink import DbEnvelopeSink + from application.utils.librarian.factory import build_components + from application.utils.librarian.queue_runner import run_librarian_queue + + cfg = load_config() + snapshot = { + k: getattr(cfg, k) + for k in ( + "cre_summary", + "dual_index", + "use_rrf", + "context_enrich", + "margin_gamma", + "hub_leaf_cage", + "focus_query", + "hybrid_beta", + "hybrid_gamma", + "cre_text_enrich", + "prior_cage", + "pref_inject", + "prefer_audit_ids", + ) + } + database = get_database() + components = build_components(database, config=cfg) + sink = DbEnvelopeSink(database.session, RUN_ID) + summary = run_librarian_queue( + database.session, + RUN_ID, + components, + cfg, + at=datetime.now(timezone.utc), + sink=sink, + dry_run=False, + ) + payload = ( + summary + if isinstance(summary, dict) + else getattr(summary, "__dict__", {"raw": str(summary)}) + ) + return {"config": snapshot, "summary": payload} + + +def score_b2(out_path: Path) -> Dict[str, Any]: + cmd = [ + sys.executable, + str(ROOT / "scripts" / "oie_owasp_eval" / "run_b2_pr_mappings.py"), + "--score-only", + "--run-id", + RUN_ID, + "--out", + str(out_path), + "--exclude-fixtures", + "owasp_asvs_5_0_provisional", + ] + # Default excludes agentic unless --include-agentic. + env = os.environ.copy() + env.setdefault("PYTHONPATH", str(ROOT)) + proc = subprocess.run(cmd, cwd=str(ROOT), env=env, check=False) + if proc.returncode != 0 or not out_path.is_file(): + return {"status": "failed", "returncode": proc.returncode} + report = json.loads(out_path.read_text()) + return {"status": "ok", "returncode": 0, "report": report} + + +def headline_from_b2(report: Mapping[str, Any]) -> Dict[str, Any]: + by_resource = dict(report.get("by_resource") or {}) + hits = scorable = 0 + families: Dict[str, Any] = {} + for label, bucket in by_resource.items(): + if label == AGENTIC_LABEL: + continue + families[label] = bucket + hits += int(bucket.get("hits") or 0) + scorable += int(bucket.get("scorable") or 0) + accuracy = round((hits / scorable) if scorable else 0.0, 4) + agentic = by_resource.get(AGENTIC_LABEL) + return { + "hits": hits, + "scorable": scorable, + "accuracy": accuracy, + "by_resource": families, + "excluded_resources": [AGENTIC_LABEL], + "agentic_stub": dict(agentic) if agentic is not None else None, + "footnote": ( + "Agentic AI stub has no hub Links; exclude from headline denominator." + ), + } + + +def write_axes_preamble(path: Path, *, completed: int = 0) -> None: + n = total_combos() + lines = [ + "# Exhaustive B2 switch grid", + "", + f"**Shared harvest:** `{RUN_ID}` (Module C re-score per combo)", + "**Gold:** canonical (`local_gold: false`); Agentic excluded from headline", + f"**Device:** `CRE_LIBRARIAN_DEVICE=cpu`; `CRE_LIBRARIAN_SHORTLIST_JUDGE=0` " + f"(fixed; Gemini judge hangs blocked judge-ON factorial)", + f"**Combos:** {n} full factorial ({completed} completed)", + "", + "## Axes", + "", + "| axis | env / levels | note |", + "|---|---|---|", + ] + for ax in AXES: + levels = ", ".join(str(t[0]) for t in ax["levels"]) + env = ax["env"] or "CRE_LIBRARIAN_HYBRID_BETA + HYBRID_GAMMA" + lines.append(f"| `{ax['key']}` | `{env}` → {levels} | {ax['note']} |") + lines.extend( + [ + "", + "## Fixed (not grid axes)", + "", + "- `PRIOR_CAGE=1`, `PREF_INJECT=1`, `PREFER_AUDIT_IDS=1` (Lawrence defaults)", + "- `STANDARD_RETRIEVAL` off", + "- Trained cross-encoder deferred (not in this grid)", + "- ASVS5 harvest excluded (chapter HTML OOM risk)", + "", + f"Total = {' × '.join(str(len(a['levels'])) for a in AXES)} = **{n}**", + "", + "## Ranked results (hits/scorable)", + "", + ] + ) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(lines) + "\n") + + +def load_completed() -> Dict[str, Dict[str, Any]]: + out: Dict[str, Dict[str, Any]] = {} + if not GRID_DIR.is_dir(): + return out + for path in GRID_DIR.glob("g_*.json"): + # Skip scoring side-cars: g_.b2_report.json + if path.name.endswith(".b2_report.json") or ".b2_report." in path.name: + continue + try: + data = json.loads(path.read_text()) + except json.JSONDecodeError: + continue + if data.get("headline") and data.get("status") == "ok": + out[path.stem] = data + return out + + +def rebuild_master(rows: Sequence[Mapping[str, Any]]) -> List[Dict[str, Any]]: + ranked: List[Dict[str, Any]] = sorted( + [dict(r) for r in rows], + key=lambda r: ( + -float((r.get("headline") or {}).get("accuracy") or 0.0), + -int((r.get("headline") or {}).get("hits") or 0), + str(r.get("experiment_id") or ""), + ), + ) + # Collect family labels + family_labels: List[str] = [] + seen: set = set() + for r in ranked: + for lab in (r.get("headline") or {}).get("by_resource") or {}: + if lab not in seen: + seen.add(lab) + family_labels.append(lab) + + fieldnames = [ + "rank", + "experiment_id", + "hits", + "scorable", + "accuracy", + "notes", + *[ax["key"] for ax in AXES], + *[f"fam::{lab}" for lab in family_labels], + ] + GRID_DIR.mkdir(parents=True, exist_ok=True) + with MASTER_CSV.open("w", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=fieldnames) + w.writeheader() + for i, r in enumerate(ranked, 1): + h = r.get("headline") or {} + vec = r.get("axis_vector") or {} + row: Dict[str, Any] = { + "rank": i, + "experiment_id": r.get("experiment_id"), + "hits": h.get("hits"), + "scorable": h.get("scorable"), + "accuracy": h.get("accuracy"), + "notes": "; ".join(r.get("notes") or []), + } + for ax in AXES: + row[ax["key"]] = vec.get(ax["key"]) + by = h.get("by_resource") or {} + for lab in family_labels: + b = by.get(lab) or {} + row[f"fam::{lab}"] = f"{b.get('hits')}/{b.get('scorable')}" if b else "" + w.writerow(row) + + # comparison.md with preamble + top table + write_axes_preamble(GRID_COMPARISON_MD, completed=len(ranked)) + lines = GRID_COMPARISON_MD.read_text().rstrip() + "\n\n" + lines += ( + "| rank | id | accuracy | hits/scorable | flags | notes |\n" + "|---:|---|---:|---:|---|---|\n" + ) + for i, r in enumerate(ranked[:50], 1): + h = r.get("headline") or {} + flags = r.get("flags") or {} + flag_s = ", ".join(f"{k}={v}" for k, v in sorted(flags.items())) or "(none)" + notes = "; ".join(r.get("notes") or []) or "—" + lines += ( + f"| {i} | `{r.get('experiment_id')}` | {h.get('accuracy')} | " + f"{h.get('hits')}/{h.get('scorable')} | `{flag_s}` | {notes} |\n" + ) + if len(ranked) > 50: + lines += f"\n_… {len(ranked) - 50} more rows in `master.csv`_\n" + # Per-family for top 10 + if family_labels and ranked: + lines += "\n## Top-10 per-family\n\n" + lines += ( + "| rank | id | " + " | ".join(f"`{lab}`" for lab in family_labels) + " |\n" + ) + lines += "|---:|---|" + "|".join(["---:"] * len(family_labels)) + "|\n" + for i, r in enumerate(ranked[:10], 1): + by = (r.get("headline") or {}).get("by_resource") or {} + cells = [] + for lab in family_labels: + b = by.get(lab) or {} + if not b: + cells.append("—") + else: + cells.append(f"{b.get('hits')}/{b.get('scorable')}") + lines += ( + f"| {i} | `{r.get('experiment_id')}` | " + " | ".join(cells) + " |\n" + ) + lines += f"\n_Updated {datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%MZ')}_\n" + GRID_COMPARISON_MD.write_text(lines) + + # Pointer summary in parent comparison.md + top = ranked[:5] if ranked else [] + summary_lines = [ + "# OIE experiment comparison (exhaustive switch grid)", + "", + f"Full factorial grid: **{len(ranked)} / {total_combos()}** completed.", + f"Artifacts: `tmp/oie_owasp_eval/experiments/grid/` " + f"(`master.csv`, `comparison.md`, per-combo `g_*.json`).", + "", + "## Top 5 (canonical gold, agentic excluded)", + "", + "| rank | id | accuracy | hits/scorable | axis vector |", + "|---:|---|---:|---:|---|", + ] + for i, r in enumerate(top, 1): + h = r.get("headline") or {} + vec = r.get("axis_vector") or {} + vec_s = ", ".join(f"{k}={v}" for k, v in vec.items()) + summary_lines.append( + f"| {i} | `{r.get('experiment_id')}` | {h.get('accuracy')} | " + f"{h.get('hits')}/{h.get('scorable')} | `{vec_s}` |" + ) + summary_lines.extend( + [ + "", + "## Axes / combo count", + "", + f"See [grid/comparison.md](grid/comparison.md). " f"N = {total_combos()}.", + "", + f"_Updated {datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%MZ')}_", + "", + ] + ) + COMPARISON_MD.write_text("\n".join(summary_lines)) + return ranked + + +def promote_winners(ranked: Sequence[Mapping[str, Any]]) -> None: + if not ranked: + return + best = ranked[0] + h = best.get("headline") or {} + vec = best.get("axis_vector") or {} + flags = best.get("flags") or {} + yaml_flags: Dict[str, str] = {} + for k, v in flags.items(): + if k in ("CRE_LIBRARIAN_HYBRID_BETA", "CRE_LIBRARIAN_HYBRID_GAMMA"): + if vec.get("hybrid") == "nameheavy": + yaml_flags[k] = str(v) + continue + if k == "CRE_LIBRARIAN_MARGIN_GAMMA": + yaml_flags[k] = str(v) + continue + if str(v) in ("1", "true", "True"): + yaml_flags[k] = "1" + + if yaml_flags: + flag_block = "".join(f' {k}: "{v}"\n' for k, v in sorted(yaml_flags.items())) + flags_section = "flags:\n" + flag_block + else: + flags_section = "flags: {}\n" + STACK_YAML.write_text( + "experiment_id: stack_winners\n" + "description: >\n" + f" Exhaustive switch grid winner ({datetime.now(timezone.utc).date()}): " + f"{best.get('experiment_id')} → {h.get('hits')}/{h.get('scorable')} " + f"({h.get('accuracy')}). Full factorial over retrieval/precision toggles; " + "trained CE deferred. No live Links writes.\n" + f"{flags_section}" + "include_agentic: false\n" + ) + + top5 = ranked[:5] + body = [ + "# OIE experiment winners (exhaustive switch grid)", + "", + f"**Date:** {datetime.now(timezone.utc).strftime('%Y-%m-%d')}", + "**Worktree:** `OpenCRE-wt-oie-1088-harness`", + f"**Shared pipeline knowledge:** `{RUN_ID}` " + "(Module C re-run per combo; CE on CPU; shortlist judge off — " + "Gemini hangs blocked judge-ON factorial)", + "**Gold:** canonical (`local_gold: false`); Agentic stub excluded", + f"**Grid:** {len(ranked)} / {total_combos()} combos " + f"(see `tmp/oie_owasp_eval/experiments/grid/`)", + "", + "## Best combo", + "", + f"- **id:** `{best.get('experiment_id')}`", + f"- **score:** {h.get('hits')}/{h.get('scorable')} ({h.get('accuracy')})", + f"- **axes:** `{json.dumps(vec)}`", + f"- **env flags:** `{json.dumps(flags)}`", + "", + "## Top 5", + "", + "| rank | id | hits/scorable | accuracy | notes |", + "|---:|---|---:|---:|---|", + ] + for i, r in enumerate(top5, 1): + hh = r.get("headline") or {} + notes = "; ".join(r.get("notes") or []) or "—" + body.append( + f"| {i} | `{r.get('experiment_id')}` | " + f"{hh.get('hits')}/{hh.get('scorable')} | {hh.get('accuracy')} | {notes} |" + ) + body.extend( + [ + "", + "## Promoted stack_winners.yaml flags", + "", + "```", + yaml.safe_dump({"flags": yaml_flags}, default_flow_style=False).rstrip(), + "```", + "", + "## No-ops / caveats", + "", + "- `USE_RRF=1` without `DUAL_INDEX=1` is a no-op " + "(copied from rrf=0 sibling; still in master.csv).", + "- ASVS5 harvest still excluded (chapter HTML OOM); not in denominator.", + "- Trained CE deferred — not in this grid.", + "- `SHORTLIST_JUDGE=0` fixed (Gemini judge hangs blocked judge-ON factorial).", + "", + "## Parallel Module C (isolated DBs)", + "", + "4 workers × `cre_grid_w0`…`cre_grid_w3` (~500MB TEMPLATE clones of `cre`).", + "", + "```bash", + "WORKERS=4 scripts/oie_owasp_eval/clone_grid_worker_dbs.sh", + "python -u scripts/oie_owasp_eval/run_exhaustive_switch_grid_parallel.py " + "--workers 4 --clone", + "# re-attach aggregator if it exits while workers keep running:", + "python -u scripts/oie_owasp_eval/run_exhaustive_switch_grid_parallel.py " + "--monitor-only", + "```", + "", + "Do not write live Links from this matrix.", + "", + ] + ) + WINNERS_MD.write_text("\n".join(body)) + + +def combo_id_from_vector(vector: Mapping[str, Any]) -> str: + """Build stable combo id from axis vector tags.""" + return "g_" + "_".join( + [ + str(vector["cre_summary"]), + str(vector["dual_index"]), + str(vector["use_rrf"]), + str(vector["context_enrich"]), + str(vector["margin_gamma"]), + str(vector["hub_leaf_cage"]), + str(vector["focus_query"]), + str(vector["hybrid"]), + str(vector["cre_text_enrich"]), + ] + ) + + +def sibling_rrf_off_id(combo_id: str, vector: Mapping[str, Any]) -> Optional[str]: + """If this combo is RRF-without-dual, return the id with use_rrf=0.""" + if vector.get("use_rrf") != "1" or vector.get("dual_index") != "0": + return None + off = dict(vector) + off["use_rrf"] = "0" + return combo_id_from_vector(off) + + +def rrf_on_twin_id(vector: Mapping[str, Any]) -> Optional[str]: + """If dual=0 and rrf=0, return the pure-noop twin with use_rrf=1.""" + if vector.get("dual_index") != "0" or vector.get("use_rrf") != "0": + return None + on = dict(vector) + on["use_rrf"] = "1" + return combo_id_from_vector(on) + + +def copy_noop_from( + combo_id: str, + flags: Dict[str, str], + vector: Dict[str, Any], + notes: List[str], + source: Mapping[str, Any], +) -> Optional[Dict[str, Any]]: + """Clone a scored sibling for known no-op flag vectors (no Module C).""" + if not claim_combo(combo_id): + return None + payload = dict(source) + payload["experiment_id"] = combo_id + payload["created_at"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + payload["axis_vector"] = vector + payload["flags"] = flags + payload["notes"] = list(notes) + [f"copied_noop_from={source.get('experiment_id')}"] + payload["elapsed_sec"] = 0.0 + payload["copied_from"] = source.get("experiment_id") + payload["status"] = source.get("status") or "ok" + # Do not copy b2_report bytes; headline already in json. + write_combo_json_atomic(combo_id, payload) + print( + f"NOOP_COPY {combo_id} <- {source.get('experiment_id')} " + f"{(payload.get('headline') or {}).get('hits')}/" + f"{(payload.get('headline') or {}).get('scorable')}", + flush=True, + ) + return payload + + +def run_one( + combo_id: str, flags: Dict[str, str], vector: Dict[str, Any], notes: List[str] +) -> Dict[str, Any]: + t0 = time.time() + conn = parse_db_url() + print(f"======== {combo_id} db={conn['dbname']} ========", flush=True) + print(f"flags {flags}", flush=True) + reset_queues() + mc = run_module_c(flags) + b2_path = GRID_DIR / f"{combo_id}.b2_report.json" + scored = score_b2(b2_path) + elapsed = round(time.time() - t0, 1) + payload: Dict[str, Any] = { + "experiment_id": combo_id, + "created_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "run_id": RUN_ID, + "worker_db": conn["dbname"], + "local_gold": False, + "exclude_agentic_from_headline": True, + "axis_vector": vector, + "flags": flags, + "notes": notes, + "fixed_env": FIXED_ENV, + "module_c": { + "config": mc.get("config"), + "summary": { + k: (mc.get("summary") or {}).get(k) + for k in ( + "decisions_total", + "linked", + "review_required", + "errors", + "consumed", + ) + if isinstance(mc.get("summary"), dict) + }, + }, + "elapsed_sec": elapsed, + "status": scored.get("status"), + "b2": None, + "headline": None, + "asvs5": { + "status": "blocked", + "reason": "excluded from B2 harvest (chapter HTML OOMs Module C)", + }, + } + if scored.get("status") == "ok": + report = scored["report"] + payload["b2"] = { + "status": "ok", + "report_path": str(b2_path.relative_to(ROOT)), + "hits": report.get("hits"), + "scorable": report.get("scorable"), + "accuracy": report.get("accuracy"), + "by_resource": report.get("by_resource"), + } + payload["headline"] = headline_from_b2(report) + print( + f"RESULT {combo_id} " + f"{payload['headline']['hits']}/{payload['headline']['scorable']} " + f"({payload['headline']['accuracy']}) in {elapsed}s", + flush=True, + ) + else: + payload["b2"] = {"status": "failed", "returncode": scored.get("returncode")} + print(f"FAIL {combo_id} rc={scored.get('returncode')}", flush=True) + write_combo_json_atomic(combo_id, payload) + return payload + + +def refresh_artifacts_locked( + completed_rows: Sequence[Mapping[str, Any]] +) -> List[Dict[str, Any]]: + """Rebuild master/comparison/winners under mkdir lock (parallel-safe).""" + if not try_acquire_artifacts_lock(): + print("ARTIFACTS_LOCK_BUSY skipping rebuild", flush=True) + return [] + try: + ranked = rebuild_master(completed_rows) + promote_winners(ranked) + return ranked + finally: + release_artifacts_lock() + + +def write_state( + *, + completed: int, + total: int, + last: str, + module_c_runs: int = 0, + noop_copied: int = 0, + worker_id: Optional[int] = None, + workers: Optional[int] = None, + status: str = "in_progress", +) -> None: + payload: Dict[str, Any] = { + "updated_at": datetime.now(timezone.utc).isoformat(), + "completed": completed, + "total": total, + "last": last, + "module_c_runs": module_c_runs, + "noop_copied": noop_copied, + "status": status, + } + if worker_id is not None: + payload["worker_id"] = worker_id + if workers is not None: + payload["workers"] = workers + STATE_PATH.write_text(json.dumps(payload, indent=2) + "\n") + + +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dry-run-count", action="store_true") + parser.add_argument("--limit", type=int, default=0, help="Stop after N new combos") + parser.add_argument("--only", nargs="*", default=[], help="Only these combo ids") + parser.add_argument( + "--refresh-only", action="store_true", help="Rebuild CSV/md only" + ) + parser.add_argument( + "--force", + action="store_true", + help="Re-run even if combo json exists", + ) + parser.add_argument( + "--worker-id", + type=int, + default=None, + help="0-based worker index; with --workers, shard combo indices", + ) + parser.add_argument( + "--workers", + type=int, + default=1, + help="Total parallel workers (default 1 = sequential)", + ) + parser.add_argument( + "--skip-artifacts", + action="store_true", + help="Do not rebuild master.csv/comparison/winners (aggregator does it)", + ) + parser.add_argument( + "--db-name", + type=str, + default=None, + help="Override Postgres database name (sets DEV_DATABASE_URL host/user defaults)", + ) + args = parser.parse_args(list(argv) if argv is not None else None) + + os.chdir(ROOT) + if args.db_name: + os.environ["DEV_DATABASE_URL"] = ( + f"postgresql://cre:password@127.0.0.1:5432/{args.db_name}" + ) + os.environ.setdefault("DEV_DATABASE_URL", DEFAULT_DB_URL) + os.environ.setdefault("FLASK_CONFIG", "development") + os.environ.setdefault("NO_LOAD_GRAPH_DB", "1") + os.environ.setdefault("PYTHONPATH", str(ROOT)) + os.environ.setdefault( + "CRE_LIBRARIAN_CRE_SUMMARY_CACHE", str(ROOT / "tmp" / "oie_cre_summaries") + ) + for k, v in FIXED_ENV.items(): + os.environ.setdefault(k, v) + + workers = max(1, int(args.workers or 1)) + worker_id = args.worker_id + if worker_id is not None and not (0 <= worker_id < workers): + print(f"BAD_WORKER_ID {worker_id} workers={workers}", flush=True) + return 2 + skip_artifacts = bool(args.skip_artifacts) or ( + worker_id is not None and workers > 1 + ) + + GRID_DIR.mkdir(parents=True, exist_ok=True) + LOCKS_DIR.mkdir(parents=True, exist_ok=True) + n = total_combos() + conn = parse_db_url() + print( + f"GRID_TOTAL={n} worker={worker_id}/{workers} db={conn['dbname']} " + f"skip_artifacts={skip_artifacts}", + flush=True, + ) + if worker_id is None: + write_axes_preamble(GRID_COMPARISON_MD, completed=0) + COMPARISON_MD.write_text( + "# OIE experiment comparison (exhaustive switch grid)\n\n" + f"Starting full factorial: **0 / {n}**.\n\n" + "Axes documented in `grid/comparison.md`.\n" + ) + + if args.dry_run_count: + for i, (cid, flags, vec, notes) in enumerate(iter_combos(), 1): + if i <= 3 or i == n: + print(i, cid, vec, notes) + print(f"TOTAL={n}") + return 0 + + if args.refresh_only: + completed = list(load_completed().values()) + ranked = refresh_artifacts_locked(completed) + print(f"REFRESHed {len(completed)} rows ranked={len(ranked)}", flush=True) + return 0 + + completed_map = load_completed() + print(f"RESUME already_ok={len(completed_map)}", flush=True) + ran = 0 + noop_copied = 0 + only = set(args.only or []) + + def _write_state(last: str) -> None: + # Always re-scan disk so parallel workers see each other's progress. + done = len(load_completed()) + write_state( + completed=done, + total=n, + last=last, + module_c_runs=ran, + noop_copied=noop_copied, + worker_id=worker_id, + workers=workers if worker_id is not None else None, + ) + + def _maybe_rebuild() -> None: + if skip_artifacts: + return + # Refresh map from disk before rebuild (noop twins from other workers). + fresh = load_completed() + completed_map.clear() + completed_map.update(fresh) + refresh_artifacts_locked(list(completed_map.values())) + + def _try_noop_copy( + combo_id: str, + flags: Dict[str, str], + vector: Dict[str, Any], + notes: List[str], + ) -> Optional[Dict[str, Any]]: + """Instant-copy RRF-without-dual from rrf=0 sibling when available.""" + sib = sibling_rrf_off_id(combo_id, vector) + if not sib: + return None + # Sibling may have been written by another worker — re-read disk. + if sib not in completed_map: + completed_map.update(load_completed()) + if sib not in completed_map: + return None + return copy_noop_from(combo_id, flags, vector, notes, completed_map[sib]) + + def _in_shard(combo_index: int) -> bool: + if worker_id is None or workers <= 1: + return True + return (combo_index % workers) == worker_id + + # Eager pass: materialize every RRF-without-dual noop whose sibling is done. + # All workers may attempt (claim_combo serializes); keeps noops off Module C. + if not args.force: + eager = 0 + for combo_id, flags, vector, notes in iter_combos(): + if only and combo_id not in only: + continue + if combo_id in completed_map: + continue + payload = _try_noop_copy(combo_id, flags, vector, notes) + if payload is None: + continue + completed_map[combo_id] = payload + noop_copied += 1 + eager += 1 + if eager: + _maybe_rebuild() + _write_state(f"eager_noop×{eager}") + print( + f"EAGER_NOOP_COPY count={eager} completed={len(load_completed())}", + flush=True, + ) + + try: + for combo_index, (combo_id, flags, vector, notes) in enumerate(iter_combos()): + if only and combo_id not in only: + continue + if not args.force and combo_id in completed_map: + continue + # Known no-op: any worker may copy (claim); do not require shard. + is_rrf_noop = ( + vector.get("use_rrf") == "1" and vector.get("dual_index") == "0" + ) + if not is_rrf_noop and not _in_shard(combo_index): + continue + # Re-check disk (another worker may have finished this id). + if not args.force: + completed_map.update(load_completed()) + if combo_id in completed_map: + continue + + payload = ( + None if args.force else _try_noop_copy(combo_id, flags, vector, notes) + ) + if payload is not None: + completed_map[combo_id] = payload + noop_copied += 1 + else: + if is_rrf_noop and not args.force: + # Sibling not ready yet; shard owner of twin will copy later, + # or we skip until sibling lands. Do not Module-C a known noop. + continue + if not _in_shard(combo_index): + continue + if not claim_combo(combo_id): + print(f"SKIP_CLAIMED {combo_id}", flush=True) + continue + try: + payload = run_one(combo_id, flags, vector, notes) + except Exception: # noqa: BLE001 — keep walking the grid + traceback.print_exc() + print(f"COMBO_FAIL {combo_id}", flush=True) + release_combo_claim(combo_id) + ran += 1 + continue + if payload.get("status") == "ok": + completed_map[combo_id] = payload + twin_id = rrf_on_twin_id(vector) + if ( + twin_id + and twin_id not in completed_map + and (not only or twin_id in only) + ): + twin_vec = dict(vector) + twin_vec["use_rrf"] = "1" + twin_flags = dict(flags) + twin_flags["CRE_LIBRARIAN_USE_RRF"] = "1" + twin_notes = list(notes) + ["noop: USE_RRF without DUAL_INDEX"] + twin_payload = copy_noop_from( + twin_id, + twin_flags, + twin_vec, + twin_notes, + payload, + ) + if twin_payload is not None: + completed_map[twin_id] = twin_payload + noop_copied += 1 + else: + # Failed score — release claim so resume can retry. + release_combo_claim(combo_id) + ran += 1 + _maybe_rebuild() + _write_state(combo_id) + if args.limit and ran >= args.limit: + print(f"LIMIT reached ({args.limit})", flush=True) + break + except Exception: # noqa: BLE001 + traceback.print_exc() + if not skip_artifacts: + refresh_artifacts_locked(list(load_completed().values())) + print("GRID_PARTIAL_FAIL", flush=True) + return 1 + + if not skip_artifacts: + refresh_artifacts_locked(list(load_completed().values())) + done = len(load_completed()) + write_state( + completed=done, + total=n, + last="worker_done" if worker_id is not None else "GRID_DONE", + module_c_runs=ran, + noop_copied=noop_copied, + worker_id=worker_id, + workers=workers if worker_id is not None else None, + status="done" if done >= n else "in_progress", + ) + print( + f"GRID_DONE completed={done}/{n} newly_ran={ran} noop_copied={noop_copied} " + f"worker={worker_id}", + flush=True, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/oie_owasp_eval/run_exhaustive_switch_grid_parallel.py b/scripts/oie_owasp_eval/run_exhaustive_switch_grid_parallel.py new file mode 100755 index 000000000..aa6c479d2 --- /dev/null +++ b/scripts/oie_owasp_eval/run_exhaustive_switch_grid_parallel.py @@ -0,0 +1,435 @@ +#!/usr/bin/env python3 +"""Launch isolated parallel Module C switch-grid workers. + +Each worker gets its own Postgres DB (cre_grid_w0 …) cloned from ``cre`` so +decision_queue DELETE/reset never races. Combo JSON writes use mkdir locks in +``grid/.locks/``. This process: + +1. Optionally clones worker DBs (``--clone``). +2. Spawns N worker processes (``run_exhaustive_switch_grid.py --worker-id``). +3. Aggregates status → ``grid/_state.json``, ``status.txt``, master/comparison. + +Re-attach without respawning:: + + python -u scripts/oie_owasp_eval/run_exhaustive_switch_grid_parallel.py --monitor-only + +Example:: + + python -u scripts/oie_owasp_eval/run_exhaustive_switch_grid_parallel.py \\ + --workers 4 --clone +""" + +from __future__ import annotations + +import argparse +import json +import os +import signal +import subprocess +import sys +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence + +ROOT = Path(__file__).resolve().parents[2] +EXP_ART = ROOT / "tmp" / "oie_owasp_eval" / "experiments" +GRID_DIR = EXP_ART / "grid" +STATE_PATH = GRID_DIR / "_state.json" +STATUS_TXT = EXP_ART / "status.txt" +STATUS_LOG = EXP_ART / "status.log" +PARALLEL_DIR = EXP_ART / "parallel" +CLONE_SCRIPT = ROOT / "scripts" / "oie_owasp_eval" / "clone_grid_worker_dbs.sh" +WORKER_SCRIPT = ROOT / "scripts" / "oie_owasp_eval" / "run_exhaustive_switch_grid.py" +DB_PREFIX = "cre_grid_w" + + +def _iso() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def count_completed() -> tuple[int, int]: + sys.path.insert(0, str(ROOT / "scripts" / "oie_owasp_eval")) + from run_exhaustive_switch_grid import load_completed, total_combos # noqa: WPS433 + + return len(load_completed()), total_combos() + + +def refresh_artifacts() -> int: + sys.path.insert(0, str(ROOT / "scripts" / "oie_owasp_eval")) + from run_exhaustive_switch_grid import ( # noqa: WPS433 + load_completed, + refresh_artifacts_locked, + total_combos, + write_state, + ) + + rows = list(load_completed().values()) + refresh_artifacts_locked(rows) + n = total_combos() + write_state( + completed=len(rows), + total=n, + last="aggregator", + status="done" if len(rows) >= n else "in_progress", + workers=None, + ) + return len(rows) + + +def write_status_line( + *, completed: int, total: int, last: str, workers_alive: int +) -> None: + pct = f"{(100.0 * completed / total):.1f}" if total else "0.0" + line = ( + f"{_iso()} {completed}/{total} ({pct}%) last={last} " + f"running={'yes' if workers_alive else 'no'} parallel_workers={workers_alive}" + ) + STATUS_TXT.write_text(line + "\n") + with STATUS_LOG.open("a") as fh: + fh.write(line + "\n") + print(line, flush=True) + + +def clone_dbs(workers: int, force: bool) -> None: + env = os.environ.copy() + env["WORKERS"] = str(workers) + env["FORCE"] = "1" if force else "0" + env.setdefault("PGPASSWORD", "password") + print(f"CLONE workers={workers} force={force}", flush=True) + subprocess.run(["zsh", str(CLONE_SCRIPT)], cwd=str(ROOT), env=env, check=True) + + +def count_worker_procs() -> int: + try: + out = subprocess.check_output( + ["pgrep", "-f", r"run_exhaustive_switch_grid\.py --worker-id"], + text=True, + ) + return len([ln for ln in out.splitlines() if ln.strip()]) + except subprocess.CalledProcessError: + return 0 + + +def spawn_worker( + worker_id: int, + workers: int, + *, + log_path: Path, +) -> subprocess.Popen: + db_name = f"{DB_PREFIX}{worker_id}" + env = os.environ.copy() + env["DEV_DATABASE_URL"] = f"postgresql://cre:password@127.0.0.1:5432/{db_name}" + env["PYTHONPATH"] = str(ROOT) + env["FLASK_CONFIG"] = "development" + env["NO_LOAD_GRAPH_DB"] = "1" + env["CRE_LIBRARIAN_DEVICE"] = "cpu" + env["CRE_LIBRARIAN_SHORTLIST_JUDGE"] = "0" + env["TOKENIZERS_PARALLELISM"] = "false" + env.setdefault( + "CRE_LIBRARIAN_CRE_SUMMARY_CACHE", str(ROOT / "tmp" / "oie_cre_summaries") + ) + env.setdefault("PGPASSWORD", "password") + cmd = [ + sys.executable, + "-u", + str(WORKER_SCRIPT), + "--worker-id", + str(worker_id), + "--workers", + str(workers), + "--db-name", + db_name, + "--skip-artifacts", + ] + log_path.parent.mkdir(parents=True, exist_ok=True) + log_fh = open(log_path, "a", buffering=1) + log_fh.write(f"\n===== WORKER {worker_id} start {_iso()} db={db_name} =====\n") + log_fh.flush() + proc = subprocess.Popen( + cmd, + cwd=str(ROOT), + env=env, + stdout=log_fh, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + proc._log_fh = log_fh # type: ignore[attr-defined] + return proc + + +def load_oie_env() -> None: + oie_env = ROOT / "tmp" / "oie.env" + if not oie_env.is_file(): + return + for line in oie_env.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, _, v = line.partition("=") + os.environ.setdefault(k.strip(), v.strip().strip("'").strip('"')) + + +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--workers", type=int, default=4) + parser.add_argument("--clone", action="store_true", help="Clone cre → cre_grid_w*") + parser.add_argument( + "--force-clone", + action="store_true", + help="Drop and recreate worker DBs", + ) + parser.add_argument( + "--aggregate-only", + action="store_true", + help="Only rebuild artifacts / status and exit", + ) + parser.add_argument( + "--monitor-only", + action="store_true", + help="Do not spawn workers; poll status until done / workers exit", + ) + parser.add_argument( + "--poll-sec", + type=float, + default=30.0, + help="Aggregator poll interval", + ) + args = parser.parse_args(list(argv) if argv is not None else None) + + os.chdir(ROOT) + PARALLEL_DIR.mkdir(parents=True, exist_ok=True) + GRID_DIR.mkdir(parents=True, exist_ok=True) + load_oie_env() + + if args.aggregate_only: + done = refresh_artifacts() + write_status_line( + completed=done, total=512, last="aggregate_only", workers_alive=0 + ) + print(f"AGGREGATE_ONLY completed={done}", flush=True) + return 0 + + workers = max(1, int(args.workers)) + if args.clone or args.force_clone: + clone_dbs(workers, force=bool(args.force_clone)) + + procs: List[subprocess.Popen] = [] + pid_map: Dict[int, int] = {} + + if args.monitor_only: + meta_path = PARALLEL_DIR / "launcher.json" + if meta_path.is_file(): + try: + prev = json.loads(meta_path.read_text()) + pid_map = {int(k): int(v) for k, v in (prev.get("pids") or {}).items()} + except (json.JSONDecodeError, OSError, ValueError): + pid_map = {} + print( + f"MONITOR_ONLY existing_workers={count_worker_procs()} known_pids={pid_map}", + flush=True, + ) + (PARALLEL_DIR / "monitor.json").write_text( + json.dumps( + { + "started_at": _iso(), + "mode": "monitor_only", + "workers": workers, + "pids": pid_map, + "dbs": [f"{DB_PREFIX}{i}" for i in range(workers)], + "pipeline_run_id": "orch-exp-baseline-20260921", + }, + indent=2, + ) + + "\n" + ) + (PARALLEL_DIR / "monitor.pid").write_text(f"{os.getpid()}\n") + else: + for i in range(workers): + db = f"{DB_PREFIX}{i}" + chk = subprocess.run( + [ + "psql", + "-h", + "127.0.0.1", + "-U", + "cre", + "-d", + db, + "-tAc", + "SELECT count(*) FROM knowledge_queue " + "WHERE pipeline_run_id='orch-exp-baseline-20260921'", + ], + env={ + **os.environ, + "PGPASSWORD": os.environ.get("PGPASSWORD", "password"), + }, + capture_output=True, + text=True, + check=False, + ) + if chk.returncode != 0: + print( + f"MISSING_DB {db}: run with --clone. stderr={chk.stderr}", + flush=True, + ) + return 2 + print(f"DB_OK {db} kq={chk.stdout.strip()}", flush=True) + + for i in range(workers): + log = PARALLEL_DIR / f"worker_{i}.log" + proc = spawn_worker(i, workers, log_path=log) + procs.append(proc) + pid_map[i] = proc.pid + print( + f"SPAWNED worker={i} pid={proc.pid} db={DB_PREFIX}{i} log={log}", + flush=True, + ) + + (PARALLEL_DIR / "launcher.json").write_text( + json.dumps( + { + "started_at": _iso(), + "workers": workers, + "pids": pid_map, + "dbs": [f"{DB_PREFIX}{i}" for i in range(workers)], + "pipeline_run_id": "orch-exp-baseline-20260921", + }, + indent=2, + ) + + "\n" + ) + (PARALLEL_DIR / "launcher.pid").write_text(f"{os.getpid()}\n") + + stop = {"flag": False} + + def _handle_sig(_signum: int, _frame: Any) -> None: + stop["flag"] = True + print("AGGREGATOR_SIGNAL shutting down…", flush=True) + for p in procs: + if p.poll() is None: + try: + os.killpg(os.getpgid(p.pid), signal.SIGTERM) + except OSError: + p.terminate() + + signal.signal(signal.SIGTERM, _handle_sig) + signal.signal(signal.SIGINT, _handle_sig) + + try: + completed, total = count_completed() + except Exception: # noqa: BLE001 + completed, total = 0, 512 + alive_n = ( + count_worker_procs() + if args.monitor_only + else sum(1 for p in procs if p.poll() is None) + ) + write_status_line( + completed=completed, + total=total, + last="monitor_start" if args.monitor_only else "parallel_start", + workers_alive=alive_n, + ) + try: + refresh_artifacts() + except Exception: # noqa: BLE001 + print("AGGREGATOR_REFRESH_FAIL initial", flush=True) + + while not stop["flag"]: + try: + alive_n = ( + count_worker_procs() + if args.monitor_only + else sum(1 for p in procs if p.poll() is None) + ) + try: + completed, total = count_completed() + except Exception: # noqa: BLE001 + print("COUNT_FAIL", flush=True) + last = "parallel" + if STATE_PATH.is_file(): + try: + last = str(json.loads(STATE_PATH.read_text()).get("last") or last) + except (json.JSONDecodeError, OSError): + pass + write_status_line( + completed=completed, + total=total, + last=last, + workers_alive=alive_n, + ) + print( + f"HEARTBEAT completed={completed}/{total} alive={alive_n}", flush=True + ) + try: + refresh_artifacts() + print("REFRESH_OK", flush=True) + except Exception: # noqa: BLE001 + import traceback + + traceback.print_exc() + print("AGGREGATOR_REFRESH_FAIL", flush=True) + + if completed >= total: + print(f"PARALLEL_DONE {completed}/{total}", flush=True) + break + if alive_n == 0: + print("ALL_WORKERS_EXITED", flush=True) + time.sleep(2) + try: + refresh_artifacts() + except Exception: # noqa: BLE001 + pass + completed, total = count_completed() + write_status_line( + completed=completed, + total=total, + last="workers_exited", + workers_alive=0, + ) + break + except Exception: # noqa: BLE001 + import traceback + + traceback.print_exc() + print("MONITOR_LOOP_FAIL continuing", flush=True) + time.sleep(max(5.0, float(args.poll_sec))) + + for p in procs: + if p.poll() is None: + try: + os.killpg(os.getpgid(p.pid), signal.SIGTERM) + except OSError: + p.terminate() + try: + p.wait(timeout=30) + except subprocess.TimeoutExpired: + try: + os.killpg(os.getpgid(p.pid), signal.SIGKILL) + except OSError: + p.kill() + fh = getattr(p, "_log_fh", None) + if fh is not None: + try: + fh.close() + except OSError: + pass + + completed, total = count_completed() + try: + refresh_artifacts() + except Exception: # noqa: BLE001 + pass + write_status_line( + completed=completed, + total=total, + last="parallel_exit", + workers_alive=count_worker_procs() if args.monitor_only else 0, + ) + print(f"LAUNCHER_EXIT completed={completed}/{total}", flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/oie_owasp_eval/run_experiment.py b/scripts/oie_owasp_eval/run_experiment.py new file mode 100644 index 000000000..f863cca9b --- /dev/null +++ b/scripts/oie_owasp_eval/run_experiment.py @@ -0,0 +1,718 @@ +#!/usr/bin/env python3 +"""OIE experiment runner: named flag configs → scorers → comparison table. + +Same canonical answer key every run (``local_gold: false`` for gate reports). +One lever per config; writes ``tmp/oie_owasp_eval/experiments/.json`` and +refreshes ``comparison.md``. + +Usage: + PYTHONPATH=. python scripts/oie_owasp_eval/run_experiment.py \\ + --config scripts/oie_owasp_eval/experiments/baseline.yaml + + PYTHONPATH=. python scripts/oie_owasp_eval/run_experiment.py \\ + --experiment-id cre_summary --flags CRE_LIBRARIAN_CRE_SUMMARY=1 + + PYTHONPATH=. python scripts/oie_owasp_eval/run_experiment.py \\ + --config scripts/oie_owasp_eval/experiments/baseline.yaml --dry-run + + PYTHONPATH=. python scripts/oie_owasp_eval/run_experiment.py \\ + --config scripts/oie_owasp_eval/experiments/baseline.yaml \\ + --score-only --run-id orch-b2- +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT_DIR = Path(__file__).resolve().parent +EXPERIMENTS_DIR = SCRIPT_DIR / "experiments" +ART = ROOT / "tmp" / "oie_owasp_eval" +EXP_ART = ART / "experiments" +COMPARISON_MD = EXP_ART / "comparison.md" + +# Experiment levers from the OIE prod-accuracy plan (defaults stay off in prod). +KNOWN_FLAGS: Tuple[str, ...] = ( + "CRE_LIBRARIAN_CRE_SUMMARY", + "CRE_LIBRARIAN_DUAL_INDEX", + "CRE_LIBRARIAN_USE_RRF", + "CRE_LIBRARIAN_CONTEXT_ENRICH", + "CRE_LIBRARIAN_MARGIN_GAMMA", + "CRE_LIBRARIAN_HUB_LEAF_CAGE", + "CRE_LIBRARIAN_CROSSENCODER_MODEL", +) + +# Previously unwired levers; Phase 2 landed them in config_loader. Keep empty +# unless a future experiment flag is env-only before wiring. +FLAGS_NOT_YET_WIRED: Tuple[str, ...] = () + +ASVS5_FIXTURE = ( + ROOT + / "application" + / "tests" + / "fixtures" + / "owasp_mappings" + / "owasp_asvs_5_0_provisional.json" +) +ASVS5_SCORER_CANDIDATES = ( + SCRIPT_DIR / "run_asvs5_score.py", + SCRIPT_DIR / "score_asvs5.py", + SCRIPT_DIR / "run_b2_asvs5.py", +) + + +def _load_dotenv() -> None: + for env_path in (ROOT / ".env", ROOT / "tmp" / "oie.env"): + if not env_path.is_file(): + continue + for line in env_path.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, _, v = line.partition("=") + if env_path.name == "oie.env" and ( + k.startswith("CRE_LIBRARIAN_") or k.startswith("DEV_DATABASE") + ): + os.environ[k.strip()] = v.strip() + else: + os.environ.setdefault(k.strip(), v.strip().strip('"').strip("'")) + + +def parse_flag_pairs(pairs: Sequence[str]) -> Dict[str, str]: + out: Dict[str, str] = {} + for raw in pairs: + if "=" not in raw: + raise ValueError(f"flag must be KEY=VAL, got {raw!r}") + k, _, v = raw.partition("=") + key = k.strip() + if not key: + raise ValueError(f"empty flag key in {raw!r}") + out[key] = v.strip() + return out + + +def load_config(path: Path) -> Dict[str, Any]: + data = yaml.safe_load(path.read_text()) or {} + if not isinstance(data, dict): + raise ValueError(f"config {path} must be a mapping") + return data + + +def resolve_experiment( + *, + config_path: Optional[Path], + experiment_id: Optional[str], + flag_pairs: Sequence[str], +) -> Tuple[str, Dict[str, str], Dict[str, Any]]: + """Return (experiment_id, flags, raw_config_meta).""" + meta: Dict[str, Any] = {} + flags: Dict[str, str] = {} + exp_id = (experiment_id or "").strip() + + if config_path is not None: + meta = load_config(config_path) + exp_id = str(meta.get("experiment_id") or exp_id or config_path.stem).strip() + raw_flags = meta.get("flags") or {} + if not isinstance(raw_flags, Mapping): + raise ValueError(f"{config_path}: flags must be a mapping") + flags = {str(k): "" if v is None else str(v) for k, v in raw_flags.items()} + + cli_flags = parse_flag_pairs(flag_pairs) + flags.update(cli_flags) + + if not exp_id: + raise ValueError( + "experiment_id required (--experiment-id or config experiment_id)" + ) + + return exp_id, flags, meta + + +def apply_flags(flags: Mapping[str, str]) -> List[str]: + """Set os.environ from flag vector; return warnings for unwired flags.""" + warnings: List[str] = [] + for key, val in flags.items(): + os.environ[key] = str(val) + if key in FLAGS_NOT_YET_WIRED: + warnings.append( + f"{key} set in env but not yet read by librarian config_loader" + ) + return warnings + + +def clear_known_experiment_flags() -> None: + """Unset known experiment levers so baseline is not polluted by the shell.""" + for key in KNOWN_FLAGS: + os.environ.pop(key, None) + + +def headline_from_b2( + report: Mapping[str, Any], + *, + exclude_agentic: bool, +) -> Dict[str, Any]: + """Build headline totals; agentic stub footnoted / excludeable.""" + by_resource = dict(report.get("by_resource") or {}) + agentic_label = "OWASP Agentic AI (stub, no hub Links)" + agentic = by_resource.get(agentic_label) + footnote = ( + "Agentic AI stub has no hub Links; exclude from headline denominator " + "(footnote only)." + ) + + if exclude_agentic and agentic is not None: + hits = scorable = 0 + families: Dict[str, Any] = {} + for label, bucket in by_resource.items(): + if label == agentic_label: + continue + families[label] = bucket + hits += int(bucket.get("hits") or 0) + scorable += int(bucket.get("scorable") or 0) + accuracy = round((hits / scorable) if scorable else 0.0, 4) + return { + "hits": hits, + "scorable": scorable, + "accuracy": accuracy, + "by_resource": families, + "excluded_resources": [agentic_label], + "agentic_stub": { + **dict(agentic), + "footnote": footnote, + }, + "footnote": footnote, + } + + return { + "hits": int(report.get("hits") or 0), + "scorable": int(report.get("scorable") or 0), + "accuracy": float(report.get("accuracy") or 0.0), + "by_resource": by_resource, + "excluded_resources": [], + "agentic_stub": ( + {**dict(agentic), "footnote": footnote} if agentic is not None else None + ), + "footnote": footnote if agentic is not None else None, + } + + +def discover_asvs5_hook() -> Dict[str, Any]: + """ASVS5 is primarily a B2 harness arm; dedicated scorer is optional.""" + scorers = [p for p in ASVS5_SCORER_CANDIDATES if p.is_file()] + fixture_ok = ASVS5_FIXTURE.is_file() + sources_dir = SCRIPT_DIR / "fixtures" / "b2_sources" / "owasp_asvs_5_0_provisional" + sources_ok = sources_dir.is_dir() and any(sources_dir.glob("*.txt")) + base: Dict[str, Any] = { + "fixture": (str(ASVS5_FIXTURE.relative_to(ROOT)) if fixture_ok else None), + "b2_harness_arm": True, + "sources_dir": str(sources_dir.relative_to(ROOT)), + "sources_present": sources_ok, + } + if scorers and fixture_ok: + return { + **base, + "status": "available", + "scorer": str(scorers[0].relative_to(ROOT)), + } + if fixture_ok and not sources_ok: + return { + **base, + "status": "blocked", + "reason": ( + "ASVS5 provisional fixture present and listed in B2 harness, " + "but no harvested b2_sources/owasp_asvs_5_0_provisional/*.txt " + "(rows stay unaligned until Test A harvest)" + ), + } + reasons: List[str] = [] + if not fixture_ok: + reasons.append(f"missing fixture {ASVS5_FIXTURE.relative_to(ROOT)}") + if not scorers: + reasons.append( + "no dedicated ASVS5 scorer (B2 harness arm still applies when fixture+sources exist)" + ) + return {**base, "status": "unavailable", "reason": "; ".join(reasons)} + + +def invoke_b2( + *, + run_id: str, + score_only: bool, + keep_all_knowledge: bool, + include_agentic: bool, + out_path: Path, + extra_env: Mapping[str, str], + exclude_fixtures: Optional[Sequence[str]] = None, +) -> Tuple[int, Path]: + """Subprocess B2 with canonical gold forced (no --local-gold).""" + cmd = [ + sys.executable, + str(SCRIPT_DIR / "run_b2_pr_mappings.py"), + "--out", + str(out_path), + ] + if score_only: + cmd.extend(["--score-only", "--run-id", run_id]) + elif run_id: + cmd.extend(["--run-id", run_id]) + if keep_all_knowledge: + cmd.append("--keep-all-knowledge") + if include_agentic: + cmd.append("--include-agentic") + if exclude_fixtures: + cmd.append("--exclude-fixtures") + cmd.extend(list(exclude_fixtures)) + # Never pass --local-gold: gate reports stay on canonical fixtures. + + env = os.environ.copy() + env.update({k: str(v) for k, v in extra_env.items()}) + env.setdefault("PYTHONPATH", str(ROOT)) + if "PYTHONPATH" in env and str(ROOT) not in env["PYTHONPATH"].split(os.pathsep): + env["PYTHONPATH"] = str(ROOT) + os.pathsep + env["PYTHONPATH"] + + proc = subprocess.run(cmd, cwd=str(ROOT), env=env, check=False) + return proc.returncode, out_path + + +def invoke_b1_hook(*, run_id: str, out_path: Path) -> Dict[str, Any]: + scorer = SCRIPT_DIR / "score_b1_opencre_api.py" + if not scorer.is_file(): + return {"status": "unavailable", "reason": "score_b1_opencre_api.py missing"} + if not run_id: + return {"status": "skipped", "reason": "no run_id for B1"} + cmd = [ + sys.executable, + str(scorer), + "--run-id", + run_id, + "--out", + str(out_path), + ] + env = os.environ.copy() + env.setdefault("PYTHONPATH", str(ROOT)) + proc = subprocess.run(cmd, cwd=str(ROOT), env=env, check=False) + result: Dict[str, Any] = { + "status": "ok" if proc.returncode == 0 else "failed", + "returncode": proc.returncode, + "report_path": str(out_path.relative_to(ROOT)) if out_path.is_file() else None, + } + if out_path.is_file(): + try: + payload = json.loads(out_path.read_text()) + result["summary"] = { + k: payload.get(k) + for k in ("hits", "scorable", "accuracy", "by_resource") + if k in payload + } + except json.JSONDecodeError: + result["summary"] = None + return result + + +def invoke_asvs5_hook(*, run_id: str, flags: Mapping[str, str]) -> Dict[str, Any]: + info = discover_asvs5_hook() + if info["status"] != "available": + return info + # Hook reserved for when ASVS5 scorer lands; do not invent a long job here. + return { + "status": "skipped", + "reason": ( + "ASVS5 scorer present but experiment runner only hooks it; " + "invoke scorer separately until matrix job wires it" + ), + "scorer": info.get("scorer"), + "fixture": info.get("fixture"), + "run_id": run_id or None, + "flags": dict(flags), + } + + +def write_experiment_json(path: Path, payload: Mapping[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2) + "\n") + + +def _family_bucket(headline: Mapping[str, Any], label: str) -> Dict[str, Any]: + by = headline.get("by_resource") or {} + bucket = by.get(label) + return dict(bucket) if isinstance(bucket, Mapping) else {} + + +def rebuild_comparison_md(art_dir: Path, out_path: Path) -> None: + rows: List[Dict[str, Any]] = [] + for path in sorted(art_dir.glob("*.json")): + # Skip nested scorer dumps (*.b2_report.json / *.b1_report.json). + if path.name.endswith("_report.json"): + continue + try: + data = json.loads(path.read_text()) + except json.JSONDecodeError: + continue + if not isinstance(data, dict) or not data.get("experiment_id"): + continue + rows.append(data) + + lines = [ + "# OIE experiment comparison", + "", + "Canonical gold (`local_gold: false`). Agentic stub footnoted / excluded " + "from headline unless an experiment sets `include_agentic: true`.", + "", + "## Headline", + "", + "| experiment_id | accuracy | hits/scorable | flags | notes |", + "|---|---:|---:|---|---|", + ] + family_labels: List[str] = [] + seen_families: set = set() + for data in rows: + exp_id = data.get("experiment_id", path_stem_fallback(data)) + headline = data.get("headline") or {} + for label in headline.get("by_resource") or {}: + if label not in seen_families: + seen_families.add(label) + family_labels.append(label) + acc = headline.get("accuracy") + hits = headline.get("hits") + scorable = headline.get("scorable") + if acc is None and data.get("dry_run"): + acc_s, hs = "—", "dry-run" + elif acc is None and (data.get("b2") or {}).get("status") == "blocked": + acc_s, hs = "—", "blocked" + else: + acc_s = f"{acc:.4f}" if isinstance(acc, (int, float)) else "—" + hs = ( + f"{hits}/{scorable}" + if hits is not None and scorable is not None + else "—" + ) + flags = data.get("flags") or {} + flag_s = ", ".join(f"{k}={v}" for k, v in sorted(flags.items())) or "(none)" + notes: List[str] = [] + if data.get("dry_run"): + notes.append("dry-run") + if (data.get("b2") or {}).get("status") == "blocked": + notes.append(str((data.get("b2") or {}).get("reason") or "blocked")) + if headline.get("footnote"): + notes.append("agentic excluded") + if data.get("flag_warnings"): + notes.append("unwired flags") + b1 = data.get("b1") or {} + if b1.get("status") and b1.get("status") not in ("skipped", None): + notes.append(f"b1={b1.get('status')}") + asvs5 = data.get("asvs5") or {} + if asvs5.get("status") == "unavailable": + notes.append("asvs5 n/a") + elif asvs5.get("status") == "blocked": + notes.append(f"asvs5 blocked: {asvs5.get('reason', '')}") + elif asvs5.get("status") and asvs5.get("status") != "skipped": + notes.append(f"asvs5={asvs5.get('status')}") + lines.append( + f"| `{exp_id}` | {acc_s} | {hs} | `{flag_s}` | {'; '.join(notes) or '—'} |" + ) + + if family_labels: + lines.extend( + [ + "", + "## Per-family accuracy (agentic excluded from headline)", + "", + "| experiment_id | " + + " | ".join(f"`{lab}`" for lab in family_labels) + + " |", + "|---|" + "|".join(["---:"] * len(family_labels)) + "|", + ] + ) + for data in rows: + exp_id = data.get("experiment_id", "?") + headline = data.get("headline") or {} + cells: List[str] = [] + for lab in family_labels: + bucket = _family_bucket(headline, lab) + if not bucket: + cells.append("—") + continue + h = bucket.get("hits") + s = bucket.get("scorable") + a = bucket.get("accuracy") + if isinstance(a, (int, float)) and s is not None: + cells.append(f"{a:.2f} ({h}/{s})") + elif s is not None: + cells.append(f"{h}/{s}") + else: + cells.append("—") + lines.append(f"| `{exp_id}` | " + " | ".join(cells) + " |") + + lines.extend( + [ + "", + "## Blockers / provenance", + "", + "- Score-only reuse of one `run_id` only re-scores the **same** " + "Module C decisions; flag ablations require a fresh pipeline run " + "(queues are cleared each B2 invoke).", + "- ASVS5 provisional gold is in the B2 harness list when the " + "fixture exists; without `b2_sources/owasp_asvs_5_0_provisional/` " + "texts those rows stay unaligned (not in scorable denominator).", + "", + f"_Updated {datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%MZ')}_", + "", + ] + ) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text("\n".join(lines)) + + +def path_stem_fallback(_data: Mapping[str, Any]) -> str: + return "?" + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument( + "--config", + type=Path, + help="YAML under scripts/oie_owasp_eval/experiments/*.yaml", + ) + p.add_argument("--experiment-id", default="", help="Override / set experiment id") + p.add_argument( + "--flags", + nargs="*", + default=[], + metavar="KEY=VAL", + help="Extra CRE_LIBRARIAN_* flags (override config)", + ) + p.add_argument( + "--dry-run", + action="store_true", + help="Load config, apply flag vector in-process, write stub JSON + comparison.md", + ) + p.add_argument( + "--score-only", + action="store_true", + help="Pass through to B2 (requires --run-id)", + ) + p.add_argument("--run-id", default="", help="Pipeline run id for B2/B1") + p.add_argument( + "--keep-all-knowledge", + action="store_true", + help="Pass through to B2", + ) + p.add_argument( + "--include-agentic", + action="store_true", + help="Include agentic stub in B2 denominator (default: exclude / footnote)", + ) + p.add_argument( + "--run-b1", + action="store_true", + help="Also invoke B1 scorer when a run_id is available", + ) + p.add_argument( + "--run-asvs5", + action="store_true", + help="Attempt ASVS5 hook when scorer/fixture exist", + ) + p.add_argument( + "--include-asvs5", + action="store_true", + help=( + "Include ASVS5 provisional in B2 harvest+score. Default excludes it " + "(chapter-level HTML sources OOM Module C until proper md harvest)." + ), + ) + p.add_argument( + "--skip-b2", + action="store_true", + help="Do not invoke B2 (still write experiment JSON / comparison)", + ) + return p + + +def main(argv: Optional[Sequence[str]] = None) -> int: + args = build_parser().parse_args(list(argv) if argv is not None else None) + _load_dotenv() + + config_path = args.config + if config_path is not None: + if not config_path.is_file(): + # Allow bare names relative to experiments/ + alt = EXPERIMENTS_DIR / config_path + if alt.is_file(): + config_path = alt + elif (EXPERIMENTS_DIR / f"{config_path}.yaml").is_file(): + config_path = EXPERIMENTS_DIR / f"{config_path}.yaml" + else: + print(f"config not found: {args.config}", file=sys.stderr) + return 2 + config_path = config_path.resolve() + + try: + exp_id, flags, meta = resolve_experiment( + config_path=config_path, + experiment_id=args.experiment_id or None, + flag_pairs=args.flags, + ) + except ValueError as exc: + print(str(exc), file=sys.stderr) + return 2 + + include_agentic = bool(args.include_agentic) or bool(meta.get("include_agentic")) + exclude_agentic = not include_agentic + + clear_known_experiment_flags() + flag_warnings = apply_flags(flags) + + run_id = (args.run_id or str(meta.get("run_id") or "")).strip() + if not run_id and not args.dry_run and not args.score_only and not args.skip_b2: + run_id = "orch-exp-" + datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + + payload: Dict[str, Any] = { + "experiment_id": exp_id, + "created_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "flags": flags, + "flag_warnings": flag_warnings, + "local_gold": False, + "exclude_agentic_from_headline": exclude_agentic, + "config_path": ( + str(config_path.relative_to(ROOT)) if config_path is not None else None + ), + "description": meta.get("description"), + "dry_run": bool(args.dry_run), + "run_id": run_id or None, + "b2": None, + "headline": None, + "b1": {"status": "skipped", "reason": "--run-b1 not set"}, + "asvs5": discover_asvs5_hook(), + "decision_fp": None, + } + + if "CRE_LIBRARIAN_MARGIN_GAMMA" in flags: + payload["decision_fp"] = { + "status": "pending", + "note": ( + "Margin experiment should report B2 hit and decision-FP; " + "FP metric hook not yet implemented in B2 scorer" + ), + } + + EXP_ART.mkdir(parents=True, exist_ok=True) + out_json = EXP_ART / f"{exp_id}.json" + b2_out = EXP_ART / f"{exp_id}.b2_report.json" + + if args.dry_run: + payload["headline"] = { + "hits": None, + "scorable": None, + "accuracy": None, + "by_resource": {}, + "excluded_resources": ( + ["OWASP Agentic AI (stub, no hub Links)"] if exclude_agentic else [] + ), + "footnote": ( + "Agentic AI stub has no hub Links; exclude from headline denominator " + "(footnote only)." + if exclude_agentic + else None + ), + } + payload["b2"] = {"status": "skipped", "reason": "dry-run"} + if args.run_asvs5: + payload["asvs5"] = invoke_asvs5_hook(run_id=run_id, flags=flags) + write_experiment_json(out_json, payload) + rebuild_comparison_md(EXP_ART, COMPARISON_MD) + print(json.dumps(payload, indent=2)) + print(f"wrote {out_json}", flush=True) + print(f"wrote {COMPARISON_MD}", flush=True) + return 0 + + if not args.skip_b2: + exclude_fixtures: List[str] = [] + if not args.include_asvs5: + exclude_fixtures.append("owasp_asvs_5_0_provisional") + payload["asvs5"] = { + **discover_asvs5_hook(), + "status": "blocked", + "reason": ( + "excluded from B2 harvest for matrix stability " + "(chapter HTML OOMs Module C; pass --include-asvs5 to force)" + ), + } + rc, report_path = invoke_b2( + run_id=run_id, + score_only=bool(args.score_only), + keep_all_knowledge=bool(args.keep_all_knowledge), + include_agentic=include_agentic, + out_path=b2_out, + extra_env=flags, + exclude_fixtures=exclude_fixtures, + ) + b2_block: Dict[str, Any] = { + "status": "ok" if rc == 0 else "failed", + "returncode": rc, + "report_path": ( + str(report_path.relative_to(ROOT)) if report_path.is_file() else None + ), + "local_gold": False, + } + if report_path.is_file(): + report = json.loads(report_path.read_text()) + b2_block["hits"] = report.get("hits") + b2_block["scorable"] = report.get("scorable") + b2_block["accuracy"] = report.get("accuracy") + b2_block["by_resource"] = report.get("by_resource") + b2_block["agentic_footnote"] = report.get("agentic_footnote") + payload["run_id"] = report.get("run_id") or run_id + payload["headline"] = headline_from_b2( + report, exclude_agentic=exclude_agentic + ) + # Surface ASVS5 provisional family from B2 when present. + asvs_label = "OWASP ASVS 5.0 (provisional v4→v5 gold)" + by_res = report.get("by_resource") or {} + if asvs_label in by_res: + payload["asvs5"] = { + **discover_asvs5_hook(), + "status": "b2_arm", + "family": by_res[asvs_label], + } + else: + payload["headline"] = None + payload["b2"] = b2_block + if rc != 0 and not report_path.is_file(): + write_experiment_json(out_json, payload) + rebuild_comparison_md(EXP_ART, COMPARISON_MD) + print(f"B2 failed rc={rc}; wrote partial {out_json}", file=sys.stderr) + return rc or 2 + else: + payload["b2"] = {"status": "skipped", "reason": "--skip-b2"} + + effective_run_id = str(payload.get("run_id") or run_id or "") + if args.run_b1: + b1_out = EXP_ART / f"{exp_id}.b1_report.json" + payload["b1"] = invoke_b1_hook(run_id=effective_run_id, out_path=b1_out) + if args.run_asvs5 and (payload.get("asvs5") or {}).get("status") != "b2_arm": + payload["asvs5"] = invoke_asvs5_hook(run_id=effective_run_id, flags=flags) + + write_experiment_json(out_json, payload) + rebuild_comparison_md(EXP_ART, COMPARISON_MD) + print(json.dumps({k: payload[k] for k in payload if k != "b2"}, indent=2)) + if payload.get("b2"): + print(json.dumps({"b2": payload["b2"]}, indent=2)) + print(f"wrote {out_json}", flush=True) + print(f"wrote {COMPARISON_MD}", flush=True) + if payload.get("b2", {}).get("status") == "failed": + return int(payload["b2"].get("returncode") or 2) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/oie_owasp_eval/run_module_c.py b/scripts/oie_owasp_eval/run_module_c.py new file mode 100644 index 000000000..0d28aafb7 --- /dev/null +++ b/scripts/oie_owasp_eval/run_module_c.py @@ -0,0 +1,348 @@ +#!/usr/bin/env python3 +"""Run Module C on dual-classify keepers → decision_queue (Module D handoff). + +Copies CRE embeddings from ``standards_cache.sqlite``, seeds ``knowledge_queue`` +from the expensive-gate keepers (or intersection), runs C.0–C.4 with live +Gemini query embeddings + a vector-score reranker (no torch cross-encoder +required in the agent sandbox), and writes a JSON report of links / reviews. +""" + +from __future__ import annotations + +import json +import os +import shutil +import sqlite3 +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Set + +ROOT = Path(__file__).resolve().parents[2] +ART = ROOT / "tmp" / "oie_owasp_eval" + + +def _load_dotenv() -> None: + env_path = ROOT / ".env" + if not env_path.is_file(): + return + for line in env_path.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, _, v = line.partition("=") + os.environ.setdefault(k.strip(), v.strip().strip('"').strip("'")) + + +def _seed_knowledge(session: Any, run_id: str, keepers: List[Dict[str, Any]]) -> int: + from application.database.db import KnowledgeQueueItem, generate_uuid + from application.utils.noise_filter.hashing import compute_content_hash + + session.query(KnowledgeQueueItem).filter_by(pipeline_run_id=run_id).delete( + synchronize_session=False + ) + n = 0 + for k in keepers: + text = k.get("text") or k.get("_full_text") or k.get("text_preview") or "" + if not text.strip(): + continue + chunk_id = k["chunk_id"] + # chk:art:OWASP/Repo:path:idx → art:OWASP/Repo:path + artifact_id = k.get("artifact_id") + if not artifact_id: + body = chunk_id[4:] if chunk_id.startswith("chk:") else chunk_id + artifact_id = body.rsplit(":", 1)[0] + session.add( + KnowledgeQueueItem( + id=generate_uuid(), + content_hash=compute_content_hash(text), + chunk_id=chunk_id, + artifact_id=artifact_id, + pipeline_run_id=run_id, + schema_version="0.2.0", + source_type="github", + source_repo=k.get("source_repo") or "OWASP/unknown", + source_commit_sha="tarball00000000000000000000000000000001", + source_committed_at="2026-08-29T18:00:00Z", + locator_kind="repo_path", + locator_path=k.get("locator_path") or "", + span_index=0, + span_total=1, + text=text, + llm_label=k.get("llm_label") or "KNOWLEDGE", + confidence=float(k.get("confidence") or 0.9), + created_at=datetime.now(timezone.utc), + ) + ) + n += 1 + session.commit() + return n + + +def _load_keepers_with_full_text() -> List[Dict[str, Any]]: + """Intersection of cheap∩expensive keepers, with full text from eval_expensive.""" + cheap = json.loads((ART / "classify_cheap.json").read_text()) + exp = json.loads((ART / "classify_expensive.json").read_text()) + cheap_ids = {k["chunk_id"] for k in cheap["knowledge"]["would_flow_to_c"]} + exp_map = {k["chunk_id"]: k for k in exp["knowledge"]["would_flow_to_c"]} + both_ids = cheap_ids & set(exp_map) + + # Pull full text from expensive sqlite knowledge_queue + exp_db = ART / "eval_expensive.sqlite" + full: Dict[str, str] = {} + if exp_db.is_file(): + con = sqlite3.connect(exp_db) + for cid, text in con.execute( + "select chunk_id, text from knowledge_queue" + ).fetchall(): + full[cid] = text or "" + con.close() + + keepers = [] + for cid in sorted(both_ids): + k = dict(exp_map[cid]) + k["_full_text"] = full.get(cid) or k.get("text_preview") or "" + k["text"] = k["_full_text"] + keepers.append(k) + return keepers + + +def _cre_lookup(session: Any) -> Dict[str, Dict[str, str]]: + from application.database.db import CRE + + out: Dict[str, Dict[str, str]] = {} + for row in session.query(CRE).all(): + out[row.id] = { + "id": row.id, + "external_id": getattr(row, "external_id", None) or "", + "name": row.name or "", + } + return out + + +def main() -> int: + _load_dotenv() + os.environ.setdefault("FLASK_CONFIG", "development") + os.environ.setdefault("NO_LOAD_GRAPH_DB", "1") + os.environ["CRE_LIBRARIAN_RETRIEVER_BACKEND"] = "in_memory" + sys.path.insert(0, str(ROOT)) + + src = ROOT / "standards_cache.sqlite" + dst = ART / "eval_module_c.sqlite" + if not src.is_file(): + print("missing standards_cache.sqlite", file=sys.stderr) + return 2 + + print(f"copy {src} → {dst} …", flush=True) + if dst.exists(): + dst.unlink() + shutil.copy2(src, dst) + + run_id = "owasp-eval-20260829T192100Z-module-c" + keepers = _load_keepers_with_full_text() + # Cap for cost/time in agent: prefer 80 stratified by repo if huge + if len(keepers) > 80: + by_repo: Dict[str, List[Dict[str, Any]]] = {} + for k in keepers: + by_repo.setdefault(k.get("source_repo") or "?", []).append(k) + sampled: List[Dict[str, Any]] = [] + per = max(2, 80 // max(len(by_repo), 1)) + for repo, rows in sorted(by_repo.items(), key=lambda kv: -len(kv[1])): + sampled.extend(rows[:per]) + if len(sampled) >= 80: + break + keepers = sampled[:80] + print( + f"sampled {len(keepers)} keepers for live C (from intersection)", flush=True + ) + else: + print(f"using {len(keepers)} intersection keepers", flush=True) + + from application import sqla + from application.cmd.cre_main import db_connect + from application.defs import cre_defs as defs + from application.utils.librarian.candidate_retriever import ( + CandidatePool, + RetrieverBackend, + build_retriever, + ) + from application.utils.librarian.config_loader import load_config + from application.utils.librarian.envelope_sink import DbEnvelopeSink + from application.utils.librarian.factory import LibrarianComponents, build_scaler + from application.utils.librarian.queue_runner import run_librarian_queue + from application.utils.librarian.schemas import CreCandidate, RetrievalAudit + + db_url = f"sqlite:///{dst.resolve()}" + database = db_connect(db_url) + sqla.create_all() + + seeded = _seed_knowledge(sqla.session, run_id, keepers) + print(f"seeded knowledge_queue={seeded}", flush=True) + + cre_emb = database.get_embeddings_by_doc_type(defs.Credoctypes.CRE.value) + cre_texts = database.get_embedding_contents_by_doc_type(defs.Credoctypes.CRE.value) + print(f"CRE hub vectors={len(cre_emb)}", flush=True) + + from application.prompt_client import prompt_client + + ph = prompt_client.PromptHandler(database=database) + embed_fn = ph.get_text_embeddings + + cfg = load_config() + + pool = CandidatePool.from_mapping(cre_emb) + retriever = build_retriever( + RetrieverBackend.in_memory, + embed_fn=embed_fn, + top_k=cfg.top_k_retrieval, + threshold=cfg.link_threshold, + pool=pool, + connection=None, + ) + + class _VectorReranker: + """Sandbox stand-in for CrossEncoder: promote retrieval order with logits.""" + + def rerank(self, text: str, audit: RetrievalAudit) -> RetrievalAudit: + reranked = [] + for i, c in enumerate(audit.candidates[: cfg.top_k_rerank]): + # High logit for top retrieval hits so temperature scaler can fire. + score = 12.0 - i * 1.5 + if c.score_vector is not None: + score = float(c.score_vector) * 20.0 + reranked.append( + CreCandidate( + cre_id=c.cre_id, score_rerank=score, score_vector=c.score_vector + ) + ) + return audit.model_copy(update={"reranked": reranked}) + + components = LibrarianComponents( + retriever=retriever, + reranker=_VectorReranker(), + scaler=build_scaler(cfg), + known_cre_ids=frozenset(cre_emb.keys()), + cre_membership=frozenset(cre_emb.keys()), + ) + + sink = DbEnvelopeSink(sqla.session, run_id) + at = datetime.now(timezone.utc) + summary = run_librarian_queue( + sqla.session, + run_id, + components, + cfg, + at=at, + sink=sink, + dry_run=False, + ) + print(summary.to_json(), flush=True) + + # Report decision_queue + CRE names + from application.database.db import DecisionQueueItem, KnowledgeQueueItem + + cre_meta = _cre_lookup(sqla.session) + decisions = ( + sqla.session.query(DecisionQueueItem).filter_by(pipeline_run_id=run_id).all() + ) + links = [] + reviews = [] + for row in decisions: + env = row.envelope if isinstance(row.envelope, dict) else {} + cre_id = None + if isinstance(env.get("links"), list) and env["links"]: + cre_id = env["links"][0].get("cre_id") + if isinstance(env.get("suggested_links"), list) and env["suggested_links"]: + cre_id = cre_id or env["suggested_links"][0].get("cre_id") + meta = cre_meta.get(cre_id or "", {}) + entry = { + "chunk_id": row.chunk_id, + "status": row.status, + "reason_code": row.reason_code, + "confidence": row.confidence, + "cre_id": cre_id, + "cre_external_id": meta.get("external_id"), + "cre_name": meta.get("name"), + "source_label": row.source_label, + "text_preview": ( + (env.get("knowledge") or {}).get("text") + if isinstance(env.get("knowledge"), dict) + else "" + )[:160], + } + if row.status == "linked": + links.append(entry) + else: + reviews.append(entry) + + consumed = ( + sqla.session.query(KnowledgeQueueItem) + .filter( + KnowledgeQueueItem.pipeline_run_id == run_id, + KnowledgeQueueItem.consumed_at.isnot(None), + ) + .count() + ) + + report = { + "run_id": run_id, + "db": str(dst), + "summary": json.loads(summary.to_json()), + "seeded_keepers": seeded, + "decision_queue_rows": len(decisions), + "knowledge_consumed": consumed, + "module_d_handoff": { + "table": "decision_queue", + "rows": len(decisions), + "note": ( + "Module D reads decision_queue envelopes (LinkProposal / ReviewItem). " + "Graph writes are W8b / Module D — not executed here." + ), + }, + "links": links[:60], + "reviews": reviews[:40], + "link_count": len(links), + "review_count": len(reviews), + "cre_hub_size": len(cre_emb), + "reranker": "vector_score_stand_in (no sentence-transformers in sandbox)", + "limitations": [ + "Cross-encoder replaced by vector-score logits — rankings approximate live C.2", + "NullSafetyGuard → status may be degraded (safety unevaluated)", + "Sampled intersection keepers if >80", + ], + } + (ART / "module_c_report.json").write_text(json.dumps(report, indent=2) + "\n") + + # Update eval state + state_path = ART / "state.json" + if state_path.is_file(): + st = json.loads(state_path.read_text()) + st["phases"]["module_c"] = { + "status": "ok", + "detail": ( + f"seeded={seeded} decisions={len(decisions)} " + f"links≈{len(links)} reviews≈{len(reviews)}" + ), + "at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + } + st["module_c_run_id"] = run_id + state_path.write_text(json.dumps(st, indent=2) + "\n") + + print( + json.dumps( + { + "seeded": seeded, + "decisions": len(decisions), + "links": len(links), + "reviews": len(reviews), + "consumed": consumed, + "report": str(ART / "module_c_report.json"), + }, + indent=2, + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/oie_owasp_eval/run_official_orchestrator_local.py b/scripts/oie_owasp_eval/run_official_orchestrator_local.py new file mode 100644 index 000000000..9c776f421 --- /dev/null +++ b/scripts/oie_owasp_eval/run_official_orchestrator_local.py @@ -0,0 +1,437 @@ +#!/usr/bin/env python3 +"""TEMP local helper: drive A→B→C via the official ``run_oie_pipeline``. + +Not production. Isolates a copy of ``standards_cache.sqlite`` (CRE hub), +optionally harvests a tiny ASVS slice, then runs Modules B and C through +``application.utils.oie_orchestrator.run_oie_pipeline`` — the same entry +``scripts/run_oie_pipeline.py`` / ``make oie-pipeline`` use. + +A backends + git — real ``run_harvester`` (needs git clone; Cursor sandbox often blocks) + tarball — injects a tarball harvester that still hands off via ``harvest_input`` + (default; works when ``.git`` writes are forbidden) + +Examples + # Small end-to-end (recommended first run) + PYTHONPATH=. python scripts/oie_owasp_eval/run_official_orchestrator_local.py \\ + --a-mode tarball --max-chunks 20 + + # True Module A (Terminal.app / outside sandbox) + PYTHONPATH=. python scripts/oie_owasp_eval/run_official_orchestrator_local.py \\ + --a-mode git + + # Re-run B+C only against an existing DB + run_id + PYTHONPATH=. python scripts/oie_owasp_eval/run_official_orchestrator_local.py \\ + --skip-a --db tmp/oie_owasp_eval/orchestrator_full.sqlite --run-id +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import sys +import tarfile +import urllib.request +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, List, Optional, Tuple + +ROOT = Path(__file__).resolve().parents[2] +ART = ROOT / "tmp" / "oie_owasp_eval" +DEFAULT_DB = ART / "orchestrator_full.sqlite" +DEFAULT_REPOS_YAML = ART / "orchestrator_repos.yaml" +TARBALL_CACHE = ART / "tarballs" + + +def _load_dotenv() -> None: + env_path = ROOT / ".env" + if not env_path.is_file(): + return + for line in env_path.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, _, v = line.partition("=") + os.environ.setdefault(k.strip(), v.strip().strip('"').strip("'")) + + +def _now_run_id() -> str: + return "orch-local-" + datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + + +def _sqlite_url(path: Path) -> str: + return f"sqlite:///{path.resolve()}" + + +def _write_tiny_repos_yaml(path: Path) -> None: + """One ASVS include so git-mode A stays cheap.""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + """\ +repositories: + - id: owasp-asvs-local-slice + type: github + enabled: true + owner: OWASP + repo: ASVS + branch: master + paths: + include: + - "4.0/en/0x11-V2-Authentication.md" + exclude: + - "**/archive/**" + chunking: + strategy: markdown_heading + max_tokens: 1200 + overlap_tokens: 100 + polling: + mode: incremental + interval_minutes: 60 +""" + ) + + +def _preflight(*, need_cross_encoder: bool) -> List[str]: + problems: List[str] = [] + if not ( + os.environ.get("GEMINI_API_KEY") + or os.environ.get("OPENAI_API_KEY") + or os.environ.get("LLM_API_KEY") + ): + problems.append( + "No embedding/LLM key in env (need GEMINI_API_KEY or OPENAI_API_KEY " + "for Module B classify + Module C query embeddings)." + ) + if need_cross_encoder: + try: + import sentence_transformers # noqa: F401 + except ImportError: + problems.append( + "sentence-transformers missing — Module C's real C.2 CrossEncoder " + "will fail. Install with: pip install 'sentence-transformers>=5,<6' " + "(see requirements-dev.txt; intentionally not on prod requirements)." + ) + return problems + + +def _prepare_db(*, src: Path, dst: Path, reuse: bool) -> str: + if not src.is_file(): + raise SystemExit(f"CRE hub sqlite not found: {src}") + dst.parent.mkdir(parents=True, exist_ok=True) + if dst.is_file() and reuse: + print(f"reusing DB {dst}", flush=True) + else: + print(f"copy {src} → {dst} …", flush=True) + shutil.copy2(src, dst) + return _sqlite_url(dst) + + +def _download_asvs_tarball() -> Path: + TARBALL_CACHE.mkdir(parents=True, exist_ok=True) + dest = TARBALL_CACHE / "ASVS.tgz" + if dest.is_file() and dest.stat().st_size > 1000: + try: + with tarfile.open(dest, "r:gz") as tf: + tf.getmembers()[:1] + return dest + except Exception: + dest.unlink(missing_ok=True) + url = "https://codeload.github.com/OWASP/ASVS/tar.gz/refs/heads/master" + print(f"GET {url}", flush=True) + urllib.request.urlretrieve(url, dest) + return dest + + +def _tarball_harvester( + session: Any, + pipeline_run_id: str, + *, + dry_run: bool = False, + sync_repos: bool = True, # unused; signature matches run_harvester + max_chunks: int = 20, + include_suffix: str = "4.0/en/0x11-V2-Authentication.md", +) -> Any: + """Injected Module A: same harvest_input contract, no git.""" + from application.utils.harvester.chunk_pipeline import DocumentChunkPipeline + from application.utils.harvester.harvest_writer import write_harvest_input + from application.utils.harvester.heading_extractor import HeadingExtractor + from application.utils.harvester.models import Document, Locator, SourceInfo + from application.utils.harvester.pipeline import RunSummary + from application.utils.harvester.schemas import ChunkingConfig + + tgz = _download_asvs_tarball() + files: List[Tuple[str, str]] = [] + with tarfile.open(tgz, "r:gz") as tf: + for member in tf.getmembers(): + if not member.isfile(): + continue + parts = Path(member.name).parts + if len(parts) < 2: + continue + rel = str(Path(*parts[1:])) + if not rel.endswith(include_suffix) and include_suffix not in rel: + continue + extracted = tf.extractfile(member) + if extracted is None: + continue + text = extracted.read().decode("utf-8", errors="replace") + if len(text.strip()) < 40: + continue + files.append((rel, text)) + + chunking = ChunkingConfig( + strategy="markdown_heading", max_tokens=1200, overlap_tokens=100 + ) + pipeline = DocumentChunkPipeline(chunking=chunking) + headings = HeadingExtractor() + committed_at = datetime.now(timezone.utc) + sha = "tarball00000000000000000000000000000001" + records = [] + for rel, text in files: + doc = Document( + schema_version="0.2.0", + artifact_id=f"art:OWASP/ASVS:{rel}", + pipeline_run_id=pipeline_run_id, + text=text, + heading_structure=headings.extract(text), + source=SourceInfo( + type="github", + repository="OWASP/ASVS", + commit_sha=sha, + committed_at=committed_at, + ), + locator=Locator(kind="repo_path", id=rel, path=rel), + ) + records.extend(pipeline.chunk(doc)) + + if max_chunks > 0: + records = records[:max_chunks] + + summary = RunSummary( + run_id=pipeline_run_id, + repositories=1 if files else 0, + files_seen=len(files), + files_retained=len(files), + documents_emitted=len(files), + dry_run=dry_run, + ) + if dry_run: + summary.chunks_written = len(records) + summary.status = "ok" + return summary + + written = write_harvest_input(session, pipeline_run_id, records) + summary.chunks_written = written + summary.status = "ok" if written else "degraded: no chunks" + print( + f"tarball A: files={len(files)} chunks_capped={len(records)} " + f"written={written}", + flush=True, + ) + return summary + + +def _queue_snapshot(session: Any, run_id: str) -> dict: + from application.database.db import ( + DecisionQueueItem, + HarvestInput, + KnowledgeQueueItem, + ) + + harvest = session.query(HarvestInput).filter_by(pipeline_run_id=run_id).count() + knowledge = ( + session.query(KnowledgeQueueItem).filter_by(pipeline_run_id=run_id).count() + ) + knowledge_consumed = ( + session.query(KnowledgeQueueItem) + .filter( + KnowledgeQueueItem.pipeline_run_id == run_id, + KnowledgeQueueItem.consumed_at.isnot(None), + ) + .count() + ) + decisions = session.query(DecisionQueueItem).filter_by(pipeline_run_id=run_id).all() + linked = sum(1 for d in decisions if d.status == "linked") + review = sum(1 for d in decisions if d.status == "review_required") + top = sorted( + ( + { + "chunk_id": d.chunk_id, + "status": d.status, + "reason_code": d.reason_code, + "confidence": d.confidence, + } + for d in decisions + ), + key=lambda x: -(x["confidence"] or 0), + )[:15] + return { + "harvest_input": harvest, + "knowledge_queue": knowledge, + "knowledge_consumed": knowledge_consumed, + "decision_queue": len(decisions), + "linked": linked, + "review_required": review, + "top_decisions": top, + } + + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser( + description="TEMP: official OIE orchestrator A→B→C on a local sqlite copy" + ) + parser.add_argument( + "--hub-db", + type=Path, + default=ROOT / "standards_cache.sqlite", + help="source sqlite with CRE embeddings (copied unless --reuse-db / --cache-file)", + ) + parser.add_argument( + "--db", + type=Path, + default=DEFAULT_DB, + help="destination sqlite path when not using --cache-file", + ) + parser.add_argument( + "--cache-file", + default="", + help="SQLAlchemy URL (e.g. postgresql://cre:password@127.0.0.1:5432/cre). " + "When set, skips sqlite hub copy.", + ) + parser.add_argument("--reuse-db", action="store_true") + parser.add_argument("--run-id", default="") + parser.add_argument( + "--a-mode", + choices=("tarball", "git"), + default="tarball", + help="Module A backend (default tarball for sandbox)", + ) + parser.add_argument("--skip-a", action="store_true") + parser.add_argument("--skip-b", action="store_true") + parser.add_argument("--skip-c", action="store_true") + parser.add_argument("--dry-run", action="store_true") + parser.add_argument( + "--no-sync-repos", + action="store_true", + help="git mode: skip clone/fetch (use HARVESTER_CACHE_DIR as-is)", + ) + parser.add_argument( + "--max-chunks", + type=int, + default=20, + help="tarball A only: cap harvest_input rows (0 = no cap)", + ) + parser.add_argument( + "--repos-yaml", + type=Path, + default=DEFAULT_REPOS_YAML, + help="git mode: repos.yaml (default: tiny ASVS slice under tmp/)", + ) + parser.add_argument( + "--allow-missing-cross-encoder", + action="store_true", + help="skip sentence-transformers preflight (C will error at build_components)", + ) + args = parser.parse_args(argv) + + os.environ.setdefault("FLASK_CONFIG", "development") + os.environ.setdefault("NO_LOAD_GRAPH_DB", "1") + os.environ.setdefault("CRE_LIBRARIAN_RETRIEVER_BACKEND", "in_memory") + _load_dotenv() + + problems = _preflight(need_cross_encoder=not args.allow_missing_cross_encoder) + if problems and not args.skip_c: + print("PREFLIGHT FAILED:", file=sys.stderr) + for p in problems: + print(f" - {p}", file=sys.stderr) + return 2 + + if args.cache_file.strip(): + cache_file = args.cache_file.strip() + print(f"using cache_file={cache_file}", flush=True) + db_label = cache_file + else: + cache_file = _prepare_db(src=args.hub_db, dst=args.db, reuse=args.reuse_db) + db_label = str(args.db.resolve()) + run_id = (args.run_id or "").strip() or _now_run_id() + + run_harvester_fn = None + if not args.skip_a: + if args.a_mode == "tarball": + + def run_harvester_fn(session, rid, *, dry_run=False, sync_repos=True): + return _tarball_harvester( + session, + rid, + dry_run=dry_run, + sync_repos=sync_repos, + max_chunks=args.max_chunks, + ) + + else: + _write_tiny_repos_yaml(args.repos_yaml) + from application.utils.harvester.pipeline import run_harvester + + repos_yaml = args.repos_yaml + + def run_harvester_fn(session, rid, *, dry_run=False, sync_repos=True): + return run_harvester( + session, + rid, + repos_yaml=repos_yaml, + dry_run=dry_run, + sync_repos=sync_repos, + ) + + from application import sqla + from application.cmd.cre_main import db_connect + from application.utils.oie_orchestrator import run_oie_pipeline + + print( + json.dumps( + { + "run_id": run_id, + "cache_file": cache_file, + "a_mode": "skipped" if args.skip_a else args.a_mode, + "orchestrator": "application.utils.oie_orchestrator.run_oie_pipeline", + }, + indent=2, + ), + flush=True, + ) + + result = run_oie_pipeline( + cache_file=cache_file, + pipeline_run_id=run_id, + skip_a=args.skip_a, + skip_b=args.skip_b, + skip_c=args.skip_c, + dry_run=args.dry_run, + sync_repos=not args.no_sync_repos, + stop_on_error=True, + run_harvester_fn=run_harvester_fn, + ) + + db_connect(cache_file) + snap = _queue_snapshot(sqla.session, run_id) + out = { + "orchestrator": json.loads(result.to_json()), + "queues": snap, + "db": db_label, + "fix": ( + "Fit CRE_LIBRARIAN_TEMPERATURE via " + "scripts/evaluate_librarian.py --use_live_embeddings " + "before trusting auto-links; τ=0.8 assumes calibrated C.2 logits." + ), + } + report_path = ART / f"orchestrator_full_{run_id}.json" + report_path.write_text(json.dumps(out, indent=2) + "\n") + print(json.dumps(out, indent=2)) + print(f"wrote {report_path}", flush=True) + return 0 if result.to_dict()["ok"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/oie_owasp_eval/run_phase.py b/scripts/oie_owasp_eval/run_phase.py new file mode 100644 index 000000000..f7fe62dca --- /dev/null +++ b/scripts/oie_owasp_eval/run_phase.py @@ -0,0 +1,829 @@ +#!/usr/bin/env python3 +"""Unattended OIE OWASP eval — advance one phase per invocation. + +Phases write JSON artifacts under tmp/oie_owasp_eval/ for the agent loop / canvas. +Does not commit. Loads GEMINI_API_KEY from .env via python-dotenv if present. +""" + +from __future__ import annotations + +import json +import logging +import os +import subprocess +import sys +import traceback +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional + +ROOT = Path(__file__).resolve().parents[2] +ART = ROOT / "tmp" / "oie_owasp_eval" +STATE_PATH = ART / "state.json" +WAVE1 = [ + # Core standards already wired or high-value for CRE + ("ASVS", "master", ["4.0/en/**/*.md", "5.0/en/**/*.md"]), + ("CheatSheetSeries", "master", ["cheatsheets/**/*.md"]), + ("wstg", "master", ["document/**/*.md"]), + ("SAMM", "master", ["**/*.md"]), + ("Top10", "master", ["**/*.md"]), + ("API-Security", "master", ["**/*.md"]), + ("mastg", "master", ["Document/**/*.md", "document/**/*.md", "**/*.md"]), + ("masvs", "master", ["**/*.md"]), + ("AISVS", "main", ["**/*.md"]), + ( + "www-project-top-10-for-large-language-model-applications", + "main", + ["**/*.md"], + ), + ("www-project-proactive-controls", "main", ["**/*.md"]), + ("www-project-ai-testing-guide", "main", ["**/*.md"]), + ("www-project-ai-security-and-privacy-guide", "main", ["**/*.md"]), + ("www-project-devsecops-guideline", "main", ["**/*.md"]), + ("secure-coding-practices-quick-reference-guide", "master", ["**/*.md"]), + ("DevSecOpsGuideline", "main", ["**/*.md"]), + ("MCP-Security-Testing-Guide", "main", ["**/*.md"]), + ("Software-Component-Verification-Standard", "main", ["**/*.md"]), + ("IoT-Security-Verification-Standard-ISVS", "master", ["**/*.md"]), + ("cornucopia", "master", ["**/*.md"]), +] + +logging.basicConfig(level=logging.INFO, format="%(levelname)s:%(name)s:%(message)s") +logger = logging.getLogger("oie_owasp_eval") + + +def _now() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _load_dotenv() -> None: + env_path = ROOT / ".env" + if not env_path.is_file(): + return + for line in env_path.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, _, v = line.partition("=") + k, v = k.strip(), v.strip().strip('"').strip("'") + os.environ.setdefault(k, v) + + +def _load_state() -> Dict[str, Any]: + return json.loads(STATE_PATH.read_text()) + + +def _save_state(state: Dict[str, Any]) -> None: + state["updated_at"] = _now() + STATE_PATH.write_text(json.dumps(state, indent=2) + "\n") + (ART / "STATUS.md").write_text( + f"# OIE OWASP eval status\n\n" + f"- run_id: `{state['run_id']}`\n" + f"- phase: **{state['phase']}**\n" + f"- status: **{state['status']}**\n" + f"- updated: {state['updated_at']}\n" + f"- last_error: {state.get('last_error') or 'none'}\n" + f"- human_gates: {json.dumps(state.get('human_gates') or [])}\n\n" + f"## Phases\n\n" + + "\n".join( + f"- `{k}`: {v.get('status')}" + + (f" — {v.get('detail', '')}" if v.get("detail") else "") + for k, v in state["phases"].items() + ) + + "\n" + ) + + +def _set_phase( + state: Dict[str, Any], + name: str, + status: str, + *, + detail: str = "", + make_current: bool = True, +) -> None: + state["phases"][name] = { + "status": status, + "detail": detail, + "at": _now(), + } + if make_current: + state["phase"] = name + _save_state(state) + + +def _repos_yaml_for(entries: List[tuple], path: Path) -> None: + lines = ["repositories:\n"] + for owner_repo, branch, includes in entries: + rid = owner_repo.lower().replace("_", "-") + lines.append(f" - id: owasp-{rid}\n") + lines.append(" type: github\n") + lines.append(" enabled: true\n") + lines.append(" owner: OWASP\n") + lines.append(f" repo: {owner_repo}\n") + lines.append(f" branch: {branch}\n") + lines.append(" paths:\n") + lines.append(" include:\n") + for inc in includes: + lines.append(f' - "{inc}"\n') + lines.append(" exclude:\n") + lines.append(' - "**/archive/**"\n') + lines.append(' - "**/.github/**"\n') + lines.append(' - "**/node_modules/**"\n') + lines.append(" chunking:\n") + lines.append(" strategy: markdown_heading\n") + lines.append(" max_tokens: 1200\n") + lines.append(" overlap_tokens: 100\n") + lines.append(" polling:\n") + lines.append(" mode: incremental\n") + lines.append(" interval_minutes: 1440\n") + lines.append("\n") + path.write_text("".join(lines)) + + +def phase_bootstrap(state: Dict[str, Any]) -> None: + _set_phase(state, "bootstrap", "running", detail="checking ollama + gemini key") + _load_dotenv() + if not os.environ.get("GEMINI_API_KEY"): + state["human_gates"].append( + { + "id": "missing_gemini_key", + "ask": "GEMINI_API_KEY missing from .env — add it to continue cloud classify/judge", + } + ) + state["status"] = "blocked" + _set_phase(state, "bootstrap", "blocked", detail="GEMINI_API_KEY missing") + return + + # Ensure ollama is up (best-effort). + try: + subprocess.run( + ["ollama", "list"], + check=True, + capture_output=True, + text=True, + timeout=30, + ) + except Exception as exc: # noqa: BLE001 + state["human_gates"].append( + { + "id": "ollama_down", + "ask": f"Start `ollama serve` (need qwen2.5:14b). Error: {exc}", + } + ) + state["status"] = "blocked" + _set_phase(state, "bootstrap", "blocked", detail=str(exc)) + return + + ART.mkdir(parents=True, exist_ok=True) + _set_phase(state, "bootstrap", "ok", detail="keys + ollama ok") + state["status"] = "running" + state["phase"] = "generate_repos_yaml" + _save_state(state) + + +def phase_generate_repos_yaml(state: Dict[str, Any]) -> None: + _set_phase(state, "generate_repos_yaml", "running") + wave1_path = ART / "repos_wave1.yaml" + _repos_yaml_for(WAVE1, wave1_path) + + # Wave2: remaining candidates with broad **/*.md + cands_path = ART / "candidates.json" + wave2_entries: List[tuple] = [] + if cands_path.is_file(): + cands = json.loads(cands_path.read_text()) + wave1_names = {e[0] for e in WAVE1} + for r in cands: + name = r["name"] + if name in wave1_names: + continue + # Prefer main; harvester will fail soft per-repo if wrong. + wave2_entries.append((name, "main", ["**/*.md"])) + wave2_path = ART / "repos_wave2.yaml" + _repos_yaml_for(wave2_entries[:250], wave2_path) # hard cap for laptop + + meta = { + "wave1_repos": len(WAVE1), + "wave2_repos": min(len(wave2_entries), 250), + "wave1_yaml": str(wave1_path.relative_to(ROOT)), + "wave2_yaml": str(wave2_path.relative_to(ROOT)), + } + (ART / "repos_meta.json").write_text(json.dumps(meta, indent=2) + "\n") + _set_phase( + state, + "generate_repos_yaml", + "ok", + detail=f"wave1={meta['wave1_repos']} wave2={meta['wave2_repos']}", + ) + state["phase"] = "harvest_wave1" + _save_state(state) + + +def _run_harvester(run_id: str, repos_yaml: Path, db_url: str) -> Dict[str, Any]: + env = os.environ.copy() + env.setdefault("FLASK_CONFIG", "development") + env.setdefault("NO_LOAD_GRAPH_DB", "1") + env["PYTHONPATH"] = str(ROOT) + cmd = [ + sys.executable, + str(ROOT / "cre.py"), + "--run_harvester", + "--run_id", + run_id, + "--cache_file", + db_url, + "--harvester_repos_yaml", + str(repos_yaml), + ] + # Allow long clone/sync + proc = subprocess.run( + cmd, + cwd=str(ROOT), + env=env, + capture_output=True, + text=True, + timeout=60 * 60 * 6, + ) + out = (proc.stdout or "").strip().splitlines() + summary: Dict[str, Any] = {"exit_code": proc.returncode, "raw_tail": out[-5:]} + for line in reversed(out): + line = line.strip() + if line.startswith("{") and "run_id" in line: + try: + summary["summary"] = json.loads(line) + except json.JSONDecodeError: + pass + break + if proc.returncode != 0: + summary["stderr_tail"] = (proc.stderr or "")[-2000:] + return summary + + +def phase_harvest_wave1(state: Dict[str, Any]) -> None: + """Prefer tarball harvest — agent sandboxes often forbid writing ``.git/``.""" + _set_phase(state, "harvest_wave1", "running", detail="tarball download + chunking") + env = os.environ.copy() + env.setdefault("FLASK_CONFIG", "development") + env.setdefault("NO_LOAD_GRAPH_DB", "1") + env["PYTHONPATH"] = str(ROOT) + proc = subprocess.run( + [sys.executable, str(ROOT / "scripts/oie_owasp_eval/tarball_harvest.py")], + cwd=str(ROOT), + env=env, + capture_output=True, + text=True, + timeout=60 * 60 * 2, + ) + (ART / "harvest_wave1.log").write_text( + (proc.stdout or "") + "\n" + (proc.stderr or "") + ) + # tarball_harvest updates state.json itself on success + state = _load_state() + if proc.returncode != 0 and state.get("phase") == "harvest_wave1": + state["status"] = "error" + state["last_error"] = (proc.stderr or proc.stdout or "")[-2000:] + _set_phase(state, "harvest_wave1", "error", detail="tarball harvest failed") + return + # Ensure we advanced + state = _load_state() + if state.get("phase") == "classify_dual": + return + state["phase"] = "classify_dual" + state["status"] = "running" + _save_state(state) + + +def _snapshot_harvest(db_url: str, run_id: str) -> Dict[str, Any]: + os.environ.setdefault("FLASK_CONFIG", "development") + os.environ.setdefault("NO_LOAD_GRAPH_DB", "1") + sys.path.insert(0, str(ROOT)) + from application.cmd.cre_main import db_connect + from application import sqla + from application.database.db import HarvestInput + + db_connect(db_url) + rows = sqla.session.query(HarvestInput).filter_by(pipeline_run_id=run_id).all() + by_repo: Dict[str, int] = {} + samples: List[Dict[str, Any]] = [] + for row in rows: + payload = row.payload or {} + repo = payload.get("source_repo") or "unknown" + by_repo[repo] = by_repo.get(repo, 0) + 1 + if len(samples) < 25: + samples.append( + { + "chunk_id": payload.get("chunk_id"), + "source_repo": repo, + "locator_path": payload.get("locator_path") + or (payload.get("locator") or {}).get("path"), + "text_preview": (payload.get("text") or "")[:180], + "status": row.status, + } + ) + return { + "run_id": run_id, + "total_rows": len(rows), + "by_repo": dict(sorted(by_repo.items(), key=lambda kv: -kv[1])), + "samples": samples, + } + + +def _reset_harvest_pending(db_url: str, run_id: str) -> None: + sys.path.insert(0, str(ROOT)) + from application.cmd.cre_main import db_connect + from application import sqla + from application.database.db import HarvestInput + + db_connect(db_url) + ( + sqla.session.query(HarvestInput) + .filter_by(pipeline_run_id=run_id) + .update({"status": "pending"}, synchronize_session=False) + ) + sqla.session.commit() + + +def _run_noise_filter(db_url: str, run_id: str, model: str) -> Dict[str, Any]: + env = os.environ.copy() + # Ensure .env keys are present for LiteLLM (cre.py child). + _load_dotenv() + for key in ("GEMINI_API_KEY", "GOOGLE_API_KEY", "CRE_NOISE_FILTER_LLM_MODEL"): + if key in os.environ: + env[key] = os.environ[key] + env.setdefault("FLASK_CONFIG", "development") + env.setdefault("NO_LOAD_GRAPH_DB", "1") + env["PYTHONPATH"] = str(ROOT) + env["CRE_NOISE_FILTER_LLM_MODEL"] = model + # Smaller batches for local model / flaky pro + if model.startswith("ollama/"): + env["CRE_NOISE_FILTER_BATCH_SIZE"] = "4" + env["CRE_NOISE_FILTER_MAX_CHARS"] = "1200" + elif "pro" in model: + env["CRE_NOISE_FILTER_BATCH_SIZE"] = "5" + env["CRE_LLM_MAX_RETRIES"] = "4" + env["CRE_LLM_RETRY_SLEEP_SECONDS"] = "20" + else: + env["CRE_NOISE_FILTER_BATCH_SIZE"] = "10" + cmd = [ + sys.executable, + str(ROOT / "cre.py"), + "--run_noise_filter", + "--run_id", + run_id, + "--cache_file", + db_url, + ] + proc = subprocess.run( + cmd, + cwd=str(ROOT), + env=env, + capture_output=True, + text=True, + timeout=60 * 60 * 8, + ) + out = (proc.stdout or "").strip().splitlines() + summary: Dict[str, Any] = {"exit_code": proc.returncode, "model": model} + for line in reversed(out): + if line.strip().startswith("{") and "run_id" in line: + try: + summary["summary"] = json.loads(line.strip()) + except json.JSONDecodeError: + pass + break + if proc.returncode != 0: + summary["stderr_tail"] = (proc.stderr or "")[-2000:] + return summary + + +def _snapshot_knowledge(db_url: str, run_id: str) -> Dict[str, Any]: + sys.path.insert(0, str(ROOT)) + from application.cmd.cre_main import db_connect + from application import sqla + from application.database.db import KnowledgeQueueItem + + db_connect(db_url) + rows = ( + sqla.session.query(KnowledgeQueueItem).filter_by(pipeline_run_id=run_id).all() + ) + by_label: Dict[str, int] = {} + by_repo: Dict[str, int] = {} + keepers: List[Dict[str, Any]] = [] + for row in rows: + by_label[row.llm_label] = by_label.get(row.llm_label, 0) + 1 + by_repo[row.source_repo] = by_repo.get(row.source_repo, 0) + 1 + keepers.append( + { + "chunk_id": row.chunk_id, + "source_repo": row.source_repo, + "locator_path": row.locator_path, + "llm_label": row.llm_label, + "confidence": row.confidence, + "text_preview": (row.text or "")[:200], + } + ) + return { + "run_id": run_id, + "total": len(rows), + "by_label": by_label, + "by_repo": dict(sorted(by_repo.items(), key=lambda kv: -kv[1])), + "would_flow_to_c": keepers, + } + + +def _sample_harvest_for_eval( + db_url: str, run_id: str, *, per_repo: int = 25, max_total: int = 400 +) -> str: + """Copy a stratified sample of harvest rows to a new run_id for dual classify.""" + import copy + from collections import defaultdict + + sys.path.insert(0, str(ROOT)) + from application import sqla + from application.cmd.cre_main import db_connect + from application.database.db import HarvestInput + + db_connect(db_url) + rows = sqla.session.query(HarvestInput).filter_by(pipeline_run_id=run_id).all() + by_repo: Dict[str, list] = defaultdict(list) + for row in rows: + repo = ( + (row.payload or {}).get("source_repo") + or ((row.payload or {}).get("source") or {}).get("repo") + or "unknown" + ) + by_repo[repo].append(row) + + sample_run = run_id + "-sample" + # wipe prior sample + ( + sqla.session.query(HarvestInput) + .filter_by(pipeline_run_id=sample_run) + .delete(synchronize_session=False) + ) + picked = 0 + for repo, repo_rows in sorted(by_repo.items(), key=lambda kv: -len(kv[1])): + if picked >= max_total: + break + take = min(per_repo, max_total - picked, len(repo_rows)) + # spread: first, middle, last-ish + idxs = sorted( + { + 0, + len(repo_rows) // 2, + len(repo_rows) - 1, + *[i * max(len(repo_rows) // take, 1) for i in range(take)], + } + )[:take] + for i in idxs: + row = repo_rows[i] + payload = copy.deepcopy(row.payload or {}) + payload["pipeline_run_id"] = sample_run + sqla.session.add( + HarvestInput( + pipeline_run_id=sample_run, status="pending", payload=payload + ) + ) + picked += 1 + sqla.session.commit() + meta = { + "source_run": run_id, + "sample_run": sample_run, + "source_total": len(rows), + "sample_total": picked, + "per_repo": per_repo, + "by_repo_source": {k: len(v) for k, v in by_repo.items()}, + } + (ART / "classify_sample_meta.json").write_text(json.dumps(meta, indent=2) + "\n") + return sample_run + + +def phase_classify_dual(state: Dict[str, Any]) -> None: + _load_dotenv() + _set_phase(state, "classify_dual", "running", detail="sample → qwen → gemini") + db_url = state["db_url"] + base_run = state["wave1_run_id"] + local_model = state["models"]["local"] + cloud_model = state["models"]["cloud"] + + sample_run = _sample_harvest_for_eval(db_url, base_run) + state["sample_run_id"] = sample_run + _save_state(state) + + local = _run_noise_filter(db_url, sample_run, local_model) + local_snap = _snapshot_knowledge(db_url, sample_run) + (ART / "classify_qwen.json").write_text( + json.dumps({"result": local, "knowledge": local_snap}, indent=2) + "\n" + ) + + gem_run = sample_run + "-gemini" + _clone_harvest_run(db_url, sample_run, gem_run) + # Reset sample rows already marked processed so clone is the only gemini input + gem = _run_noise_filter(db_url, gem_run, cloud_model) + gem_snap = _snapshot_knowledge(db_url, gem_run) + (ART / "classify_gemini.json").write_text( + json.dumps({"result": gem, "knowledge": gem_snap}, indent=2) + "\n" + ) + + qwen_ids = {k["chunk_id"] for k in local_snap["would_flow_to_c"]} + gem_ids = {k["chunk_id"] for k in gem_snap["would_flow_to_c"]} + diff = { + "qwen_only": sorted(qwen_ids - gem_ids), + "gemini_only": sorted(gem_ids - qwen_ids), + "both": sorted(qwen_ids & gem_ids), + "qwen_count": len(qwen_ids), + "gemini_count": len(gem_ids), + "agreement_rate": ( + round(len(qwen_ids & gem_ids) / max(len(qwen_ids | gem_ids), 1), 4) + ), + "note": "Dual classify runs on stratified sample; see classify_sample_meta.json", + } + (ART / "classify_diff.json").write_text(json.dumps(diff, indent=2) + "\n") + + state["gemini_run_id"] = gem_run + _set_phase( + state, + "classify_dual", + "ok", + detail=( + f"qwen={diff['qwen_count']} gemini={diff['gemini_count']} " + f"agree={diff['agreement_rate']}" + ), + ) + state["phase"] = "judge" + _save_state(state) + + +def _clone_harvest_run(db_url: str, src_run: str, dst_run: str) -> None: + sys.path.insert(0, str(ROOT)) + from application.cmd.cre_main import db_connect + from application import sqla + from application.database.db import HarvestInput + import copy + + db_connect(db_url) + rows = sqla.session.query(HarvestInput).filter_by(pipeline_run_id=src_run).all() + for row in rows: + payload = copy.deepcopy(row.payload or {}) + payload["pipeline_run_id"] = dst_run + sqla.session.add( + HarvestInput( + pipeline_run_id=dst_run, + status="pending", + payload=payload, + ) + ) + sqla.session.commit() + + +def phase_judge(state: Dict[str, Any]) -> None: + """Ask Gemini-pro to flag nonsensical keepers / improvement areas.""" + _load_dotenv() + _set_phase(state, "judge", "running") + gem = json.loads((ART / "classify_gemini.json").read_text()) + qwen = json.loads((ART / "classify_qwen.json").read_text()) + diff = json.loads((ART / "classify_diff.json").read_text()) + harvest = json.loads((ART / "harvest_wave1_snapshot.json").read_text()) + + keepers = gem["knowledge"]["would_flow_to_c"][:80] + disagreements = [] + qwen_map = {k["chunk_id"]: k for k in qwen["knowledge"]["would_flow_to_c"]} + gem_map = {k["chunk_id"]: k for k in gem["knowledge"]["would_flow_to_c"]} + for cid in (diff["qwen_only"] + diff["gemini_only"])[:40]: + disagreements.append( + { + "chunk_id": cid, + "in_qwen": cid in qwen_map, + "in_gemini": cid in gem_map, + "sample": qwen_map.get(cid) or gem_map.get(cid), + } + ) + + prompt = { + "task": ( + "You are reviewing OpenCRE Module B noise-filter output for an OWASP " + "harvest. Flag keepers that should NOT become CRE knowledge (noise, " + "meta, marketing, chapter logistics without security content, empty " + "stubs, license-only, contributor lists). Suggest concrete pipeline " + "improvements for Module A path filters and Module B prompts." + ), + "harvest_totals": harvest.get("by_repo"), + "agreement": { + "qwen_count": diff["qwen_count"], + "gemini_count": diff["gemini_count"], + "agreement_rate": diff["agreement_rate"], + }, + "sample_keepers_gemini": keepers, + "disagreements": disagreements, + } + + from application.prompt_client.litellm_router import ( + completion as litellm_completion, + extract_content_text, + ) + + resp = litellm_completion( + model=state["models"]["judge"], + temperature=0.1, + messages=[ + { + "role": "system", + "content": ( + "Respond with JSON only: {" + '"nonsensical_keepers":[{"chunk_id":"","reason":""}],' + '"false_drops_suspected":[{"chunk_id":"","reason":""}],' + '"path_filter_improvements":[""],' + '"prompt_improvements":[""],' + '"orchestrator_improvements":[""],' + '"summary":""}' + ), + }, + {"role": "user", "content": json.dumps(prompt)[:120000]}, + ], + response_format={"type": "json_object"}, + ) + text = extract_content_text(resp, strict=False) + try: + verdict = json.loads(text) + except json.JSONDecodeError: + verdict = {"raw": text, "summary": "judge returned non-JSON"} + + (ART / "judge_report.json").write_text(json.dumps(verdict, indent=2) + "\n") + _set_phase(state, "judge", "ok", detail=(verdict.get("summary") or "")[:200]) + state["phase"] = "next_steps_doc" + # Skip wave2 auto for now unless human wants — mark pending optional + state["phases"]["harvest_wave2"] = { + "status": "deferred", + "detail": "Wave2 (250 repos) deferred until you approve overnight cost/time", + "at": _now(), + } + state["human_gates"].append( + { + "id": "approve_wave2", + "ask": ( + "Wave1 judge done. Approve Wave2 harvest of ~250 more OWASP " + "www-project repos? Reply: approve wave2 | skip wave2" + ), + } + ) + _save_state(state) + + +def phase_next_steps_doc(state: Dict[str, Any]) -> None: + _set_phase(state, "next_steps_doc", "running") + judge = {} + if (ART / "judge_report.json").is_file(): + judge = json.loads((ART / "judge_report.json").read_text()) + diff = {} + if (ART / "classify_diff.json").is_file(): + diff = json.loads((ART / "classify_diff.json").read_text()) + harvest = {} + if (ART / "harvest_wave1_snapshot.json").is_file(): + harvest = json.loads((ART / "harvest_wave1_snapshot.json").read_text()) + + doc = f"""# Next steps — OIE → MCP / agent for human Q&A + +**Audience:** OpenCRE maintainers. **Status:** draft from eval run `{state['run_id']}` ({_now()}). + +## What we just proved + +- Module A can harvest configured OWASP markdown into `harvest_input`. +- Module B (local Qwen 14B vs Gemini 2.5 Pro) can label keepers that would enter `knowledge_queue` (Module C input). +- Wave1 harvest rows: **{harvest.get('total_rows', 'n/a')}**; Qwen keepers: **{diff.get('qwen_count', 'n/a')}**; Gemini keepers: **{diff.get('gemini_count', 'n/a')}**; agreement: **{diff.get('agreement_rate', 'n/a')}**. +- Judge summary: {judge.get('summary', '(pending)')} + +## Product shape: “ask OpenCRE anything” + +Humans will ask two different classes of questions: + +| Class | Example | Backing | +|-------|---------|---------| +| **Normative / CRE** | “How should I store passwords?” | CRE graph + Module C/D links + standards chunks | +| **Procedural / community** | “I’m in place X — when is the nearest OWASP meeting?” | Chapter calendars, Meetup/ICS feeds, chapter repos — **not** CRE nodes | + +Do **not** force procedural answers through the Librarian→CRE path. Add a **router** in the agent/MCP layer. + +## Proposed MCP server: `opencre` + +Tools (v1): + +1. `cre_search` — semantic + keyword over CRE / standards (existing chat/embeddings). +2. `standard_lookup` — ASVS/WSTG/CheatSheet section by id/path. +3. `pipeline_status` — last OIE run_id, queue depths, degraded flags. +4. `harvest_diff` — “what would Module C see for run R?” +5. `chapter_events_near` — geocode + nearest upcoming OWASP chapter events (new). +6. `ask` — natural-language entry that **routes** to 1–5. + +Resources: + +- `opencre://cre/{{id}}` +- `opencre://run/{{pipeline_run_id}}/knowledge` +- `opencre://chapters/events?lat=&lon=` + +## Architecture + +``` +Human / IDE agent + │ MCP (stdio or HTTP) + ▼ +opencre-mcp router + ├─ intent: security knowledge → PromptHandler / CRE + optional OIE citations + ├─ intent: pipeline ops → read harvest_input / knowledge_queue / decision_queue + └─ intent: community logistics → chapter calendar provider (Meetup GraphQL, + chapter site ICS, or curated OWASP chapter JSON — start curated) +``` + +Nightly Cursor Automation / cron: + +1. `make oie-pipeline` (A→B→C) for configured `repos.yaml` +2. Publish run summary JSON to `tmp/` or S3; MCP `pipeline_status` reads it +3. Optional: Slack digest of “new KNOWLEDGE chunks this week” + +## Implementation sequence + +1. **Ship Wave1 repos.yaml** curated list into `application/utils/harvester/repos.yaml` after path-filter nits from judge. +2. **MCP skeleton** under `scripts/mcp_opencre/` (repo already notes MCP in requirements-dev) — tools 1, 3, 4 first. +3. **Router prompt** with hard rule: logistics ≠ CRE. +4. **Chapter events v0:** curated JSON of major chapters + Meetup links; `chapter_events_near` uses Haversine; no scrape war. +5. **Eval harness** (this run) as `make oie-owasp-eval` for dual-model CI smoke on a tiny fixture. + +## Judge-driven improvements to fold in + +{json.dumps(judge.get('path_filter_improvements') or [], indent=2)} + +{json.dumps(judge.get('prompt_improvements') or [], indent=2)} + +{json.dumps(judge.get('orchestrator_improvements') or [], indent=2)} + +## Human gates remaining + +{json.dumps(state.get('human_gates') or [], indent=2)} + +## Out of scope for MCP v1 + +- Writing CRE links from chat (Module D human review stays) +- Running paid Gemini classify on every IDE keystroke +- Scraping every OWASP chapter website without ToS review +""" + + out = ROOT / "docs" / "gsoc_2026_module_a" / "next_steps_mcp_agent.md" + out.write_text(doc) + (ART / "next_steps_mcp_agent.md").write_text(doc) + _set_phase(state, "next_steps_doc", "ok", detail=str(out.relative_to(ROOT))) + state["phase"] = "done" + state["phases"]["done"] = { + "status": "ok", + "detail": "observe artifacts under tmp/oie_owasp_eval; approve wave2 if desired", + "at": _now(), + } + state["status"] = "awaiting_human" if state.get("human_gates") else "done" + _save_state(state) + + +PHASES = { + "bootstrap": phase_bootstrap, + "generate_repos_yaml": phase_generate_repos_yaml, + "harvest_wave1": phase_harvest_wave1, + "classify_dual": phase_classify_dual, + "judge": phase_judge, + "next_steps_doc": phase_next_steps_doc, +} + + +def main() -> int: + ART.mkdir(parents=True, exist_ok=True) + if not STATE_PATH.is_file(): + print("missing state.json", file=sys.stderr) + return 2 + state = _load_state() + if state.get("status") == "blocked": + print(json.dumps({"blocked": True, "gates": state.get("human_gates")})) + return 3 + phase = state.get("phase") or "bootstrap" + if phase == "done": + print(json.dumps({"done": True, "state": state})) + return 0 + if phase == "harvest_wave2": + print( + json.dumps({"waiting": "approve_wave2", "gates": state.get("human_gates")}) + ) + return 0 + fn = PHASES.get(phase) + if not fn: + print(f"unknown phase {phase}", file=sys.stderr) + return 2 + try: + fn(state) + except Exception as exc: # noqa: BLE001 + state = _load_state() + state["status"] = "error" + state["last_error"] = f"{exc}\n{traceback.format_exc()[-1500:]}" + _set_phase(state, phase, "error", detail=str(exc)) + print(state["last_error"], file=sys.stderr) + return 1 + state = _load_state() + print(json.dumps({"phase": state["phase"], "status": state["status"]}, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/oie_owasp_eval/run_qwen_outside_sandbox.sh b/scripts/oie_owasp_eval/run_qwen_outside_sandbox.sh new file mode 100755 index 000000000..8a04c2f6b --- /dev/null +++ b/scripts/oie_owasp_eval/run_qwen_outside_sandbox.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Run this in Terminal.app / iTerm (NOT Cursor agent sandbox). +set -euo pipefail +cd "$(dirname "$0")/../.." +source venv/bin/activate +export PYTHONPATH=. +export FLASK_CONFIG=development +export NO_LOAD_GRAPH_DB=1 +export CRE_NOISE_FILTER_LLM_MODEL="${CRE_NOISE_FILTER_LLM_MODEL:-ollama/qwen2.5:14b}" +export CRE_NOISE_FILTER_BATCH_SIZE=4 +RUN_ID=$(python -c "import json; print(json.load(open('tmp/oie_owasp_eval/state.json'))['sample_run_id'])") +DB=$(python -c "import json; print(json.load(open('tmp/oie_owasp_eval/state.json'))['db_url'])") +# Reset sample to pending +python - < +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +from collections import defaultdict +from pathlib import Path +from typing import Any, Dict, List, Optional, Set, Tuple +from urllib.parse import quote, unquote, urlparse + +import requests + +ROOT = Path(__file__).resolve().parents[2] +ART = ROOT / "tmp" / "oie_owasp_eval" + +RESOURCE_SPECS = ( + { + "api_name": "ASVS", + "repo_substr": "ASVS", + "label": "ASVS", + "grain": "chunk", + }, + { + "api_name": "OWASP Cheat Sheets", + "repo_substr": "CheatSheetSeries", + "label": "CheatSheetSeries", + "grain": "sheet", + }, +) + +_SECTION_ID_RE = re.compile( + r"\b(?:V)?(\d+\.\d+(?:\.\d+)?)\b|\*\*(\d+\.\d+(?:\.\d+)?)\*\*", + re.IGNORECASE, +) + + +def _session() -> requests.Session: + s = requests.Session() + s.headers["User-Agent"] = "OpenCRE-OIE-B1/1.0" + s.trust_env = False + return s + + +def fetch_standard_nodes(base_url: str, api_name: str) -> List[Dict[str, Any]]: + base = base_url.rstrip("/") + url = f"{base}/rest/v1/Standard/{quote(api_name)}" + sess = _session() + r = sess.get(url, timeout=60) + r.raise_for_status() + payload = r.json() + nodes = list(payload.get("standards") or []) + total = int(payload.get("total_pages") or 1) + for page in range(2, total + 1): + rr = sess.get(url, params={"page": page}, timeout=60) + rr.raise_for_status() + nodes.extend(rr.json().get("standards") or []) + return nodes + + +def human_cre_ids(node: Dict[str, Any]) -> Set[str]: + out: Set[str] = set() + for link in node.get("links") or []: + doc = link.get("document") or {} + if str(doc.get("doctype") or "").upper() == "CRE": + cid = str(doc.get("id") or "").strip() + if cid: + out.add(cid) + return out + + +def section_keys_for_node(node: Dict[str, Any]) -> Set[str]: + """Keys used to align a harvest chunk to this API node.""" + keys: Set[str] = set() + nid = str(node.get("id") or "") + parts = nid.split(":") + if len(parts) >= 2 and parts[0].upper() in ("ASVS", "OWASP CHEAT SHEETS"): + keys.add(parts[1].strip().lower()) + section = str(node.get("section") or "").strip().lower() + if section: + keys.add(section[:120]) + for m in _SECTION_ID_RE.finditer(section): + keys.add((m.group(1) or m.group(2) or "").lower()) + href = str(node.get("hyperlink") or "") + if href: + keys.add(href.lower()) + path = urlparse(href).path.lower() + keys.add(path) + frag = urlparse(href).fragment.lower() + if frag: + keys.add(frag) + base = Path(unquote(path)).name.lower() + if base: + keys.add(base) + keys.add(base.replace(".html", "").replace(".md", "").replace("_", " ")) + keys.add(base.replace(".html", "").replace(".md", "")) + keys.discard("") + return keys + + +def build_gold_index(nodes: List[Dict[str, Any]]) -> Dict[str, Set[str]]: + key_to_cres: Dict[str, Set[str]] = defaultdict(set) + for node in nodes: + cres = human_cre_ids(node) + if not cres: + continue + for key in section_keys_for_node(node): + key_to_cres[key].update(cres) + return key_to_cres + + +def sheet_stem(locator_path: str) -> str: + name = Path(locator_path or "").name.lower() + return name.replace(".md", "").replace(".html", "") + + +def chunk_keys(text: str, locator_path: str, heading_path: List[str]) -> Set[str]: + keys: Set[str] = set() + path = (locator_path or "").lower() + if path: + keys.add(path) + keys.add(Path(path).name.lower()) + stem = sheet_stem(path) + keys.add(stem) + keys.add(stem.replace("_", " ").replace("-", " ")) + keys.add(stem.replace("_", "-")) + for h in heading_path or []: + keys.add(str(h).strip().lower()) + for m in _SECTION_ID_RE.finditer(text or ""): + sid = (m.group(1) or m.group(2) or "").lower() + keys.add(sid) + keys.add(f"v{sid}") + for m in re.finditer(r"\b(\d+\.\d+\.\d+)\b", text or ""): + keys.add(m.group(1).lower()) + keys.discard("") + return keys + + +def top2_cre_external_ids( + envelope: Dict[str, Any], uuid_to_ext: Dict[str, str] +) -> List[str]: + """Prefer retrieval.reranked order; fall back to suggested/links.""" + ordered: List[str] = [] + + def add(cre_id: Optional[str]) -> None: + if not cre_id: + return + ext = uuid_to_ext.get(cre_id, cre_id) + if ext not in ordered: + ordered.append(ext) + + retrieval = envelope.get("retrieval") or {} + for key in ("reranked", "candidates"): + for cand in retrieval.get(key) or []: + if isinstance(cand, dict): + add(cand.get("cre_id")) + if len(ordered) >= 2: + return ordered[:2] + for field in ("links", "suggested_links"): + for link in envelope.get(field) or []: + if isinstance(link, dict): + add(link.get("cre_id")) + if len(ordered) >= 2: + return ordered[:2] + return ordered[:2] + + +def match_gold(keys: Set[str], key_to_cres: Dict[str, Set[str]]) -> Set[str]: + gold: Set[str] = set() + for k in keys: + if k in key_to_cres: + gold |= key_to_cres[k] + for gk, cres in key_to_cres.items(): + if len(k) >= 8 and (k in gk or gk in k): + gold |= cres + return gold + + +def _resource_for(repo: str, path: str, chunk_id: str) -> Optional[Dict[str, Any]]: + for spec in RESOURCE_SPECS: + if spec["repo_substr"] in (repo or path or chunk_id): + return spec + return None + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--base-url", + default=os.environ.get("OPENCRE_BASE_URL", "https://opencre.org"), + ) + parser.add_argument( + "--cache-file", + default=os.environ.get( + "DEV_DATABASE_URL", "postgresql://cre:password@127.0.0.1:5432/cre" + ), + ) + parser.add_argument("--run-id", required=True) + parser.add_argument( + "--out", + type=Path, + default=ART / "b1_accuracy_report.json", + ) + args = parser.parse_args() + + os.environ.setdefault("FLASK_CONFIG", "development") + os.environ.setdefault("NO_LOAD_GRAPH_DB", "1") + + gold_by_resource: Dict[str, Dict[str, Set[str]]] = {} + for spec in RESOURCE_SPECS: + nodes = fetch_standard_nodes(args.base_url, spec["api_name"]) + gold_by_resource[spec["label"]] = build_gold_index(nodes) + print( + f"gold {spec['label']}: nodes={len(nodes)} " + f"index_keys={len(gold_by_resource[spec['label']])}", + flush=True, + ) + + from application import sqla + from application.cmd.cre_main import db_connect + from application.database.db import CRE, DecisionQueueItem + + db_connect(args.cache_file) + uuid_to_ext = { + row.id: (row.external_id or row.id) + for row in sqla.session.query(CRE.id, CRE.external_id).all() + } + + decisions = ( + sqla.session.query(DecisionQueueItem) + .filter_by(pipeline_run_id=args.run_id) + .all() + ) + + # Sheet-level aggregation for Cheat Sheets + sheet_pred: Dict[str, Set[str]] = defaultdict(set) + sheet_gold: Dict[str, Set[str]] = {} + sheet_meta: Dict[str, Dict[str, Any]] = {} + + chunk_scorable = 0 + chunk_hits = 0 + details: List[Dict[str, Any]] = [] + unaligned = 0 + + for row in decisions: + env = row.envelope if isinstance(row.envelope, dict) else {} + knowledge = env.get("knowledge") or {} + text = knowledge.get("text") or "" + loc = knowledge.get("locator") or {} + path = loc.get("path") or loc.get("id") or "" + src = knowledge.get("source") or {} + repo = str(src.get("repository") or "") + chunk_id = row.chunk_id or "" + if not repo and chunk_id.startswith("chk:art:"): + body = chunk_id[len("chk:art:") :] + art = body.rsplit(":", 1)[0] + if art.startswith("OWASP/"): + bits = art.split(":", 1) + repo = bits[0] + if len(bits) > 1 and not path: + path = bits[1] + + spec = _resource_for(repo, path, chunk_id) + if not spec: + continue + + keys = chunk_keys(text, path, []) + gold = match_gold(keys, gold_by_resource[spec["label"]]) + top2 = top2_cre_external_ids(env, uuid_to_ext) + top2_ext = [t for t in top2 if t] + + if spec["grain"] == "sheet": + stem = sheet_stem(path) or sheet_stem(chunk_id) + if not stem: + unaligned += 1 + continue + sheet_pred[stem].update(top2_ext) + if stem not in sheet_gold: + sheet_keys = { + stem, + stem.replace("_", " "), + stem.replace("_", "-"), + f"{stem}.md", + f"{stem}.html", + path.lower() if path else "", + } + sheet_keys.discard("") + sheet_gold[stem] = match_gold( + sheet_keys, gold_by_resource[spec["label"]] + ) + sheet_meta[stem] = { + "resource": spec["label"], + "path": path, + "grain": "sheet", + } + continue + + # ASVS chunk grain + if not gold: + unaligned += 1 + details.append( + { + "chunk_id": chunk_id, + "resource": spec["label"], + "grain": "chunk", + "aligned": False, + "top2": top2_ext, + "gold_size": 0, + "hit": False, + } + ) + continue + + chunk_scorable += 1 + hit = any(t in gold for t in top2_ext) + if hit: + chunk_hits += 1 + details.append( + { + "chunk_id": chunk_id, + "resource": spec["label"], + "grain": "chunk", + "aligned": True, + "top2": top2_ext, + "gold_sample": sorted(gold)[:8], + "hit": hit, + "keys_sample": sorted(keys)[:8], + } + ) + + sheet_scorable = 0 + sheet_hits = 0 + for stem, pred in sorted(sheet_pred.items()): + gold = sheet_gold.get(stem) or set() + meta = sheet_meta.get(stem) or {} + if not gold: + unaligned += 1 + details.append( + { + "sheet": stem, + "resource": meta.get("resource", "CheatSheetSeries"), + "grain": "sheet", + "aligned": False, + "predicted": sorted(pred)[:12], + "gold_size": 0, + "hit": False, + } + ) + continue + sheet_scorable += 1 + hit = bool(pred & gold) + if hit: + sheet_hits += 1 + details.append( + { + "sheet": stem, + "resource": meta.get("resource", "CheatSheetSeries"), + "grain": "sheet", + "aligned": True, + "predicted": sorted(pred)[:12], + "gold_sample": sorted(gold)[:12], + "hit": hit, + "path": meta.get("path"), + } + ) + + scorable = chunk_scorable + sheet_scorable + hits = chunk_hits + sheet_hits + rate = (hits / scorable) if scorable else 0.0 + report = { + "run_id": args.run_id, + "base_url": args.base_url, + "rule": { + "ASVS": "chunk: ≥1 of top-2 suggested CRE external_ids ∈ section gold", + "CheatSheetSeries": ( + "sheet: union of top-2 across chunks; hit if ∩ sheet gold nonempty" + ), + }, + "resources": [s["label"] for s in RESOURCE_SPECS], + "decisions_total": len(decisions), + "scorable": scorable, + "unaligned": unaligned, + "hits": hits, + "accuracy": round(rate, 4), + "pass_gt_60": rate > 0.60, + "by_grain": { + "ASVS_chunk": { + "scorable": chunk_scorable, + "hits": chunk_hits, + "accuracy": round( + (chunk_hits / chunk_scorable) if chunk_scorable else 0.0, 4 + ), + }, + "CheatSheetSeries_sheet": { + "scorable": sheet_scorable, + "hits": sheet_hits, + "accuracy": round( + (sheet_hits / sheet_scorable) if sheet_scorable else 0.0, 4 + ), + }, + }, + "details": details[:200], + } + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps({k: report[k] for k in report if k != "details"}, indent=2)) + print(f"wrote {args.out}", flush=True) + return 0 if scorable else 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/oie_owasp_eval/score_b2_hop_distance.py b/scripts/oie_owasp_eval/score_b2_hop_distance.py new file mode 100644 index 000000000..609cdd366 --- /dev/null +++ b/scripts/oie_owasp_eval/score_b2_hop_distance.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +"""Score cached B2 reports with CRE hop-distance buckets (GA-style weights). + +Reads existing ``*.b2_report.json`` ``details`` (gold/predicted); does not re-run +Module C. Builds/caches CRE Related+Contains adjacency from local Postgres. + +Usage: + PYTHONPATH=. python scripts/oie_owasp_eval/score_b2_hop_distance.py + PYTHONPATH=. python scripts/oie_owasp_eval/score_b2_hop_distance.py --force +""" + +from __future__ import annotations + +import argparse +import csv +import json +import os +import sys +from pathlib import Path +from typing import Any, Dict, List, Optional + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT_DIR = Path(__file__).resolve().parent +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(SCRIPT_DIR)) + +from hop_distance import ( # noqa: E402 + BUCKETS, + build_adjacency_from_db, + load_adjacency, + save_adjacency, + score_details, +) + +EXP = ROOT / "tmp" / "oie_owasp_eval" / "experiments" +OUT_DIR = EXP / "hop_analysis" +ADJ_PATH = OUT_DIR / "cre_adjacency.json.gz" +HIGHLIGHT = ( + "g_1_0_0_0_0.85_0_0_lawrence_0", + "stack_winners", + "cre_summary", + "baseline", +) + + +def _connect_session(db_url: Optional[str]) -> Any: + url = ( + db_url + or os.environ.get("SQLALCHEMY_DATABASE_URI") + or os.environ.get("DATABASE_URL") + or "postgresql://cre:password@127.0.0.1:5432/cre" + ) + # Heroku-style postgres:// → postgresql:// + if url.startswith("postgres://"): + url = "postgresql://" + url[len("postgres://") :] + + from sqlalchemy import create_engine + from sqlalchemy.orm import sessionmaker + + engine = create_engine(url) + Session = sessionmaker(bind=engine) + return Session() + + +def ensure_adjacency(*, force: bool, db_url: Optional[str]) -> Any: + if ADJ_PATH.is_file() and not force: + print(f"Loading adjacency cache {ADJ_PATH}") + return load_adjacency(ADJ_PATH) + print(f"Building CRE adjacency from DB → {ADJ_PATH}") + session = _connect_session(db_url) + try: + adj = build_adjacency_from_db(session) + finally: + session.close() + if len(adj.known_ids) < 10: + raise SystemExit( + f"CRE adjacency too small ({len(adj.known_ids)} ids) — wrong DB?" + ) + save_adjacency(adj, ADJ_PATH) + n_edges = sum(len(adj.neighbors(n)) for n in adj.known_ids) + print(f" known_cres={len(adj.known_ids)} directed_edges={n_edges}") + return adj + + +def discover_reports(glob_pat: str) -> List[Path]: + paths = sorted(EXP.glob(glob_pat)) + # Also grid subdir if pattern is only top-level + if "grid" not in glob_pat: + paths += sorted((EXP / "grid").glob("*.b2_report.json")) + # de-dupe + seen = set() + out: List[Path] = [] + for p in paths: + if p.resolve() in seen: + continue + seen.add(p.resolve()) + out.append(p) + return out + + +def score_report(path: Path, adj: Any, *, force: bool) -> Optional[Dict[str, Any]]: + stem = path.name.replace(".b2_report.json", "") + out_path = OUT_DIR / f"{stem}.hop.json" + if out_path.is_file() and not force: + return json.loads(out_path.read_text(encoding="utf-8")) + + report = json.loads(path.read_text(encoding="utf-8")) + details = report.get("details") or [] + if not details: + print(f" skip {path.name}: no details") + return None + + scored = score_details(details, adj) + payload = { + "source_report": str(path.relative_to(ROOT)), + "experiment_id": stem, + "b2_exact_hits": report.get("hits"), + "b2_scorable": report.get("scorable"), + "b2_accuracy": report.get("accuracy"), + "scorable": scored["scorable"], + "counts": scored["counts"], + "rates": scored["rates"], + "pred_ids_total": scored["pred_ids_total"], + "invented_ids_total": scored["invented_ids_total"], + "rows": scored["rows"], + } + out_path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + return payload + + +def write_summary(results: List[Dict[str, Any]]) -> None: + results = sorted( + results, + key=lambda r: ( + -float((r.get("rates") or {}).get("distance1_hit") or 0), + -float((r.get("rates") or {}).get("exact") or 0), + r.get("experiment_id") or "", + ), + ) + csv_path = OUT_DIR / "summary.csv" + md_path = OUT_DIR / "SUMMARY.md" + fields = [ + "experiment_id", + "scorable", + "exact", + "one_hop_related", + "one_hop_contains", + "multi_hop", + "hallucination", + "irrelevant", + "pct_exact", + "pct_one_hop_related", + "pct_one_hop_contains", + "pct_multi_hop", + "pct_hallucination", + "pct_irrelevant", + "pct_distance1_hit", + "pct_within_2_hops", + "pct_within_3_hops", + "pct_ga_path_hit", + "pct_invented_pred", + "b2_accuracy", + ] + with csv_path.open("w", encoding="utf-8", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=fields) + w.writeheader() + for r in results: + c = r.get("counts") or {} + rates = r.get("rates") or {} + w.writerow( + { + "experiment_id": r.get("experiment_id"), + "scorable": r.get("scorable"), + **{b: c.get(b, 0) for b in BUCKETS}, + "pct_exact": rates.get("exact"), + "pct_one_hop_related": rates.get("one_hop_related"), + "pct_one_hop_contains": rates.get("one_hop_contains"), + "pct_multi_hop": rates.get("multi_hop"), + "pct_hallucination": rates.get("hallucination"), + "pct_irrelevant": rates.get("irrelevant"), + "pct_distance1_hit": rates.get("distance1_hit"), + "pct_within_2_hops": rates.get("within_2_hops"), + "pct_within_3_hops": rates.get("within_3_hops"), + "pct_ga_path_hit": rates.get("ga_path_hit"), + "pct_invented_pred": rates.get("invented_pred"), + "b2_accuracy": r.get("b2_accuracy"), + } + ) + + lines = [ + "# B2 hop-distance analysis", + "", + "Exclusive buckets per scorable section (canonical gold). " + "Weights from gap_analysis PENALTIES (RELATED=2, CONTAINS_UP=2, CONTAINS_DOWN=1).", + "", + "- **exact** — predicted ∩ gold", + "- **one_hop_related** / **one_hop_contains** — distance-1 CRE edge", + "- **multi_hop** — path length ≥ 2 (GA-useful but weaker)", + "- **hallucination** — all predicted CRE ids invented (not in DB)", + "- **irrelevant** — known CREs but no path to gold", + "", + f"- `distance1_hit` = exact + 1-hop related + 1-hop contains (good enough for precise-ish RAG / soft link)", + f"- `within_2_hops` / `within_3_hops` = hop-capped soft hits (prefer these over unbounded `ga_path_hit`)", + f"- `ga_path_hit` = any CRE path (often ~100% because the CRE Related+Contains graph is one component)", + "", + f"Reports scored: **{len(results)}**", + f"Adjacency cache: `{ADJ_PATH.relative_to(ROOT)}`", + "", + "## Highlights", + "", + "| id | exact | d1 | ≤2 hops | ≤3 hops | related1 | contains1 | multi | halluc |", + "|---|---:|---:|---:|---:|---:|---:|---:|---:|", + ] + by_id = {r["experiment_id"]: r for r in results} + for hid in HIGHLIGHT: + r = by_id.get(hid) + if not r: + continue + c = r["counts"] + rates = r["rates"] + lines.append( + f"| `{hid}` | {rates['exact']:.1%} | {rates['distance1_hit']:.1%} | " + f"{rates.get('within_2_hops', 0):.1%} | {rates.get('within_3_hops', 0):.1%} | " + f"{c['one_hop_related']} | {c['one_hop_contains']} | " + f"{c['multi_hop']} | {c['hallucination']} |" + ) + lines += [ + "", + "## Top 15 by distance1_hit", + "", + "| rank | id | exact | d1 | ≤2 | ≤3 | halluc |", + "|---:|---|---:|---:|---:|---:|---:|", + ] + for i, r in enumerate(results[:15], 1): + rates = r["rates"] + lines.append( + f"| {i} | `{r['experiment_id']}` | {rates['exact']:.1%} | " + f"{rates['distance1_hit']:.1%} | {rates.get('within_2_hops', 0):.1%} | " + f"{rates.get('within_3_hops', 0):.1%} | {rates['hallucination']:.1%} |" + ) + md_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + print(f"Wrote {csv_path}") + print(f"Wrote {md_path}") + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument( + "--reports-glob", + default="*.b2_report.json", + help="Glob under tmp/oie_owasp_eval/experiments/ (grid/ also scanned)", + ) + ap.add_argument("--force", action="store_true", help="Rebuild adj + rescore") + ap.add_argument("--db-url", default=None, help="Postgres URL for CRE graph") + ap.add_argument( + "--limit", + type=int, + default=0, + help="Score only first N reports (debug)", + ) + args = ap.parse_args() + + OUT_DIR.mkdir(parents=True, exist_ok=True) + adj = ensure_adjacency(force=args.force, db_url=args.db_url) + reports = discover_reports(args.reports_glob) + if args.limit > 0: + reports = reports[: args.limit] + print(f"Scoring {len(reports)} reports…") + + results: List[Dict[str, Any]] = [] + for i, path in enumerate(reports, 1): + if i % 50 == 0 or i == 1: + print(f" [{i}/{len(reports)}] {path.name}") + payload = score_report(path, adj, force=args.force) + if payload: + # drop rows from summary aggregate memory — keep counts only in csv; + # full rows already on disk + slim = {k: v for k, v in payload.items() if k != "rows"} + results.append(slim) + + write_summary(results) + + # Print winner spotlight + for hid in HIGHLIGHT[:2]: + p = OUT_DIR / f"{hid}.hop.json" + if not p.is_file(): + continue + data = json.loads(p.read_text(encoding="utf-8")) + rates = data["rates"] + counts = data["counts"] + print( + f"\n{hid}: exact={rates['exact']:.1%} d1={rates['distance1_hit']:.1%} " + f"≤2={rates.get('within_2_hops', 0):.1%} ≤3={rates.get('within_3_hops', 0):.1%} " + f"rel1={counts['one_hop_related']} contains1={counts['one_hop_contains']} " + f"multi={counts['multi_hop']} halluc={counts['hallucination']} " + f"/ {data['scorable']}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/oie_owasp_eval/seed_hub_links_from_b2_gold.py b/scripts/oie_owasp_eval/seed_hub_links_from_b2_gold.py new file mode 100644 index 000000000..5f445898f --- /dev/null +++ b/scripts/oie_owasp_eval/seed_hub_links_from_b2_gold.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +"""Seed hub Nodes+Links from B2 gold JSON (production map-once helper ONLY). + +WARNING: Do **not** use this before a B2 accuracy run. Seeding gold then +scoring against the same gold is circular (proves Link-prior, not transfer +from related standards). + +For eval (path B): leave target families unmapped; Module C must use organic +hub Links (Top10 2021, CCM, ASVS, LLM/AIX, …) + edition remap / control-name. + +This script is for simulating a human map-once into the hub after review. +Gold rows come from ``application.utils.mapping_fixtures``. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any, Dict, List, Sequence, Tuple + +from application.utils.mapping_fixtures import load_owasp_mapping_fixture + +ROOT = Path(__file__).resolve().parents[2] + +# (fixture stem, Node.name, version string) +SEED_SPECS: List[Tuple[str, str, str]] = [ + ("owasp_kubernetes_top10_2022", "OWASP Kubernetes Top Ten 2022", "2022"), + ("owasp_kubernetes_top10_2025", "OWASP Kubernetes Top Ten 2025", "2025"), + ("owasp_top10_2025", "OWASP Top 10 2025", "2025"), + ("owasp_api_top10_2023", "OWASP API Security Top 10 2023", "2023"), +] + + +def _load_dotenv() -> None: + for env_path in (ROOT / ".env", ROOT / "tmp" / "oie.env"): + if not env_path.is_file(): + continue + for line in env_path.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, _, v = line.partition("=") + if env_path.name == "oie.env" and ( + k.startswith("CRE_LIBRARIAN_") or k.startswith("DEV_DATABASE") + ): + os.environ[k.strip()] = v.strip() + else: + os.environ.setdefault(k.strip(), v.strip().strip('"').strip("'")) + + +def seed_standard( + database: Any, + *, + name: str, + version: str, + rows: Sequence[Dict[str, Any]], + dry_run: bool, +) -> Dict[str, int]: + from application.database.db import CRE, Links + from application.defs import cre_defs + + stats = { + "sections": 0, + "nodes_upserted": 0, + "links_added": 0, + "links_existed": 0, + "missing_cres": 0, + "empty_sections": 0, + } + for row in rows: + sid = str(row.get("section_id") or "").strip() + title = str(row.get("section") or sid).strip() + href = str(row.get("hyperlink") or "").strip() + cre_ids = [str(c) for c in (row.get("cre_ids") or []) if c] + if not sid: + continue + stats["sections"] += 1 + if not cre_ids: + stats["empty_sections"] += 1 + continue + + if dry_run: + for ext in cre_ids: + cre = database.session.query(CRE).filter(CRE.external_id == ext).first() + if cre is None: + stats["missing_cres"] += 1 + else: + stats["links_added"] += 1 + stats["nodes_upserted"] += 1 + continue + + std = cre_defs.Standard( + name=name, + section=title, + sectionID=sid, + subsection="", + version=version, + hyperlink=href, + ) + dbnode = database.add_node(std) + if dbnode is None: + continue + stats["nodes_upserted"] += 1 + + for ext in cre_ids: + cre = database.session.query(CRE).filter(CRE.external_id == ext).first() + if cre is None: + stats["missing_cres"] += 1 + continue + existed = ( + database.session.query(Links) + .filter(Links.cre == cre.id, Links.node == dbnode.id) + .first() + ) + database.add_link(cre, dbnode, cre_defs.LinkTypes.LinkedTo) + if existed: + stats["links_existed"] += 1 + else: + stats["links_added"] += 1 + return stats + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--cache-file", default="") + args = parser.parse_args() + + os.environ.setdefault("FLASK_CONFIG", "development") + os.environ.setdefault("NO_LOAD_GRAPH_DB", "1") + _load_dotenv() + cache = ( + args.cache_file + or os.environ.get("DEV_DATABASE_URL") + or os.environ.get("CRE_CACHE_FILE") + or "" + ) + if not cache: + print("DEV_DATABASE_URL / --cache-file required", file=sys.stderr) + return 2 + + from application import sqla + from application.cmd.cre_main import db_connect + from application.database import db as db_module + + db_connect(cache) + database = db_module.Node_collection() + + report: Dict[str, Any] = {"dry_run": args.dry_run, "standards": []} + for fixture_name, name, version in SEED_SPECS: + try: + rows = load_owasp_mapping_fixture(fixture_name) + except FileNotFoundError: + report["standards"].append( + {"name": name, "version": version, "error": "mapping fixture missing"} + ) + continue + stats = seed_standard( + database, name=name, version=version, rows=rows, dry_run=args.dry_run + ) + report["standards"].append( + { + "name": name, + "version": version, + "source": fixture_name, + **stats, + } + ) + + if not args.dry_run: + sqla.session.commit() + + print(json.dumps(report, indent=2), flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_oie_e2e_smoke.py b/scripts/run_oie_e2e_smoke.py index 09ee451e0..91c7dfdc6 100644 --- a/scripts/run_oie_e2e_smoke.py +++ b/scripts/run_oie_e2e_smoke.py @@ -136,6 +136,7 @@ def run_c(pipeline_run_id, **kwargs): reranker=_Reranker(), scaler=_Scaler(), known_cre_ids=frozenset({"616-305"}), + cre_membership=frozenset({"616-305"}), ) return run_librarian_queue( sqla.session, diff --git a/scripts/setup_oie.sh b/scripts/setup_oie.sh new file mode 100755 index 000000000..6ee5245e9 --- /dev/null +++ b/scripts/setup_oie.sh @@ -0,0 +1,274 @@ +#!/usr/bin/env bash +# setup_oie — bring up local Postgres for OpenCRE OIE (A→B→C) and sync the CRE hub. +# +# Reusable local bootstrap. Idempotent where possible. +# +# ./scripts/setup_oie.sh --source local +# ./scripts/setup_oie.sh --source upstream # Heroku APP_NAME (default opencreorg) +# ./scripts/setup_oie.sh --source local --calibrate # also fit CRE_LIBRARIAN_TEMPERATURE +# ./scripts/setup_oie.sh --source local --install-ml +# +# Writes env knobs to tmp/oie.env (source before running the orchestrator). +# +# Env overrides: +# PG_URL / DEV_DATABASE_URL local Postgres URL +# SQLITE_HUB path to standards_cache.sqlite +# APP_NAME / HEROKU_APP upstream Heroku app (default opencreorg) +# OIE_ENV_FILE output env file (default tmp/oie.env) + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${ROOT}" + +DB_SCRIPT_NAME="setup_oie" +# shellcheck source=scripts/db/common.sh +source "${ROOT}/scripts/db/common.sh" + +PG_URL="${PG_URL:-${DEV_DATABASE_URL:-postgresql://cre:password@127.0.0.1:5432/cre}}" +SQLITE_HUB="${SQLITE_HUB:-${ROOT}/standards_cache.sqlite}" +OIE_ENV_FILE="${OIE_ENV_FILE:-${ROOT}/tmp/oie.env}" +OIE_DIR="${ROOT}/tmp/oie_owasp_eval" +SOURCE="local" +DO_CALIBRATE=0 +DO_INSTALL_ML=0 +SKIP_DOCKER=0 +SKIP_SYNC=0 +FORCE_SYNC=0 +MIN_CRE_EMB="${MIN_CRE_EMB:-400}" + +usage() { + sed -n '2,20p' "$0" | sed 's/^# \{0,1\}//' + cat <&2; exit 1; } + +while [[ $# -gt 0 ]]; do + case "$1" in + --source) SOURCE="${2:-}"; shift 2 ;; + --sqlite) SQLITE_HUB="${2:-}"; shift 2 ;; + --pg-url) PG_URL="${2:-}"; shift 2 ;; + --install-ml) DO_INSTALL_ML=1; shift ;; + --calibrate) DO_CALIBRATE=1; shift ;; + --skip-docker) SKIP_DOCKER=1; shift ;; + --skip-sync) SKIP_SYNC=1; shift ;; + --force-sync) FORCE_SYNC=1; shift ;; + -h|--help) usage; exit 0 ;; + *) die "unknown arg: $1 (try --help)" ;; + esac +done + +[[ "${SOURCE}" == "local" || "${SOURCE}" == "upstream" ]] || die "--source must be local|upstream" + +ensure_venv() { + if [[ -d "${ROOT}/venv" ]]; then + # shellcheck disable=SC1091 + source "${ROOT}/venv/bin/activate" + else + die "venv missing — run make install-python first" + fi +} + +ensure_docker_postgres() { + if [[ "${SKIP_DOCKER}" == "1" ]]; then + log "skip-docker: not starting cre-postgres" + return + fi + if ! docker info >/dev/null 2>&1; then + if command -v colima >/dev/null 2>&1; then + log "docker not ready — starting colima" + colima start + else + die "docker is not running and colima is not installed" + fi + fi + log "starting cre-postgres (make docker-postgres)" + make docker-postgres +} + +migrate_local() { + log "flask db upgrade → ${PG_URL}" + export FLASK_APP="${ROOT}/cre.py" + export FLASK_CONFIG="${FLASK_CONFIG:-development}" + export NO_LOAD_GRAPH_DB=1 + export DEV_DATABASE_URL="${PG_URL}" + flask db upgrade +} + +cre_emb_count() { + psql "${PG_URL}" -Atc \ + "SELECT count(*) FROM embeddings WHERE doc_type = 'CRE' AND embedding_vec IS NOT NULL" \ + 2>/dev/null || echo 0 +} + +sync_hub() { + if [[ "${SKIP_SYNC}" == "1" ]]; then + log "skip-sync" + return + fi + local count + count="$(cre_emb_count)" + if [[ "${FORCE_SYNC}" != "1" && "${count}" -ge "${MIN_CRE_EMB}" ]]; then + log "CRE hub already populated (${count} embeddings ≥ ${MIN_CRE_EMB}); skip sync (use --force-sync to refresh)" + return + fi + + ensure_venv + mkdir -p "${OIE_DIR}" + if [[ "${SOURCE}" == "local" ]]; then + [[ -f "${SQLITE_HUB}" ]] || die "sqlite hub not found: ${SQLITE_HUB}" + log "syncing CRE hub from local sqlite ${SQLITE_HUB}" + PYTHONPATH="${ROOT}" python "${ROOT}/scripts/oie_sync_cre_hub.py" \ + --from-sqlite "${SQLITE_HUB}" \ + --to-postgres "${PG_URL}" + else + ensure_heroku_auth + local up_url + up_url="$(resolve_app_database_url)" + log "syncing CRE hub from upstream Heroku app ${APP_NAME} (read-only)" + PYTHONPATH="${ROOT}" python "${ROOT}/scripts/oie_sync_cre_hub.py" \ + --from-postgres "${up_url}" \ + --to-postgres "${PG_URL}" + fi +} + +install_ml() { + [[ "${DO_INSTALL_ML}" == "1" ]] || return 0 + ensure_venv + log "pip install sentence-transformers (C.2 cross-encoder; not on prod requirements.txt)" + pip install 'sentence-transformers>=5,<6' +} + +write_env_file() { + local temperature="${1:-1.0}" + mkdir -p "$(dirname "${OIE_ENV_FILE}")" + cat >"${OIE_ENV_FILE}" </dev/null; then + die "sentence-transformers required for --calibrate (pass --install-ml)" + fi + # Load API keys from repo .env if present (do not overwrite existing exports). + if [[ -f "${ROOT}/.env" ]]; then + set -a + # shellcheck disable=SC1091 + source <(grep -E '^(GEMINI_API_KEY|OPENAI_API_KEY|LLM_API_KEY|HF_TOKEN)=' "${ROOT}/.env" | sed 's/^/export /' || true) + set +a + fi + [[ -n "${GEMINI_API_KEY:-}${OPENAI_API_KEY:-}${LLM_API_KEY:-}" ]] || \ + die "need GEMINI_API_KEY or OPENAI_API_KEY in env/.env for live calibration" + + export FLASK_CONFIG=development + export NO_LOAD_GRAPH_DB=1 + export DEV_DATABASE_URL="${PG_URL}" + export CRE_LIBRARIAN_RETRIEVER_BACKEND=pgvector + export CRE_LIBRARIAN_LINK_THRESHOLD=0.8 + + mkdir -p "${OIE_DIR}" + local logf="${OIE_DIR}/evaluate_librarian_live.log" + log "fitting CRE_LIBRARIAN_TEMPERATURE (live embeddings) — log: ${logf}" + set +e + PYTHONPATH="${ROOT}" python -u "${ROOT}/scripts/evaluate_librarian.py" \ + --dataset "${ROOT}/application/tests/librarian/fixtures/golden_dataset.json" \ + --use_live_embeddings \ + --cache_file "${PG_URL}" \ + 2>&1 | tee "${logf}" + local rc=${PIPESTATUS[0]} + set -e + + local t + t="$(rg -o 'fitted T=[0-9.]+' "${logf}" | tail -1 | sed 's/fitted T=//' || true)" + if [[ -z "${t}" ]]; then + write_env_file "1.0" + die "calibration did not print fitted T=… (rc=${rc}); see ${logf}" + fi + write_env_file "${t}" + # Also stash machine-readable + printf '%s\n' "${t}" >"${OIE_DIR}/fitted_temperature.txt" + log "fitted T=${t} (also ${OIE_DIR}/fitted_temperature.txt)" + [[ "${rc}" -eq 0 ]] || log "WARNING: evaluate_librarian exited ${rc} — T still written; check ECE gate in log" +} + +print_next_steps() { + cat </dev/null 2>&1 || die "psql required" + command -v flask >/dev/null 2>&1 || die "flask required (venv)" + + migrate_local + sync_hub + ensure_oie_base_metadata + install_ml + calibrate_temperature + print_next_steps + + local n + n="$(cre_emb_count)" + log "done — CRE embeddings with vec: ${n}" + [[ "${n}" -ge 1 ]] || die "CRE hub empty after setup" +} + +main From 25f07e16869259189c9220bba6b30bed75f9063a Mon Sep 17 00:00:00 2001 From: Spyros Date: Mon, 5 Oct 2026 11:01:21 +0100 Subject: [PATCH 2/2] Address review feedback on B2 harness scripts. - Redact database passwords in orchestrator, B2 and setup_oie output; write tmp/oie.env with mode 600. - Keep the existing --db for --skip-a and a user-supplied --repos-yaml. - Create the artifact dir before writing orchestrator reports. - Clear per-model knowledge rows between Qwen and Gemini classify runs. - Key the switch-grid embed cache by model, honour --force with stale-lock recovery, write explicit zero flags to stack_winners.yaml, and tolerate LibrarianConfig fields that are env-only. - Match B1 sheet-grain gold exactly instead of by substring. --- application/tests/db_url_test.py | 31 ++++++++ application/tests/oie_eval_scripts_test.py | 78 +++++++++++++++++++ application/utils/db_url.py | 22 ++++++ scripts/oie_owasp_eval/run_b2_pr_mappings.py | 7 +- .../run_exhaustive_switch_grid.py | 46 ++++++++--- .../run_official_orchestrator_local.py | 19 +++-- scripts/oie_owasp_eval/run_phase.py | 20 +++++ .../run_qwen_outside_sandbox.sh | 3 +- .../oie_owasp_eval/score_b1_opencre_api.py | 15 +++- scripts/setup_oie.sh | 7 +- 10 files changed, 225 insertions(+), 23 deletions(-) create mode 100644 application/tests/db_url_test.py create mode 100644 application/tests/oie_eval_scripts_test.py create mode 100644 application/utils/db_url.py diff --git a/application/tests/db_url_test.py b/application/tests/db_url_test.py new file mode 100644 index 000000000..c8d1fbd82 --- /dev/null +++ b/application/tests/db_url_test.py @@ -0,0 +1,31 @@ +import unittest + +from application.utils.db_url import redact_db_url + + +class RedactDbUrlTest(unittest.TestCase): + def test_masks_password_and_keeps_rest(self) -> None: + self.assertEqual( + redact_db_url("postgresql://cre:s3cret@127.0.0.1:5432/cre?sslmode=require"), + "postgresql://cre:***@127.0.0.1:5432/cre?sslmode=require", + ) + + def test_url_without_password_is_unchanged(self) -> None: + for url in ( + "postgresql://127.0.0.1/cre", + "postgresql://cre@127.0.0.1/cre", + "sqlite:///tmp/x.sqlite", + "standards_cache.sqlite", + "", + ): + self.assertEqual(redact_db_url(url), url) + + def test_password_with_special_characters(self) -> None: + self.assertEqual( + redact_db_url("postgres://u:p%40ss:word@db.example.com/x"), + "postgres://u:***@db.example.com/x", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/tests/oie_eval_scripts_test.py b/application/tests/oie_eval_scripts_test.py new file mode 100644 index 000000000..bc4d5c2b9 --- /dev/null +++ b/application/tests/oie_eval_scripts_test.py @@ -0,0 +1,78 @@ +import importlib.util +import json +import os +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[2] +EVAL_DIR = ROOT / "scripts" / "oie_owasp_eval" + + +def _load(name: str): + spec = importlib.util.spec_from_file_location( + f"oie_eval_{name}", EVAL_DIR / f"{name}.py" + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class MatchGoldTest(unittest.TestCase): + def setUp(self) -> None: + self.mod = _load("score_b1_opencre_api") + self.gold = { + "authentication cheat sheet": {"cre-authn"}, + "multifactor authentication cheat sheet": {"cre-mfa"}, + } + + def test_fuzzy_match_bleeds_into_longer_sheet_name(self) -> None: + got = self.mod.match_gold({"authentication cheat sheet"}, self.gold) + self.assertEqual(got, {"cre-authn", "cre-mfa"}) + + def test_exact_match_keeps_sheets_separate(self) -> None: + got = self.mod.match_gold({"authentication cheat sheet"}, self.gold, exact=True) + self.assertEqual(got, {"cre-authn"}) + + +class ClaimComboTest(unittest.TestCase): + def setUp(self) -> None: + self.mod = _load("run_exhaustive_switch_grid") + self.tmp = tempfile.TemporaryDirectory() + grid = Path(self.tmp.name) + patcher_grid = patch.object(self.mod, "GRID_DIR", grid) + patcher_locks = patch.object(self.mod, "LOCKS_DIR", grid / ".locks") + patcher_grid.start() + patcher_locks.start() + self.addCleanup(patcher_grid.stop) + self.addCleanup(patcher_locks.stop) + self.addCleanup(self.tmp.cleanup) + self.grid = grid + + def _write_ok(self, combo_id: str) -> None: + (self.grid / f"{combo_id}.json").write_text( + json.dumps({"status": "ok", "headline": {"hits": 1}}) + ) + + def test_completed_combo_is_skipped_unless_forced(self) -> None: + self._write_ok("c1") + self.assertFalse(self.mod.claim_combo("c1")) + self.assertTrue(self.mod.claim_combo("c1", force=True)) + + def test_live_claim_blocks_second_worker(self) -> None: + self.assertTrue(self.mod.claim_combo("c2")) + self.assertFalse(self.mod.claim_combo("c2")) + self.assertFalse(self.mod.claim_combo("c2", force=True)) + + def test_claim_left_by_dead_worker_is_reclaimed(self) -> None: + lock = self.grid / ".locks" / "c3" + lock.mkdir(parents=True) + (lock / "pid").write_text("999999999\n") + self.assertTrue(self.mod.claim_combo("c3")) + self.assertEqual((lock / "pid").read_text().strip(), str(os.getpid())) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/utils/db_url.py b/application/utils/db_url.py new file mode 100644 index 000000000..f86976bc2 --- /dev/null +++ b/application/utils/db_url.py @@ -0,0 +1,22 @@ +"""Helpers for handling database URLs in logs and reports.""" + +from __future__ import annotations + +from urllib.parse import urlsplit, urlunsplit + + +def redact_db_url(url: str) -> str: + """Replace the password in ``scheme://user:password@host/db`` with ``***``. + + Anything that is not a URL with credentials (file paths, bare names) is + returned unchanged. + """ + try: + parts = urlsplit(url) + except ValueError: + return "" + userinfo, sep, hostport = parts.netloc.rpartition("@") + if not sep or ":" not in userinfo: + return url + user = userinfo.split(":", 1)[0] + return urlunsplit(parts._replace(netloc=f"{user}:***@{hostport}")) diff --git a/scripts/oie_owasp_eval/run_b2_pr_mappings.py b/scripts/oie_owasp_eval/run_b2_pr_mappings.py index 9291ed12c..b71e3dc8f 100644 --- a/scripts/oie_owasp_eval/run_b2_pr_mappings.py +++ b/scripts/oie_owasp_eval/run_b2_pr_mappings.py @@ -789,7 +789,12 @@ def main() -> int: run_id = args.run_id or ( "orch-b2-" + datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") ) - print(json.dumps({"run_id": run_id, "cache": cache}, indent=2), flush=True) + from application.utils.db_url import redact_db_url + + print( + json.dumps({"run_id": run_id, "cache": redact_db_url(cache)}, indent=2), + flush=True, + ) orch = run_pipeline(run_id, cache, keep_all_knowledge=args.keep_all_knowledge) (ART / "b2_phase_status.json").write_text(json.dumps(orch, indent=2) + "\n") print(json.dumps(orch, indent=2), flush=True) diff --git a/scripts/oie_owasp_eval/run_exhaustive_switch_grid.py b/scripts/oie_owasp_eval/run_exhaustive_switch_grid.py index 07b7e856a..b01331284 100644 --- a/scripts/oie_owasp_eval/run_exhaustive_switch_grid.py +++ b/scripts/oie_owasp_eval/run_exhaustive_switch_grid.py @@ -249,13 +249,30 @@ def reset_queues() -> None: ) -def claim_combo(combo_id: str) -> bool: +def _lock_owner_alive(lock: Path) -> bool: + try: + pid = int((lock / "pid").read_text().strip() or 0) + except (OSError, ValueError): + return False + if pid <= 0: + return False + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except OSError: + return True + return True + + +def claim_combo(combo_id: str, *, force: bool = False) -> bool: """Exclusive claim via mkdir lockdir so parallel workers never double-write. - Returns False if another worker holds the claim or a completed JSON exists. + Returns False if another live worker holds the claim or, unless ``force``, + a completed JSON exists. Claims left by dead workers are reclaimed. """ out = GRID_DIR / f"{combo_id}.json" - if out.is_file(): + if out.is_file() and not force: try: data = json.loads(out.read_text()) if data.get("status") == "ok" and data.get("headline"): @@ -264,6 +281,8 @@ def claim_combo(combo_id: str) -> bool: pass LOCKS_DIR.mkdir(parents=True, exist_ok=True) lock = LOCKS_DIR / combo_id + if lock.is_dir() and not _lock_owner_alive(lock): + release_combo_claim(combo_id) try: lock.mkdir() (lock / "pid").write_text(f"{os.getpid()}\n") @@ -366,7 +385,9 @@ def install_query_embed_disk_cache() -> None: def _cached(self: Any, text: Any) -> Any: if not isinstance(text, str): return original(self, text) - key = hashlib.sha256(text.encode("utf-8")).hexdigest() + key = hashlib.sha256( + f"{getattr(self, 'embed_model', '')}\n{text}".encode("utf-8") + ).hexdigest() path = EMBED_CACHE_DIR / f"{key}.json" if path.is_file(): try: @@ -429,7 +450,7 @@ def run_module_c(flags: Mapping[str, str]) -> Dict[str, Any]: cfg = load_config() snapshot = { - k: getattr(cfg, k) + k: getattr(cfg, k, os.environ.get(f"CRE_LIBRARIAN_{k.upper()}")) for k in ( "cre_summary", "dual_index", @@ -714,8 +735,9 @@ def promote_winners(ranked: Sequence[Mapping[str, Any]]) -> None: if k == "CRE_LIBRARIAN_MARGIN_GAMMA": yaml_flags[k] = str(v) continue - if str(v) in ("1", "true", "True"): - yaml_flags[k] = "1" + # Write explicit zeros too: cre_summary/dual_index default to on, and the + # experiment runner clears the environment before applying winners. + yaml_flags[k] = "1" if str(v) in ("1", "true", "True") else "0" if yaml_flags: flag_block = "".join(f' {k}: "{v}"\n' for k, v in sorted(yaml_flags.items())) @@ -895,10 +917,12 @@ def run_one( "summary": { k: (mc.get("summary") or {}).get(k) for k in ( - "decisions_total", + "read", "linked", - "review_required", - "errors", + "review", + "skipped", + "errored", + "persisted", "consumed", ) if isinstance(mc.get("summary"), dict) @@ -1175,7 +1199,7 @@ def _in_shard(combo_index: int) -> bool: continue if not _in_shard(combo_index): continue - if not claim_combo(combo_id): + if not claim_combo(combo_id, force=args.force): print(f"SKIP_CLAIMED {combo_id}", flush=True) continue try: diff --git a/scripts/oie_owasp_eval/run_official_orchestrator_local.py b/scripts/oie_owasp_eval/run_official_orchestrator_local.py index 9c776f421..639abff5e 100644 --- a/scripts/oie_owasp_eval/run_official_orchestrator_local.py +++ b/scripts/oie_owasp_eval/run_official_orchestrator_local.py @@ -336,6 +336,8 @@ def main(argv: Optional[List[str]] = None) -> int: ) args = parser.parse_args(argv) + from application.utils.db_url import redact_db_url + os.environ.setdefault("FLASK_CONFIG", "development") os.environ.setdefault("NO_LOAD_GRAPH_DB", "1") os.environ.setdefault("CRE_LIBRARIAN_RETRIEVER_BACKEND", "in_memory") @@ -350,10 +352,15 @@ def main(argv: Optional[List[str]] = None) -> int: if args.cache_file.strip(): cache_file = args.cache_file.strip() - print(f"using cache_file={cache_file}", flush=True) - db_label = cache_file + db_label = redact_db_url(cache_file) + print(f"using cache_file={db_label}", flush=True) else: - cache_file = _prepare_db(src=args.hub_db, dst=args.db, reuse=args.reuse_db) + if args.skip_a and not args.db.is_file(): + print(f"--skip-a requires an existing --db: {args.db}", file=sys.stderr) + return 2 + cache_file = _prepare_db( + src=args.hub_db, dst=args.db, reuse=args.reuse_db or args.skip_a + ) db_label = str(args.db.resolve()) run_id = (args.run_id or "").strip() or _now_run_id() @@ -371,7 +378,8 @@ def run_harvester_fn(session, rid, *, dry_run=False, sync_repos=True): ) else: - _write_tiny_repos_yaml(args.repos_yaml) + if args.repos_yaml.resolve() == DEFAULT_REPOS_YAML.resolve(): + _write_tiny_repos_yaml(args.repos_yaml) from application.utils.harvester.pipeline import run_harvester repos_yaml = args.repos_yaml @@ -393,7 +401,7 @@ def run_harvester_fn(session, rid, *, dry_run=False, sync_repos=True): json.dumps( { "run_id": run_id, - "cache_file": cache_file, + "cache_file": db_label, "a_mode": "skipped" if args.skip_a else args.a_mode, "orchestrator": "application.utils.oie_orchestrator.run_oie_pipeline", }, @@ -427,6 +435,7 @@ def run_harvester_fn(session, rid, *, dry_run=False, sync_repos=True): ), } report_path = ART / f"orchestrator_full_{run_id}.json" + ART.mkdir(parents=True, exist_ok=True) report_path.write_text(json.dumps(out, indent=2) + "\n") print(json.dumps(out, indent=2)) print(f"wrote {report_path}", flush=True) diff --git a/scripts/oie_owasp_eval/run_phase.py b/scripts/oie_owasp_eval/run_phase.py index f7fe62dca..baa9fbeac 100644 --- a/scripts/oie_owasp_eval/run_phase.py +++ b/scripts/oie_owasp_eval/run_phase.py @@ -516,6 +516,10 @@ def phase_classify_dual(state: Dict[str, Any]) -> None: ) gem_run = sample_run + "-gemini" + # content_hash is globally unique: rows kept by the first model would make the + # second model's identical chunks dedupe away and skew the agreement rate. + _clear_knowledge(db_url, sample_run) + _clear_knowledge(db_url, gem_run) _clone_harvest_run(db_url, sample_run, gem_run) # Reset sample rows already marked processed so clone is the only gemini input gem = _run_noise_filter(db_url, gem_run, cloud_model) @@ -553,6 +557,19 @@ def phase_classify_dual(state: Dict[str, Any]) -> None: _save_state(state) +def _clear_knowledge(db_url: str, run_id: str) -> None: + sys.path.insert(0, str(ROOT)) + from application.cmd.cre_main import db_connect + from application import sqla + from application.database.db import KnowledgeQueueItem + + db_connect(db_url) + sqla.session.query(KnowledgeQueueItem).filter_by(pipeline_run_id=run_id).delete( + synchronize_session=False + ) + sqla.session.commit() + + def _clone_harvest_run(db_url: str, src_run: str, dst_run: str) -> None: sys.path.insert(0, str(ROOT)) from application.cmd.cre_main import db_connect @@ -561,6 +578,9 @@ def _clone_harvest_run(db_url: str, src_run: str, dst_run: str) -> None: import copy db_connect(db_url) + sqla.session.query(HarvestInput).filter_by(pipeline_run_id=dst_run).delete( + synchronize_session=False + ) rows = sqla.session.query(HarvestInput).filter_by(pipeline_run_id=src_run).all() for row in rows: payload = copy.deepcopy(row.payload or {}) diff --git a/scripts/oie_owasp_eval/run_qwen_outside_sandbox.sh b/scripts/oie_owasp_eval/run_qwen_outside_sandbox.sh index 8a04c2f6b..f511b7eb5 100755 --- a/scripts/oie_owasp_eval/run_qwen_outside_sandbox.sh +++ b/scripts/oie_owasp_eval/run_qwen_outside_sandbox.sh @@ -14,10 +14,11 @@ DB=$(python -c "import json; print(json.load(open('tmp/oie_owasp_eval/state.json python - < None: return ordered[:2] -def match_gold(keys: Set[str], key_to_cres: Dict[str, Set[str]]) -> Set[str]: +def match_gold( + keys: Set[str], key_to_cres: Dict[str, Set[str]], *, exact: bool = False +) -> Set[str]: + """Union the gold CREs of every matching key. + + ``exact`` disables substring matching, which would otherwise let + ``authentication cheat sheet`` pull in ``multifactor authentication cheat sheet``. + """ gold: Set[str] = set() for k in keys: if k in key_to_cres: gold |= key_to_cres[k] + if exact: + continue for gk, cres in key_to_cres.items(): - if len(k) >= 8 and (k in gk or gk in k): + if len(k) >= 8 and len(gk) >= 8 and (k in gk or gk in k): gold |= cres return gold @@ -306,7 +315,7 @@ def main() -> int: } sheet_keys.discard("") sheet_gold[stem] = match_gold( - sheet_keys, gold_by_resource[spec["label"]] + sheet_keys, gold_by_resource[spec["label"]], exact=True ) sheet_meta[stem] = { "resource": spec["label"], diff --git a/scripts/setup_oie.sh b/scripts/setup_oie.sh index 6ee5245e9..d95691eb8 100755 --- a/scripts/setup_oie.sh +++ b/scripts/setup_oie.sh @@ -26,6 +26,8 @@ DB_SCRIPT_NAME="setup_oie" source "${ROOT}/scripts/db/common.sh" PG_URL="${PG_URL:-${DEV_DATABASE_URL:-postgresql://cre:password@127.0.0.1:5432/cre}}" +redact_url() { sed -E 's#(://[^:/@]+):[^@/]*@#\1:***@#' <<<"$1"; } + SQLITE_HUB="${SQLITE_HUB:-${ROOT}/standards_cache.sqlite}" OIE_ENV_FILE="${OIE_ENV_FILE:-${ROOT}/tmp/oie.env}" OIE_DIR="${ROOT}/tmp/oie_owasp_eval" @@ -44,7 +46,7 @@ usage() { Options: --source local|upstream Hub source (default: local sqlite) --sqlite PATH Local hub sqlite (default: ./standards_cache.sqlite) - --pg-url URL Local Postgres (default: ${PG_URL}) + --pg-url URL Local Postgres (default: $(redact_url "${PG_URL}")) --install-ml pip install sentence-transformers (Module C.2) --calibrate Fit CRE_LIBRARIAN_TEMPERATURE; write into env file --skip-docker Assume cre-postgres already running @@ -101,7 +103,7 @@ ensure_docker_postgres() { } migrate_local() { - log "flask db upgrade → ${PG_URL}" + log "flask db upgrade → $(redact_url "${PG_URL}")" export FLASK_APP="${ROOT}/cre.py" export FLASK_CONFIG="${FLASK_CONFIG:-development}" export NO_LOAD_GRAPH_DB=1 @@ -156,6 +158,7 @@ install_ml() { write_env_file() { local temperature="${1:-1.0}" mkdir -p "$(dirname "${OIE_ENV_FILE}")" + install -m 600 /dev/null "${OIE_ENV_FILE}" cat >"${OIE_ENV_FILE}" <