feat(oie): full-pipeline runner, docs, and experimental-flag notes

.env.example

@@ -52,14 +52,25 @@ GOOGLE_CLIENT_SECRET=your-client-secret
 GOOGLE_SECRET_JSON=path/to/secret.json
 LOGIN_ALLOWED_DOMAINS=example.com
 
-# GCP
+# GCP / Gemini (optional — any LiteLLM provider works for Module B)
 
 GCP_NATIVE=false
 GEMINI_API_KEY=your-gemini-api-key
 VERTEX_CHAT_MODEL=gemini-2.5-flash
 VERTEX_EMBED_CONTENT_MODEL=gemini-embedding-001
 
-# Noise / Relevance Filter (Module B)
+# Other chat providers (LiteLLM). Set the key that matches CRE_NOISE_FILTER_LLM_MODEL.
+# OPENAI_API_KEY=sk-...
+# ANTHROPIC_API_KEY=sk-ant-...
+# GROQ_API_KEY=gsk_...
+# MISTRAL_API_KEY=...
+# OPENROUTER_API_KEY=...
+
+# Noise / Relevance Filter (Module B) — LiteLLM model id (provider/model)
+# Examples: gemini/gemini-2.5-flash-lite | openai/gpt-4o-mini |
+#           anthropic/claude-haiku-4-5 | groq/llama-3.3-70b-versatile
+# Any chat model LiteLLM supports; mid-size models usually work best
+# (cheap + reliable for this classify gate — avoid tiny completion stubs).
 
 CRE_NOISE_FILTER_LLM_MODEL=gemini/gemini-2.5-flash-lite
 CRE_NOISE_FILTER_BATCH_SIZE=10

.gitignore

@@ -79,6 +79,8 @@ standards_cache.sqlite
 !docs/gsoc_2026_module_b/module_c_contract.md
 !docs/gsoc_2026_module_b/module_b_runbook.md
 !docs/gsoc_2026_module_b/final_metrics.md
+!docs/oie/*.md
+!scripts/oie_owasp_eval/experiments/*.md
 
 ### Dev DBDumps
 *.sql

Makefile

@@ -141,8 +141,12 @@ install-python:
 	make install-deps-python &&\
 	playwright install  # Python embeddings/scraping (prompt_client); NOT frontend e2e — keep when migrating to Cypress
 
-install-typescript:
-	yarn add webpack && cd application/frontend && yarn build
+# Yarn install only. Do not `yarn add` or `yarn build` here: webpack is
+# already in package.json, and a local build rewrites
+# application/frontend/www/bundle.js (tracked so Heroku can serve the UI
+# without a Node compile step). That was dirtying contributor trees after
+# `make install`. Rebuild with `make frontend` when you change the UI.
+install-typescript: install-deps-typescript
 
 install: install-typescript install-python migrate-upgrade
 
@@ -185,6 +189,22 @@ oie-pipeline:
 	[ -d "./venv" ] && . ./venv/bin/activate &&\
 	PYTHONPATH=. python scripts/run_oie_pipeline.py --cache_file "$(or $(CACHE_FILE),sqlite:///$(CURDIR)/standards_cache.sqlite)" $(OIE_ARGS)
 
+# Deterministic OIE base metadata (Node then CRE document_metadata.oie). Fill-if-missing.
+# Example:
+#   make oie-tag-base CACHE_FILE=postgresql://cre:password@127.0.0.1:5432/cre_prodclone
+oie-tag-base:
+	[ -d "./venv" ] && . ./venv/bin/activate &&\
+	PYTHONPATH=. python scripts/oie_ensure_base_metadata.py \
+		--cache-file "$(or $(CACHE_FILE),$(or $(DEV_DATABASE_URL),postgresql://cre:password@127.0.0.1:5432/cre))" \
+		$(OIE_TAG_ARGS)
+
+# Local OIE bootstrap: start cre-postgres, migrate, sync CRE hub (local|upstream).
+# Examples:
+#   make setup-oie
+#   make setup-oie SETUP_OIE_ARGS='--source upstream --install-ml --calibrate'
+setup-oie:
+	bash ./scripts/setup_oie.sh $(SETUP_OIE_ARGS)
+
 oie-e2e-smoke:
 	[ -d "./venv" ] && . ./venv/bin/activate &&\
 	PYTHONPATH=. python scripts/run_oie_e2e_smoke.py

application/tests/librarian/config_loader_test.py

@@ -42,6 +42,16 @@ def test_defaults_when_env_unset(self):
         self.assertAlmostEqual(cfg.hybrid_gamma, 0.70)
         self.assertAlmostEqual(cfg.hybrid_beta, HYBRID_BETA)
         self.assertAlmostEqual(cfg.hybrid_gamma, HYBRID_GAMMA)
+        self.assertTrue(cfg.leaf_drilldown)
+        self.assertEqual(cfg.leaf_drilldown_resources, ())
+        self.assertEqual(cfg.leaf_drilldown_force_resources, ())
+        self.assertEqual(cfg.leaf_drilldown_min_children, 3)
+        self.assertEqual(cfg.leaf_drilldown_min_sections, 20)
+        self.assertFalse(cfg.leaf_drilldown_keep_hub)
+        self.assertFalse(cfg.leaf_drilldown_hub_first)
+        self.assertEqual(cfg.umbrella_promote_cap, 8)
+        self.assertEqual(cfg.shortlist_judge_max_picks, 3)
+        self.assertIsNone(cfg.margin_gamma)
 
     def test_config_is_frozen(self):
         with mock.patch.dict(os.environ, {}, clear=True):
@@ -114,6 +124,51 @@ def test_link_threshold_above_one_raises(self):
             with self.assertRaises(ValueError):
                 load_config()
 
+    def test_empty_env_string_means_all_resources(self):
+        with mock.patch.dict(
+            os.environ,
+            {"CRE_LIBRARIAN_LEAF_DRILLDOWN_RESOURCES": ""},
+            clear=True,
+        ):
+            cfg = load_config()
+        self.assertEqual(cfg.leaf_drilldown_resources, ())
+
+    def test_leaf_drilldown_gate_env_overrides(self):
+        with mock.patch.dict(
+            os.environ,
+            {
+                "CRE_LIBRARIAN_LEAF_DRILLDOWN_RESOURCES": "asvs",
+                "CRE_LIBRARIAN_LEAF_DRILLDOWN_FORCE_RESOURCES": "api",
+                "CRE_LIBRARIAN_LEAF_DRILLDOWN_MIN_CHILDREN": "5",
+                "CRE_LIBRARIAN_LEAF_DRILLDOWN_MIN_SECTIONS": "50",
+                "CRE_LIBRARIAN_LEAF_DRILLDOWN_KEEP_HUB": "1",
+                "CRE_LIBRARIAN_LEAF_DRILLDOWN_HUB_FIRST": "true",
+                "CRE_LIBRARIAN_UMBRELLA_PROMOTE_CAP": "8",
+                "CRE_LIBRARIAN_SHORTLIST_JUDGE_MAX_PICKS": "3",
+                "CRE_LIBRARIAN_MARGIN_GAMMA": "0.85",
+            },
+            clear=True,
+        ):
+            cfg = load_config()
+        self.assertEqual(cfg.leaf_drilldown_resources, ("asvs",))
+        self.assertEqual(cfg.leaf_drilldown_force_resources, ("api",))
+        self.assertEqual(cfg.leaf_drilldown_min_children, 5)
+        self.assertEqual(cfg.leaf_drilldown_min_sections, 50)
+        self.assertTrue(cfg.leaf_drilldown_keep_hub)
+        self.assertTrue(cfg.leaf_drilldown_hub_first)
+        self.assertEqual(cfg.umbrella_promote_cap, 8)
+        self.assertEqual(cfg.shortlist_judge_max_picks, 3)
+        self.assertAlmostEqual(cfg.margin_gamma or 0.0, 0.85)
+
+    def test_leaf_drilldown_resources_star_means_all(self):
+        with mock.patch.dict(
+            os.environ,
+            {"CRE_LIBRARIAN_LEAF_DRILLDOWN_RESOURCES": "*"},
+            clear=True,
+        ):
+            cfg = load_config()
+        self.assertEqual(cfg.leaf_drilldown_resources, ())
+
     def test_non_positive_temperature_raises(self):
         """T divides the logits, so zero or negative is undefined, not merely a
         bad setting — the same guard TemperatureScaler applies."""