From 5657668dbcf597428d92939ebb8167c5c009ff99 Mon Sep 17 00:00:00 2001
From: Spyros <morfeas3000@gmail.com>
Date: Tue, 6 Oct 2026 20:38:55 +0100
Subject: [PATCH 1/6] Parse OWASP event pages and harvest event repos

Event repos (www-event-*) are Jekyll sites: front matter, a schema.org JSON-LD
block and Liquid templating around the real prose. event_page turns that into a
short title/dates/location/description header plus the markdown body, used both
at harvest time and when a filed event node is embedded. Events are now
harvestable; pages with no content are skipped without blocking the checkpoint.
---
 application/prompt_client/prompt_client.py    |   3 +-
 .../tests/harvester_test/event_page_test.py   | 145 ++++++++++++++
 .../fixtures/event_index.md.txt               |  45 +++++
 .../harvester_test/harvest_event_test.py      | 134 +++++++++++++
 .../tests/harvester_test/selection_test.py    |   6 +-
 application/utils/harvester/event_page.py     | 185 ++++++++++++++++++
 application/utils/harvester/pipeline.py       |   5 +
 application/utils/harvester/schemas.py        |  10 +-
 8 files changed, 526 insertions(+), 7 deletions(-)
 create mode 100644 application/tests/harvester_test/event_page_test.py
 create mode 100644 application/tests/harvester_test/fixtures/event_index.md.txt
 create mode 100644 application/tests/harvester_test/harvest_event_test.py
 create mode 100644 application/utils/harvester/event_page.py

diff --git a/application/prompt_client/prompt_client.py b/application/prompt_client/prompt_client.py
index 8aee6d607..1f4852ba8 100644
--- a/application/prompt_client/prompt_client.py
+++ b/application/prompt_client/prompt_client.py
@@ -10,6 +10,7 @@
 from urllib.parse import urljoin, urlparse
 
 from application.prompt_client import embed_alignment, litellm_router
+from application.utils.harvester.event_page import clean_embedding_text
 
 from scipy import sparse
 from sklearn.metrics.pairwise import cosine_similarity
@@ -315,7 +316,7 @@ def get_content(self, url) -> Optional[str]:
         if is_plain_text_embed_url(url):
             text = _fetch_plain_http_text(url)
             if text:
-                return text
+                return clean_embedding_text(url, text)
             logger.warning(
                 "Plain-text URL %s: empty HTTP body, falling through to Playwright",
                 url,
diff --git a/application/tests/harvester_test/event_page_test.py b/application/tests/harvester_test/event_page_test.py
new file mode 100644
index 000000000..bbdf733f5
--- /dev/null
+++ b/application/tests/harvester_test/event_page_test.py
@@ -0,0 +1,145 @@
+"""Jekyll event pages: front matter + JSON-LD become clean prose; templating is dropped."""
+
+import unittest
+from pathlib import Path
+
+from unittest.mock import patch
+
+from application.prompt_client import prompt_client
+from application.utils.harvester.event_page import (
+    clean_embedding_text,
+    prepare_event_page,
+)
+
+FIXTURE = Path(__file__).parent / "fixtures" / "event_index.md.txt"
+
+
+class PrepareEventPageTests(unittest.TestCase):
+    def setUp(self) -> None:
+        self.text = FIXTURE.read_text(encoding="utf-8")
+        self.out = prepare_event_page(self.text)
+
+    def test_title_comes_from_the_event_name(self) -> None:
+        self.assertTrue(
+            self.out.startswith("# AppSec Days - Summer of Security 2020\n"), self.out
+        )
+
+    def test_event_facts_are_stated_as_prose(self) -> None:
+        self.assertIn("Dates: 2020-06-23 to 2020-08-26", self.out)
+        self.assertIn("Location: https://appsecdays.org/", self.out)
+        self.assertIn("virtual AppSec Days", self.out)
+
+    def test_markup_noise_is_removed(self) -> None:
+        for noise in (
+            "schema.org",
+            "@context",
+            "application/ld+json",
+            "layout: event",
+            "{%",
+            "{{",
+            "<!--",
+            "<ul>",
+            "OnlineEventAttendanceMode",
+        ):
+            self.assertNotIn(noise, self.out)
+
+    def test_markdown_body_is_kept(self) -> None:
+        self.assertIn("### Training Sessions Include", self.out)
+        self.assertIn("threat modeling with the OWASP Top 10", self.out)
+        self.assertIn("https://appsecdays.org/sponsors/swag/", self.out)
+
+    def test_falls_back_to_front_matter_title_without_json_ld(self) -> None:
+        out = prepare_event_page(
+            "---\ntitle: Local Meetup\nlayout: event\n---\n\nCome and learn about XSS.\n"
+        )
+        self.assertTrue(out.startswith("# Local Meetup\n"), out)
+        self.assertIn("Come and learn about XSS.", out)
+        self.assertNotIn("layout", out)
+
+    def test_malformed_json_ld_is_dropped_not_fatal(self) -> None:
+        out = prepare_event_page(
+            '---\ntitle: T\n---\n<script type="application/ld+json">{not json</script>\nBody text here.\n'
+        )
+        self.assertIn("Body text here.", out)
+        self.assertNotIn("not json", out)
+
+    def test_raw_newlines_inside_json_strings_are_tolerated(self) -> None:
+        out = prepare_event_page(
+            '<script type="application/ld+json">{"@type": "Event", "name": "N",\n'
+            ' "description": "line one\n        line two", "startDate": "2020-01-01"}'
+            "</script>\nBody.\n"
+        )
+        self.assertIn("# N", out)
+        self.assertIn("line one line two", out)
+
+    def test_json_ld_graph_and_list_forms(self) -> None:
+        out = prepare_event_page(
+            '<script type="application/ld+json">'
+            '{"@graph": [{"@type": "Organization", "name": "Org"},'
+            '{"@type": "Event", "name": "Graph Event", "startDate": "2025-01-02",'
+            ' "location": {"@type": "Place", "name": "Lisbon", "address": "Rua 1"}}]}'
+            "</script>\nBody.\n"
+        )
+        self.assertIn("# Graph Event", out)
+        self.assertIn("Dates: 2025-01-02", out)
+        self.assertIn("Location: Lisbon", out)
+
+    def test_page_with_no_content_is_blank(self) -> None:
+        self.assertEqual(
+            prepare_event_page("---\nlayout: event\n---\n{% include x.md %}\n"), ""
+        )
+
+    def test_plain_markdown_passes_through(self) -> None:
+        self.assertEqual(
+            prepare_event_page("# Hello\n\nWorld.\n").strip(), "# Hello\n\nWorld."
+        )
+
+
+RAW = (
+    "https://raw.githubusercontent.com/OWASP/www-event-2020-07-virtual/master/index.md"
+)
+
+
+class CleanEmbeddingTextTests(unittest.TestCase):
+    def setUp(self) -> None:
+        self.text = FIXTURE.read_text(encoding="utf-8")
+
+    def test_event_repo_pages_are_cleaned(self) -> None:
+        out = clean_embedding_text(RAW, self.text)
+        self.assertIn("Dates: 2020-06-23 to 2020-08-26", out)
+        self.assertNotIn("ld+json", out)
+
+    def test_github_blob_and_raw_forms_of_event_repos_match(self) -> None:
+        for url in (
+            "https://github.com/OWASP/www-event-2024-global-appsec-lisbon/raw/main/index.md",
+            "https://raw.githubusercontent.com/OWASP/www-event-2024-global-appsec-lisbon/main/index.md",
+        ):
+            self.assertNotIn("ld+json", clean_embedding_text(url, self.text))
+
+    def test_other_repos_and_non_markdown_are_untouched(self) -> None:
+        for url in (
+            "https://raw.githubusercontent.com/OWASP/ASVS/master/README.md",
+            "https://raw.githubusercontent.com/someone/www-event-x/main/index.md",
+            "https://raw.githubusercontent.com/OWASP/www-event-x/main/data.json",
+            "https://example.com/OWASP/www-event-x/main/index.md",
+        ):
+            self.assertEqual(clean_embedding_text(url, self.text), self.text, url)
+
+    def test_empty_cleaned_text_falls_back_to_the_original(self) -> None:
+        text = "---\nlayout: event\n---\n{% include x.md %}\n"
+        self.assertEqual(clean_embedding_text(RAW, text), text)
+
+    def test_get_content_cleans_event_pages(self) -> None:
+        emb = prompt_client.in_memory_embeddings.__new__(
+            prompt_client.in_memory_embeddings
+        )
+        with patch.object(
+            prompt_client, "_fetch_plain_http_text", return_value=self.text
+        ):
+            out = emb.get_content(RAW)
+        self.assertIn("Dates: 2020-06-23", out)
+        self.assertNotIn("schema.org", out)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/application/tests/harvester_test/fixtures/event_index.md.txt b/application/tests/harvester_test/fixtures/event_index.md.txt
new file mode 100644
index 000000000..faa3e4f84
--- /dev/null
+++ b/application/tests/harvester_test/fixtures/event_index.md.txt
@@ -0,0 +1,45 @@
+---
+
+title: AppSec Days - July 2020
+layout: event
+
+---
+<script type="application/ld+json">
+{
+  "@context": "https://schema.org",
+  "@type": "Event",
+  "name": " AppSec Days - Summer of Security 2020",
+  "description": "The OWASP Foundation is hosting virtual AppSec Days on the last Tuesday and Wednesday in June, July, and August.",
+  "startDate": "2020-06-23",
+  "endDate": "2020-08-26",
+  "eventAttendanceMode": "https://schema.org/OnlineEventAttendanceMode",
+  "location": {
+    "@type": "VirtualLocation",
+    "url": "https://appsecdays.org/"
+  },
+  "offers": [{
+    "@type": "Offer",
+    "name": "Training Courses",
+    "price": "495",
+    "priceCurrency": "USD",
+    "url": "https://appsecdays.org/register/"
+  }]
+}
+</script>
+
+<!-- rebuild 13 -->
+
+{{ site.data.event-details.pitch | markdownify }}
+
+ **[Check out our sponsors](https://appsecdays.org/sponsors/swag/)**
+
+### Training Sessions Include
+
+* Learn threat modeling with the OWASP Top 10.
+{% if site.data.trainings.count > 0 %}
+<ul>
+  {% for trainer in trainings %}
+    <li><a href="{{trainer.URL}}">{{ trainer.Title }}</a></li>
+  {% endfor %}
+</ul>
+{% endif %}
diff --git a/application/tests/harvester_test/harvest_event_test.py b/application/tests/harvester_test/harvest_event_test.py
new file mode 100644
index 000000000..75b75a848
--- /dev/null
+++ b/application/tests/harvester_test/harvest_event_test.py
@@ -0,0 +1,134 @@
+"""Event repos are harvested through the event-page parser."""
+
+from cre_logging import get_logger
+
+logger = get_logger(__name__)
+
+import tempfile
+import unittest
+from datetime import datetime, timezone
+from pathlib import Path
+from unittest.mock import patch
+
+from application import create_app, sqla
+from application.database import db
+from application.utils.harvester import pipeline as harvest_pipeline
+from application.utils.harvester.schemas import HARVESTABLE_KINDS
+
+HEAD = "abc1234def5678"
+FIXTURE = Path(__file__).parent / "fixtures" / "event_index.md.txt"
+
+REPOS_YAML = """
+repositories:
+  - id: owasp-www-event-demo
+    kind: event
+    enabled: true
+    type: github
+    owner: OWASP
+    repo: www-event-demo
+    branch: main
+    paths: {include: ["index.md"]}
+    chunking: {strategy: markdown_heading, max_tokens: 1000}
+    polling: {mode: full, interval_minutes: 1440}
+"""
+
+
+def _client(files):
+    class FakeClient:
+        def __init__(self, *args, **kwargs) -> None:
+            pass
+
+        def sync(self) -> None:
+            pass
+
+        def get_current_commit_sha(self) -> str:
+            return HEAD
+
+        def get_file_at_commit(self, sha: str, path: str) -> str:
+            return files[path]
+
+        def get_local_path(self) -> Path:
+            return Path("/nonexistent")
+
+    return FakeClient
+
+
+class HarvestEventTests(unittest.TestCase):
+    def setUp(self) -> None:
+        self.app = create_app(mode="test")
+        self.ctx = self.app.app_context()
+        self.ctx.push()
+        sqla.create_all()
+        self.tmp = tempfile.TemporaryDirectory()
+        self.yaml = Path(self.tmp.name) / "repos.yaml"
+        self.yaml.write_text(REPOS_YAML)
+
+    def tearDown(self) -> None:
+        sqla.session.remove()
+        sqla.drop_all()
+        self.ctx.pop()
+        self.tmp.cleanup()
+
+    def _run(self, files):
+        patches = [
+            patch.object(harvest_pipeline, "GitRepositoryClient", _client(files)),
+            patch.object(
+                harvest_pipeline.ChangeDetector,
+                "get_files_at_commit",
+                lambda self, sha: list(files),
+            ),
+            patch.object(
+                harvest_pipeline,
+                "_commit_timestamp",
+                lambda client, sha: datetime(2026, 1, 1, tzinfo=timezone.utc),
+            ),
+        ]
+        for p in patches:
+            p.start()
+            self.addCleanup(p.stop)
+        return harvest_pipeline.run_harvester(
+            sqla.session,
+            "run-1",
+            repos_yaml=self.yaml,
+            sync_repos=False,
+            kinds=HARVESTABLE_KINDS,
+        )
+
+    def _texts(self):
+        return [row.payload["text"] for row in sqla.session.query(db.HarvestInput)]
+
+    def test_events_are_harvestable(self) -> None:
+        self.assertIn("event", HARVESTABLE_KINDS)
+
+    def test_event_page_is_parsed_before_chunking(self) -> None:
+        summary = self._run({"index.md": FIXTURE.read_text(encoding="utf-8")})
+        self.assertEqual(summary.errors, 0)
+        self.assertEqual(summary.repositories, 1)
+        self.assertGreater(summary.chunks_written, 0)
+        joined = "\n".join(self._texts())
+        self.assertIn("Summer of Security 2020", joined)
+        self.assertIn("2020-06-23", joined)
+        self.assertIn("threat modeling", joined)
+        for noise in ("schema.org", "ld+json", "{%", "{{", "layout: event"):
+            self.assertNotIn(noise, joined)
+
+    def test_page_with_no_content_is_skipped_and_checkpoint_still_advances(
+        self,
+    ) -> None:
+        from application.utils.harvester.checkpoint_store import CheckpointStore
+
+        summary = self._run(
+            {"index.md": "---\nlayout: event\n---\n{% include x.md %}\n"}
+        )
+        self.assertEqual(summary.errors, 0)
+        self.assertEqual(summary.chunks_written, 0)
+        self.assertEqual(
+            CheckpointStore(session=sqla.session)
+            .load("owasp-www-event-demo")
+            .last_processed_commit,
+            HEAD,
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/application/tests/harvester_test/selection_test.py b/application/tests/harvester_test/selection_test.py
index 1764349fa..c4a38e042 100644
--- a/application/tests/harvester_test/selection_test.py
+++ b/application/tests/harvester_test/selection_test.py
@@ -127,7 +127,7 @@ def test_default_kinds_are_the_curated_standards_only(self) -> None:
         sel = select_repositories(cfg.repositories, self.store, now=NOW)
         self.assertEqual([r.repo for r in sel.selected], ["std"])
 
-    def test_harvestable_kinds_exclude_chapter_and_event(self) -> None:
+    def test_harvestable_kinds_exclude_only_chapters(self) -> None:
         cfg = _repos(
             _entry("std", kind="standard"),
             _entry("proj", kind="project"),
@@ -138,7 +138,9 @@ def test_harvestable_kinds_exclude_chapter_and_event(self) -> None:
         sel = select_repositories(
             cfg.repositories, self.store, kinds=HARVESTABLE_KINDS, now=NOW
         )
-        self.assertEqual([r.repo for r in sel.selected], ["std", "proj", "other"])
+        self.assertEqual(
+            [r.repo for r in sel.selected], ["std", "proj", "other", "evt"]
+        )
 
     def test_explicit_kinds_filter(self) -> None:
         cfg = _repos(_entry("std", kind="standard"), _entry("proj", kind="project"))
diff --git a/application/utils/harvester/event_page.py b/application/utils/harvester/event_page.py
new file mode 100644
index 000000000..0bc93d14d
--- /dev/null
+++ b/application/utils/harvester/event_page.py
@@ -0,0 +1,185 @@
+"""Turn an OWASP event page (a Jekyll ``index.md``) into plain markdown prose.
+
+Event repos (``www-event-*``) keep their facts in YAML front matter and a
+schema.org JSON-LD ``<script>`` block, and wrap the rest in Liquid templating.
+Fed to a chunker as-is, that markup would dominate the embedding text, so this
+extracts the event's name, dates, location and description into a short header
+and keeps only the real markdown body.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any, Dict, List, Optional
+from urllib.parse import urlparse
+
+import yaml
+
+_FRONT_MATTER = re.compile(r"\A\s*---[ \t]*\n(.*?)\n---[ \t]*(?:\n|\Z)", re.DOTALL)
+_JSON_LD = re.compile(
+    r"<script\b[^>]*type\s*=\s*[\"']application/ld\+json[\"'][^>]*>(.*?)</script\s*>",
+    re.DOTALL | re.IGNORECASE,
+)
+_OTHER_SCRIPT_OR_STYLE = re.compile(
+    r"<(script|style)\b[^>]*>.*?</\1\s*>", re.DOTALL | re.IGNORECASE
+)
+_HTML_COMMENT = re.compile(r"<!--.*?-->", re.DOTALL)
+_LIQUID_BLOCK = re.compile(
+    r"\{%-?\s*(for|if|unless|case|capture)\b.*?%\}.*?\{%-?\s*end\1\s*-?%\}",
+    re.DOTALL,
+)
+_LIQUID_TAG = re.compile(r"\{%.*?%\}", re.DOTALL)
+_LIQUID_OUTPUT = re.compile(r"\{\{.*?\}\}", re.DOTALL)
+_BLANK_RUNS = re.compile(r"\n{3,}")
+
+
+def _front_matter_title(text: str) -> tuple[Optional[str], str]:
+    match = _FRONT_MATTER.match(text)
+    if not match:
+        return None, text
+    title: Optional[str] = None
+    try:
+        data = yaml.safe_load(match.group(1))
+    except yaml.YAMLError:
+        data = None
+    if isinstance(data, dict) and isinstance(data.get("title"), str):
+        title = data["title"].strip() or None
+    return title, text[match.end() :]
+
+
+def _event_objects(node: Any) -> List[Dict[str, Any]]:
+    if isinstance(node, list):
+        return [e for item in node for e in _event_objects(item)]
+    if not isinstance(node, dict):
+        return []
+    found: List[Dict[str, Any]] = []
+    kind = node.get("@type")
+    kinds = kind if isinstance(kind, list) else [kind]
+    if any(isinstance(k, str) and k.endswith("Event") for k in kinds):
+        found.append(node)
+    found.extend(_event_objects(node.get("@graph")))
+    return found
+
+
+def _text(value: Any) -> str:
+    return " ".join(value.split()) if isinstance(value, str) else ""
+
+
+def _location(value: Any) -> str:
+    if isinstance(value, str):
+        return _text(value)
+    if isinstance(value, list):
+        return ", ".join(p for p in (_location(v) for v in value) if p)
+    if not isinstance(value, dict):
+        return ""
+    parts = [_text(value.get("name"))]
+    address = value.get("address")
+    if isinstance(address, dict):
+        parts.append(
+            ", ".join(
+                _text(address.get(k))
+                for k in (
+                    "streetAddress",
+                    "addressLocality",
+                    "addressRegion",
+                    "addressCountry",
+                )
+                if _text(address.get(k))
+            )
+        )
+    else:
+        parts.append(_text(address))
+    if not any(parts):
+        parts.append(_text(value.get("url")))
+    return ", ".join(p for p in parts if p)
+
+
+def _header(event: Dict[str, Any], fallback_title: Optional[str]) -> str:
+    name = _text(event.get("name")) or fallback_title or ""
+    lines: List[str] = [f"# {name}"] if name else []
+    facts: List[str] = []
+    start, end = _text(event.get("startDate")), _text(event.get("endDate"))
+    if start and end and end != start:
+        facts.append(f"Dates: {start} to {end}")
+    elif start or end:
+        facts.append(f"Dates: {start or end}")
+    where = _location(event.get("location"))
+    if where:
+        facts.append(f"Location: {where}")
+    if facts:
+        lines.append("\n".join(facts))
+    description = _text(event.get("description"))
+    if description:
+        lines.append(description)
+    return "\n\n".join(lines)
+
+
+def _strip_templating(text: str) -> str:
+    text = _OTHER_SCRIPT_OR_STYLE.sub("", text)
+    text = _HTML_COMMENT.sub("", text)
+    previous = None
+    while previous != text:
+        previous = text
+        text = _LIQUID_BLOCK.sub("", text)
+    text = _LIQUID_TAG.sub("", text)
+    text = _LIQUID_OUTPUT.sub("", text)
+    return text
+
+
+def prepare_event_page(text: str) -> str:
+    """Return clean markdown for an event page, or ``""`` if it has no content."""
+    title, body = _front_matter_title(text)
+
+    events: List[Dict[str, Any]] = []
+
+    def collect(match: "re.Match[str]") -> str:
+        try:
+            # Hand-written JSON-LD often has raw newlines inside strings.
+            events.extend(_event_objects(json.loads(match.group(1), strict=False)))
+        except ValueError:
+            pass
+        return ""
+
+    body = _JSON_LD.sub(collect, body)
+    body = _BLANK_RUNS.sub("\n\n", _strip_templating(body)).strip()
+
+    header = ""
+    if events:
+        header = _header(events[0], title)
+    elif title and body:
+        header = f"# {title}"
+
+    parts = [p for p in (header, body) if p]
+    return "\n\n".join(parts) + "\n" if parts else ""
+
+
+def _event_repo_markdown(url: str) -> bool:
+    """True for a GitHub raw/blob URL of a markdown file in an OWASP ``www-event-*`` repo."""
+    try:
+        parsed = urlparse((url or "").strip())
+    except ValueError:
+        return False
+    if parsed.scheme not in ("http", "https"):
+        return False
+    host = (parsed.hostname or "").lower()
+    if host not in ("raw.githubusercontent.com", "github.com", "www.github.com"):
+        return False
+    parts = [p for p in parsed.path.split("/") if p]
+    return (
+        len(parts) >= 3
+        and parts[0].lower() == "owasp"
+        and parts[1].lower().startswith("www-event-")
+        and parts[-1].lower().endswith(".md")
+    )
+
+
+def clean_embedding_text(url: str, text: str) -> str:
+    """Clean fetched page text before embedding it, for event-repo pages only.
+
+    The same markup that is dropped at harvest time would otherwise dominate the
+    embedding of a filed event node. Falls back to ``text`` if nothing is left.
+    """
+    if not _event_repo_markdown(url):
+        return text
+    return prepare_event_page(text) or text
diff --git a/application/utils/harvester/pipeline.py b/application/utils/harvester/pipeline.py
index 336714427..cd0537bfa 100644
--- a/application/utils/harvester/pipeline.py
+++ b/application/utils/harvester/pipeline.py
@@ -21,6 +21,7 @@
 from application.utils.harvester.chunk_pipeline import DocumentChunkPipeline
 from application.utils.harvester.config_loader import load_repo_config
 from application.utils.harvester.document_builder import DocumentBuilder
+from application.utils.harvester.event_page import prepare_event_page
 from application.utils.harvester.file_filter import FileFilter
 from application.utils.harvester.git_repository_client import GitRepositoryClient
 from application.utils.harvester.harvest_writer import write_harvest_input
@@ -196,6 +197,10 @@ def _harvest_repository(
     documents: list[Document] = []
     for path in candidates:
         text = client.get_file_at_commit(head, path)
+        if repo_cfg.kind == "event":
+            text = prepare_event_page(text)
+            if not text.strip():
+                continue
         block = DiffBlock(
             file_path=path,
             added_lines=[],
diff --git a/application/utils/harvester/schemas.py b/application/utils/harvester/schemas.py
index 1057c6454..da7bd3c6e 100644
--- a/application/utils/harvester/schemas.py
+++ b/application/utils/harvester/schemas.py
@@ -23,11 +23,13 @@ def validate_cron_line(value: Optional[str]) -> Optional[str]:
         )
     return text
 
-# What an OWASP org repo is for. ``standard``/``project``/``other`` feed the
-# knowledge-graph expansion path (harvest -> filter -> Librarian); ``chapter``
-# and ``event`` repos only feed the OWASP agent's metadata index.
+
+# What an OWASP org repo is for. ``standard``/``project``/``other``/``event``
+# feed the knowledge-graph expansion path (harvest -> filter -> Librarian; event
+# pages are cleaned by ``event_page`` first); ``chapter`` repos only feed the
+# OWASP agent's metadata index.
 RepoKind = Literal["standard", "project", "chapter", "event", "other"]
-HARVESTABLE_KINDS: tuple[str, ...] = ("standard", "project", "other")
+HARVESTABLE_KINDS: tuple[str, ...] = ("standard", "project", "other", "event")
 # What a harvest run visits when the caller names no kinds: the curated standards
 # only, as before repos.yaml covered the whole org. Scheduled runs ask for
 # HARVESTABLE_KINDS explicitly.

From 1ebd0f833e82ca1cf8da34510a200459d55e73a2 Mon Sep 17 00:00:00 2001
From: Spyros <morfeas3000@gmail.com>
Date: Tue, 6 Oct 2026 20:38:55 +0100
Subject: [PATCH 2/6] Embed auto-filed graph nodes in the CRE expansion job

New embed_nodes stage (after gap analysis) embeds auto-filed nodes that have no
embedding yet, so similarity search and chat can see them. The pending set is
derived from the graph, so a failed run is retried next time; failures degrade
the job without touching filed nodes or edges. OIE_EMBED_FILED_NODES=0 turns it
off.
---
 .../tests/oie_scheduler/graph_filer_test.py   | 36 +++++++
 .../tests/oie_scheduler/runner_test.py        | 96 ++++++++++++++++++-
 .../utils/oie_scheduler/graph_filer.py        | 27 ++++++
 application/utils/oie_scheduler/jobs.py       | 47 ++++++++-
 application/utils/oie_scheduler/runner.py     |  2 +
 5 files changed, 205 insertions(+), 3 deletions(-)

diff --git a/application/tests/oie_scheduler/graph_filer_test.py b/application/tests/oie_scheduler/graph_filer_test.py
index 702af890a..02977c88f 100644
--- a/application/tests/oie_scheduler/graph_filer_test.py
+++ b/application/tests/oie_scheduler/graph_filer_test.py
@@ -56,6 +56,42 @@ def _envelope(
     }
 
 
+class PendingEmbeddingsTest(unittest.TestCase):
+    def setUp(self) -> None:
+        self.app = create_app(mode="test")
+        self.ctx = self.app.app_context()
+        self.ctx.push()
+        sqla.create_all()
+
+    def tearDown(self) -> None:
+        sqla.session.remove()
+        sqla.drop_all()
+        self.ctx.pop()
+
+    def _node(self, name, tags):
+        node = db.Node(name=name, section="s", ntype="Standard", tags=tags)
+        sqla.session.add(node)
+        sqla.session.commit()
+        return node.id
+
+    def test_only_auto_filed_nodes_without_embeddings_are_pending(self) -> None:
+        filed = self._node("OWASP/a", f"x,{graph_filer.AUTO_FILED_TAG}")
+        embedded = self._node("OWASP/b", graph_filer.AUTO_FILED_TAG)
+        self._node("ASVS", "curated")
+        sqla.session.add(
+            db.Embeddings(doc_type="Standard", node_id=embedded, embedding_vec="[1]")
+        )
+        sqla.session.commit()
+        self.assertEqual(
+            graph_filer.auto_filed_nodes_without_embeddings(sqla.session), [filed]
+        )
+
+    def test_nothing_pending_on_an_empty_graph(self) -> None:
+        self.assertEqual(
+            graph_filer.auto_filed_nodes_without_embeddings(sqla.session), []
+        )
+
+
 class GraphFilerTest(unittest.TestCase):
     def setUp(self) -> None:
         self.app = create_app(mode="test")
diff --git a/application/tests/oie_scheduler/runner_test.py b/application/tests/oie_scheduler/runner_test.py
index ad2d6e858..7a77d932f 100644
--- a/application/tests/oie_scheduler/runner_test.py
+++ b/application/tests/oie_scheduler/runner_test.py
@@ -37,6 +37,7 @@ def fn(ctx):
     "librarian": _ok("librarian"),
     "file_graph": _ok("file_graph"),
     "gap_analysis": _ok("gap_analysis"),
+    "embed_nodes": _ok("embed_nodes"),
 }
 
 
@@ -193,8 +194,12 @@ def setUp(self) -> None:
         )
         sqla.session.commit()
         self.ga_calls = []
+        self.embed_calls = []
 
-    def _expansion(self, **cfg):
+    def _embed(self, collection, node_ids):
+        self.embed_calls.append(list(node_ids))
+
+    def _expansion(self, now=NOW, **cfg):
         cfg.setdefault("filing_enabled", True)
         overrides = {
             "noise_filter": _ok("noise_filter"),
@@ -204,10 +209,11 @@ def _expansion(self, **cfg):
             self.collection,
             "cre_expansion",
             cache_file="db-url",
-            now=NOW,
+            now=now,
             config=SchedulerConfig(**cfg),
             overrides=overrides,
             ga_fn=self.ga_calls.append,
+            embed_fn=self._embed,
         )
 
     def test_filing_edges_triggers_gap_analysis_once(self) -> None:
@@ -234,6 +240,7 @@ def test_no_new_edges_means_no_gap_analysis(self) -> None:
                 "librarian": _ok("librarian"),
             },
             ga_fn=self.ga_calls.append,
+            embed_fn=self._embed,
         )
         by_name = {s.name: s for s in out.stages}
         self.assertEqual(by_name["gap_analysis"].status, "skipped")
@@ -247,6 +254,81 @@ def test_kill_switch_skips_filing_and_gap_analysis(self) -> None:
         self.assertEqual(self.ga_calls, [])
         self.assertIsNone(sqla.session.query(db.DecisionQueueItem).one().consumed_at)
 
+    def _filed_node_id(self):
+        return sqla.session.query(db.Node).one().id
+
+    def test_filed_nodes_are_embedded(self) -> None:
+        out = self._expansion()
+        by_name = {s.name: s for s in out.stages}
+        self.assertEqual(by_name["embed_nodes"].status, "ok")
+        self.assertEqual(by_name["embed_nodes"].summary["pending"], 1)
+        self.assertEqual(self.embed_calls, [[self._filed_node_id()]])
+
+    def test_embeds_after_gap_analysis_so_it_cannot_delay_it(self) -> None:
+        out = self._expansion()
+        names = [s.name for s in out.stages]
+        self.assertLess(names.index("gap_analysis"), names.index("embed_nodes"))
+
+    def test_node_that_already_has_an_embedding_is_left_alone(self) -> None:
+        self._expansion()
+        self.embed_calls.clear()
+        sqla.session.add(
+            db.Embeddings(
+                doc_type="Standard",
+                node_id=self._filed_node_id(),
+                embedding_vec="[0.1,0.2]",
+            )
+        )
+        sqla.session.commit()
+        out = self._expansion(now=NOW + timedelta(days=1))
+        by_name = {s.name: s for s in out.stages}
+        self.assertEqual(by_name["embed_nodes"].status, "skipped")
+        self.assertEqual(self.embed_calls, [])
+
+    def test_curated_nodes_are_never_embedded_by_the_job(self) -> None:
+        sqla.session.add(db.Node(name="ASVS", section="V1", ntype="Standard", tags=""))
+        sqla.session.commit()
+        self._expansion()
+        self.assertEqual(len(self.embed_calls), 1)
+        self.assertEqual(len(self.embed_calls[0]), 1)
+
+    def test_embedding_can_be_switched_off(self) -> None:
+        out = self._expansion(embed_filed_nodes=False)
+        by_name = {s.name: s for s in out.stages}
+        self.assertEqual(by_name["embed_nodes"].status, "skipped")
+        self.assertEqual(self.embed_calls, [])
+
+    def test_embedding_failure_degrades_the_job_but_keeps_the_graph_edges(
+        self,
+    ) -> None:
+        def boom(collection, node_ids):
+            raise RuntimeError("no LLM credentials")
+
+        out = run_job(
+            self.collection,
+            "cre_expansion",
+            cache_file="db-url",
+            now=NOW,
+            config=SchedulerConfig(),
+            overrides={
+                "noise_filter": _ok("noise_filter"),
+                "librarian": _ok("librarian"),
+            },
+            ga_fn=self.ga_calls.append,
+            embed_fn=boom,
+        )
+        by_name = {s.name: s for s in out.stages}
+        self.assertEqual(by_name["embed_nodes"].status, "degraded")
+        self.assertIn("no LLM credentials", by_name["embed_nodes"].detail)
+        self.assertEqual(by_name["file_graph"].summary["links_added"], 1)
+        self.assertEqual(out.status, "degraded")
+
+    def test_nothing_filed_means_nothing_to_embed(self) -> None:
+        out = self._expansion(filing_enabled=False)
+        by_name = {s.name: s for s in out.stages}
+        self.assertEqual(by_name["embed_nodes"].status, "skipped")
+        self.assertEqual(self.embed_calls, [])
+
     def test_high_floor_drops_the_link_and_skips_gap_analysis(self) -> None:
         out = self._expansion(file_floor=0.99)
         by_name = {s.name: s for s in out.stages}
@@ -401,6 +483,16 @@ def fake(*, repo_cfg, **_):
         self.assertEqual(out.status, "degraded")
 
 
+class SchedulerConfigEnvTest(unittest.TestCase):
+    def test_embedding_filed_nodes_defaults_on_and_has_a_kill_switch(self) -> None:
+        with patch.dict("os.environ", {}, clear=False) as env:
+            env.pop("OIE_EMBED_FILED_NODES", None)
+            self.assertTrue(SchedulerConfig.from_env().embed_filed_nodes)
+        for off in ("0", "false", "OFF", "no"):
+            with patch.dict("os.environ", {"OIE_EMBED_FILED_NODES": off}):
+                self.assertFalse(SchedulerConfig.from_env().embed_filed_nodes)
+
+
 class HealthTest(SchedulerTestBase):
     def test_never_ran_is_unhealthy(self) -> None:
         report = health.evaluate_health(sqla.session, NOW)
diff --git a/application/utils/oie_scheduler/graph_filer.py b/application/utils/oie_scheduler/graph_filer.py
index 1dbb19a69..f7f5b7cf5 100644
--- a/application/utils/oie_scheduler/graph_filer.py
+++ b/application/utils/oie_scheduler/graph_filer.py
@@ -25,6 +25,7 @@
 from datetime import datetime
 from typing import Any, Collection, Dict, List, Optional
 
+import sqlalchemy as sa
 from pydantic import ValidationError
 
 from application.database import db
@@ -63,6 +64,32 @@ def to_dict(self) -> Dict[str, Any]:
         return out
 
 
+def auto_filed_nodes_without_embeddings(session: Any) -> List[str]:
+    """Ids of auto-filed nodes that have no embedding row yet.
+
+    Derived from the graph rather than from one run's filer result, so a node
+    whose embedding failed (or whose run crashed) is picked up by the next run.
+    """
+    rows = (
+        session.query(db.Node.id, db.Node.tags)
+        .filter(db.Node.tags.like(f"%{AUTO_FILED_TAG}%"))
+        .filter(~sqla_exists_embedding())
+        .order_by(db.Node.id)
+        .all()
+    )
+    return [
+        node_id
+        for node_id, tags in rows
+        if AUTO_FILED_TAG in [t.strip() for t in (tags or "").split(",")]
+    ]
+
+
+def sqla_exists_embedding() -> Any:
+    return (
+        sa.select(db.Embeddings.id).where(db.Embeddings.node_id == db.Node.id).exists()
+    )
+
+
 def standard_name_for_repo(repo: str) -> str:
     return repo if "/" in repo else f"OWASP/{repo}"
 
diff --git a/application/utils/oie_scheduler/jobs.py b/application/utils/oie_scheduler/jobs.py
index da48b942e..55b6bf183 100644
--- a/application/utils/oie_scheduler/jobs.py
+++ b/application/utils/oie_scheduler/jobs.py
@@ -47,6 +47,7 @@ class SchedulerConfig:
     file_floor: float = 0.9
     # Default off: Module C decisions stay for review until an operator enables filing.
     filing_enabled: bool = False
+    embed_filed_nodes: bool = True
     repos_yaml: Optional[str] = None
     agent_db: Optional[str] = None
 
@@ -63,6 +64,8 @@ def from_env(cls) -> "SchedulerConfig":
             metadata_max_repos=int(os.getenv("OIE_METADATA_MAX_REPOS", "50")),
             file_floor=floor,
             filing_enabled=raw_filing in ("1", "true", "yes", "on"),
+            embed_filed_nodes=os.getenv("OIE_EMBED_FILED_NODES", "1").strip().lower()
+            not in ("0", "false", "no", "off"),
             repos_yaml=os.getenv("OIE_REPOS_YAML") or None,
             agent_db=os.getenv("OWASP_AGENT_DB") or None,
         )
@@ -99,6 +102,7 @@ class JobContext:
     config: SchedulerConfig = field(default_factory=SchedulerConfig)
     overrides: Dict[str, StageFn] = field(default_factory=dict)
     ga_fn: Optional[Callable[[str], None]] = None
+    embed_fn: Optional[Callable[[Any, List[str]], None]] = None
     #: Results one stage hands to a later one (e.g. the filer's graph change).
     state: Dict[str, Any] = field(default_factory=dict)
 
@@ -335,6 +339,46 @@ def stage_gap_analysis(ctx: JobContext) -> StageResult:
     )
 
 
+def _default_embed(collection: Any, node_ids: List[str]) -> None:
+    from application.prompt_client import prompt_client
+
+    embeddings = prompt_client.PromptHandler(database=collection).embeddings_instance
+    embeddings.setup_playwright()
+    try:
+        embeddings.generate_embeddings(collection, node_ids)
+    finally:
+        embeddings.teardown_playwright()
+
+
+def stage_embed_nodes(ctx: JobContext) -> StageResult:
+    """Embed auto-filed nodes so similarity search and chat can see them.
+
+    A failure here (no LLM credentials, provider outage) degrades the job but
+    leaves the filed nodes and edges in place; the next run retries because the
+    pending set is derived from the graph, not from this run.
+    """
+    if not ctx.config.embed_filed_nodes:
+        return StageResult("embed_nodes", "skipped", "OIE_EMBED_FILED_NODES is off")
+    if ctx.dry_run:
+        return StageResult("embed_nodes", "skipped", "dry run")
+    pending = graph_filer.auto_filed_nodes_without_embeddings(ctx.session)
+    if not pending:
+        return StageResult("embed_nodes", "skipped", "no filed nodes need embeddings")
+    summary = {"pending": len(pending)}
+    try:
+        (ctx.embed_fn or _default_embed)(ctx.collection, pending)
+    except Exception as exc:  # noqa: BLE001 -- provider/credential failures vary
+        ctx.session.rollback()
+        logger.exception("embedding filed nodes failed")
+        return StageResult(
+            "embed_nodes", "degraded", f"{type(exc).__name__}: {exc}", summary
+        )
+    summary["embedded"] = len(pending) - len(
+        graph_filer.auto_filed_nodes_without_embeddings(ctx.session)
+    )
+    return StageResult("embed_nodes", "ok", "", summary)
+
+
 def run_stage(name: str, fn: StageFn, ctx: JobContext) -> StageResult:
     """Run one stage, turning an exception into an ``error`` result."""
     started = time.monotonic()
@@ -364,12 +408,13 @@ def run_stage(name: str, fn: StageFn, ctx: JobContext) -> StageResult:
     "cre_expansion": JobSpec(
         name="cre_expansion",
         interval_minutes=1440,
-        description="Noise filter, Librarian, graph filing and gap analysis",
+        description="Noise filter, Librarian, graph filing, gap analysis and embeddings",
         stages=(
             ("noise_filter", stage_noise_filter),
             ("librarian", stage_librarian),
             ("file_graph", stage_file_graph),
             ("gap_analysis", stage_gap_analysis),
+            ("embed_nodes", stage_embed_nodes),
         ),
     ),
 }
diff --git a/application/utils/oie_scheduler/runner.py b/application/utils/oie_scheduler/runner.py
index 39c9151dc..ca9459f8b 100644
--- a/application/utils/oie_scheduler/runner.py
+++ b/application/utils/oie_scheduler/runner.py
@@ -68,6 +68,7 @@ def run_job(
     config: Optional[SchedulerConfig] = None,
     overrides: Optional[Dict[str, StageFn]] = None,
     ga_fn: Optional[Callable[[str], None]] = None,
+    embed_fn: Optional[Callable[[Any, List[str]], None]] = None,
 ) -> RunOutcome:
     if job_name not in JOBS:
         raise ValueError(f"unknown job {job_name!r}; expected one of {sorted(JOBS)}")
@@ -110,6 +111,7 @@ def run_job(
                 config=config or SchedulerConfig.from_env(),
                 overrides=dict(overrides or {}),
                 ga_fn=ga_fn,
+                embed_fn=embed_fn,
             )
             stages: List[StageResult] = []
             error: Optional[str] = None

From 1210ac2af6ee4321c72c404e464df005359c5b84 Mon Sep 17 00:00:00 2001
From: Spyros <morfeas3000@gmail.com>
Date: Tue, 6 Oct 2026 20:48:36 +0100
Subject: [PATCH 3/6] Make event page parsing linear-time and clean Playwright
 fallback text

Page content is untrusted: replace unbounded lazy regexes with str.find scans,
bounded tag regexes and a single-pass Liquid block stripper, and cap page size.
The Playwright fallback in get_content now cleans event pages too.
---
 application/prompt_client/prompt_client.py    |   2 +-
 .../tests/harvester_test/event_page_test.py   |  32 +++++
 application/utils/harvester/event_page.py     | 124 +++++++++++++-----
 3 files changed, 124 insertions(+), 34 deletions(-)

diff --git a/application/prompt_client/prompt_client.py b/application/prompt_client/prompt_client.py
index 1f4852ba8..4e0494527 100644
--- a/application/prompt_client/prompt_client.py
+++ b/application/prompt_client/prompt_client.py
@@ -345,7 +345,7 @@ def get_content(self, url) -> Optional[str]:
                 logger.info(f"loading page {url}")
                 page.goto(url)
                 text = page.locator("body").inner_text()
-                return text
+                return clean_embedding_text(url, text)
             except requests.exceptions.RequestException as e:
                 logger.error(
                     f"Error fetching content for URL: {url} - {str(e)} (attempt {attempts}/9)"
diff --git a/application/tests/harvester_test/event_page_test.py b/application/tests/harvester_test/event_page_test.py
index bbdf733f5..a184a7c30 100644
--- a/application/tests/harvester_test/event_page_test.py
+++ b/application/tests/harvester_test/event_page_test.py
@@ -95,6 +95,38 @@ def test_plain_markdown_passes_through(self) -> None:
         )
 
 
+class AdversarialInputTests(unittest.TestCase):
+    """Page content is untrusted repo text; unclosed constructs must not blow up."""
+
+    def test_unterminated_markup_is_linear_enough(self) -> None:
+        import time
+
+        for text in (
+            "{% if x %} text\n" * 20000,
+            '<script type="application/ld+json">' * 5000,
+            "{% if a %}" * 5000 + "{% endif %}" * 5000,
+            "{{ " * 50000,
+            "<!-- " * 50000,
+            "{% " * 50000,
+            "<script " * 50000,
+        ):
+            started = time.monotonic()
+            prepare_event_page(text)
+            self.assertLess(time.monotonic() - started, 5.0, text[:20])
+
+    def test_unterminated_block_keeps_the_prose_around_it(self) -> None:
+        out = prepare_event_page("Before.\n{% if x %}\nAfter.\n")
+        self.assertIn("Before.", out)
+        self.assertIn("After.", out)
+
+    def test_nested_blocks_are_removed_whole(self) -> None:
+        out = prepare_event_page(
+            "Keep.\n{% if a %}\n{% for b in c %}x{% endfor %}\ngone\n{% endif %}\nKeep too.\n"
+        )
+        self.assertNotIn("gone", out)
+        self.assertIn("Keep too.", out)
+
+
 RAW = (
     "https://raw.githubusercontent.com/OWASP/www-event-2020-07-virtual/master/index.md"
 )
diff --git a/application/utils/harvester/event_page.py b/application/utils/harvester/event_page.py
index 0bc93d14d..3a9159422 100644
--- a/application/utils/harvester/event_page.py
+++ b/application/utils/harvester/event_page.py
@@ -16,21 +16,17 @@
 
 import yaml
 
+# Page content is untrusted repo text, so every scan below is linear: delimiters
+# are located with ``str.find`` or bounded regexes, never unbounded ``.*?`` that
+# can rescan to the end of the page from each unmatched opener.
+MAX_PAGE_CHARS = 300_000
 _FRONT_MATTER = re.compile(r"\A\s*---[ \t]*\n(.*?)\n---[ \t]*(?:\n|\Z)", re.DOTALL)
-_JSON_LD = re.compile(
-    r"<script\b[^>]*type\s*=\s*[\"']application/ld\+json[\"'][^>]*>(.*?)</script\s*>",
-    re.DOTALL | re.IGNORECASE,
-)
-_OTHER_SCRIPT_OR_STYLE = re.compile(
-    r"<(script|style)\b[^>]*>.*?</\1\s*>", re.DOTALL | re.IGNORECASE
-)
-_HTML_COMMENT = re.compile(r"<!--.*?-->", re.DOTALL)
-_LIQUID_BLOCK = re.compile(
-    r"\{%-?\s*(for|if|unless|case|capture)\b.*?%\}.*?\{%-?\s*end\1\s*-?%\}",
-    re.DOTALL,
-)
-_LIQUID_TAG = re.compile(r"\{%.*?%\}", re.DOTALL)
-_LIQUID_OUTPUT = re.compile(r"\{\{.*?\}\}", re.DOTALL)
+_SCRIPT_OR_STYLE_OPEN = re.compile(r"<(script|style)\b([^>]{0,500})>", re.IGNORECASE)
+_LD_JSON_ATTR = re.compile(r"type\s*=\s*[\"']application/ld\+json[\"']", re.IGNORECASE)
+_LIQUID_TAG = re.compile(r"\{%-?((?:(?!%\}).){0,300})%\}", re.DOTALL)
+_LIQUID_OUTPUT = re.compile(r"\{\{(?:(?!\}\}).){0,300}\}\}", re.DOTALL)
+_LIQUID_BLOCK_OPEN = re.compile(r"\s*-?\s*(for|if|unless|case|capture)\b")
+_LIQUID_BLOCK_CLOSE = re.compile(r"\s*-?\s*end(for|if|unless|case|capture)\s*-?\s*\Z")
 _BLANK_RUNS = re.compile(r"\n{3,}")
 
 
@@ -115,33 +111,95 @@ def _header(event: Dict[str, Any], fallback_title: Optional[str]) -> str:
     return "\n\n".join(lines)
 
 
+def _strip_delimited(text: str, open_: str, close: str) -> str:
+    """Remove ``open_ ... close`` spans; an unterminated opener is left alone."""
+    out: List[str] = []
+    pos = 0
+    while True:
+        i = text.find(open_, pos)
+        if i < 0:
+            break
+        j = text.find(close, i + len(open_))
+        if j < 0:
+            break
+        out.append(text[pos:i])
+        pos = j + len(close)
+    out.append(text[pos:])
+    return "".join(out)
+
+
+def _strip_liquid_blocks(text: str) -> str:
+    """Drop whole ``{% if/for/... %}...{% end... %}`` blocks (outermost), in one pass."""
+    spans: List[tuple[int, int]] = []
+    stack: List[tuple[str, int]] = []
+    for match in _LIQUID_TAG.finditer(text):
+        body = match.group(1)
+        opened = _LIQUID_BLOCK_OPEN.match(body)
+        closed = _LIQUID_BLOCK_CLOSE.match(body)
+        if opened:
+            stack.append((opened.group(1), match.start()))
+        elif closed:
+            names = [name for name, _ in stack]
+            if closed.group(1) in names:
+                depth = len(names) - 1 - names[::-1].index(closed.group(1))
+                start = stack[depth][1]
+                del stack[depth:]
+                if not stack:
+                    spans.append((start, match.end()))
+    out: List[str] = []
+    pos = 0
+    for start, end in spans:
+        out.append(text[pos:start])
+        pos = end
+    out.append(text[pos:])
+    return "".join(out)
+
+
 def _strip_templating(text: str) -> str:
-    text = _OTHER_SCRIPT_OR_STYLE.sub("", text)
-    text = _HTML_COMMENT.sub("", text)
-    previous = None
-    while previous != text:
-        previous = text
-        text = _LIQUID_BLOCK.sub("", text)
+    text = _strip_delimited(text, "<!--", "-->")
+    text = _strip_liquid_blocks(text)
     text = _LIQUID_TAG.sub("", text)
-    text = _LIQUID_OUTPUT.sub("", text)
-    return text
+    return _LIQUID_OUTPUT.sub("", text)
+
+
+def _extract_scripts(text: str, events: List[Dict[str, Any]]) -> str:
+    """Remove ``<script>``/``<style>`` blocks, collecting schema.org Event JSON-LD."""
+    out: List[str] = []
+    lowered = text.lower()
+    pos = 0
+    scan = 0
+    while True:
+        match = _SCRIPT_OR_STYLE_OPEN.search(text, scan)
+        if not match:
+            break
+        name = match.group(1).lower()
+        close = lowered.find(f"</{name}", match.end())
+        if close < 0:
+            scan = match.end()
+            continue
+        end = lowered.find(">", close)
+        if end < 0:
+            break
+        if name == "script" and _LD_JSON_ATTR.search(match.group(2)):
+            try:
+                # Hand-written JSON-LD often has raw newlines inside strings.
+                events.extend(
+                    _event_objects(json.loads(text[match.end() : close], strict=False))
+                )
+            except ValueError:
+                pass
+        out.append(text[pos : match.start()])
+        pos = scan = end + 1
+    out.append(text[pos:])
+    return "".join(out)
 
 
 def prepare_event_page(text: str) -> str:
     """Return clean markdown for an event page, or ``""`` if it has no content."""
-    title, body = _front_matter_title(text)
+    title, body = _front_matter_title(text[:MAX_PAGE_CHARS])
 
     events: List[Dict[str, Any]] = []
-
-    def collect(match: "re.Match[str]") -> str:
-        try:
-            # Hand-written JSON-LD often has raw newlines inside strings.
-            events.extend(_event_objects(json.loads(match.group(1), strict=False)))
-        except ValueError:
-            pass
-        return ""
-
-    body = _JSON_LD.sub(collect, body)
+    body = _extract_scripts(body, events)
     body = _BLANK_RUNS.sub("\n\n", _strip_templating(body)).strip()
 
     header = ""

From bc13dc6504316855134044361ba20b570d0f1ec6 Mon Sep 17 00:00:00 2001
From: Spyros <morfeas3000@gmail.com>
Date: Tue, 6 Oct 2026 21:07:04 +0100
Subject: [PATCH 4/6] Make event-repo harvesting opt-in (OIE_HARVEST_EVENTS)

Event pages are metadata, so the scheduled harvest skips them unless
OIE_HARVEST_EVENTS=1. The parser and explicit --harvester_kinds event still work.
---
 .../harvester_test/harvest_event_test.py      | 12 +++--
 .../tests/harvester_test/selection_test.py    |  6 +--
 .../tests/oie_scheduler/runner_test.py        | 45 +++++++++++++++++++
 application/utils/harvester/schemas.py        | 12 ++---
 application/utils/oie_scheduler/jobs.py       | 13 +++++-
 5 files changed, 73 insertions(+), 15 deletions(-)

diff --git a/application/tests/harvester_test/harvest_event_test.py b/application/tests/harvester_test/harvest_event_test.py
index 75b75a848..8edfff7f7 100644
--- a/application/tests/harvester_test/harvest_event_test.py
+++ b/application/tests/harvester_test/harvest_event_test.py
@@ -13,7 +13,10 @@
 from application import create_app, sqla
 from application.database import db
 from application.utils.harvester import pipeline as harvest_pipeline
-from application.utils.harvester.schemas import HARVESTABLE_KINDS
+from application.utils.harvester.schemas import (
+    HARVESTABLE_KINDS,
+    OPTIONAL_HARVEST_KINDS,
+)
 
 HEAD = "abc1234def5678"
 FIXTURE = Path(__file__).parent / "fixtures" / "event_index.md.txt"
@@ -91,14 +94,15 @@ def _run(self, files):
             "run-1",
             repos_yaml=self.yaml,
             sync_repos=False,
-            kinds=HARVESTABLE_KINDS,
+            kinds=HARVESTABLE_KINDS + OPTIONAL_HARVEST_KINDS,
         )
 
     def _texts(self):
         return [row.payload["text"] for row in sqla.session.query(db.HarvestInput)]
 
-    def test_events_are_harvestable(self) -> None:
-        self.assertIn("event", HARVESTABLE_KINDS)
+    def test_events_are_opt_in(self) -> None:
+        self.assertNotIn("event", HARVESTABLE_KINDS)
+        self.assertIn("event", OPTIONAL_HARVEST_KINDS)
 
     def test_event_page_is_parsed_before_chunking(self) -> None:
         summary = self._run({"index.md": FIXTURE.read_text(encoding="utf-8")})
diff --git a/application/tests/harvester_test/selection_test.py b/application/tests/harvester_test/selection_test.py
index c4a38e042..1764349fa 100644
--- a/application/tests/harvester_test/selection_test.py
+++ b/application/tests/harvester_test/selection_test.py
@@ -127,7 +127,7 @@ def test_default_kinds_are_the_curated_standards_only(self) -> None:
         sel = select_repositories(cfg.repositories, self.store, now=NOW)
         self.assertEqual([r.repo for r in sel.selected], ["std"])
 
-    def test_harvestable_kinds_exclude_only_chapters(self) -> None:
+    def test_harvestable_kinds_exclude_chapter_and_event(self) -> None:
         cfg = _repos(
             _entry("std", kind="standard"),
             _entry("proj", kind="project"),
@@ -138,9 +138,7 @@ def test_harvestable_kinds_exclude_only_chapters(self) -> None:
         sel = select_repositories(
             cfg.repositories, self.store, kinds=HARVESTABLE_KINDS, now=NOW
         )
-        self.assertEqual(
-            [r.repo for r in sel.selected], ["std", "proj", "other", "evt"]
-        )
+        self.assertEqual([r.repo for r in sel.selected], ["std", "proj", "other"])
 
     def test_explicit_kinds_filter(self) -> None:
         cfg = _repos(_entry("std", kind="standard"), _entry("proj", kind="project"))
diff --git a/application/tests/oie_scheduler/runner_test.py b/application/tests/oie_scheduler/runner_test.py
index 7a77d932f..92a3234f4 100644
--- a/application/tests/oie_scheduler/runner_test.py
+++ b/application/tests/oie_scheduler/runner_test.py
@@ -432,6 +432,43 @@ def fake(*, repo_cfg, **_):
         self.assertEqual(harvest.summary["deferred"], 1)
         self.assertEqual(harvest.summary["repository_ids"], ["owasp-p1", "owasp-p2"])
 
+    def _event_repos(self):
+        self.repos.write_text(
+            self.repos.read_text().replace(
+                "repositories:",
+                "  event:\n"
+                "    type: github\n"
+                "    owner: OWASP\n"
+                "    branch: main\n"
+                "    paths: {include: [index.md]}\n"
+                "    chunking: {strategy: markdown_heading, max_tokens: 1000}\n"
+                "    polling: {mode: full, interval_minutes: 1440}\n"
+                "repositories:\n  - {id: owasp-e1, kind: event, repo: www-event-1}",
+            )
+        )
+
+    def _harvested(self, **cfg):
+        visited = []
+
+        def fake(*, repo_cfg, **_):
+            visited.append(repo_cfg.id)
+            return 1
+
+        with patch.object(harvest_pipeline, "_harvest_repository", fake):
+            self.run_job(
+                overrides={"agent_sync": _ok("agent_sync")},
+                config=SchedulerConfig(repos_yaml=str(self.repos), **cfg),
+            )
+        return visited
+
+    def test_event_repos_are_not_harvested_by_default(self) -> None:
+        self._event_repos()
+        self.assertNotIn("owasp-e1", self._harvested())
+
+    def test_event_repos_are_harvested_when_opted_in(self) -> None:
+        self._event_repos()
+        self.assertIn("owasp-e1", self._harvested(harvest_events=True))
+
     def _agent_sync(self, agent_db):
         from application.utils.owasp_agent import index_store
         from application.utils.owasp_agent import sync as agent_sync
@@ -484,6 +521,14 @@ def fake(*, repo_cfg, **_):
 
 
 class SchedulerConfigEnvTest(unittest.TestCase):
+    def test_event_harvest_defaults_off_and_is_opt_in(self) -> None:
+        with patch.dict("os.environ", {}, clear=False) as env:
+            env.pop("OIE_HARVEST_EVENTS", None)
+            self.assertFalse(SchedulerConfig.from_env().harvest_events)
+        for on in ("1", "true", "ON", "yes"):
+            with patch.dict("os.environ", {"OIE_HARVEST_EVENTS": on}):
+                self.assertTrue(SchedulerConfig.from_env().harvest_events)
+
     def test_embedding_filed_nodes_defaults_on_and_has_a_kill_switch(self) -> None:
         with patch.dict("os.environ", {}, clear=False) as env:
             env.pop("OIE_EMBED_FILED_NODES", None)
diff --git a/application/utils/harvester/schemas.py b/application/utils/harvester/schemas.py
index da7bd3c6e..6c9b90cf5 100644
--- a/application/utils/harvester/schemas.py
+++ b/application/utils/harvester/schemas.py
@@ -24,12 +24,14 @@ def validate_cron_line(value: Optional[str]) -> Optional[str]:
     return text
 
 
-# What an OWASP org repo is for. ``standard``/``project``/``other``/``event``
-# feed the knowledge-graph expansion path (harvest -> filter -> Librarian; event
-# pages are cleaned by ``event_page`` first); ``chapter`` repos only feed the
-# OWASP agent's metadata index.
+# What an OWASP org repo is for. ``standard``/``project``/``other`` feed the
+# knowledge-graph expansion path (harvest -> filter -> Librarian). ``chapter``
+# repos only feed the OWASP agent's metadata index. ``event`` pages are metadata
+# too, so harvesting them into the expansion path is opt-in
+# (``OPTIONAL_HARVEST_KINDS``; cleaned by ``event_page`` first).
 RepoKind = Literal["standard", "project", "chapter", "event", "other"]
-HARVESTABLE_KINDS: tuple[str, ...] = ("standard", "project", "other", "event")
+HARVESTABLE_KINDS: tuple[str, ...] = ("standard", "project", "other")
+OPTIONAL_HARVEST_KINDS: tuple[str, ...] = ("event",)
 # What a harvest run visits when the caller names no kinds: the curated standards
 # only, as before repos.yaml covered the whole org. Scheduled runs ask for
 # HARVESTABLE_KINDS explicitly.
diff --git a/application/utils/oie_scheduler/jobs.py b/application/utils/oie_scheduler/jobs.py
index 55b6bf183..b16347783 100644
--- a/application/utils/oie_scheduler/jobs.py
+++ b/application/utils/oie_scheduler/jobs.py
@@ -47,6 +47,7 @@ class SchedulerConfig:
     file_floor: float = 0.9
     # Default off: Module C decisions stay for review until an operator enables filing.
     filing_enabled: bool = False
+    harvest_events: bool = False
     embed_filed_nodes: bool = True
     repos_yaml: Optional[str] = None
     agent_db: Optional[str] = None
@@ -64,6 +65,8 @@ def from_env(cls) -> "SchedulerConfig":
             metadata_max_repos=int(os.getenv("OIE_METADATA_MAX_REPOS", "50")),
             file_floor=floor,
             filing_enabled=raw_filing in ("1", "true", "yes", "on"),
+            harvest_events=os.getenv("OIE_HARVEST_EVENTS", "0").strip().lower()
+            in ("1", "true", "yes", "on"),
             embed_filed_nodes=os.getenv("OIE_EMBED_FILED_NODES", "1").strip().lower()
             not in ("0", "false", "no", "off"),
             repos_yaml=os.getenv("OIE_REPOS_YAML") or None,
@@ -196,14 +199,20 @@ def stage_agent_sync(ctx: JobContext) -> StageResult:
 
 def stage_harvest(ctx: JobContext) -> StageResult:
     from application.utils.harvester.pipeline import run_harvester
-    from application.utils.harvester.schemas import HARVESTABLE_KINDS
+    from application.utils.harvester.schemas import (
+        HARVESTABLE_KINDS,
+        OPTIONAL_HARVEST_KINDS,
+    )
 
+    kinds = HARVESTABLE_KINDS
+    if ctx.config.harvest_events:
+        kinds = HARVESTABLE_KINDS + OPTIONAL_HARVEST_KINDS
     summary = run_harvester(
         ctx.session,
         ctx.run_id,
         repos_yaml=ctx.config.repos_yaml,
         dry_run=ctx.dry_run,
-        kinds=HARVESTABLE_KINDS,
+        kinds=kinds,
         only_due=True,
         max_repos=ctx.config.harvest_max_repos,
         now=ctx.now,

From fcefa22a6fe702fa45b9aa42c2462d26e49ee29a Mon Sep 17 00:00:00 2001
From: Spyros <morfeas3000@gmail.com>
Date: Thu, 22 Oct 2026 14:14:01 +0100
Subject: [PATCH 5/6] style: format files flagged by CI (black, prettier)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
---
 .../frontend/src/pages/Admin/Admin.tsx        |  8 +-
 application/tests/librarian/pipeline_test.py  | 75 ++++++++-----------
 .../tests/owasp_agent/owasp_agent_test.py     |  3 -
 application/utils/librarian/factory.py        |  5 +-
 .../utils/librarian/llm_safety_guard.py       |  4 +-
 .../utils/oie_orchestrator/pipeline.py        |  1 -
 scripts/owasp_agent_adversarial_asker.py      | 22 +++---
 scripts/owasp_agent_adversarial_batch_run.py  |  4 +-
 scripts/owasp_agent_embed_meta.py             |  7 +-
 scripts/owasp_agent_synth_eval.py             |  7 +-
 10 files changed, 58 insertions(+), 78 deletions(-)

diff --git a/application/frontend/src/pages/Admin/Admin.tsx b/application/frontend/src/pages/Admin/Admin.tsx
index 8307a9d66..60e83a064 100644
--- a/application/frontend/src/pages/Admin/Admin.tsx
+++ b/application/frontend/src/pages/Admin/Admin.tsx
@@ -714,8 +714,8 @@ function PipelineTab({ origin }: { origin: string }) {
       <p className="admin-help">
         Look for <code>oie</code>/<code>started</code> (flags + skip_reason) and <code>oie_trace</code>{' '}
         (engine, graph_path, visited). The admin default runs Module B, C, and C.1 (graph filer). Skipped
-        stages mean the run passed the matching <code>skip_*</code> flag true. Module A with chunks=0
-        usually means harvester checkpoints were already at HEAD (no file diffs).
+        stages mean the run passed the matching <code>skip_*</code> flag true. Module A with chunks=0 usually
+        means harvester checkpoints were already at HEAD (no file diffs).
       </p>
       <table className="admin-table">
         <thead>
@@ -1058,8 +1058,8 @@ function ConfigTab({ origin }: { origin: string }) {
       <p className="admin-help">
         CRE explorer graph comes from upstream at <code>make install</code> / <code>make dev</code> (skip with{' '}
         <code>SKIP_UPSTREAM_SYNC=1</code>). Golden-set harvest runs A → B → C → C.1. C.1 only auto-files
-        high-confidence links when the toggle below is on (default off — review{' '}
-        <code>decision_queue</code> first). OWASP agent sync is separate — progress shows under Pipeline.
+        high-confidence links when the toggle below is on (default off — review <code>decision_queue</code>{' '}
+        first). OWASP agent sync is separate — progress shows under Pipeline.
       </p>
       <p>
         <Checkbox
diff --git a/application/tests/librarian/pipeline_test.py b/application/tests/librarian/pipeline_test.py
index cfb3808ce..a12674d89 100644
--- a/application/tests/librarian/pipeline_test.py
+++ b/application/tests/librarian/pipeline_test.py
@@ -126,9 +126,8 @@ def retrieve(self, text):
             _Scaler(0.95),
             threshold=0.8,
             pipeline_run_id=RUN,
-        
-        safety_guard=PassThroughSafetyGuard(),
-    )
+            safety_guard=PassThroughSafetyGuard(),
+        )
         env = pipeline.run(at=AT).envelopes[0]
         self.assertIsInstance(env, ReviewItem)
         # Empty cage → CRE_GAP (coverage proposal), not a silent no_candidates.
@@ -165,9 +164,8 @@ def retrieve(self, text):
             known_cre_ids=frozenset({"616-305"}),
             cre_id_map={"616-305": "uuid-616"},
             cre_membership=frozenset({"uuid-616", "616-305"}),
-        
-        safety_guard=PassThroughSafetyGuard(),
-    )
+            safety_guard=PassThroughSafetyGuard(),
+        )
         result = pipeline.run(at=AT)
         self.assertEqual(result.stats.linked, 1)
         env = result.envelopes[0]
@@ -195,9 +193,8 @@ def retrieve(self, text):
             pipeline_run_id=RUN,
             known_cre_ids=frozenset({"616-305", "177-260"}),
             cre_membership=frozenset({"616-305", "177-260"}),
-        
-        safety_guard=PassThroughSafetyGuard(),
-    )
+            safety_guard=PassThroughSafetyGuard(),
+        )
         result = pipeline.run(at=AT)
         self.assertEqual(result.stats.review, 1)
         env = result.envelopes[0]
@@ -225,9 +222,8 @@ def test_bogus_cre_id_cannot_produce_link_proposal(self):
             threshold=0.8,
             pipeline_run_id=RUN,
             cre_membership=frozenset({"616-305"}),
-        
-        safety_guard=PassThroughSafetyGuard(),
-    )
+            safety_guard=PassThroughSafetyGuard(),
+        )
         result = pipeline.run(at=AT)
         self.assertEqual(result.stats.linked, 0)
         self.assertEqual(result.stats.review, 1)
@@ -245,9 +241,8 @@ def test_valid_membership_still_auto_links(self):
             threshold=0.8,
             pipeline_run_id=RUN,
             cre_membership=frozenset({"616-305"}),
-        
-        safety_guard=PassThroughSafetyGuard(),
-    )
+            safety_guard=PassThroughSafetyGuard(),
+        )
         result = pipeline.run(at=AT)
         self.assertEqual(result.stats.linked, 1)
         env = result.envelopes[0]
@@ -267,9 +262,8 @@ def test_review_suggested_links_drop_invalid_ids(self):
             threshold=0.8,
             pipeline_run_id=RUN,
             cre_membership=frozenset({"616-305"}),
-        
-        safety_guard=PassThroughSafetyGuard(),
-    )
+            safety_guard=PassThroughSafetyGuard(),
+        )
         result = pipeline.run(at=AT)
         env = result.envelopes[0]
         self.assertIsInstance(env, ReviewItem)
@@ -300,9 +294,8 @@ def _run_with(self, failing_component, rows):
             parts["scaler"],
             threshold=0.8,
             pipeline_run_id=RUN,
-        
-        safety_guard=PassThroughSafetyGuard(),
-    ).run(at=AT)
+            safety_guard=PassThroughSafetyGuard(),
+        ).run(at=AT)
 
     def test_retriever_failure_is_contained(self):
         result = self._run_with("retriever", [_row()])
@@ -336,9 +329,8 @@ def rerank(self, text, audit):
             _Scaler(0.95),
             threshold=0.8,
             pipeline_run_id=RUN,
-        
-        safety_guard=PassThroughSafetyGuard(),
-    ).run(at=AT)
+            safety_guard=PassThroughSafetyGuard(),
+        ).run(at=AT)
 
         self.assertEqual(result.stats.total, 3)
         self.assertEqual(result.stats.errored, 1)
@@ -368,9 +360,8 @@ def _run(self, rows, scaler_confidence=0.95):
             _Scaler(scaler_confidence),
             threshold=0.8,
             pipeline_run_id=RUN,
-        
-        safety_guard=PassThroughSafetyGuard(),
-    ).run(at=AT)
+            safety_guard=PassThroughSafetyGuard(),
+        ).run(at=AT)
 
     def test_outcome_per_row_carries_the_queue_id(self):
         result = self._run([_row(row_id="a"), _row(row_id="b")])
@@ -392,9 +383,8 @@ def test_errored_row_is_not_finished(self):
             _Scaler(0.95),
             threshold=0.8,
             pipeline_run_id=RUN,
-        
-        safety_guard=PassThroughSafetyGuard(),
-    ).run(at=AT)
+            safety_guard=PassThroughSafetyGuard(),
+        ).run(at=AT)
         self.assertEqual([o.status.value for o in result.outcomes], ["errored"])
         self.assertEqual(result.finished_row_ids(), [])
 
@@ -455,9 +445,8 @@ def llm(system: str, user: str) -> str:
             threshold=0.8,
             pipeline_run_id=RUN,
             shortlist_llm_fn=llm,
-        
-        safety_guard=PassThroughSafetyGuard(),
-    )
+            safety_guard=PassThroughSafetyGuard(),
+        )
         result = pipeline.run(at=AT)
         env = result.envelopes[0]
         self.assertIsInstance(env, ReviewItem)
@@ -479,9 +468,8 @@ def llm(system: str, user: str) -> str:
             threshold=0.8,
             pipeline_run_id=RUN,
             shortlist_llm_fn=llm,
-        
-        safety_guard=PassThroughSafetyGuard(),
-    ).run(at=AT)
+            safety_guard=PassThroughSafetyGuard(),
+        ).run(at=AT)
         self.assertEqual(result.stats.linked, 1)
         self.assertNotIn("shortlist-judge", result.envelopes[0].retrieval.retriever)
 
@@ -538,9 +526,8 @@ def _score_fn(pairs):
             leaf_drilldown_resources=("asvs", "aisvs"),
             leaf_drilldown_min_children=3,
             leaf_drilldown_min_sections=0,
-        
-        safety_guard=PassThroughSafetyGuard(),
-    ).run(at=AT)
+            safety_guard=PassThroughSafetyGuard(),
+        ).run(at=AT)
         env = result.envelopes[0]
         ids = [c.cre_id for c in env.retrieval.reranked]
         self.assertEqual(ids[:2], [hub, "noise"])
@@ -596,9 +583,8 @@ def _score_fn(pairs):
             leaf_drilldown_force_resources=("api",),
             leaf_drilldown_min_children=3,
             leaf_drilldown_min_sections=20,
-        
-        safety_guard=PassThroughSafetyGuard(),
-    ).run(at=AT)
+            safety_guard=PassThroughSafetyGuard(),
+        ).run(at=AT)
         env = result.envelopes[0]
         ids = [c.cre_id for c in env.retrieval.reranked]
         self.assertEqual(ids[0], leaf)
@@ -658,9 +644,8 @@ def _score_fn(pairs):
             leaf_drilldown_resources=(),
             leaf_drilldown_min_children=3,
             leaf_drilldown_min_sections=20,
-        
-        safety_guard=PassThroughSafetyGuard(),
-    ).run(at=AT)
+            safety_guard=PassThroughSafetyGuard(),
+        ).run(at=AT)
         env = result.envelopes[0]
         ids = [c.cre_id for c in env.retrieval.reranked]
         self.assertEqual(ids[0], leaf)
diff --git a/application/tests/owasp_agent/owasp_agent_test.py b/application/tests/owasp_agent/owasp_agent_test.py
index 2a89fcdc8..33e302117 100644
--- a/application/tests/owasp_agent/owasp_agent_test.py
+++ b/application/tests/owasp_agent/owasp_agent_test.py
@@ -969,9 +969,6 @@ def test_clarify_followup_uses_history(self) -> None:
             self.assertIn("morocco", resp["response"].lower())
 
 
-
-
-
 class TestIndexStoreDefaults(unittest.TestCase):
     def test_override_env_wins_over_app_db(self) -> None:
         with patch.dict(
diff --git a/application/utils/librarian/factory.py b/application/utils/librarian/factory.py
index d9c0b5c41..7f3bab231 100644
--- a/application/utils/librarian/factory.py
+++ b/application/utils/librarian/factory.py
@@ -54,7 +54,7 @@ class LibrarianComponents:
     shortlist_llm_fn: Optional[Callable[[str, str], str]] = None
     #: C.4 safety guard (None → pipeline keeps NullSafetyGuard).
     safety_guard: Optional[Any] = None
-#: Contains parent→children index for leaf drill-down (None = skip).
+    #: Contains parent→children index for leaf drill-down (None = skip).
     parent_index: Optional[Any] = None
     #: When True, promote better Contains children over hub umbrellas after C.2.
     leaf_drilldown: bool = True
@@ -72,6 +72,7 @@ class LibrarianComponents:
     shortlist_judge_max_picks: int = 3
     margin_gamma: Optional[float] = None
 
+
 def build_scaler(config: Optional[LibrarianConfig] = None) -> Scaler:
     """The C.3 calibrator at the configured temperature.
 
@@ -343,7 +344,7 @@ def build_components(
         cre_prior=cre_prior,
         shortlist_llm_fn=shortlist_llm,
         safety_guard=safety_guard,
-parent_index=parent_index,
+        parent_index=parent_index,
         leaf_drilldown=config.leaf_drilldown,
         leaf_drilldown_resources=config.leaf_drilldown_resources,
         leaf_drilldown_force_resources=config.leaf_drilldown_force_resources,
diff --git a/application/utils/librarian/llm_safety_guard.py b/application/utils/librarian/llm_safety_guard.py
index dd24456f8..3aace1fd5 100644
--- a/application/utils/librarian/llm_safety_guard.py
+++ b/application/utils/librarian/llm_safety_guard.py
@@ -71,7 +71,9 @@ def default_safety_litellm_fn(model: Optional[str] = None) -> LlmFn:
         "CRE_LIBRARIAN_SAFETY_GUARD_MODEL",
         os.environ.get(
             "CRE_LIBRARIAN_SHORTLIST_JUDGE_MODEL",
-            os.environ.get("CRE_NOISE_FILTER_LLM_MODEL", "gemini/gemini-2.5-flash-lite"),
+            os.environ.get(
+                "CRE_NOISE_FILTER_LLM_MODEL", "gemini/gemini-2.5-flash-lite"
+            ),
         ),
     )
     return system_user_fn(model_name, temperature=0.0)
diff --git a/application/utils/oie_orchestrator/pipeline.py b/application/utils/oie_orchestrator/pipeline.py
index 1bef6243a..d0cd94975 100644
--- a/application/utils/oie_orchestrator/pipeline.py
+++ b/application/utils/oie_orchestrator/pipeline.py
@@ -158,7 +158,6 @@ def _refuse_c1_for_safety_gap(c_stage: "StageResult") -> Optional[str]:
     return None
 
 
-
 def _connect(cache_file: str) -> Any:
     from application import sqla
     from application.cmd.cre_main import db_connect
diff --git a/scripts/owasp_agent_adversarial_asker.py b/scripts/owasp_agent_adversarial_asker.py
index 340ef8f02..067b46c78 100644
--- a/scripts/owasp_agent_adversarial_asker.py
+++ b/scripts/owasp_agent_adversarial_asker.py
@@ -614,10 +614,9 @@ def ask_inprocess(q: str) -> str:
     os.environ.setdefault("OWASP_AGENT_ENABLED", "1")
     sys.path.insert(0, str(_REPO))
     if _INPROCESS_APP is None:
-        db_url = (
-            (os.environ.get("DATABASE_URL") or "").strip()
-            or (os.environ.get("DEV_DATABASE_URL") or "").strip()
-        )
+        db_url = (os.environ.get("DATABASE_URL") or "").strip() or (
+            os.environ.get("DEV_DATABASE_URL") or ""
+        ).strip()
         if db_url and not (os.environ.get("SQLALCHEMY_DATABASE_URI") or "").strip():
             try:
                 from application.utils.postgres_url import (
@@ -773,9 +772,7 @@ async def _start_async(self) -> None:
         from mcp.client.stdio import stdio_client
 
         transport = (os.environ.get("OWASP_ADV_MCP_TRANSPORT") or "staging_ssh").strip()
-        params = (
-            _local_mcp_params() if transport == "local" else _staging_ssh_params()
-        )
+        params = _local_mcp_params() if transport == "local" else _staging_ssh_params()
         self._cm_stdio = stdio_client(params)
         read, write = await self._cm_stdio.__aenter__()
         self._cm_session = ClientSession(read, write)
@@ -796,9 +793,7 @@ def ask(self, q: str) -> str:
 
     async def _ask_async(self, q: str) -> str:
         corpus = (os.environ.get("OWASP_ADV_CORPUS") or "auto").strip()
-        result = await self._session.call_tool(
-            "ask", {"prompt": q, "corpus": corpus}
-        )
+        result = await self._session.call_tool("ask", {"prompt": q, "corpus": corpus})
         if getattr(result, "isError", False):
             texts = []
             for c in result.content or []:
@@ -955,7 +950,12 @@ def main() -> int:
         bank = [c for c in bank if c.cat in allow]
         if not bank:
             print(
-                json.dumps({"error": "empty bank after OWASP_ADV_CATS filter", "cats": sorted(allow)}),
+                json.dumps(
+                    {
+                        "error": "empty bank after OWASP_ADV_CATS filter",
+                        "cats": sorted(allow),
+                    }
+                ),
                 flush=True,
             )
             return 2
diff --git a/scripts/owasp_agent_adversarial_batch_run.py b/scripts/owasp_agent_adversarial_batch_run.py
index e0eb285ed..44a3002e3 100644
--- a/scripts/owasp_agent_adversarial_batch_run.py
+++ b/scripts/owasp_agent_adversarial_batch_run.py
@@ -82,9 +82,7 @@ def main() -> int:
             log.flush()
             print(done, end="", flush=True)
             t = batch_end + 1
-        log.write(
-            f"ALL_BATCHES_DONE {time.strftime('%H:%M:%S', time.gmtime())}\n"
-        )
+        log.write(f"ALL_BATCHES_DONE {time.strftime('%H:%M:%S', time.gmtime())}\n")
         log.flush()
     print("ALL_BATCHES_DONE", flush=True)
     return 0
diff --git a/scripts/owasp_agent_embed_meta.py b/scripts/owasp_agent_embed_meta.py
index 5518666a6..43cdb1c25 100644
--- a/scripts/owasp_agent_embed_meta.py
+++ b/scripts/owasp_agent_embed_meta.py
@@ -23,10 +23,9 @@
 
 
 def _db_url() -> str:
-    raw = (
-        (os.environ.get("DATABASE_URL") or "").strip()
-        or (os.environ.get("DEV_DATABASE_URL") or "").strip()
-    )
+    raw = (os.environ.get("DATABASE_URL") or "").strip() or (
+        os.environ.get("DEV_DATABASE_URL") or ""
+    ).strip()
     if not raw:
         raise SystemExit("DATABASE_URL / DEV_DATABASE_URL required")
     try:
diff --git a/scripts/owasp_agent_synth_eval.py b/scripts/owasp_agent_synth_eval.py
index bebdafe64..9e7f06525 100644
--- a/scripts/owasp_agent_synth_eval.py
+++ b/scripts/owasp_agent_synth_eval.py
@@ -175,10 +175,9 @@ def ask_completion_inprocess(
     os.environ["NO_LOGIN"] = "1"
     os.environ.setdefault("OWASP_AGENT_ENABLED", "1")
     # Prefer Postgres URL from the container env for SQLAlchemy (psycopg2 pin).
-    db_url = (
-        (os.environ.get("DATABASE_URL") or "").strip()
-        or (os.environ.get("DEV_DATABASE_URL") or "").strip()
-    )
+    db_url = (os.environ.get("DATABASE_URL") or "").strip() or (
+        os.environ.get("DEV_DATABASE_URL") or ""
+    ).strip()
     if db_url and not (os.environ.get("SQLALCHEMY_DATABASE_URI") or "").strip():
         try:
             from application.utils.postgres_url import (

From 3351aebe49f2353e233c8e4adc3a68f4fbbd9f50 Mon Sep 17 00:00:00 2001
From: Spyros <morfeas3000@gmail.com>
Date: Thu, 22 Oct 2026 17:18:32 +0100
Subject: [PATCH 6/6] test: embedding-failure scheduler test needs graph filing
 enabled now that it defaults off

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
---
 application/tests/oie_scheduler/runner_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/application/tests/oie_scheduler/runner_test.py b/application/tests/oie_scheduler/runner_test.py
index 92a3234f4..d1c8fc498 100644
--- a/application/tests/oie_scheduler/runner_test.py
+++ b/application/tests/oie_scheduler/runner_test.py
@@ -309,7 +309,7 @@ def boom(collection, node_ids):
             "cre_expansion",
             cache_file="db-url",
             now=NOW,
-            config=SchedulerConfig(),
+            config=SchedulerConfig(filing_enabled=True),
             overrides={
                 "noise_filter": _ok("noise_filter"),
                 "librarian": _ok("librarian"),