OIE: parse event pages and embed auto-filed nodes

application/prompt_client/prompt_client.py

@@ -10,6 +10,7 @@
 from urllib.parse import urljoin, urlparse
 
 from application.prompt_client import embed_alignment, litellm_router
+from application.utils.harvester.event_page import clean_embedding_text
 
 from scipy import sparse
 from sklearn.metrics.pairwise import cosine_similarity
@@ -315,7 +316,7 @@ def get_content(self, url) -> Optional[str]:
         if is_plain_text_embed_url(url):
             text = _fetch_plain_http_text(url)
             if text:
-                return text
+                return clean_embedding_text(url, text)
             logger.warning(
                 "Plain-text URL %s: empty HTTP body, falling through to Playwright",
                 url,
@@ -344,7 +345,7 @@ def get_content(self, url) -> Optional[str]:
                 logger.info(f"loading page {url}")
                 page.goto(url)
                 text = page.locator("body").inner_text()
-                return text
+                return clean_embedding_text(url, text)
             except requests.exceptions.RequestException as e:
                 logger.error(
                     f"Error fetching content for URL: {url} - {str(e)} (attempt {attempts}/9)"

application/tests/harvester_test/event_page_test.py

@@ -0,0 +1,177 @@
+"""Jekyll event pages: front matter + JSON-LD become clean prose; templating is dropped."""
+
+import unittest
+from pathlib import Path
+
+from unittest.mock import patch
+
+from application.prompt_client import prompt_client
+from application.utils.harvester.event_page import (
+    clean_embedding_text,
+    prepare_event_page,
+)
+
+FIXTURE = Path(__file__).parent / "fixtures" / "event_index.md.txt"
+
+
+class PrepareEventPageTests(unittest.TestCase):
+    def setUp(self) -> None:
+        self.text = FIXTURE.read_text(encoding="utf-8")
+        self.out = prepare_event_page(self.text)
+
+    def test_title_comes_from_the_event_name(self) -> None:
+        self.assertTrue(
+            self.out.startswith("# AppSec Days - Summer of Security 2020\n"), self.out
+        )
+
+    def test_event_facts_are_stated_as_prose(self) -> None:
+        self.assertIn("Dates: 2020-06-23 to 2020-08-26", self.out)
+        self.assertIn("Location: https://appsecdays.org/", self.out)
+        self.assertIn("virtual AppSec Days", self.out)
+
+    def test_markup_noise_is_removed(self) -> None:
+        for noise in (
+            "schema.org",
+            "@context",
+            "application/ld+json",
+            "layout: event",
+            "{%",
+            "{{",
+            "<!--",
+            "<ul>",
+            "OnlineEventAttendanceMode",
+        ):
+            self.assertNotIn(noise, self.out)
+
+    def test_markdown_body_is_kept(self) -> None:
+        self.assertIn("### Training Sessions Include", self.out)
+        self.assertIn("threat modeling with the OWASP Top 10", self.out)
+        self.assertIn("https://appsecdays.org/sponsors/swag/", self.out)
+
+    def test_falls_back_to_front_matter_title_without_json_ld(self) -> None:
+        out = prepare_event_page(
+            "---\ntitle: Local Meetup\nlayout: event\n---\n\nCome and learn about XSS.\n"
+        )
+        self.assertTrue(out.startswith("# Local Meetup\n"), out)
+        self.assertIn("Come and learn about XSS.", out)
+        self.assertNotIn("layout", out)
+
+    def test_malformed_json_ld_is_dropped_not_fatal(self) -> None:
+        out = prepare_event_page(
+            '---\ntitle: T\n---\n<script type="application/ld+json">{not json</script>\nBody text here.\n'
+        )
+        self.assertIn("Body text here.", out)
+        self.assertNotIn("not json", out)
+
+    def test_raw_newlines_inside_json_strings_are_tolerated(self) -> None:
+        out = prepare_event_page(
+            '<script type="application/ld+json">{"@type": "Event", "name": "N",\n'
+            ' "description": "line one\n        line two", "startDate": "2020-01-01"}'
+            "</script>\nBody.\n"
+        )
+        self.assertIn("# N", out)
+        self.assertIn("line one line two", out)
+
+    def test_json_ld_graph_and_list_forms(self) -> None:
+        out = prepare_event_page(
+            '<script type="application/ld+json">'
+            '{"@graph": [{"@type": "Organization", "name": "Org"},'
+            '{"@type": "Event", "name": "Graph Event", "startDate": "2025-01-02",'
+            ' "location": {"@type": "Place", "name": "Lisbon", "address": "Rua 1"}}]}'
+            "</script>\nBody.\n"
+        )
+        self.assertIn("# Graph Event", out)
+        self.assertIn("Dates: 2025-01-02", out)
+        self.assertIn("Location: Lisbon", out)
+
+    def test_page_with_no_content_is_blank(self) -> None:
+        self.assertEqual(
+            prepare_event_page("---\nlayout: event\n---\n{% include x.md %}\n"), ""
+        )
+
+    def test_plain_markdown_passes_through(self) -> None:
+        self.assertEqual(
+            prepare_event_page("# Hello\n\nWorld.\n").strip(), "# Hello\n\nWorld."
+        )
+
+
+class AdversarialInputTests(unittest.TestCase):
+    """Page content is untrusted repo text; unclosed constructs must not blow up."""
+
+    def test_unterminated_markup_is_linear_enough(self) -> None:
+        import time
+
+        for text in (
+            "{% if x %} text\n" * 20000,
+            '<script type="application/ld+json">' * 5000,
+            "{% if a %}" * 5000 + "{% endif %}" * 5000,
+            "{{ " * 50000,
+            "<!-- " * 50000,
+            "{% " * 50000,
+            "<script " * 50000,
+        ):
+            started = time.monotonic()
+            prepare_event_page(text)
+            self.assertLess(time.monotonic() - started, 5.0, text[:20])
+
+    def test_unterminated_block_keeps_the_prose_around_it(self) -> None:
+        out = prepare_event_page("Before.\n{% if x %}\nAfter.\n")
+        self.assertIn("Before.", out)
+        self.assertIn("After.", out)
+
+    def test_nested_blocks_are_removed_whole(self) -> None:
+        out = prepare_event_page(
+            "Keep.\n{% if a %}\n{% for b in c %}x{% endfor %}\ngone\n{% endif %}\nKeep too.\n"
+        )
+        self.assertNotIn("gone", out)
+        self.assertIn("Keep too.", out)
+
+
+RAW = (
+    "https://raw.githubusercontent.com/OWASP/www-event-2020-07-virtual/master/index.md"
+)
+
+
+class CleanEmbeddingTextTests(unittest.TestCase):
+    def setUp(self) -> None:
+        self.text = FIXTURE.read_text(encoding="utf-8")
+
+    def test_event_repo_pages_are_cleaned(self) -> None:
+        out = clean_embedding_text(RAW, self.text)
+        self.assertIn("Dates: 2020-06-23 to 2020-08-26", out)
+        self.assertNotIn("ld+json", out)
+
+    def test_github_blob_and_raw_forms_of_event_repos_match(self) -> None:
+        for url in (
+            "https://github.com/OWASP/www-event-2024-global-appsec-lisbon/raw/main/index.md",
+            "https://raw.githubusercontent.com/OWASP/www-event-2024-global-appsec-lisbon/main/index.md",
+        ):
+            self.assertNotIn("ld+json", clean_embedding_text(url, self.text))
+
+    def test_other_repos_and_non_markdown_are_untouched(self) -> None:
+        for url in (
+            "https://raw.githubusercontent.com/OWASP/ASVS/master/README.md",
+            "https://raw.githubusercontent.com/someone/www-event-x/main/index.md",
+            "https://raw.githubusercontent.com/OWASP/www-event-x/main/data.json",
+            "https://example.com/OWASP/www-event-x/main/index.md",
+        ):
+            self.assertEqual(clean_embedding_text(url, self.text), self.text, url)
+
+    def test_empty_cleaned_text_falls_back_to_the_original(self) -> None:
+        text = "---\nlayout: event\n---\n{% include x.md %}\n"
+        self.assertEqual(clean_embedding_text(RAW, text), text)
+
+    def test_get_content_cleans_event_pages(self) -> None:
+        emb = prompt_client.in_memory_embeddings.__new__(
+            prompt_client.in_memory_embeddings
+        )
+        with patch.object(
+            prompt_client, "_fetch_plain_http_text", return_value=self.text
+        ):
+            out = emb.get_content(RAW)
+        self.assertIn("Dates: 2020-06-23", out)
+        self.assertNotIn("schema.org", out)
+
+
+if __name__ == "__main__":
+    unittest.main()

application/tests/harvester_test/fixtures/event_index.md.txt

@@ -0,0 +1,45 @@
+---
+
+title: AppSec Days - July 2020
+layout: event
+
+---
+<script type="application/ld+json">
+{
+  "@context": "https://schema.org",
+  "@type": "Event",
+  "name": " AppSec Days - Summer of Security 2020",
+  "description": "The OWASP Foundation is hosting virtual AppSec Days on the last Tuesday and Wednesday in June, July, and August.",
+  "startDate": "2020-06-23",
+  "endDate": "2020-08-26",
+  "eventAttendanceMode": "https://schema.org/OnlineEventAttendanceMode",
+  "location": {
+    "@type": "VirtualLocation",
+    "url": "https://appsecdays.org/"
+  },
+  "offers": [{
+    "@type": "Offer",
+    "name": "Training Courses",
+    "price": "495",
+    "priceCurrency": "USD",
+    "url": "https://appsecdays.org/register/"
+  }]
+}
+</script>
+
+<!-- rebuild 13 -->
+
+{{ site.data.event-details.pitch | markdownify }}
+
+ **[Check out our sponsors](https://appsecdays.org/sponsors/swag/)**
+
+### Training Sessions Include
+
+* Learn threat modeling with the OWASP Top 10.
+{% if site.data.trainings.count > 0 %}
+<ul>
+  {% for trainer in trainings %}
+    <li><a href="{{trainer.URL}}">{{ trainer.Title }}</a></li>
+  {% endfor %}
+</ul>
+{% endif %}

application/tests/harvester_test/harvest_event_test.py

@@ -0,0 +1,138 @@
+"""Event repos are harvested through the event-page parser."""
+
+from cre_logging import get_logger
+
+logger = get_logger(__name__)
+
+import tempfile
+import unittest
+from datetime import datetime, timezone
+from pathlib import Path
+from unittest.mock import patch
+
+from application import create_app, sqla
+from application.database import db
+from application.utils.harvester import pipeline as harvest_pipeline
+from application.utils.harvester.schemas import (
+    HARVESTABLE_KINDS,
+    OPTIONAL_HARVEST_KINDS,
+)
+
+HEAD = "abc1234def5678"
+FIXTURE = Path(__file__).parent / "fixtures" / "event_index.md.txt"
+
+REPOS_YAML = """
+repositories:
+  - id: owasp-www-event-demo
+    kind: event
+    enabled: true
+    type: github
+    owner: OWASP
+    repo: www-event-demo
+    branch: main
+    paths: {include: ["index.md"]}
+    chunking: {strategy: markdown_heading, max_tokens: 1000}
+    polling: {mode: full, interval_minutes: 1440}
+"""
+
+
+def _client(files):
+    class FakeClient:
+        def __init__(self, *args, **kwargs) -> None:
+            pass
+
+        def sync(self) -> None:
+            pass
+
+        def get_current_commit_sha(self) -> str:
+            return HEAD
+
+        def get_file_at_commit(self, sha: str, path: str) -> str:
+            return files[path]
+
+        def get_local_path(self) -> Path:
+            return Path("/nonexistent")
+
+    return FakeClient
+
+
+class HarvestEventTests(unittest.TestCase):
+    def setUp(self) -> None:
+        self.app = create_app(mode="test")
+        self.ctx = self.app.app_context()
+        self.ctx.push()
+        sqla.create_all()
+        self.tmp = tempfile.TemporaryDirectory()
+        self.yaml = Path(self.tmp.name) / "repos.yaml"
+        self.yaml.write_text(REPOS_YAML)
+
+    def tearDown(self) -> None:
+        sqla.session.remove()
+        sqla.drop_all()
+        self.ctx.pop()
+        self.tmp.cleanup()
+
+    def _run(self, files):
+        patches = [
+            patch.object(harvest_pipeline, "GitRepositoryClient", _client(files)),
+            patch.object(
+                harvest_pipeline.ChangeDetector,
+                "get_files_at_commit",
+                lambda self, sha: list(files),
+            ),
+            patch.object(
+                harvest_pipeline,
+                "_commit_timestamp",
+                lambda client, sha: datetime(2026, 1, 1, tzinfo=timezone.utc),
+            ),
+        ]
+        for p in patches:
+            p.start()
+            self.addCleanup(p.stop)
+        return harvest_pipeline.run_harvester(
+            sqla.session,
+            "run-1",
+            repos_yaml=self.yaml,
+            sync_repos=False,
+            kinds=HARVESTABLE_KINDS + OPTIONAL_HARVEST_KINDS,
+        )
+
+    def _texts(self):
+        return [row.payload["text"] for row in sqla.session.query(db.HarvestInput)]
+
+    def test_events_are_opt_in(self) -> None:
+        self.assertNotIn("event", HARVESTABLE_KINDS)
+        self.assertIn("event", OPTIONAL_HARVEST_KINDS)
+
+    def test_event_page_is_parsed_before_chunking(self) -> None:
+        summary = self._run({"index.md": FIXTURE.read_text(encoding="utf-8")})
+        self.assertEqual(summary.errors, 0)
+        self.assertEqual(summary.repositories, 1)
+        self.assertGreater(summary.chunks_written, 0)
+        joined = "\n".join(self._texts())
+        self.assertIn("Summer of Security 2020", joined)
+        self.assertIn("2020-06-23", joined)
+        self.assertIn("threat modeling", joined)
+        for noise in ("schema.org", "ld+json", "{%", "{{", "layout: event"):
+            self.assertNotIn(noise, joined)
+
+    def test_page_with_no_content_is_skipped_and_checkpoint_still_advances(
+        self,
+    ) -> None:
+        from application.utils.harvester.checkpoint_store import CheckpointStore
+
+        summary = self._run(
+            {"index.md": "---\nlayout: event\n---\n{% include x.md %}\n"}
+        )
+        self.assertEqual(summary.errors, 0)
+        self.assertEqual(summary.chunks_written, 0)
+        self.assertEqual(
+            CheckpointStore(session=sqla.session)
+            .load("owasp-www-event-demo")
+            .last_processed_commit,
+            HEAD,
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()