diff --git a/application/prompt_client/prompt_client.py b/application/prompt_client/prompt_client.py
index 8aee6d607..4e0494527 100644
--- a/application/prompt_client/prompt_client.py
+++ b/application/prompt_client/prompt_client.py
@@ -10,6 +10,7 @@
from urllib.parse import urljoin, urlparse
from application.prompt_client import embed_alignment, litellm_router
+from application.utils.harvester.event_page import clean_embedding_text
from scipy import sparse
from sklearn.metrics.pairwise import cosine_similarity
@@ -315,7 +316,7 @@ def get_content(self, url) -> Optional[str]:
if is_plain_text_embed_url(url):
text = _fetch_plain_http_text(url)
if text:
- return text
+ return clean_embedding_text(url, text)
logger.warning(
"Plain-text URL %s: empty HTTP body, falling through to Playwright",
url,
@@ -344,7 +345,7 @@ def get_content(self, url) -> Optional[str]:
logger.info(f"loading page {url}")
page.goto(url)
text = page.locator("body").inner_text()
- return text
+ return clean_embedding_text(url, text)
except requests.exceptions.RequestException as e:
logger.error(
f"Error fetching content for URL: {url} - {str(e)} (attempt {attempts}/9)"
diff --git a/application/tests/harvester_test/event_page_test.py b/application/tests/harvester_test/event_page_test.py
new file mode 100644
index 000000000..a184a7c30
--- /dev/null
+++ b/application/tests/harvester_test/event_page_test.py
@@ -0,0 +1,177 @@
+"""Jekyll event pages: front matter + JSON-LD become clean prose; templating is dropped."""
+
+import unittest
+from pathlib import Path
+
+from unittest.mock import patch
+
+from application.prompt_client import prompt_client
+from application.utils.harvester.event_page import (
+ clean_embedding_text,
+ prepare_event_page,
+)
+
+FIXTURE = Path(__file__).parent / "fixtures" / "event_index.md.txt"
+
+
+class PrepareEventPageTests(unittest.TestCase):
+ def setUp(self) -> None:
+ self.text = FIXTURE.read_text(encoding="utf-8")
+ self.out = prepare_event_page(self.text)
+
+ def test_title_comes_from_the_event_name(self) -> None:
+ self.assertTrue(
+ self.out.startswith("# AppSec Days - Summer of Security 2020\n"), self.out
+ )
+
+ def test_event_facts_are_stated_as_prose(self) -> None:
+ self.assertIn("Dates: 2020-06-23 to 2020-08-26", self.out)
+ self.assertIn("Location: https://appsecdays.org/", self.out)
+ self.assertIn("virtual AppSec Days", self.out)
+
+ def test_markup_noise_is_removed(self) -> None:
+ for noise in (
+ "schema.org",
+ "@context",
+ "application/ld+json",
+ "layout: event",
+ "{%",
+ "{{",
+ "
+
+{{ site.data.event-details.pitch | markdownify }}
+
+ **[Check out our sponsors](https://appsecdays.org/sponsors/swag/)**
+
+### Training Sessions Include
+
+* Learn threat modeling with the OWASP Top 10.
+{% if site.data.trainings.count > 0 %}
+
+{% endif %}
diff --git a/application/tests/harvester_test/harvest_event_test.py b/application/tests/harvester_test/harvest_event_test.py
new file mode 100644
index 000000000..8edfff7f7
--- /dev/null
+++ b/application/tests/harvester_test/harvest_event_test.py
@@ -0,0 +1,138 @@
+"""Event repos are harvested through the event-page parser."""
+
+from cre_logging import get_logger
+
+logger = get_logger(__name__)
+
+import tempfile
+import unittest
+from datetime import datetime, timezone
+from pathlib import Path
+from unittest.mock import patch
+
+from application import create_app, sqla
+from application.database import db
+from application.utils.harvester import pipeline as harvest_pipeline
+from application.utils.harvester.schemas import (
+ HARVESTABLE_KINDS,
+ OPTIONAL_HARVEST_KINDS,
+)
+
+HEAD = "abc1234def5678"
+FIXTURE = Path(__file__).parent / "fixtures" / "event_index.md.txt"
+
+REPOS_YAML = """
+repositories:
+ - id: owasp-www-event-demo
+ kind: event
+ enabled: true
+ type: github
+ owner: OWASP
+ repo: www-event-demo
+ branch: main
+ paths: {include: ["index.md"]}
+ chunking: {strategy: markdown_heading, max_tokens: 1000}
+ polling: {mode: full, interval_minutes: 1440}
+"""
+
+
+def _client(files):
+ class FakeClient:
+ def __init__(self, *args, **kwargs) -> None:
+ pass
+
+ def sync(self) -> None:
+ pass
+
+ def get_current_commit_sha(self) -> str:
+ return HEAD
+
+ def get_file_at_commit(self, sha: str, path: str) -> str:
+ return files[path]
+
+ def get_local_path(self) -> Path:
+ return Path("/nonexistent")
+
+ return FakeClient
+
+
+class HarvestEventTests(unittest.TestCase):
+ def setUp(self) -> None:
+ self.app = create_app(mode="test")
+ self.ctx = self.app.app_context()
+ self.ctx.push()
+ sqla.create_all()
+ self.tmp = tempfile.TemporaryDirectory()
+ self.yaml = Path(self.tmp.name) / "repos.yaml"
+ self.yaml.write_text(REPOS_YAML)
+
+ def tearDown(self) -> None:
+ sqla.session.remove()
+ sqla.drop_all()
+ self.ctx.pop()
+ self.tmp.cleanup()
+
+ def _run(self, files):
+ patches = [
+ patch.object(harvest_pipeline, "GitRepositoryClient", _client(files)),
+ patch.object(
+ harvest_pipeline.ChangeDetector,
+ "get_files_at_commit",
+ lambda self, sha: list(files),
+ ),
+ patch.object(
+ harvest_pipeline,
+ "_commit_timestamp",
+ lambda client, sha: datetime(2026, 1, 1, tzinfo=timezone.utc),
+ ),
+ ]
+ for p in patches:
+ p.start()
+ self.addCleanup(p.stop)
+ return harvest_pipeline.run_harvester(
+ sqla.session,
+ "run-1",
+ repos_yaml=self.yaml,
+ sync_repos=False,
+ kinds=HARVESTABLE_KINDS + OPTIONAL_HARVEST_KINDS,
+ )
+
+ def _texts(self):
+ return [row.payload["text"] for row in sqla.session.query(db.HarvestInput)]
+
+ def test_events_are_opt_in(self) -> None:
+ self.assertNotIn("event", HARVESTABLE_KINDS)
+ self.assertIn("event", OPTIONAL_HARVEST_KINDS)
+
+ def test_event_page_is_parsed_before_chunking(self) -> None:
+ summary = self._run({"index.md": FIXTURE.read_text(encoding="utf-8")})
+ self.assertEqual(summary.errors, 0)
+ self.assertEqual(summary.repositories, 1)
+ self.assertGreater(summary.chunks_written, 0)
+ joined = "\n".join(self._texts())
+ self.assertIn("Summer of Security 2020", joined)
+ self.assertIn("2020-06-23", joined)
+ self.assertIn("threat modeling", joined)
+ for noise in ("schema.org", "ld+json", "{%", "{{", "layout: event"):
+ self.assertNotIn(noise, joined)
+
+ def test_page_with_no_content_is_skipped_and_checkpoint_still_advances(
+ self,
+ ) -> None:
+ from application.utils.harvester.checkpoint_store import CheckpointStore
+
+ summary = self._run(
+ {"index.md": "---\nlayout: event\n---\n{% include x.md %}\n"}
+ )
+ self.assertEqual(summary.errors, 0)
+ self.assertEqual(summary.chunks_written, 0)
+ self.assertEqual(
+ CheckpointStore(session=sqla.session)
+ .load("owasp-www-event-demo")
+ .last_processed_commit,
+ HEAD,
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/application/tests/oie_scheduler/graph_filer_test.py b/application/tests/oie_scheduler/graph_filer_test.py
index 702af890a..02977c88f 100644
--- a/application/tests/oie_scheduler/graph_filer_test.py
+++ b/application/tests/oie_scheduler/graph_filer_test.py
@@ -56,6 +56,42 @@ def _envelope(
}
+class PendingEmbeddingsTest(unittest.TestCase):
+ def setUp(self) -> None:
+ self.app = create_app(mode="test")
+ self.ctx = self.app.app_context()
+ self.ctx.push()
+ sqla.create_all()
+
+ def tearDown(self) -> None:
+ sqla.session.remove()
+ sqla.drop_all()
+ self.ctx.pop()
+
+ def _node(self, name, tags):
+ node = db.Node(name=name, section="s", ntype="Standard", tags=tags)
+ sqla.session.add(node)
+ sqla.session.commit()
+ return node.id
+
+ def test_only_auto_filed_nodes_without_embeddings_are_pending(self) -> None:
+ filed = self._node("OWASP/a", f"x,{graph_filer.AUTO_FILED_TAG}")
+ embedded = self._node("OWASP/b", graph_filer.AUTO_FILED_TAG)
+ self._node("ASVS", "curated")
+ sqla.session.add(
+ db.Embeddings(doc_type="Standard", node_id=embedded, embedding_vec="[1]")
+ )
+ sqla.session.commit()
+ self.assertEqual(
+ graph_filer.auto_filed_nodes_without_embeddings(sqla.session), [filed]
+ )
+
+ def test_nothing_pending_on_an_empty_graph(self) -> None:
+ self.assertEqual(
+ graph_filer.auto_filed_nodes_without_embeddings(sqla.session), []
+ )
+
+
class GraphFilerTest(unittest.TestCase):
def setUp(self) -> None:
self.app = create_app(mode="test")
diff --git a/application/tests/oie_scheduler/runner_test.py b/application/tests/oie_scheduler/runner_test.py
index ad2d6e858..d1c8fc498 100644
--- a/application/tests/oie_scheduler/runner_test.py
+++ b/application/tests/oie_scheduler/runner_test.py
@@ -37,6 +37,7 @@ def fn(ctx):
"librarian": _ok("librarian"),
"file_graph": _ok("file_graph"),
"gap_analysis": _ok("gap_analysis"),
+ "embed_nodes": _ok("embed_nodes"),
}
@@ -193,8 +194,12 @@ def setUp(self) -> None:
)
sqla.session.commit()
self.ga_calls = []
+ self.embed_calls = []
- def _expansion(self, **cfg):
+ def _embed(self, collection, node_ids):
+ self.embed_calls.append(list(node_ids))
+
+ def _expansion(self, now=NOW, **cfg):
cfg.setdefault("filing_enabled", True)
overrides = {
"noise_filter": _ok("noise_filter"),
@@ -204,10 +209,11 @@ def _expansion(self, **cfg):
self.collection,
"cre_expansion",
cache_file="db-url",
- now=NOW,
+ now=now,
config=SchedulerConfig(**cfg),
overrides=overrides,
ga_fn=self.ga_calls.append,
+ embed_fn=self._embed,
)
def test_filing_edges_triggers_gap_analysis_once(self) -> None:
@@ -234,6 +240,7 @@ def test_no_new_edges_means_no_gap_analysis(self) -> None:
"librarian": _ok("librarian"),
},
ga_fn=self.ga_calls.append,
+ embed_fn=self._embed,
)
by_name = {s.name: s for s in out.stages}
self.assertEqual(by_name["gap_analysis"].status, "skipped")
@@ -247,6 +254,81 @@ def test_kill_switch_skips_filing_and_gap_analysis(self) -> None:
self.assertEqual(self.ga_calls, [])
self.assertIsNone(sqla.session.query(db.DecisionQueueItem).one().consumed_at)
+ def _filed_node_id(self):
+ return sqla.session.query(db.Node).one().id
+
+ def test_filed_nodes_are_embedded(self) -> None:
+ out = self._expansion()
+ by_name = {s.name: s for s in out.stages}
+ self.assertEqual(by_name["embed_nodes"].status, "ok")
+ self.assertEqual(by_name["embed_nodes"].summary["pending"], 1)
+ self.assertEqual(self.embed_calls, [[self._filed_node_id()]])
+
+ def test_embeds_after_gap_analysis_so_it_cannot_delay_it(self) -> None:
+ out = self._expansion()
+ names = [s.name for s in out.stages]
+ self.assertLess(names.index("gap_analysis"), names.index("embed_nodes"))
+
+ def test_node_that_already_has_an_embedding_is_left_alone(self) -> None:
+ self._expansion()
+ self.embed_calls.clear()
+ sqla.session.add(
+ db.Embeddings(
+ doc_type="Standard",
+ node_id=self._filed_node_id(),
+ embedding_vec="[0.1,0.2]",
+ )
+ )
+ sqla.session.commit()
+ out = self._expansion(now=NOW + timedelta(days=1))
+ by_name = {s.name: s for s in out.stages}
+ self.assertEqual(by_name["embed_nodes"].status, "skipped")
+ self.assertEqual(self.embed_calls, [])
+
+ def test_curated_nodes_are_never_embedded_by_the_job(self) -> None:
+ sqla.session.add(db.Node(name="ASVS", section="V1", ntype="Standard", tags=""))
+ sqla.session.commit()
+ self._expansion()
+ self.assertEqual(len(self.embed_calls), 1)
+ self.assertEqual(len(self.embed_calls[0]), 1)
+
+ def test_embedding_can_be_switched_off(self) -> None:
+ out = self._expansion(embed_filed_nodes=False)
+ by_name = {s.name: s for s in out.stages}
+ self.assertEqual(by_name["embed_nodes"].status, "skipped")
+ self.assertEqual(self.embed_calls, [])
+
+ def test_embedding_failure_degrades_the_job_but_keeps_the_graph_edges(
+ self,
+ ) -> None:
+ def boom(collection, node_ids):
+ raise RuntimeError("no LLM credentials")
+
+ out = run_job(
+ self.collection,
+ "cre_expansion",
+ cache_file="db-url",
+ now=NOW,
+ config=SchedulerConfig(filing_enabled=True),
+ overrides={
+ "noise_filter": _ok("noise_filter"),
+ "librarian": _ok("librarian"),
+ },
+ ga_fn=self.ga_calls.append,
+ embed_fn=boom,
+ )
+ by_name = {s.name: s for s in out.stages}
+ self.assertEqual(by_name["embed_nodes"].status, "degraded")
+ self.assertIn("no LLM credentials", by_name["embed_nodes"].detail)
+ self.assertEqual(by_name["file_graph"].summary["links_added"], 1)
+ self.assertEqual(out.status, "degraded")
+
+ def test_nothing_filed_means_nothing_to_embed(self) -> None:
+ out = self._expansion(filing_enabled=False)
+ by_name = {s.name: s for s in out.stages}
+ self.assertEqual(by_name["embed_nodes"].status, "skipped")
+ self.assertEqual(self.embed_calls, [])
+
def test_high_floor_drops_the_link_and_skips_gap_analysis(self) -> None:
out = self._expansion(file_floor=0.99)
by_name = {s.name: s for s in out.stages}
@@ -350,6 +432,43 @@ def fake(*, repo_cfg, **_):
self.assertEqual(harvest.summary["deferred"], 1)
self.assertEqual(harvest.summary["repository_ids"], ["owasp-p1", "owasp-p2"])
+ def _event_repos(self):
+ self.repos.write_text(
+ self.repos.read_text().replace(
+ "repositories:",
+ " event:\n"
+ " type: github\n"
+ " owner: OWASP\n"
+ " branch: main\n"
+ " paths: {include: [index.md]}\n"
+ " chunking: {strategy: markdown_heading, max_tokens: 1000}\n"
+ " polling: {mode: full, interval_minutes: 1440}\n"
+ "repositories:\n - {id: owasp-e1, kind: event, repo: www-event-1}",
+ )
+ )
+
+ def _harvested(self, **cfg):
+ visited = []
+
+ def fake(*, repo_cfg, **_):
+ visited.append(repo_cfg.id)
+ return 1
+
+ with patch.object(harvest_pipeline, "_harvest_repository", fake):
+ self.run_job(
+ overrides={"agent_sync": _ok("agent_sync")},
+ config=SchedulerConfig(repos_yaml=str(self.repos), **cfg),
+ )
+ return visited
+
+ def test_event_repos_are_not_harvested_by_default(self) -> None:
+ self._event_repos()
+ self.assertNotIn("owasp-e1", self._harvested())
+
+ def test_event_repos_are_harvested_when_opted_in(self) -> None:
+ self._event_repos()
+ self.assertIn("owasp-e1", self._harvested(harvest_events=True))
+
def _agent_sync(self, agent_db):
from application.utils.owasp_agent import index_store
from application.utils.owasp_agent import sync as agent_sync
@@ -401,6 +520,24 @@ def fake(*, repo_cfg, **_):
self.assertEqual(out.status, "degraded")
+class SchedulerConfigEnvTest(unittest.TestCase):
+ def test_event_harvest_defaults_off_and_is_opt_in(self) -> None:
+ with patch.dict("os.environ", {}, clear=False) as env:
+ env.pop("OIE_HARVEST_EVENTS", None)
+ self.assertFalse(SchedulerConfig.from_env().harvest_events)
+ for on in ("1", "true", "ON", "yes"):
+ with patch.dict("os.environ", {"OIE_HARVEST_EVENTS": on}):
+ self.assertTrue(SchedulerConfig.from_env().harvest_events)
+
+ def test_embedding_filed_nodes_defaults_on_and_has_a_kill_switch(self) -> None:
+ with patch.dict("os.environ", {}, clear=False) as env:
+ env.pop("OIE_EMBED_FILED_NODES", None)
+ self.assertTrue(SchedulerConfig.from_env().embed_filed_nodes)
+ for off in ("0", "false", "OFF", "no"):
+ with patch.dict("os.environ", {"OIE_EMBED_FILED_NODES": off}):
+ self.assertFalse(SchedulerConfig.from_env().embed_filed_nodes)
+
+
class HealthTest(SchedulerTestBase):
def test_never_ran_is_unhealthy(self) -> None:
report = health.evaluate_health(sqla.session, NOW)
diff --git a/application/utils/harvester/event_page.py b/application/utils/harvester/event_page.py
new file mode 100644
index 000000000..3a9159422
--- /dev/null
+++ b/application/utils/harvester/event_page.py
@@ -0,0 +1,243 @@
+"""Turn an OWASP event page (a Jekyll ``index.md``) into plain markdown prose.
+
+Event repos (``www-event-*``) keep their facts in YAML front matter and a
+schema.org JSON-LD ``