diff --git a/application/prompt_client/prompt_client.py b/application/prompt_client/prompt_client.py index 8aee6d607..4e0494527 100644 --- a/application/prompt_client/prompt_client.py +++ b/application/prompt_client/prompt_client.py @@ -10,6 +10,7 @@ from urllib.parse import urljoin, urlparse from application.prompt_client import embed_alignment, litellm_router +from application.utils.harvester.event_page import clean_embedding_text from scipy import sparse from sklearn.metrics.pairwise import cosine_similarity @@ -315,7 +316,7 @@ def get_content(self, url) -> Optional[str]: if is_plain_text_embed_url(url): text = _fetch_plain_http_text(url) if text: - return text + return clean_embedding_text(url, text) logger.warning( "Plain-text URL %s: empty HTTP body, falling through to Playwright", url, @@ -344,7 +345,7 @@ def get_content(self, url) -> Optional[str]: logger.info(f"loading page {url}") page.goto(url) text = page.locator("body").inner_text() - return text + return clean_embedding_text(url, text) except requests.exceptions.RequestException as e: logger.error( f"Error fetching content for URL: {url} - {str(e)} (attempt {attempts}/9)" diff --git a/application/tests/harvester_test/event_page_test.py b/application/tests/harvester_test/event_page_test.py new file mode 100644 index 000000000..a184a7c30 --- /dev/null +++ b/application/tests/harvester_test/event_page_test.py @@ -0,0 +1,177 @@ +"""Jekyll event pages: front matter + JSON-LD become clean prose; templating is dropped.""" + +import unittest +from pathlib import Path + +from unittest.mock import patch + +from application.prompt_client import prompt_client +from application.utils.harvester.event_page import ( + clean_embedding_text, + prepare_event_page, +) + +FIXTURE = Path(__file__).parent / "fixtures" / "event_index.md.txt" + + +class PrepareEventPageTests(unittest.TestCase): + def setUp(self) -> None: + self.text = FIXTURE.read_text(encoding="utf-8") + self.out = prepare_event_page(self.text) + + def test_title_comes_from_the_event_name(self) -> None: + self.assertTrue( + self.out.startswith("# AppSec Days - Summer of Security 2020\n"), self.out + ) + + def test_event_facts_are_stated_as_prose(self) -> None: + self.assertIn("Dates: 2020-06-23 to 2020-08-26", self.out) + self.assertIn("Location: https://appsecdays.org/", self.out) + self.assertIn("virtual AppSec Days", self.out) + + def test_markup_noise_is_removed(self) -> None: + for noise in ( + "schema.org", + "@context", + "application/ld+json", + "layout: event", + "{%", + "{{", + " + +{{ site.data.event-details.pitch | markdownify }} + + **[Check out our sponsors](https://appsecdays.org/sponsors/swag/)** + +### Training Sessions Include + +* Learn threat modeling with the OWASP Top 10. +{% if site.data.trainings.count > 0 %} + +{% endif %} diff --git a/application/tests/harvester_test/harvest_event_test.py b/application/tests/harvester_test/harvest_event_test.py new file mode 100644 index 000000000..8edfff7f7 --- /dev/null +++ b/application/tests/harvester_test/harvest_event_test.py @@ -0,0 +1,138 @@ +"""Event repos are harvested through the event-page parser.""" + +from cre_logging import get_logger + +logger = get_logger(__name__) + +import tempfile +import unittest +from datetime import datetime, timezone +from pathlib import Path +from unittest.mock import patch + +from application import create_app, sqla +from application.database import db +from application.utils.harvester import pipeline as harvest_pipeline +from application.utils.harvester.schemas import ( + HARVESTABLE_KINDS, + OPTIONAL_HARVEST_KINDS, +) + +HEAD = "abc1234def5678" +FIXTURE = Path(__file__).parent / "fixtures" / "event_index.md.txt" + +REPOS_YAML = """ +repositories: + - id: owasp-www-event-demo + kind: event + enabled: true + type: github + owner: OWASP + repo: www-event-demo + branch: main + paths: {include: ["index.md"]} + chunking: {strategy: markdown_heading, max_tokens: 1000} + polling: {mode: full, interval_minutes: 1440} +""" + + +def _client(files): + class FakeClient: + def __init__(self, *args, **kwargs) -> None: + pass + + def sync(self) -> None: + pass + + def get_current_commit_sha(self) -> str: + return HEAD + + def get_file_at_commit(self, sha: str, path: str) -> str: + return files[path] + + def get_local_path(self) -> Path: + return Path("/nonexistent") + + return FakeClient + + +class HarvestEventTests(unittest.TestCase): + def setUp(self) -> None: + self.app = create_app(mode="test") + self.ctx = self.app.app_context() + self.ctx.push() + sqla.create_all() + self.tmp = tempfile.TemporaryDirectory() + self.yaml = Path(self.tmp.name) / "repos.yaml" + self.yaml.write_text(REPOS_YAML) + + def tearDown(self) -> None: + sqla.session.remove() + sqla.drop_all() + self.ctx.pop() + self.tmp.cleanup() + + def _run(self, files): + patches = [ + patch.object(harvest_pipeline, "GitRepositoryClient", _client(files)), + patch.object( + harvest_pipeline.ChangeDetector, + "get_files_at_commit", + lambda self, sha: list(files), + ), + patch.object( + harvest_pipeline, + "_commit_timestamp", + lambda client, sha: datetime(2026, 1, 1, tzinfo=timezone.utc), + ), + ] + for p in patches: + p.start() + self.addCleanup(p.stop) + return harvest_pipeline.run_harvester( + sqla.session, + "run-1", + repos_yaml=self.yaml, + sync_repos=False, + kinds=HARVESTABLE_KINDS + OPTIONAL_HARVEST_KINDS, + ) + + def _texts(self): + return [row.payload["text"] for row in sqla.session.query(db.HarvestInput)] + + def test_events_are_opt_in(self) -> None: + self.assertNotIn("event", HARVESTABLE_KINDS) + self.assertIn("event", OPTIONAL_HARVEST_KINDS) + + def test_event_page_is_parsed_before_chunking(self) -> None: + summary = self._run({"index.md": FIXTURE.read_text(encoding="utf-8")}) + self.assertEqual(summary.errors, 0) + self.assertEqual(summary.repositories, 1) + self.assertGreater(summary.chunks_written, 0) + joined = "\n".join(self._texts()) + self.assertIn("Summer of Security 2020", joined) + self.assertIn("2020-06-23", joined) + self.assertIn("threat modeling", joined) + for noise in ("schema.org", "ld+json", "{%", "{{", "layout: event"): + self.assertNotIn(noise, joined) + + def test_page_with_no_content_is_skipped_and_checkpoint_still_advances( + self, + ) -> None: + from application.utils.harvester.checkpoint_store import CheckpointStore + + summary = self._run( + {"index.md": "---\nlayout: event\n---\n{% include x.md %}\n"} + ) + self.assertEqual(summary.errors, 0) + self.assertEqual(summary.chunks_written, 0) + self.assertEqual( + CheckpointStore(session=sqla.session) + .load("owasp-www-event-demo") + .last_processed_commit, + HEAD, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/tests/oie_scheduler/graph_filer_test.py b/application/tests/oie_scheduler/graph_filer_test.py index 702af890a..02977c88f 100644 --- a/application/tests/oie_scheduler/graph_filer_test.py +++ b/application/tests/oie_scheduler/graph_filer_test.py @@ -56,6 +56,42 @@ def _envelope( } +class PendingEmbeddingsTest(unittest.TestCase): + def setUp(self) -> None: + self.app = create_app(mode="test") + self.ctx = self.app.app_context() + self.ctx.push() + sqla.create_all() + + def tearDown(self) -> None: + sqla.session.remove() + sqla.drop_all() + self.ctx.pop() + + def _node(self, name, tags): + node = db.Node(name=name, section="s", ntype="Standard", tags=tags) + sqla.session.add(node) + sqla.session.commit() + return node.id + + def test_only_auto_filed_nodes_without_embeddings_are_pending(self) -> None: + filed = self._node("OWASP/a", f"x,{graph_filer.AUTO_FILED_TAG}") + embedded = self._node("OWASP/b", graph_filer.AUTO_FILED_TAG) + self._node("ASVS", "curated") + sqla.session.add( + db.Embeddings(doc_type="Standard", node_id=embedded, embedding_vec="[1]") + ) + sqla.session.commit() + self.assertEqual( + graph_filer.auto_filed_nodes_without_embeddings(sqla.session), [filed] + ) + + def test_nothing_pending_on_an_empty_graph(self) -> None: + self.assertEqual( + graph_filer.auto_filed_nodes_without_embeddings(sqla.session), [] + ) + + class GraphFilerTest(unittest.TestCase): def setUp(self) -> None: self.app = create_app(mode="test") diff --git a/application/tests/oie_scheduler/runner_test.py b/application/tests/oie_scheduler/runner_test.py index ad2d6e858..d1c8fc498 100644 --- a/application/tests/oie_scheduler/runner_test.py +++ b/application/tests/oie_scheduler/runner_test.py @@ -37,6 +37,7 @@ def fn(ctx): "librarian": _ok("librarian"), "file_graph": _ok("file_graph"), "gap_analysis": _ok("gap_analysis"), + "embed_nodes": _ok("embed_nodes"), } @@ -193,8 +194,12 @@ def setUp(self) -> None: ) sqla.session.commit() self.ga_calls = [] + self.embed_calls = [] - def _expansion(self, **cfg): + def _embed(self, collection, node_ids): + self.embed_calls.append(list(node_ids)) + + def _expansion(self, now=NOW, **cfg): cfg.setdefault("filing_enabled", True) overrides = { "noise_filter": _ok("noise_filter"), @@ -204,10 +209,11 @@ def _expansion(self, **cfg): self.collection, "cre_expansion", cache_file="db-url", - now=NOW, + now=now, config=SchedulerConfig(**cfg), overrides=overrides, ga_fn=self.ga_calls.append, + embed_fn=self._embed, ) def test_filing_edges_triggers_gap_analysis_once(self) -> None: @@ -234,6 +240,7 @@ def test_no_new_edges_means_no_gap_analysis(self) -> None: "librarian": _ok("librarian"), }, ga_fn=self.ga_calls.append, + embed_fn=self._embed, ) by_name = {s.name: s for s in out.stages} self.assertEqual(by_name["gap_analysis"].status, "skipped") @@ -247,6 +254,81 @@ def test_kill_switch_skips_filing_and_gap_analysis(self) -> None: self.assertEqual(self.ga_calls, []) self.assertIsNone(sqla.session.query(db.DecisionQueueItem).one().consumed_at) + def _filed_node_id(self): + return sqla.session.query(db.Node).one().id + + def test_filed_nodes_are_embedded(self) -> None: + out = self._expansion() + by_name = {s.name: s for s in out.stages} + self.assertEqual(by_name["embed_nodes"].status, "ok") + self.assertEqual(by_name["embed_nodes"].summary["pending"], 1) + self.assertEqual(self.embed_calls, [[self._filed_node_id()]]) + + def test_embeds_after_gap_analysis_so_it_cannot_delay_it(self) -> None: + out = self._expansion() + names = [s.name for s in out.stages] + self.assertLess(names.index("gap_analysis"), names.index("embed_nodes")) + + def test_node_that_already_has_an_embedding_is_left_alone(self) -> None: + self._expansion() + self.embed_calls.clear() + sqla.session.add( + db.Embeddings( + doc_type="Standard", + node_id=self._filed_node_id(), + embedding_vec="[0.1,0.2]", + ) + ) + sqla.session.commit() + out = self._expansion(now=NOW + timedelta(days=1)) + by_name = {s.name: s for s in out.stages} + self.assertEqual(by_name["embed_nodes"].status, "skipped") + self.assertEqual(self.embed_calls, []) + + def test_curated_nodes_are_never_embedded_by_the_job(self) -> None: + sqla.session.add(db.Node(name="ASVS", section="V1", ntype="Standard", tags="")) + sqla.session.commit() + self._expansion() + self.assertEqual(len(self.embed_calls), 1) + self.assertEqual(len(self.embed_calls[0]), 1) + + def test_embedding_can_be_switched_off(self) -> None: + out = self._expansion(embed_filed_nodes=False) + by_name = {s.name: s for s in out.stages} + self.assertEqual(by_name["embed_nodes"].status, "skipped") + self.assertEqual(self.embed_calls, []) + + def test_embedding_failure_degrades_the_job_but_keeps_the_graph_edges( + self, + ) -> None: + def boom(collection, node_ids): + raise RuntimeError("no LLM credentials") + + out = run_job( + self.collection, + "cre_expansion", + cache_file="db-url", + now=NOW, + config=SchedulerConfig(filing_enabled=True), + overrides={ + "noise_filter": _ok("noise_filter"), + "librarian": _ok("librarian"), + }, + ga_fn=self.ga_calls.append, + embed_fn=boom, + ) + by_name = {s.name: s for s in out.stages} + self.assertEqual(by_name["embed_nodes"].status, "degraded") + self.assertIn("no LLM credentials", by_name["embed_nodes"].detail) + self.assertEqual(by_name["file_graph"].summary["links_added"], 1) + self.assertEqual(out.status, "degraded") + + def test_nothing_filed_means_nothing_to_embed(self) -> None: + out = self._expansion(filing_enabled=False) + by_name = {s.name: s for s in out.stages} + self.assertEqual(by_name["embed_nodes"].status, "skipped") + self.assertEqual(self.embed_calls, []) + def test_high_floor_drops_the_link_and_skips_gap_analysis(self) -> None: out = self._expansion(file_floor=0.99) by_name = {s.name: s for s in out.stages} @@ -350,6 +432,43 @@ def fake(*, repo_cfg, **_): self.assertEqual(harvest.summary["deferred"], 1) self.assertEqual(harvest.summary["repository_ids"], ["owasp-p1", "owasp-p2"]) + def _event_repos(self): + self.repos.write_text( + self.repos.read_text().replace( + "repositories:", + " event:\n" + " type: github\n" + " owner: OWASP\n" + " branch: main\n" + " paths: {include: [index.md]}\n" + " chunking: {strategy: markdown_heading, max_tokens: 1000}\n" + " polling: {mode: full, interval_minutes: 1440}\n" + "repositories:\n - {id: owasp-e1, kind: event, repo: www-event-1}", + ) + ) + + def _harvested(self, **cfg): + visited = [] + + def fake(*, repo_cfg, **_): + visited.append(repo_cfg.id) + return 1 + + with patch.object(harvest_pipeline, "_harvest_repository", fake): + self.run_job( + overrides={"agent_sync": _ok("agent_sync")}, + config=SchedulerConfig(repos_yaml=str(self.repos), **cfg), + ) + return visited + + def test_event_repos_are_not_harvested_by_default(self) -> None: + self._event_repos() + self.assertNotIn("owasp-e1", self._harvested()) + + def test_event_repos_are_harvested_when_opted_in(self) -> None: + self._event_repos() + self.assertIn("owasp-e1", self._harvested(harvest_events=True)) + def _agent_sync(self, agent_db): from application.utils.owasp_agent import index_store from application.utils.owasp_agent import sync as agent_sync @@ -401,6 +520,24 @@ def fake(*, repo_cfg, **_): self.assertEqual(out.status, "degraded") +class SchedulerConfigEnvTest(unittest.TestCase): + def test_event_harvest_defaults_off_and_is_opt_in(self) -> None: + with patch.dict("os.environ", {}, clear=False) as env: + env.pop("OIE_HARVEST_EVENTS", None) + self.assertFalse(SchedulerConfig.from_env().harvest_events) + for on in ("1", "true", "ON", "yes"): + with patch.dict("os.environ", {"OIE_HARVEST_EVENTS": on}): + self.assertTrue(SchedulerConfig.from_env().harvest_events) + + def test_embedding_filed_nodes_defaults_on_and_has_a_kill_switch(self) -> None: + with patch.dict("os.environ", {}, clear=False) as env: + env.pop("OIE_EMBED_FILED_NODES", None) + self.assertTrue(SchedulerConfig.from_env().embed_filed_nodes) + for off in ("0", "false", "OFF", "no"): + with patch.dict("os.environ", {"OIE_EMBED_FILED_NODES": off}): + self.assertFalse(SchedulerConfig.from_env().embed_filed_nodes) + + class HealthTest(SchedulerTestBase): def test_never_ran_is_unhealthy(self) -> None: report = health.evaluate_health(sqla.session, NOW) diff --git a/application/utils/harvester/event_page.py b/application/utils/harvester/event_page.py new file mode 100644 index 000000000..3a9159422 --- /dev/null +++ b/application/utils/harvester/event_page.py @@ -0,0 +1,243 @@ +"""Turn an OWASP event page (a Jekyll ``index.md``) into plain markdown prose. + +Event repos (``www-event-*``) keep their facts in YAML front matter and a +schema.org JSON-LD ``