fix(mcp): use chatbot embedding retrieval for ask security facts

.dockerignore

@@ -5,6 +5,11 @@ tmp
 .vscode
 .github
 .git
+.harvester_cache/
+logs/
+.cursor/
+.cursorignore
+*.nohup.out
 
 # This file specifies files that are *not* uploaded to Google Cloud
 # using gcloud. It follows the same syntax as .gitignore, with the addition of

.env.example

@@ -5,6 +5,10 @@
 # Never commit your real `.env` file to version control.
 # ------------------------------------------------------------------------------
 # Database
+# Local Docker (make docker-postgres / make admin-local):
+#   postgresql://cre:password@127.0.0.1:5432/cre
+# `make admin-local` starts that container when the URL is unreachable and
+# never falls back to SQLite for the admin showcase.
 
 DEV_DATABASE_URL=postgresql://user:password@localhost:5432/opencre
 
@@ -41,6 +45,27 @@ CRE_ENABLE_MYOPENCRE=false
 
 CRE_ENABLE_LOGIN=false
 
+# Experimental OIE OWASP metadata agent (chat + MCP only — never CRE graph UI).
+# Off by default. Uses the main app Postgres (DEV_DATABASE_URL / DATABASE_URL)
+# for the agent index + pgvector. Reachable from /completion and MCP owasp_meta_* tools.
+OWASP_AGENT_ENABLED=false
+
+# Module C.1 graph filing (Automatically linked to). Off by default — leave Module C
+# decisions in decision_queue for review; set true to auto-add high-confidence links.
+OIE_GRAPH_FILING_ENABLED=false
+
+# OIE scheduler: both off by default (they spend quota / write embeddings).
+# OIE_HARVEST_EVENTS=1 also harvests www-event-* repos.
+# OIE_EMBED_FILED_NODES=1 embeds nodes that Module C auto-filing created.
+OIE_HARVEST_EVENTS=false
+OIE_EMBED_FILED_NODES=false
+
+# Nest REST API key (https://nest.owasp.org — create via GraphQL while logged in)
+NEST_API_KEY=
+NEST_API_BASE=https://nest.owasp.org/api/v0
+# Set to 1 to crawl all www-chapter / www-project repos during sync (slow).
+OWASP_AGENT_CRAWL_REPOS=0
+
 # Embeddings
 
 NO_GEN_EMBEDDINGS=false
@@ -52,14 +77,25 @@ GOOGLE_CLIENT_SECRET=your-client-secret
 GOOGLE_SECRET_JSON=path/to/secret.json
 LOGIN_ALLOWED_DOMAINS=example.com
 
-# GCP
+# GCP / Gemini (optional — any LiteLLM provider works for Module B)
 
 GCP_NATIVE=false
 GEMINI_API_KEY=your-gemini-api-key
 VERTEX_CHAT_MODEL=gemini-2.5-flash
 VERTEX_EMBED_CONTENT_MODEL=gemini-embedding-001
 
-# Noise / Relevance Filter (Module B)
+# Other chat providers (LiteLLM). Set the key that matches CRE_NOISE_FILTER_LLM_MODEL.
+# OPENAI_API_KEY=sk-...
+# ANTHROPIC_API_KEY=sk-ant-...
+# GROQ_API_KEY=gsk_...
+# MISTRAL_API_KEY=...
+# OPENROUTER_API_KEY=...
+
+# Noise / Relevance Filter (Module B) — LiteLLM model id (provider/model)
+# Examples: gemini/gemini-2.5-flash-lite | openai/gpt-4o-mini |
+#           anthropic/claude-haiku-4-5 | groq/llama-3.3-70b-versatile
+# Any chat model LiteLLM supports; mid-size models usually work best
+# (cheap + reliable for this classify gate — avoid tiny completion stubs).
 
 CRE_NOISE_FILTER_LLM_MODEL=gemini/gemini-2.5-flash-lite
 CRE_NOISE_FILTER_BATCH_SIZE=10

.gitignore

@@ -72,13 +72,16 @@ standards_cache.sqlite
 !docs/Mid_eval_blog_gsoc2026/module_B_mideval_blog.md
 !docs/Final_eval_blog_gsoc2026/module_B_final_blog.md
 !application/utils/librarian/README.md
+!application/utils/owasp_agent/README.md
 !docs/gsoc_2026_module_c/*.md
 !docs/gsoc_2026_module_a/*.md
 !docs/Final_eval_blog_gsoc2026/*.md
 !docs/gsoc_2026_module_b/module_a_contract.md
 !docs/gsoc_2026_module_b/module_c_contract.md
 !docs/gsoc_2026_module_b/module_b_runbook.md
 !docs/gsoc_2026_module_b/final_metrics.md
+!docs/oie/*.md
+!scripts/oie_owasp_eval/experiments/*.md
 
 ### Dev DBDumps
 *.sql
@@ -91,6 +94,8 @@ neo4j/
 .mypy_cache
 tmp/
 
+import_telemetry.json
+
 ### CREs dir
 cres/*
 ### Local project management tooling

Dockerfile

@@ -10,7 +10,10 @@ FROM python:3.11 as run
 COPY --from=build /code /code
 WORKDIR /code
 COPY ./scripts/prod-docker-entrypoint.sh /code
-RUN pip install -r requirements.txt gunicorn
+# Staging Compose may pass requirements-dev.txt (Librarian ML / OIE Module C.2).
+# Default stays prod requirements.txt for Heroku slug size.
+ARG REQUIREMENTS_FILE=requirements.txt
+RUN pip install -r "${REQUIREMENTS_FILE}" gunicorn
 
 ENV INSECURE_REQUESTS=1
 ENV FLASK_CONFIG="production"

Makefile

@@ -1,6 +1,6 @@
 .ONESHELL:
 
-.PHONY: run test covers install-deps dev docker lint frontend clean all e2e e2e-db
+.PHONY: run test covers install-deps dev dev-flask docker lint frontend clean all e2e e2e-db admin-local owasp-agent-sync owasp-agent-merge-concepts owasp-agent-synth-eval
 
 prod-run:
 	gunicorn cre:app --log-file=-
@@ -70,15 +70,39 @@ docker-postgres:
 
 start-containers: docker-neo4j docker-redis
 
+# RQ work-horses fork after Flask/torch load. On macOS, unset device → MPS and
+# CrashReporter popups from Homebrew Python.app. Force CPU unless overridden.
+CRE_LIBRARIAN_DEVICE?=cpu
 start-worker:
-	. ./venv/bin/activate && FLASK_APP=`pwd`/cre.py python cre.py --start_worker
+	. ./venv/bin/activate && \
+	OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES \
+	CRE_LIBRARIAN_DEVICE="$(CRE_LIBRARIAN_DEVICE)" \
+	FLASK_APP=`pwd`/cre.py python cre.py --start_worker
+
+# Local OIE RQ fleet (queue ``oie``). Default 10; override with OIE_WORKER_COUNT=N.
+# Double-fork via scripts/start_oie_workers.py so workers survive make exit (SIGHUP).
+OIE_WORKER_COUNT?=10
+start-oie-workers:
+	. ./venv/bin/activate && \
+	OIE_WORKER_COUNT="$(OIE_WORKER_COUNT)" \
+	CRE_LIBRARIAN_DEVICE="$(CRE_LIBRARIAN_DEVICE)" \
+	OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES \
+	python scripts/start_oie_workers.py
 
 upstream-sync:
-	. ./venv/bin/activate && python cre.py --upstream_sync
+	. ./venv/bin/activate &&\
+	python cre.py --upstream_sync --cache_file "$(or $(DEV_DATABASE_URL),$(CRE_CACHE_FILE),$(CURDIR)/standards_cache.sqlite)"
 
 PORT?=5000
 
-dev-flask:
+# Local app against Docker Postgres: migrate, upstream-sync if cre/node empty, Flask.
+# Same entrypoint as admin showcase / OIE local (no SQLite). macOS AirPlay often
+# holds :5000 — use PORT=5001 make dev if bind fails.
+dev dev-flask admin-local:
+	bash ./scripts/run_admin_local.sh --port $(PORT) $(ADMIN_LOCAL_ARGS)
+
+# Bare Flask without Postgres bootstrap (legacy / CI helpers). Prefer `make dev`.
+dev-flask-sqlite:
 	. ./venv/bin/activate && INSECURE_REQUESTS=1 FLASK_APP=`pwd`/cre.py  FLASK_CONFIG=development flask run --port $(PORT)
 
 dev-flask-docker:
@@ -136,15 +160,28 @@ install-deps-typescript:
 install-deps: install-deps-python install-deps-typescript
 
 install-python:
-	virtualenv -p python3 venv
+	@if [ ! -d "./venv" ]; then \
+		if command -v virtualenv >/dev/null 2>&1; then \
+			virtualenv -p python3 venv; \
+		else \
+			echo "virtualenv not on PATH — using python3 -m venv"; \
+			python3 -m venv venv; \
+		fi; \
+	fi
 	. ./venv/bin/activate &&\
 	make install-deps-python &&\
 	playwright install  # Python embeddings/scraping (prompt_client); NOT frontend e2e — keep when migrating to Cypress
 
-install-typescript:
-	yarn add webpack && cd application/frontend && yarn build
+# Yarn install only. Do not `yarn add` or `yarn build` here: webpack is
+# already in package.json, and a local build rewrites
+# application/frontend/www/bundle.js (tracked so Heroku can serve the UI
+# without a Node compile step). That was dirtying contributor trees after
+# `make install`. Rebuild with `make frontend` when you change the UI.
+install-typescript: install-deps-typescript
 
-install: install-typescript install-python migrate-upgrade
+# Deps + Docker Postgres schema + CRE upstream graph when empty (see run_admin_local.sh).
+install: install-typescript install-python
+	bash ./scripts/run_admin_local.sh --migrate-only $(ADMIN_LOCAL_ARGS)
 
 docker-dev:
 	docker build -f Dockerfile-dev -t opencre-dev:$(shell git rev-parse HEAD) .
@@ -185,10 +222,39 @@ oie-pipeline:
 	[ -d "./venv" ] && . ./venv/bin/activate &&\
 	PYTHONPATH=. python scripts/run_oie_pipeline.py --cache_file "$(or $(CACHE_FILE),sqlite:///$(CURDIR)/standards_cache.sqlite)" $(OIE_ARGS)
 
+# Deterministic OIE base metadata (Node then CRE document_metadata.oie). Fill-if-missing.
+# Example:
+#   make oie-tag-base CACHE_FILE=postgresql://cre:password@127.0.0.1:5432/cre_prodclone
+oie-tag-base:
+	[ -d "./venv" ] && . ./venv/bin/activate &&\
+	PYTHONPATH=. python scripts/oie_ensure_base_metadata.py \
+		--cache-file "$(or $(CACHE_FILE),$(or $(DEV_DATABASE_URL),postgresql://cre:password@127.0.0.1:5432/cre))" \
+		$(OIE_TAG_ARGS)
+
+# Local OIE bootstrap: start cre-postgres, migrate, sync CRE hub (local|upstream).
+# Examples:
+#   make setup-oie
+#   make setup-oie SETUP_OIE_ARGS='--source upstream --install-ml --calibrate'
+setup-oie:
+	bash ./scripts/setup_oie.sh $(SETUP_OIE_ARGS)
+
 oie-e2e-smoke:
 	[ -d "./venv" ] && . ./venv/bin/activate &&\
 	PYTHONPATH=. python scripts/run_oie_e2e_smoke.py
 
+owasp-agent-sync:
+	[ -d "./venv" ] && . ./venv/bin/activate &&\
+	PYTHONPATH=. python scripts/owasp_agent_sync.py $(OWASP_AGENT_SYNC_ARGS)
+
+owasp-agent-merge-concepts:
+	[ -d "./venv" ] && . ./venv/bin/activate &&\
+	PYTHONPATH=. python scripts/owasp_agent_merge_concepts.py $(OWASP_AGENT_MERGE_ARGS)
+
+# Score fixtures/owasp_agent_synth_v2.jsonl against OWASP_AGENT_DB (see application/utils/owasp_agent/README.md).
+owasp-agent-synth-eval:
+	[ -d "./venv" ] && . ./venv/bin/activate &&\
+	OWASP_AGENT_ENABLED=1 PYTHONPATH=. python scripts/owasp_agent_synth_eval.py
+
 openapi-generate:
 	[ -d "./venv" ] && . ./venv/bin/activate &&\
 	python scripts/generate_openapi.py
@@ -240,6 +306,21 @@ monitor-chatbot-health-prod:
 		--base-url "https://opencre.org" \
 		--output-json "$(CURDIR)/tmp/prod-chatbot-health.json"
 
+# Scheduled OIE jobs (Postgres only). Cron `make oie-tick` every 10 minutes: each
+# job runs once per slot (owasp every 10 min, cre_expansion daily) and is a no-op
+# otherwise. Overrides: OIE_JOB=owasp|cre_expansion|all, OIE_FORCE=1, OIE_DRY_RUN=1.
+oie-tick:
+	@[ -d "./.venv" ] && . ./.venv/bin/activate || ([ -d "./venv" ] && . ./venv/bin/activate); \
+	export CRE_CACHE_FILE="$${CRE_CACHE_FILE:-postgresql://cre:password@127.0.0.1:5432/cre}"; \
+	python cre.py --cache_file "$$CRE_CACHE_FILE" --run_scheduled "$${OIE_JOB:-all}" \
+		$${OIE_FORCE:+--scheduled_force} $${OIE_DRY_RUN:+--scheduled_dry_run}
+
+monitor-oie-health:
+	@[ -d "./.venv" ] && . ./.venv/bin/activate || ([ -d "./venv" ] && . ./venv/bin/activate); \
+	export CRE_CACHE_FILE="$${CRE_CACHE_FILE:-postgresql://cre:password@127.0.0.1:5432/cre}"; \
+	export PYTHONPATH="$(CURDIR)"; \
+	python scripts/monitor_oie_health.py --output-json "$(CURDIR)/tmp/oie-health.json"
+
 verify-ga-parity-local:
 	@[ -d "./.venv" ] && . ./.venv/bin/activate || ([ -d "./venv" ] && . ./venv/bin/activate); \
 	export CRE_CACHE_FILE="$${CRE_CACHE_FILE:-postgresql://cre:password@127.0.0.1:5432/cre}"; \

application/__init__.py

@@ -49,6 +49,22 @@ def create_app(mode: str = "production", conf: any = None) -> Any:
         letters = string.ascii_lowercase
         app.secret_key = "".join(random.choice(letters) for i in range(20))
 
+    # JWT API-token signing secret: env, or mint+persist once on install/boot.
+    # Tests get a stable in-process default so create_app does not touch /code.
+    try:
+        from application.utils.api_tokens.secret import ensure_api_token_jwt_secret
+
+        if (
+            mode == "test"
+            and not (os.environ.get("API_TOKEN_JWT_SECRET") or "").strip()
+        ):
+            os.environ["API_TOKEN_JWT_SECRET"] = (
+                "test-only-api-token-jwt-secret-32bytes"
+            )
+        ensure_api_token_jwt_secret()
+    except Exception:
+        logger.exception("API_TOKEN_JWT_SECRET bootstrap failed")
+
     # config[mode].init_app(app)
     sqla.init_app(app=app)
     from application.web.web_main import app as app_blueprint
@@ -59,8 +75,21 @@ def create_app(mode: str = "production", conf: any = None) -> Any:
 
     init_openapi(app)
 
-    CORS(app)
-    app.config["CORS_HEADERS"] = "Content-Type"
+    # Credentialed browser calls are same-origin in prod/staging; keep CORS
+    # narrow so cross-site pages cannot drive cookie auth. Admin mutations also
+    # require CSRF (see web_main.admin_imports_enabled_required). Without an
+    # explicit allowlist, flask-cors would reflect any Origin together with
+    # credentials, so fall back to plain (credential-less) CORS as on main.
+    cors_origins = [
+        o.strip()
+        for o in (os.environ.get("CORS_ALLOWED_ORIGINS") or "").split(",")
+        if o.strip()
+    ]
+    if cors_origins:
+        CORS(app, origins=cors_origins, supports_credentials=True)
+    else:
+        CORS(app)
+    app.config["CORS_HEADERS"] = "Content-Type, X-CSRF-Token"
 
     compress.init_app(app)
     cache.init_app(app)

application/cmd/cre_main.py

@@ -880,6 +880,39 @@ def backfill_gap_analysis_only(
     logger.info("GA backfill complete: final_missing_pairs=%s", final_missing)
 
 
+def run_scheduled_cli(args: argparse.Namespace, collection: db.Node_collection) -> int:
+    """``--run_scheduled``: run the selected OIE job(s); return a process exit code.
+
+    Postgres only -- the advisory-lock lease and the run table assume it, and the
+    default ``standards_cache.sqlite`` must not silently become the scheduler's
+    database. Exit 1 when a job errored, 2 for a non-Postgres database.
+    """
+    import json
+
+    from application.utils.oie_scheduler.runner import run_due_jobs
+
+    dialect = collection.session.get_bind().dialect.name
+    if dialect != "postgresql":
+        logger.error(
+            "--run_scheduled is Postgres-only (got %s); point --cache_file at "
+            "a postgresql:// URL (make docker-postgres for a local one)",
+            dialect,
+        )
+        return 2
+    selected = args.run_scheduled
+    force = bool(getattr(args, "scheduled_force", False))
+    outcomes = run_due_jobs(
+        collection,
+        cache_file=args.cache_file,
+        jobs=None if selected == "all" else [selected],
+        force=force,
+        dry_run=bool(getattr(args, "scheduled_dry_run", False)),
+        trigger="manual" if force else "scheduled",
+    )
+    print(json.dumps([o.to_dict() for o in outcomes], indent=2))
+    return 1 if any(o.failed for o in outcomes) else 0
+
+
 def run(args: argparse.Namespace) -> None:  # pragma: no cover
     script_path = os.path.dirname(os.path.realpath(__file__))
     os.path.join(script_path, "../cres")
@@ -892,6 +925,14 @@ def run(args: argparse.Namespace) -> None:  # pragma: no cover
         logger.info("Exported %s rows to %s", rows, csv_out)
         return
 
+    if getattr(args, "run_scheduled", ""):
+        import sys
+
+        code = run_scheduled_cli(args, db_connect(args.cache_file))
+        if code:
+            sys.exit(code)
+        return
+
     if getattr(args, "run_harvester", False):
         import sys
 
@@ -905,6 +946,12 @@ def run(args: argparse.Namespace) -> None:  # pragma: no cover
             args.run_id.strip(),
             repos_yaml=repos_yaml,
             dry_run=getattr(args, "harvester_dry_run", False),
+            kinds=[
+                k.strip()
+                for k in (getattr(args, "harvester_kinds", "") or "").split(",")
+                if k.strip()
+            ]
+            or None,
         )
         print(summary.to_json())
         if summary.status == "degraded":