feat(comprehension): ingestion-time LLM digests so ask/chat answer structure + combination questions

.dockerignore

@@ -5,6 +5,11 @@ tmp
 .vscode
 .github
 .git
+.harvester_cache/
+logs/
+.cursor/
+.cursorignore
+*.nohup.out
 
 # This file specifies files that are *not* uploaded to Google Cloud
 # using gcloud. It follows the same syntax as .gitignore, with the addition of

.env.example

@@ -5,6 +5,10 @@
 # Never commit your real `.env` file to version control.
 # ------------------------------------------------------------------------------
 # Database
+# Local Docker (make docker-postgres / make admin-local):
+#   postgresql://cre:password@127.0.0.1:5432/cre
+# `make admin-local` starts that container when the URL is unreachable and
+# never falls back to SQLite for the admin showcase.
 
 DEV_DATABASE_URL=postgresql://user:password@localhost:5432/opencre
 
@@ -41,6 +45,27 @@ CRE_ENABLE_MYOPENCRE=false
 
 CRE_ENABLE_LOGIN=false
 
+# Experimental OIE OWASP metadata agent (chat + MCP only — never CRE graph UI).
+# Off by default. Uses the main app Postgres (DEV_DATABASE_URL / DATABASE_URL)
+# for the agent index + pgvector. Reachable from /completion and MCP owasp_meta_* tools.
+OWASP_AGENT_ENABLED=false
+
+# Module C.1 graph filing (Automatically linked to). Off by default — leave Module C
+# decisions in decision_queue for review; set true to auto-add high-confidence links.
+OIE_GRAPH_FILING_ENABLED=false
+
+# OIE scheduler: both off by default (they spend quota / write embeddings).
+# OIE_HARVEST_EVENTS=1 also harvests www-event-* repos.
+# OIE_EMBED_FILED_NODES=1 embeds nodes that Module C auto-filing created.
+OIE_HARVEST_EVENTS=false
+OIE_EMBED_FILED_NODES=false
+
+# Nest REST API key (https://nest.owasp.org — create via GraphQL while logged in)
+NEST_API_KEY=
+NEST_API_BASE=https://nest.owasp.org/api/v0
+# Set to 1 to crawl all www-chapter / www-project repos during sync (slow).
+OWASP_AGENT_CRAWL_REPOS=0
+
 # Embeddings
 
 NO_GEN_EMBEDDINGS=false
@@ -52,14 +77,25 @@ GOOGLE_CLIENT_SECRET=your-client-secret
 GOOGLE_SECRET_JSON=path/to/secret.json
 LOGIN_ALLOWED_DOMAINS=example.com
 
-# GCP
+# GCP / Gemini (optional — any LiteLLM provider works for Module B)
 
 GCP_NATIVE=false
 GEMINI_API_KEY=your-gemini-api-key
 VERTEX_CHAT_MODEL=gemini-2.5-flash
 VERTEX_EMBED_CONTENT_MODEL=gemini-embedding-001
 
-# Noise / Relevance Filter (Module B)
+# Other chat providers (LiteLLM). Set the key that matches CRE_NOISE_FILTER_LLM_MODEL.
+# OPENAI_API_KEY=sk-...
+# ANTHROPIC_API_KEY=sk-ant-...
+# GROQ_API_KEY=gsk_...
+# MISTRAL_API_KEY=...
+# OPENROUTER_API_KEY=...
+
+# Noise / Relevance Filter (Module B) — LiteLLM model id (provider/model)
+# Examples: gemini/gemini-2.5-flash-lite | openai/gpt-4o-mini |
+#           anthropic/claude-haiku-4-5 | groq/llama-3.3-70b-versatile
+# Any chat model LiteLLM supports; mid-size models usually work best
+# (cheap + reliable for this classify gate — avoid tiny completion stubs).
 
 CRE_NOISE_FILTER_LLM_MODEL=gemini/gemini-2.5-flash-lite
 CRE_NOISE_FILTER_BATCH_SIZE=10
@@ -86,3 +122,75 @@ CRE_LIBRARIAN_LINK_THRESHOLD=0.8
 CRE_LIBRARIAN_BATCH_SIZE=32
 CRE_LIBRARIAN_ECE_TARGET=0.10
 CRE_LIBRARIAN_CONFORMAL_ALPHA=0.10
+
+# --- OWASP agent answers: ingestion-time comprehension (#1158) ---------------------------
+# Values below are the measured winning combination on staging (full 589-question set:
+# whole-project context 57.6% correct vs 54.7% with no flags, ~20 s end to end). Every new
+# feature is off in code unless its flag is set, EXCEPT OWASP_COMPREHENSION_ENABLED and
+# OWASP_COMPREHENSION_ABSTAIN, which default to 1 in code (comprehension/answer.py): a database
+# with comprehension rows answers from them (and abstains) unless you set them to 0.
+# Retrieval depth for the OWASP agent lane (default 8) and for comprehension documents.
+OWASP_AGENT_TOP_K=16
+OWASP_COMPREHENSION_TOP_K=24
+# Answer from ingestion-time comprehension documents; say "not found" instead of guessing.
+OWASP_COMPREHENSION_ENABLED=1
+OWASP_COMPREHENSION_ABSTAIN=1
+# How many retrieved documents the answer reads (more did not help: 24 scored the same as 10).
+OWASP_COMPREHENSION_SOURCES=10
+# Search only "raw" source chunks or only "digest" documents (default all). Raw chunks alone
+# score within ~3 points of all, which is the cheap full-import option.
+OWASP_COMPREHENSION_DOCS=all
+# Model that writes the final answer ('' = the default chat model). Gemini Pro did not help.
+OWASP_COMPREHENSION_MODEL=
+# Scope retrieval to the project named in the question as a score boost. Hurt (-10 points): keep 0.
+OWASP_COMPREHENSION_SCOPE=0
+# Half-precision HNSW vector search. Needs migration e5f6a7b8c9d0 (the index); 1 cuts ~10 s
+# of database time per question on a large embeddings table.
+OWASP_PGVECTOR_HALFVEC=0
+
+# Feature flags (1 = on). Measured on the 252-question sample unless noted.
+# L4 whole-project context: when a question names exactly one project that fits, the model
+# reads the project's whole text instead of snippets (+3 points on the full set).
+# Off by default (as in code): each question reads up to LONGCTX_CHARS of text, so enable it
+# deliberately and keep LONGCTX_CHARS bounded on public chat.
+OWASP_CX_LONGCTX=0
+# Max characters of one project to read whole (code default).
+OWASP_CX_LONGCTX_CHARS=700000
+# How many named projects to read whole at once (questions naming more fall back to retrieval).
+# 1 scored 59.1% on the sample; 3 scored 56.3% (counts and multi-part got worse), so keep 1.
+OWASP_CX_LONGCTX_PROJECTS=1
+# Off: hybrid keyword search (-2.5 points, +5 s), rerank (-2), query rewrite (-3), verify
+# (+/-3, hurts with LONGCTX), per-project (0), expand (+1, noise), count (+1, slower),
+# merged single call (-13, loses citations), in-process answer cache (real traffic only).
+# Final-answer model with thinking off (latency); multi-part questions answered part by part.
+OWASP_CX_FAST=0
+# Thinking budget for the final answer: '' (model default), low, medium or high. Measured: low and
+# medium are NOT faster than the default (~7 s); only OWASP_CX_FAST=1 (thinking off, 2.5 s but
+# 49% on the sample, 54% with OWASP_COMPREHENSION_MODEL=gemini/gemini-2.5-flash-lite) is.
+OWASP_CX_THINK_EFFORT=
+OWASP_CX_DECOMPOSE=0
+OWASP_CX_HYBRID=0
+OWASP_CX_RERANK=0
+OWASP_CX_QUERY=0
+OWASP_CX_VERIFY=0
+OWASP_CX_PERPROJECT=0
+OWASP_CX_EXPAND=0
+OWASP_CX_COUNT=0
+OWASP_CX_MERGED=0
+OWASP_CX_CACHE=0
+# Cheap model used by the rerank/query/verify helpers when those flags are on.
+OWASP_CX_HELPER_MODEL=gemini/gemini-2.5-flash-lite
+
+# Ingestion (scripts/comprehend_standard.py): bulk digests and chunk enrichment use a cheap
+# model with thinking off; repo overviews use a stronger one. Run with --estimate first.
+CRE_COMPREHENSION_BULK_MODEL=gemini/gemini-2.5-flash
+CRE_COMPREHENSION_OVERVIEW_MODEL=gemini/gemini-2.5-pro
+CRE_COMPREHENSION_ENRICH_MODEL=gemini/gemini-2.5-flash-lite
+CRE_COMPREHENSION_THINKING=off
+
+# Warm the answer path in the background at server start (the first question after a restart
+# otherwise takes ~30 s). 0 disables.
+OWASP_WARMUP=1
+
+# Remote MCP server: how many questions may be answered at the same time (each in its own thread).
+OPENCRE_MCP_ASK_CONCURRENCY=8

.gitignore

@@ -72,13 +72,16 @@ standards_cache.sqlite
 !docs/Mid_eval_blog_gsoc2026/module_B_mideval_blog.md
 !docs/Final_eval_blog_gsoc2026/module_B_final_blog.md
 !application/utils/librarian/README.md
+!application/utils/owasp_agent/README.md
 !docs/gsoc_2026_module_c/*.md
 !docs/gsoc_2026_module_a/*.md
 !docs/Final_eval_blog_gsoc2026/*.md
 !docs/gsoc_2026_module_b/module_a_contract.md
 !docs/gsoc_2026_module_b/module_c_contract.md
 !docs/gsoc_2026_module_b/module_b_runbook.md
 !docs/gsoc_2026_module_b/final_metrics.md
+!docs/oie/*.md
+!scripts/oie_owasp_eval/experiments/*.md
 
 ### Dev DBDumps
 *.sql
@@ -91,6 +94,8 @@ neo4j/
 .mypy_cache
 tmp/
 
+import_telemetry.json
+
 ### CREs dir
 cres/*
 ### Local project management tooling

Dockerfile

@@ -10,7 +10,10 @@ FROM python:3.11 as run
 COPY --from=build /code /code
 WORKDIR /code
 COPY ./scripts/prod-docker-entrypoint.sh /code
-RUN pip install -r requirements.txt gunicorn
+# Staging Compose may pass requirements-dev.txt (Librarian ML / OIE Module C.2).
+# Default stays prod requirements.txt for Heroku slug size.
+ARG REQUIREMENTS_FILE=requirements.txt
+RUN pip install -r "${REQUIREMENTS_FILE}" gunicorn
 
 ENV INSECURE_REQUESTS=1
 ENV FLASK_CONFIG="production"

Makefile

@@ -1,6 +1,6 @@
 .ONESHELL:
 
-.PHONY: run test covers install-deps dev docker lint frontend clean all e2e e2e-db
+.PHONY: run test covers install-deps dev dev-flask docker lint frontend clean all e2e e2e-db admin-local owasp-agent-sync owasp-agent-merge-concepts owasp-agent-synth-eval
 
 prod-run:
 	gunicorn cre:app --log-file=-
@@ -70,15 +70,39 @@ docker-postgres:
 
 start-containers: docker-neo4j docker-redis
 
+# RQ work-horses fork after Flask/torch load. On macOS, unset device → MPS and
+# CrashReporter popups from Homebrew Python.app. Force CPU unless overridden.
+CRE_LIBRARIAN_DEVICE?=cpu
 start-worker:
-	. ./venv/bin/activate && FLASK_APP=`pwd`/cre.py python cre.py --start_worker
+	. ./venv/bin/activate && \
+	OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES \
+	CRE_LIBRARIAN_DEVICE="$(CRE_LIBRARIAN_DEVICE)" \
+	FLASK_APP=`pwd`/cre.py python cre.py --start_worker
+
+# Local OIE RQ fleet (queue ``oie``). Default 10; override with OIE_WORKER_COUNT=N.
+# Double-fork via scripts/start_oie_workers.py so workers survive make exit (SIGHUP).
+OIE_WORKER_COUNT?=10
+start-oie-workers:
+	. ./venv/bin/activate && \
+	OIE_WORKER_COUNT="$(OIE_WORKER_COUNT)" \
+	CRE_LIBRARIAN_DEVICE="$(CRE_LIBRARIAN_DEVICE)" \
+	OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES \
+	python scripts/start_oie_workers.py
 
 upstream-sync:
-	. ./venv/bin/activate && python cre.py --upstream_sync
+	. ./venv/bin/activate &&\
+	python cre.py --upstream_sync --cache_file "$(or $(DEV_DATABASE_URL),$(CRE_CACHE_FILE),$(CURDIR)/standards_cache.sqlite)"
 
 PORT?=5000
 
-dev-flask:
+# Local app against Docker Postgres: migrate, upstream-sync if cre/node empty, Flask.
+# Same entrypoint as admin showcase / OIE local (no SQLite). macOS AirPlay often
+# holds :5000 — use PORT=5001 make dev if bind fails.
+dev dev-flask admin-local:
+	bash ./scripts/run_admin_local.sh --port $(PORT) $(ADMIN_LOCAL_ARGS)
+
+# Bare Flask without Postgres bootstrap (legacy / CI helpers). Prefer `make dev`.
+dev-flask-sqlite:
 	. ./venv/bin/activate && INSECURE_REQUESTS=1 FLASK_APP=`pwd`/cre.py  FLASK_CONFIG=development flask run --port $(PORT)
 
 dev-flask-docker:
@@ -136,15 +160,28 @@ install-deps-typescript:
 install-deps: install-deps-python install-deps-typescript
 
 install-python:
-	virtualenv -p python3 venv
+	@if [ ! -d "./venv" ]; then \
+		if command -v virtualenv >/dev/null 2>&1; then \
+			virtualenv -p python3 venv; \
+		else \
+			echo "virtualenv not on PATH — using python3 -m venv"; \
+			python3 -m venv venv; \
+		fi; \
+	fi
 	. ./venv/bin/activate &&\
 	make install-deps-python &&\
 	playwright install  # Python embeddings/scraping (prompt_client); NOT frontend e2e — keep when migrating to Cypress
 
-install-typescript:
-	yarn add webpack && cd application/frontend && yarn build
+# Yarn install only. Do not `yarn add` or `yarn build` here: webpack is
+# already in package.json, and a local build rewrites
+# application/frontend/www/bundle.js (tracked so Heroku can serve the UI
+# without a Node compile step). That was dirtying contributor trees after
+# `make install`. Rebuild with `make frontend` when you change the UI.
+install-typescript: install-deps-typescript
 
-install: install-typescript install-python migrate-upgrade
+# Deps + Docker Postgres schema + CRE upstream graph when empty (see run_admin_local.sh).
+install: install-typescript install-python
+	bash ./scripts/run_admin_local.sh --migrate-only $(ADMIN_LOCAL_ARGS)
 
 docker-dev:
 	docker build -f Dockerfile-dev -t opencre-dev:$(shell git rev-parse HEAD) .
@@ -185,10 +222,39 @@ oie-pipeline:
 	[ -d "./venv" ] && . ./venv/bin/activate &&\
 	PYTHONPATH=. python scripts/run_oie_pipeline.py --cache_file "$(or $(CACHE_FILE),sqlite:///$(CURDIR)/standards_cache.sqlite)" $(OIE_ARGS)
 
+# Deterministic OIE base metadata (Node then CRE document_metadata.oie). Fill-if-missing.
+# Example:
+#   make oie-tag-base CACHE_FILE=postgresql://cre:password@127.0.0.1:5432/cre_prodclone
+oie-tag-base:
+	[ -d "./venv" ] && . ./venv/bin/activate &&\
+	PYTHONPATH=. python scripts/oie_ensure_base_metadata.py \
+		--cache-file "$(or $(CACHE_FILE),$(or $(DEV_DATABASE_URL),postgresql://cre:password@127.0.0.1:5432/cre))" \
+		$(OIE_TAG_ARGS)
+
+# Local OIE bootstrap: start cre-postgres, migrate, sync CRE hub (local|upstream).
+# Examples:
+#   make setup-oie
+#   make setup-oie SETUP_OIE_ARGS='--source upstream --install-ml --calibrate'
+setup-oie:
+	bash ./scripts/setup_oie.sh $(SETUP_OIE_ARGS)
+
 oie-e2e-smoke:
 	[ -d "./venv" ] && . ./venv/bin/activate &&\
 	PYTHONPATH=. python scripts/run_oie_e2e_smoke.py
 
+owasp-agent-sync:
+	[ -d "./venv" ] && . ./venv/bin/activate &&\
+	PYTHONPATH=. python scripts/owasp_agent_sync.py $(OWASP_AGENT_SYNC_ARGS)
+
+owasp-agent-merge-concepts:
+	[ -d "./venv" ] && . ./venv/bin/activate &&\
+	PYTHONPATH=. python scripts/owasp_agent_merge_concepts.py $(OWASP_AGENT_MERGE_ARGS)
+
+# Score fixtures/owasp_agent_synth_v2.jsonl against OWASP_AGENT_DB (see application/utils/owasp_agent/README.md).
+owasp-agent-synth-eval:
+	[ -d "./venv" ] && . ./venv/bin/activate &&\
+	OWASP_AGENT_ENABLED=1 PYTHONPATH=. python scripts/owasp_agent_synth_eval.py
+
 openapi-generate:
 	[ -d "./venv" ] && . ./venv/bin/activate &&\
 	python scripts/generate_openapi.py
@@ -240,6 +306,21 @@ monitor-chatbot-health-prod:
 		--base-url "https://opencre.org" \
 		--output-json "$(CURDIR)/tmp/prod-chatbot-health.json"
 
+# Scheduled OIE jobs (Postgres only). Cron `make oie-tick` every 10 minutes: each
+# job runs once per slot (owasp every 10 min, cre_expansion daily) and is a no-op
+# otherwise. Overrides: OIE_JOB=owasp|cre_expansion|all, OIE_FORCE=1, OIE_DRY_RUN=1.
+oie-tick:
+	@[ -d "./.venv" ] && . ./.venv/bin/activate || ([ -d "./venv" ] && . ./venv/bin/activate); \
+	export CRE_CACHE_FILE="$${CRE_CACHE_FILE:-postgresql://cre:password@127.0.0.1:5432/cre}"; \
+	python cre.py --cache_file "$$CRE_CACHE_FILE" --run_scheduled "$${OIE_JOB:-all}" \
+		$${OIE_FORCE:+--scheduled_force} $${OIE_DRY_RUN:+--scheduled_dry_run}
+
+monitor-oie-health:
+	@[ -d "./.venv" ] && . ./.venv/bin/activate || ([ -d "./venv" ] && . ./venv/bin/activate); \
+	export CRE_CACHE_FILE="$${CRE_CACHE_FILE:-postgresql://cre:password@127.0.0.1:5432/cre}"; \
+	export PYTHONPATH="$(CURDIR)"; \
+	python scripts/monitor_oie_health.py --output-json "$(CURDIR)/tmp/oie-health.json"
+
 verify-ga-parity-local:
 	@[ -d "./.venv" ] && . ./.venv/bin/activate || ([ -d "./venv" ] && . ./venv/bin/activate); \
 	export CRE_CACHE_FILE="$${CRE_CACHE_FILE:-postgresql://cre:password@127.0.0.1:5432/cre}"; \

application/__init__.py

@@ -49,6 +49,22 @@ def create_app(mode: str = "production", conf: any = None) -> Any:
         letters = string.ascii_lowercase
         app.secret_key = "".join(random.choice(letters) for i in range(20))
 
+    # JWT API-token signing secret: env, or mint+persist once on install/boot.
+    # Tests get a stable in-process default so create_app does not touch /code.
+    try:
+        from application.utils.api_tokens.secret import ensure_api_token_jwt_secret
+
+        if (
+            mode == "test"
+            and not (os.environ.get("API_TOKEN_JWT_SECRET") or "").strip()
+        ):
+            os.environ["API_TOKEN_JWT_SECRET"] = (
+                "test-only-api-token-jwt-secret-32bytes"
+            )
+        ensure_api_token_jwt_secret()
+    except Exception:
+        logger.exception("API_TOKEN_JWT_SECRET bootstrap failed")
+
     # config[mode].init_app(app)
     sqla.init_app(app=app)
     from application.web.web_main import app as app_blueprint
@@ -59,8 +75,21 @@ def create_app(mode: str = "production", conf: any = None) -> Any:
 
     init_openapi(app)
 
-    CORS(app)
-    app.config["CORS_HEADERS"] = "Content-Type"
+    # Credentialed browser calls are same-origin in prod/staging; keep CORS
+    # narrow so cross-site pages cannot drive cookie auth. Admin mutations also
+    # require CSRF (see web_main.admin_imports_enabled_required). Without an
+    # explicit allowlist, flask-cors would reflect any Origin together with
+    # credentials, so fall back to plain (credential-less) CORS as on main.
+    cors_origins = [
+        o.strip()
+        for o in (os.environ.get("CORS_ALLOWED_ORIGINS") or "").split(",")
+        if o.strip()
+    ]
+    if cors_origins:
+        CORS(app, origins=cors_origins, supports_credentials=True)
+    else:
+        CORS(app)
+    app.config["CORS_HEADERS"] = "Content-Type, X-CSRF-Token"
 
     compress.init_app(app)
     cache.init_app(app)