feat(evals): Add security-review vs wrdn-* coverage matrix

.github/workflows/ci.yml

@@ -31,6 +31,8 @@ jobs:
       - run: pnpm lint
       - name: Service and React dashboard tests
         run: pnpm --filter @sentry/warden-service-api test && pnpm --filter @sentry/warden-service test && pnpm --filter warden-service-app test
+      - name: Security eval suite unit tests
+        run: pnpm -C packages/security-evals test
       - name: Test
         env:
           VITEST_JUNIT: test-results.junit.xml

.github/workflows/security-evals.yml

@@ -0,0 +1,108 @@
+name: Security Evals
+
+# Manual only. Runs security-review and the wrdn-* security skills over the
+# sentry vulnerability corpus and hand-picked cases. See packages/security-evals.
+on:
+  workflow_dispatch:
+    inputs:
+      skills:
+        description: Comma-separated lanes (empty runs all six)
+        default: ''
+      entries:
+        description: Comma-separated entry ids or prefixes ending in * (empty runs all)
+        default: ''
+      repeats:
+        description: Passes per shard
+        default: '1'
+      model:
+        description: Analysis model
+        default: openrouter/x-ai/grok-4.5
+      effort:
+        description: Analysis effort
+        default: high
+      aux_model:
+        description: Auxiliary model
+        default: openrouter/openai/gpt-5.6-luna
+      judge_model:
+        description: Scoring model
+        default: openrouter/anthropic/claude-sonnet-4.6
+
+permissions:
+  contents: read
+
+env:
+  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
+
+jobs:
+  security-evals:
+    runs-on: ubuntu-latest
+    timeout-minutes: 360
+    env:
+      SENTRY_CLONE: ${{ github.workspace }}/../sentry
+      SENTRY_PYTHON_CLONE: ${{ github.workspace }}/../sentry-python
+      SENTRY_WIZARD_CLONE: ${{ github.workspace }}/../sentry-wizard
+      DOTAGENTS_CLONE: ${{ github.workspace }}/../dotagents
+      # Must match WARDEN_SKILLS_REF in packages/security-evals/src/options.ts.
+      WARDEN_SKILLS_REF: daf01f970a28450627a8d2d61d9915a742a735e2
+      SKILLS_CHECKOUT: packages/security-evals/.cache/warden-skills
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+      - uses: pnpm/action-setup@02f6c237bd2518259fed6c71566509edfb3f2b74 # v4
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        with:
+          node-version: '24'
+          cache: 'pnpm'
+
+      - run: pnpm install --frozen-lockfile
+
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        with:
+          repository: getsentry/warden-skills
+          ref: ${{ env.WARDEN_SKILLS_REF }}
+          path: ${{ env.SKILLS_CHECKOUT }}
+          persist-credentials: false
+
+      # Keyed on the files that pin the scanned SHAs. Older clones still restore;
+      # the suite fetches whatever SHA they lack.
+      - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: ${{ env.SENTRY_CLONE }}
+          key: sentry-blobless-${{ hashFiles('packages/docs/src/data/benchmarking/sentry-vulnerability-corpus.json', 'packages/security-evals/cases/**') }}
+          restore-keys: sentry-blobless-
+
+      - name: Prepare sentry clone
+        run: packages/security-evals/scripts/prepare-clone.sh getsentry/sentry "$SENTRY_CLONE"
+
+      - name: Prepare sentry-python clone
+        run: packages/security-evals/scripts/prepare-clone.sh getsentry/sentry-python "$SENTRY_PYTHON_CLONE"
+
+      - name: Prepare sentry-wizard clone
+        run: packages/security-evals/scripts/prepare-clone.sh getsentry/sentry-wizard "$SENTRY_WIZARD_CLONE"
+
+      - name: Prepare dotagents clone
+        run: packages/security-evals/scripts/prepare-clone.sh getsentry/dotagents "$DOTAGENTS_CLONE"
+
+      - name: Run security evals
+        env:
+          WARDEN_OPENROUTER_API_KEY: ${{ secrets.WARDEN_OPENROUTER_API_KEY }}
+          ROACH_TOKEN: ${{ secrets.ROACH_TOKEN }}
+          WARDEN_SECURITY_EVALS_REPOS: getsentry/sentry=${{ env.SENTRY_CLONE }},getsentry/sentry-python=${{ env.SENTRY_PYTHON_CLONE }},getsentry/sentry-wizard=${{ env.SENTRY_WIZARD_CLONE }},getsentry/dotagents=${{ env.DOTAGENTS_CLONE }}
+          WARDEN_SECURITY_EVALS_SKILLS_REPO: ${{ github.workspace }}/${{ env.SKILLS_CHECKOUT }}
+          WARDEN_SECURITY_EVALS_SKILLS: ${{ inputs.skills }}
+          WARDEN_SECURITY_EVALS_ENTRIES: ${{ inputs.entries }}
+          WARDEN_SECURITY_EVALS_REPEATS: ${{ inputs.repeats }}
+          WARDEN_SECURITY_EVALS_MODEL: ${{ inputs.model }}
+          WARDEN_SECURITY_EVALS_EFFORT: ${{ inputs.effort }}
+          WARDEN_SECURITY_EVALS_AUX_MODEL: ${{ inputs.aux_model }}
+          WARDEN_SECURITY_EVALS_JUDGE_MODEL: ${{ inputs.judge_model }}
+          # Each sentry worktree is large; drop it once its shard finishes.
+          WARDEN_SECURITY_EVALS_REMOVE_WORKTREES: '1'
+        run: pnpm evals:security
+
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        if: ${{ !cancelled() }}
+        with:
+          name: security-evals-results
+          path: packages/security-evals/.cache/results.json
+          if-no-files-found: warn

AGENTS.md

@@ -39,6 +39,7 @@ packages/
 │       └── examples/  # Example configurations
 ├── docs/              # Astro docs site (warden-docs), deployed via Vercel
 ├── evals/             # Private eval package: runner, scenarios, fixtures, test skills
+├── security-evals/    # Optional security skill matrix over the sentry corpus (pnpm evals:security)
 ```
 
 ## Key Conventions

package.json

@@ -11,13 +11,15 @@
     "dev": "pnpm --filter @sentry/warden dev",
     "lint": "oxlint . && pnpm --filter warden-service-app lint:types",
     "lint:fix": "oxlint --fix . && pnpm --filter warden-service-app lint:types",
-    "test": "pnpm --filter @sentry/warden-service-api test && pnpm --filter @sentry/warden-service test && pnpm --filter warden-service-app test && pnpm --filter @sentry/warden test",
+    "test": "pnpm --filter @sentry/warden-service-api test && pnpm --filter @sentry/warden-service test && pnpm --filter warden-service-app test && pnpm --filter @sentry/warden test && pnpm -C packages/security-evals test",
     "test:coverage": "pnpm --filter @sentry/warden test:coverage",
     "test:watch": "pnpm --filter @sentry/warden test:watch",
     "test:examples": "pnpm --filter @sentry/warden test:examples",
     "evals": "pnpm -C packages/evals evals",
     "evals:scaffold": "pnpm -C packages/evals scaffold",
-    "typecheck": "pnpm --filter @sentry/warden-service-api build && pnpm --filter @sentry/warden-service build && pnpm --filter warden-service-app typecheck && pnpm --filter @sentry/warden typecheck && pnpm -C packages/evals typecheck",
+    "evals:security": "pnpm -C packages/security-evals evals:security",
+    "evals:security:add": "pnpm -C packages/security-evals add-case",
+    "typecheck": "pnpm --filter @sentry/warden-service-api build && pnpm --filter @sentry/warden-service build && pnpm --filter warden-service-app typecheck && pnpm --filter @sentry/warden typecheck && pnpm -C packages/evals typecheck && pnpm -C packages/security-evals typecheck",
     "generate:jsonl-schema": "pnpm --filter @sentry/warden generate:jsonl-schema",
     "docs": "pnpm --filter warden-docs dev",
     "docs:check": "pnpm -C packages/docs check",
@@ -40,6 +42,9 @@
     "packages/evals/*.ts": [
       "oxlint --fix"
     ],
+    "packages/security-evals/**/*.ts": [
+      "oxlint --fix"
+    ],
     "packages/docs/**/*.astro": [
       "pnpm -C packages/docs build"
     ],

packages/security-evals/.gitignore

@@ -0,0 +1 @@
+.cache/

packages/security-evals/README.md

@@ -0,0 +1,185 @@
+# Security Evals
+
+Optional suite that answers one question: does each security skill catch bugs
+the others miss? It runs builtin `security-review` and the `wrdn-*` security
+skills over the same real code and scores every lane against the same list of
+known bugs.
+
+It never runs from `pnpm evals`, the `run-evals` label, or the PR Evals
+workflow. It costs real money. Run it on purpose.
+
+## What it scans
+
+- **Corpus entries**: one per finding in
+  `packages/docs/src/data/benchmarking/sentry-vulnerability-corpus.json`
+  (86 entries, 6 SHAs, 79 files).
+- **Cases**: `cases/<owning-skill>/*.json`, for bug classes the corpus barely
+  covers (authz production catches, DoS, data exfiltration, code execution).
+  `kind: "bug"` is a known bug at `ref`. `kind: "clean"` is the same code after
+  the fix; a hit there is a false positive.
+
+Every entry pins `repository`, `ref`, and `targets`. Skills scan the targets
+inside a full worktree of that ref, so Read and Grep can trace callers, base
+classes, and permission checks the way they do in production.
+
+## Running it
+
+You need a local clone of every repository the selected entries pin and of
+getsentry/warden-skills. Today that is getsentry/sentry for the corpus and most
+cases, getsentry/sentry-python for some `wrdn-pii` cases, and
+getsentry/sentry-wizard and getsentry/dotagents for some `wrdn-code-execution`
+cases. Blobless is enough:
+
+```bash
+for repo in sentry sentry-python sentry-wizard dotagents; do
+  git clone --filter=blob:none https://github.com/getsentry/$repo.git ~/src/$repo
+done
+git clone https://github.com/getsentry/warden-skills.git ~/src/warden-skills
+```
+
+```bash
+# Every lane over every entry
+pnpm evals:security --repo getsentry/sentry=$HOME/src/sentry \
+  --repo getsentry/sentry-python=$HOME/src/sentry-python \
+  --repo getsentry/sentry-wizard=$HOME/src/sentry-wizard \
+  --repo getsentry/dotagents=$HOME/src/dotagents --skills-repo ~/src/warden-skills
+
+# One candidate lane on its own cases, three passes; other lanes come from the baseline
+pnpm evals:security --repo getsentry/sentry=$HOME/src/sentry --skills-repo ~/src/warden-skills \
+  --skills wrdn-dos-review --entries 'wrdn-dos-review/*' --repeats 3
+```
+
+`pnpm evals:security --help` lists every flag. Each one maps to a
+`WARDEN_SECURITY_EVALS_*` env var. Defaults match the production lane: pi
+runtime, `openrouter/x-ai/grok-4.5` at high effort, and
+`openrouter/openai/gpt-5.6-luna` at high effort for auxiliary work. `maxTurns`
+defaults to 100, as in the corpus runbook, so a turn cap does not read as a
+miss. `--skills-repo` is only needed when a `wrdn-*` lane is selected. Repeat
+`--repo owner/name=<path>` per repository; `--sentry-repo <path>` is shorthand
+for the getsentry/sentry one.
+
+Each pinned ref gets `git fetch origin <sha>` only if its clone lacks it, then
+one `git worktree add --detach` under `.cache/`. The warden-skills worktree
+(ref in `src/options.ts`) and every skill load happen once, before any pass. A
+missing clone or a skill that will not load fails the run there, before any
+entry is scored. Each sentry SHA gets its worktree before its first pass, and
+it is reused across lanes, entries, and repeats. Past that one fetch, model
+APIs are the only network a pass touches.
+
+In CI, run the **Security Evals** workflow by hand. It checks out
+warden-skills at the pinned ref, restores a cached blobless sentry clone,
+runs the suite, and uploads `results.json`. Bumping the pin means editing
+`WARDEN_SKILLS_REF` in both `src/options.ts` and the workflow; a unit test
+fails if they drift.
+
+## How it scores
+
+Entries sharing a ref form one shard. Each pass builds one file-mode context
+over the shard's targets and runs every selected skill on it. Per lane and
+entry, only findings in the entry's target files reach the judge, which
+applies the corpus rule: same bug, roughly the same location. Another bug in
+the same file does not count. No findings in the targets is a miss with no
+judge call.
+
+A failed skill pass, failed chunk, or failed judge call records the entry as
+an **error** for that lane, never a miss. Any error fails the run and blocks
+that skill's verdict.
+
+Model calls go through Roach like `packages/evals`. Worktree content is pinned,
+so recordings replay; `roach.json` normalizes the worktree, checkout, and
+skill cache paths.
+
+## Reading results
+
+`.cache/results.json` has sorted keys and no timestamps, so two runs diff
+cleanly. The console and the job summary show:
+
+- one verdict per `wrdn-*` lane, first
+- per lane: recall, bugs found, bugs only that lane found, clean cases
+  flagged, errors, cost, and tokens
+- an entry x lane table of hits over repeats, with error counts
+
+Single runs are noisy: two production-config security-review corpus runs
+agreed on 32 of the 46 entries either one found. That is why the verdict
+wants repeats.
+
+## Proving a skill
+
+Every `wrdn-*` lane gets exactly one verdict against security-review. The
+thresholds live in `VERDICT_RULES` in `src/verdict.ts`.
+
+| verdict | when |
+| --- | --- |
+| `earns-keep` | At least one bug the skill caught in 2 of 3 repeats (or better) that security-review caught in none, both lanes at 3+ repeats. |
+| `no-unique-catches` | No such bug, nothing unsettled, and at least 5 bug cases under `cases/<skill>/` with 3+ repeats in both lanes. |
+| `insufficient-evidence` | Anything else: an errored entry in either lane, no security-review lane, fewer than 5 owned bug cases, fewer than 3 repeats, or a bug the skill caught that security-review caught only some of the time. |
+
+Corpus entries can produce an `earns-keep`, but only owned cases count toward
+the 5. `no-unique-catches` across a real case set is the evidence for
+dropping a skill. A bare zero from one pass is not.
+
+## Adding a case
+
+A good case is a real bug that was fixed, squarely in the owning skill's
+domain, and traceable inside the repository at that ref. Synthetic code and
+bugs that need another service to explain do not belong here.
+
+```bash
+pnpm evals:security:add https://github.com/getsentry/sentry/pull/<n> --skill wrdn-dos-review
+```
+
+It writes `cases/<skill>/<name>.json` (the bug, at the fix commit's parent)
+and `<name>-fixed.json` (the clean twin, at the fix). Targets are the non-test
+`.py` and `.ts` files the fix modified; it prints anything it skipped. A
+commit URL works too. The name comes from the PR title; `--name <slug>` picks
+your own. `GITHUB_TOKEN` avoids rate limits.
+
+Then:
+
+1. Write `summary` in both files: one sentence naming the bug and where it
+   lives. The judge matches findings against it.
+2. `pnpm -C packages/security-evals test`. The suite validator checks the
+   schema, unique ids, no overlap with the corpus, a known skill directory,
+   no TODO summary, a clean twin or a `noClean` reason, and that the baseline
+   only names entries that exist. It runs in CI with no model and no clone.
+3. Run it for real:
+   `pnpm evals:security --repo getsentry/sentry=<clone> --skills-repo <clone> --skills security-review,<skill> --entries <skill>/<name>,<skill>/<name>-fixed --repeats 3`.
+   A case no lane ever finds is still a valid case; say so in the PR.
+
+Drop the clean twin only when the fix is not pinnable, and say why in
+`noClean` on the bug case.
+
+## Baseline
+
+`baseline/results.json` holds lanes a run does not execute, so a comparison
+only pays for the candidate lane. Each lane records its provenance: runtime,
+model, effort, auxiliary model and effort, max turns, judge model, warden
+version, and skill ref (`builtin` or `getsentry/warden-skills@<sha>`). A lane
+whose provenance differs from the current run is dropped with a warning, and
+an entry whose ref changed is skipped. A stale baseline never gets compared
+against a new skill version. Pass `--baseline none` to skip it.
+
+It is empty right now. Fill it after a clean run:
+
+```bash
+pnpm evals:security ... --skills security-review --repeats 3 --output /tmp/sr.json
+pnpm -C packages/security-evals baseline:update /tmp/sr.json
+```
+
+`baseline:update` replaces the tallies for entries the run covered, resets a
+lane whose provenance changed, and refuses results with errors. Update it when
+you bump the warden version, the model, or the warden-skills pin, and after
+adding cases you want compared for free.
+
+## Known gaps
+
+- Owned bug cases today: wrdn-authz 5, wrdn-dos-review 6, wrdn-data-exfil 5,
+  wrdn-code-execution 3, wrdn-pii 6. A skill below 5 reads
+  `insufficient-evidence` unless it earns its keep outright. Code execution is
+  short because fixed, attacker-reachable command, template, or deserialization
+  bugs are rare in getsentry history; most "injection" fixes are workflow YAML
+  or latent hardening with no untrusted input.
+- The corpus has almost no DoS or exfiltration entries. Cases have to carry
+  those skills.
+- Next: harvest candidate cases from production Warden PR comments that led
+  to a fix. Not built.

packages/security-evals/baseline/results.json

@@ -0,0 +1,6 @@
+{
+  "entries": {},
+  "lanes": {},
+  "summary": {},
+  "version": 2
+}

packages/security-evals/cases/wrdn-authz/sentry-scim-restriction-kept.json

@@ -0,0 +1,10 @@
+{
+  "kind": "clean",
+  "summary": "OrganizationSCIMPermission authorizes any caller with member:/team: scopes once scim_enabled is set, instead of restricting SCIM endpoints to the SCIM provider's sentry-app proxy user, so managers or owners can mutate IdP-provisioned members and teams",
+  "repository": "getsentry/sentry",
+  "ref": "c347ba8739618e24da2e590ecfa3d7fc989866ca",
+  "targets": [
+    "src/sentry/core/endpoints/scim/utils.py"
+  ],
+  "source": "https://github.com/getsentry/sentry/pull/130141"
+}

packages/security-evals/cases/wrdn-authz/sentry-scim-restriction-revert.json

@@ -0,0 +1,10 @@
+{
+  "kind": "bug",
+  "summary": "OrganizationSCIMPermission authorizes any caller with member:/team: scopes once scim_enabled is set, instead of restricting SCIM endpoints to the SCIM provider's sentry-app proxy user, so managers or owners can mutate IdP-provisioned members and teams",
+  "repository": "getsentry/sentry",
+  "ref": "9f95acb5135ce59dcd9edccbbf24cb541f7f3bb2",
+  "targets": [
+    "src/sentry/core/endpoints/scim/utils.py"
+  ],
+  "source": "https://github.com/getsentry/sentry/pull/130141"
+}

packages/security-evals/cases/wrdn-authz/sentry-seer-workflow-put-drops-inaccessible-projects.json

@@ -0,0 +1,11 @@
+{
+  "kind": "bug",
+  "summary": "PUT replaces the workflow's project selection with the caller-supplied IDs after checking access only to those IDs, which deletes existing links to projects the caller cannot access and lets a member without global access convert an all_projects workflow to a narrower selection",
+  "repository": "getsentry/sentry",
+  "ref": "d74d48356f2c822f69463c41922823908683e084",
+  "targets": [
+    "src/sentry/seer/endpoints/organization_seer_workflow_configs.py"
+  ],
+  "source": "https://github.com/getsentry/sentry/pull/129734",
+  "noClean": "Production Warden catch on this pull request commit. No fixed commit has been verified, so there is no clean twin yet."
+}

packages/security-evals/cases/wrdn-authz/sentry-service-account-all-projects-team-write.json

@@ -0,0 +1,12 @@
+{
+  "kind": "bug",
+  "summary": "A service account with all_projects gets has_global_access, and _can_create_team_member allows has_global_access callers, so an org:read service account can add an organization member to any team even though all_projects is meant to grant project access only",
+  "repository": "getsentry/sentry",
+  "ref": "3e04ec0ce4b3333ec6c27ac803d6a37e77b67034",
+  "targets": [
+    "src/sentry/auth/access.py",
+    "src/sentry/core/endpoints/organization_member_team_details.py"
+  ],
+  "source": "https://github.com/getsentry/sentry/pull/129343",
+  "noClean": "Production Warden catch on this pull request commit. No fixed commit has been verified, so there is no clean twin yet."
+}