diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f33036a8..8c930c22 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,8 @@ jobs: - run: pnpm lint - name: Service and React dashboard tests run: pnpm --filter @sentry/warden-service-api test && pnpm --filter @sentry/warden-service test && pnpm --filter warden-service-app test + - name: Security eval suite unit tests + run: pnpm -C packages/security-evals test - name: Test env: VITEST_JUNIT: test-results.junit.xml diff --git a/.github/workflows/security-evals.yml b/.github/workflows/security-evals.yml new file mode 100644 index 00000000..7ce3a999 --- /dev/null +++ b/.github/workflows/security-evals.yml @@ -0,0 +1,108 @@ +name: Security Evals + +# Manual only. Runs security-review and the wrdn-* security skills over the +# sentry vulnerability corpus and hand-picked cases. See packages/security-evals. +on: + workflow_dispatch: + inputs: + skills: + description: Comma-separated lanes (empty runs all six) + default: '' + entries: + description: Comma-separated entry ids or prefixes ending in * (empty runs all) + default: '' + repeats: + description: Passes per shard + default: '1' + model: + description: Analysis model + default: openrouter/x-ai/grok-4.5 + effort: + description: Analysis effort + default: high + aux_model: + description: Auxiliary model + default: openrouter/openai/gpt-5.6-luna + judge_model: + description: Scoring model + default: openrouter/anthropic/claude-sonnet-4.6 + +permissions: + contents: read + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + +jobs: + security-evals: + runs-on: ubuntu-latest + timeout-minutes: 360 + env: + SENTRY_CLONE: ${{ github.workspace }}/../sentry + SENTRY_PYTHON_CLONE: ${{ github.workspace }}/../sentry-python + SENTRY_WIZARD_CLONE: ${{ github.workspace }}/../sentry-wizard + DOTAGENTS_CLONE: ${{ github.workspace }}/../dotagents + # Must match WARDEN_SKILLS_REF in packages/security-evals/src/options.ts. + WARDEN_SKILLS_REF: daf01f970a28450627a8d2d61d9915a742a735e2 + SKILLS_CHECKOUT: packages/security-evals/.cache/warden-skills + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - uses: pnpm/action-setup@02f6c237bd2518259fed6c71566509edfb3f2b74 # v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '24' + cache: 'pnpm' + + - run: pnpm install --frozen-lockfile + + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + repository: getsentry/warden-skills + ref: ${{ env.WARDEN_SKILLS_REF }} + path: ${{ env.SKILLS_CHECKOUT }} + persist-credentials: false + + # Keyed on the files that pin the scanned SHAs. Older clones still restore; + # the suite fetches whatever SHA they lack. + - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ${{ env.SENTRY_CLONE }} + key: sentry-blobless-${{ hashFiles('packages/docs/src/data/benchmarking/sentry-vulnerability-corpus.json', 'packages/security-evals/cases/**') }} + restore-keys: sentry-blobless- + + - name: Prepare sentry clone + run: packages/security-evals/scripts/prepare-clone.sh getsentry/sentry "$SENTRY_CLONE" + + - name: Prepare sentry-python clone + run: packages/security-evals/scripts/prepare-clone.sh getsentry/sentry-python "$SENTRY_PYTHON_CLONE" + + - name: Prepare sentry-wizard clone + run: packages/security-evals/scripts/prepare-clone.sh getsentry/sentry-wizard "$SENTRY_WIZARD_CLONE" + + - name: Prepare dotagents clone + run: packages/security-evals/scripts/prepare-clone.sh getsentry/dotagents "$DOTAGENTS_CLONE" + + - name: Run security evals + env: + WARDEN_OPENROUTER_API_KEY: ${{ secrets.WARDEN_OPENROUTER_API_KEY }} + ROACH_TOKEN: ${{ secrets.ROACH_TOKEN }} + WARDEN_SECURITY_EVALS_REPOS: getsentry/sentry=${{ env.SENTRY_CLONE }},getsentry/sentry-python=${{ env.SENTRY_PYTHON_CLONE }},getsentry/sentry-wizard=${{ env.SENTRY_WIZARD_CLONE }},getsentry/dotagents=${{ env.DOTAGENTS_CLONE }} + WARDEN_SECURITY_EVALS_SKILLS_REPO: ${{ github.workspace }}/${{ env.SKILLS_CHECKOUT }} + WARDEN_SECURITY_EVALS_SKILLS: ${{ inputs.skills }} + WARDEN_SECURITY_EVALS_ENTRIES: ${{ inputs.entries }} + WARDEN_SECURITY_EVALS_REPEATS: ${{ inputs.repeats }} + WARDEN_SECURITY_EVALS_MODEL: ${{ inputs.model }} + WARDEN_SECURITY_EVALS_EFFORT: ${{ inputs.effort }} + WARDEN_SECURITY_EVALS_AUX_MODEL: ${{ inputs.aux_model }} + WARDEN_SECURITY_EVALS_JUDGE_MODEL: ${{ inputs.judge_model }} + # Each sentry worktree is large; drop it once its shard finishes. + WARDEN_SECURITY_EVALS_REMOVE_WORKTREES: '1' + run: pnpm evals:security + + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + if: ${{ !cancelled() }} + with: + name: security-evals-results + path: packages/security-evals/.cache/results.json + if-no-files-found: warn diff --git a/AGENTS.md b/AGENTS.md index 2235d502..2e745710 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,6 +39,7 @@ packages/ │ └── examples/ # Example configurations ├── docs/ # Astro docs site (warden-docs), deployed via Vercel ├── evals/ # Private eval package: runner, scenarios, fixtures, test skills +├── security-evals/ # Optional security skill matrix over the sentry corpus (pnpm evals:security) ``` ## Key Conventions diff --git a/package.json b/package.json index e8a26a96..1ce5348a 100644 --- a/package.json +++ b/package.json @@ -11,13 +11,15 @@ "dev": "pnpm --filter @sentry/warden dev", "lint": "oxlint . && pnpm --filter warden-service-app lint:types", "lint:fix": "oxlint --fix . && pnpm --filter warden-service-app lint:types", - "test": "pnpm --filter @sentry/warden-service-api test && pnpm --filter @sentry/warden-service test && pnpm --filter warden-service-app test && pnpm --filter @sentry/warden test", + "test": "pnpm --filter @sentry/warden-service-api test && pnpm --filter @sentry/warden-service test && pnpm --filter warden-service-app test && pnpm --filter @sentry/warden test && pnpm -C packages/security-evals test", "test:coverage": "pnpm --filter @sentry/warden test:coverage", "test:watch": "pnpm --filter @sentry/warden test:watch", "test:examples": "pnpm --filter @sentry/warden test:examples", "evals": "pnpm -C packages/evals evals", "evals:scaffold": "pnpm -C packages/evals scaffold", - "typecheck": "pnpm --filter @sentry/warden-service-api build && pnpm --filter @sentry/warden-service build && pnpm --filter warden-service-app typecheck && pnpm --filter @sentry/warden typecheck && pnpm -C packages/evals typecheck", + "evals:security": "pnpm -C packages/security-evals evals:security", + "evals:security:add": "pnpm -C packages/security-evals add-case", + "typecheck": "pnpm --filter @sentry/warden-service-api build && pnpm --filter @sentry/warden-service build && pnpm --filter warden-service-app typecheck && pnpm --filter @sentry/warden typecheck && pnpm -C packages/evals typecheck && pnpm -C packages/security-evals typecheck", "generate:jsonl-schema": "pnpm --filter @sentry/warden generate:jsonl-schema", "docs": "pnpm --filter warden-docs dev", "docs:check": "pnpm -C packages/docs check", @@ -40,6 +42,9 @@ "packages/evals/*.ts": [ "oxlint --fix" ], + "packages/security-evals/**/*.ts": [ + "oxlint --fix" + ], "packages/docs/**/*.astro": [ "pnpm -C packages/docs build" ], diff --git a/packages/security-evals/.gitignore b/packages/security-evals/.gitignore new file mode 100644 index 00000000..ceddaa37 --- /dev/null +++ b/packages/security-evals/.gitignore @@ -0,0 +1 @@ +.cache/ diff --git a/packages/security-evals/README.md b/packages/security-evals/README.md new file mode 100644 index 00000000..c079fd35 --- /dev/null +++ b/packages/security-evals/README.md @@ -0,0 +1,185 @@ +# Security Evals + +Optional suite that answers one question: does each security skill catch bugs +the others miss? It runs builtin `security-review` and the `wrdn-*` security +skills over the same real code and scores every lane against the same list of +known bugs. + +It never runs from `pnpm evals`, the `run-evals` label, or the PR Evals +workflow. It costs real money. Run it on purpose. + +## What it scans + +- **Corpus entries**: one per finding in + `packages/docs/src/data/benchmarking/sentry-vulnerability-corpus.json` + (86 entries, 6 SHAs, 79 files). +- **Cases**: `cases//*.json`, for bug classes the corpus barely + covers (authz production catches, DoS, data exfiltration, code execution). + `kind: "bug"` is a known bug at `ref`. `kind: "clean"` is the same code after + the fix; a hit there is a false positive. + +Every entry pins `repository`, `ref`, and `targets`. Skills scan the targets +inside a full worktree of that ref, so Read and Grep can trace callers, base +classes, and permission checks the way they do in production. + +## Running it + +You need a local clone of every repository the selected entries pin and of +getsentry/warden-skills. Today that is getsentry/sentry for the corpus and most +cases, getsentry/sentry-python for some `wrdn-pii` cases, and +getsentry/sentry-wizard and getsentry/dotagents for some `wrdn-code-execution` +cases. Blobless is enough: + +```bash +for repo in sentry sentry-python sentry-wizard dotagents; do + git clone --filter=blob:none https://github.com/getsentry/$repo.git ~/src/$repo +done +git clone https://github.com/getsentry/warden-skills.git ~/src/warden-skills +``` + +```bash +# Every lane over every entry +pnpm evals:security --repo getsentry/sentry=$HOME/src/sentry \ + --repo getsentry/sentry-python=$HOME/src/sentry-python \ + --repo getsentry/sentry-wizard=$HOME/src/sentry-wizard \ + --repo getsentry/dotagents=$HOME/src/dotagents --skills-repo ~/src/warden-skills + +# One candidate lane on its own cases, three passes; other lanes come from the baseline +pnpm evals:security --repo getsentry/sentry=$HOME/src/sentry --skills-repo ~/src/warden-skills \ + --skills wrdn-dos-review --entries 'wrdn-dos-review/*' --repeats 3 +``` + +`pnpm evals:security --help` lists every flag. Each one maps to a +`WARDEN_SECURITY_EVALS_*` env var. Defaults match the production lane: pi +runtime, `openrouter/x-ai/grok-4.5` at high effort, and +`openrouter/openai/gpt-5.6-luna` at high effort for auxiliary work. `maxTurns` +defaults to 100, as in the corpus runbook, so a turn cap does not read as a +miss. `--skills-repo` is only needed when a `wrdn-*` lane is selected. Repeat +`--repo owner/name=` per repository; `--sentry-repo ` is shorthand +for the getsentry/sentry one. + +Each pinned ref gets `git fetch origin ` only if its clone lacks it, then +one `git worktree add --detach` under `.cache/`. The warden-skills worktree +(ref in `src/options.ts`) and every skill load happen once, before any pass. A +missing clone or a skill that will not load fails the run there, before any +entry is scored. Each sentry SHA gets its worktree before its first pass, and +it is reused across lanes, entries, and repeats. Past that one fetch, model +APIs are the only network a pass touches. + +In CI, run the **Security Evals** workflow by hand. It checks out +warden-skills at the pinned ref, restores a cached blobless sentry clone, +runs the suite, and uploads `results.json`. Bumping the pin means editing +`WARDEN_SKILLS_REF` in both `src/options.ts` and the workflow; a unit test +fails if they drift. + +## How it scores + +Entries sharing a ref form one shard. Each pass builds one file-mode context +over the shard's targets and runs every selected skill on it. Per lane and +entry, only findings in the entry's target files reach the judge, which +applies the corpus rule: same bug, roughly the same location. Another bug in +the same file does not count. No findings in the targets is a miss with no +judge call. + +A failed skill pass, failed chunk, or failed judge call records the entry as +an **error** for that lane, never a miss. Any error fails the run and blocks +that skill's verdict. + +Model calls go through Roach like `packages/evals`. Worktree content is pinned, +so recordings replay; `roach.json` normalizes the worktree, checkout, and +skill cache paths. + +## Reading results + +`.cache/results.json` has sorted keys and no timestamps, so two runs diff +cleanly. The console and the job summary show: + +- one verdict per `wrdn-*` lane, first +- per lane: recall, bugs found, bugs only that lane found, clean cases + flagged, errors, cost, and tokens +- an entry x lane table of hits over repeats, with error counts + +Single runs are noisy: two production-config security-review corpus runs +agreed on 32 of the 46 entries either one found. That is why the verdict +wants repeats. + +## Proving a skill + +Every `wrdn-*` lane gets exactly one verdict against security-review. The +thresholds live in `VERDICT_RULES` in `src/verdict.ts`. + +| verdict | when | +| --- | --- | +| `earns-keep` | At least one bug the skill caught in 2 of 3 repeats (or better) that security-review caught in none, both lanes at 3+ repeats. | +| `no-unique-catches` | No such bug, nothing unsettled, and at least 5 bug cases under `cases//` with 3+ repeats in both lanes. | +| `insufficient-evidence` | Anything else: an errored entry in either lane, no security-review lane, fewer than 5 owned bug cases, fewer than 3 repeats, or a bug the skill caught that security-review caught only some of the time. | + +Corpus entries can produce an `earns-keep`, but only owned cases count toward +the 5. `no-unique-catches` across a real case set is the evidence for +dropping a skill. A bare zero from one pass is not. + +## Adding a case + +A good case is a real bug that was fixed, squarely in the owning skill's +domain, and traceable inside the repository at that ref. Synthetic code and +bugs that need another service to explain do not belong here. + +```bash +pnpm evals:security:add https://github.com/getsentry/sentry/pull/ --skill wrdn-dos-review +``` + +It writes `cases//.json` (the bug, at the fix commit's parent) +and `-fixed.json` (the clean twin, at the fix). Targets are the non-test +`.py` and `.ts` files the fix modified; it prints anything it skipped. A +commit URL works too. The name comes from the PR title; `--name ` picks +your own. `GITHUB_TOKEN` avoids rate limits. + +Then: + +1. Write `summary` in both files: one sentence naming the bug and where it + lives. The judge matches findings against it. +2. `pnpm -C packages/security-evals test`. The suite validator checks the + schema, unique ids, no overlap with the corpus, a known skill directory, + no TODO summary, a clean twin or a `noClean` reason, and that the baseline + only names entries that exist. It runs in CI with no model and no clone. +3. Run it for real: + `pnpm evals:security --repo getsentry/sentry= --skills-repo --skills security-review, --entries /,/-fixed --repeats 3`. + A case no lane ever finds is still a valid case; say so in the PR. + +Drop the clean twin only when the fix is not pinnable, and say why in +`noClean` on the bug case. + +## Baseline + +`baseline/results.json` holds lanes a run does not execute, so a comparison +only pays for the candidate lane. Each lane records its provenance: runtime, +model, effort, auxiliary model and effort, max turns, judge model, warden +version, and skill ref (`builtin` or `getsentry/warden-skills@`). A lane +whose provenance differs from the current run is dropped with a warning, and +an entry whose ref changed is skipped. A stale baseline never gets compared +against a new skill version. Pass `--baseline none` to skip it. + +It is empty right now. Fill it after a clean run: + +```bash +pnpm evals:security ... --skills security-review --repeats 3 --output /tmp/sr.json +pnpm -C packages/security-evals baseline:update /tmp/sr.json +``` + +`baseline:update` replaces the tallies for entries the run covered, resets a +lane whose provenance changed, and refuses results with errors. Update it when +you bump the warden version, the model, or the warden-skills pin, and after +adding cases you want compared for free. + +## Known gaps + +- Owned bug cases today: wrdn-authz 5, wrdn-dos-review 6, wrdn-data-exfil 5, + wrdn-code-execution 3, wrdn-pii 6. A skill below 5 reads + `insufficient-evidence` unless it earns its keep outright. Code execution is + short because fixed, attacker-reachable command, template, or deserialization + bugs are rare in getsentry history; most "injection" fixes are workflow YAML + or latent hardening with no untrusted input. +- The corpus has almost no DoS or exfiltration entries. Cases have to carry + those skills. +- Next: harvest candidate cases from production Warden PR comments that led + to a fix. Not built. diff --git a/packages/security-evals/baseline/results.json b/packages/security-evals/baseline/results.json new file mode 100644 index 00000000..9341710a --- /dev/null +++ b/packages/security-evals/baseline/results.json @@ -0,0 +1,6 @@ +{ + "entries": {}, + "lanes": {}, + "summary": {}, + "version": 2 +} diff --git a/packages/security-evals/cases/wrdn-authz/sentry-scim-restriction-kept.json b/packages/security-evals/cases/wrdn-authz/sentry-scim-restriction-kept.json new file mode 100644 index 00000000..59ea2235 --- /dev/null +++ b/packages/security-evals/cases/wrdn-authz/sentry-scim-restriction-kept.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "OrganizationSCIMPermission authorizes any caller with member:/team: scopes once scim_enabled is set, instead of restricting SCIM endpoints to the SCIM provider's sentry-app proxy user, so managers or owners can mutate IdP-provisioned members and teams", + "repository": "getsentry/sentry", + "ref": "c347ba8739618e24da2e590ecfa3d7fc989866ca", + "targets": [ + "src/sentry/core/endpoints/scim/utils.py" + ], + "source": "https://github.com/getsentry/sentry/pull/130141" +} diff --git a/packages/security-evals/cases/wrdn-authz/sentry-scim-restriction-revert.json b/packages/security-evals/cases/wrdn-authz/sentry-scim-restriction-revert.json new file mode 100644 index 00000000..67cde197 --- /dev/null +++ b/packages/security-evals/cases/wrdn-authz/sentry-scim-restriction-revert.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "OrganizationSCIMPermission authorizes any caller with member:/team: scopes once scim_enabled is set, instead of restricting SCIM endpoints to the SCIM provider's sentry-app proxy user, so managers or owners can mutate IdP-provisioned members and teams", + "repository": "getsentry/sentry", + "ref": "9f95acb5135ce59dcd9edccbbf24cb541f7f3bb2", + "targets": [ + "src/sentry/core/endpoints/scim/utils.py" + ], + "source": "https://github.com/getsentry/sentry/pull/130141" +} diff --git a/packages/security-evals/cases/wrdn-authz/sentry-seer-workflow-put-drops-inaccessible-projects.json b/packages/security-evals/cases/wrdn-authz/sentry-seer-workflow-put-drops-inaccessible-projects.json new file mode 100644 index 00000000..cea0944e --- /dev/null +++ b/packages/security-evals/cases/wrdn-authz/sentry-seer-workflow-put-drops-inaccessible-projects.json @@ -0,0 +1,11 @@ +{ + "kind": "bug", + "summary": "PUT replaces the workflow's project selection with the caller-supplied IDs after checking access only to those IDs, which deletes existing links to projects the caller cannot access and lets a member without global access convert an all_projects workflow to a narrower selection", + "repository": "getsentry/sentry", + "ref": "d74d48356f2c822f69463c41922823908683e084", + "targets": [ + "src/sentry/seer/endpoints/organization_seer_workflow_configs.py" + ], + "source": "https://github.com/getsentry/sentry/pull/129734", + "noClean": "Production Warden catch on this pull request commit. No fixed commit has been verified, so there is no clean twin yet." +} diff --git a/packages/security-evals/cases/wrdn-authz/sentry-service-account-all-projects-team-write.json b/packages/security-evals/cases/wrdn-authz/sentry-service-account-all-projects-team-write.json new file mode 100644 index 00000000..504ea652 --- /dev/null +++ b/packages/security-evals/cases/wrdn-authz/sentry-service-account-all-projects-team-write.json @@ -0,0 +1,12 @@ +{ + "kind": "bug", + "summary": "A service account with all_projects gets has_global_access, and _can_create_team_member allows has_global_access callers, so an org:read service account can add an organization member to any team even though all_projects is meant to grant project access only", + "repository": "getsentry/sentry", + "ref": "3e04ec0ce4b3333ec6c27ac803d6a37e77b67034", + "targets": [ + "src/sentry/auth/access.py", + "src/sentry/core/endpoints/organization_member_team_details.py" + ], + "source": "https://github.com/getsentry/sentry/pull/129343", + "noClean": "Production Warden catch on this pull request commit. No fixed commit has been verified, so there is no clean twin yet." +} diff --git a/packages/security-evals/cases/wrdn-authz/sentry-service-account-org-projects-listing.json b/packages/security-evals/cases/wrdn-authz/sentry-service-account-org-projects-listing.json new file mode 100644 index 00000000..31cbe076 --- /dev/null +++ b/packages/security-evals/cases/wrdn-authz/sentry-service-account-org-projects-listing.json @@ -0,0 +1,11 @@ +{ + "kind": "bug", + "summary": "GET organization projects returns every project in the organization to a service account without global access, because the non-human auth branch does not filter by request.access.accessible_project_ids the way the agent-token branch does", + "repository": "getsentry/sentry", + "ref": "595efb30334e9427e572792031929c0298d24703", + "targets": [ + "src/sentry/core/endpoints/organization_projects.py" + ], + "source": "https://github.com/getsentry/sentry/pull/129154", + "noClean": "Production Warden catch on this pull request commit. No fixed commit has been verified, so there is no clean twin yet." +} diff --git a/packages/security-evals/cases/wrdn-authz/sentry-team-role-downgrade-fail-open-fixed.json b/packages/security-evals/cases/wrdn-authz/sentry-team-role-downgrade-fail-open-fixed.json new file mode 100644 index 00000000..64d28fa8 --- /dev/null +++ b/packages/security-evals/cases/wrdn-authz/sentry-team-role-downgrade-fail-open-fixed.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "OrganizationMemberTeamDetailsEndpoint.put treats a member with no explicit team role as freely changeable (can_set_old_role defaults to True), so a team contributor can rewrite the team role of an org admin or manager who inherits team admin from their org role, demoting them on that team", + "repository": "getsentry/sentry", + "ref": "d714026543ec355782969c8e702ca7e9d56251fd", + "targets": [ + "src/sentry/core/endpoints/organization_member_team_details.py" + ], + "source": "https://github.com/getsentry/sentry/commit/d714026543ec355782969c8e702ca7e9d56251fd" +} diff --git a/packages/security-evals/cases/wrdn-authz/sentry-team-role-downgrade-fail-open.json b/packages/security-evals/cases/wrdn-authz/sentry-team-role-downgrade-fail-open.json new file mode 100644 index 00000000..243b83ba --- /dev/null +++ b/packages/security-evals/cases/wrdn-authz/sentry-team-role-downgrade-fail-open.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "OrganizationMemberTeamDetailsEndpoint.put treats a member with no explicit team role as freely changeable (can_set_old_role defaults to True), so a team contributor can rewrite the team role of an org admin or manager who inherits team admin from their org role, demoting them on that team", + "repository": "getsentry/sentry", + "ref": "1dd5d51234118842768ed532b75fb2913b28daac", + "targets": [ + "src/sentry/core/endpoints/organization_member_team_details.py" + ], + "source": "https://github.com/getsentry/sentry/commit/d714026543ec355782969c8e702ca7e9d56251fd" +} diff --git a/packages/security-evals/cases/wrdn-code-execution/dotagents-git-argument-injection-fixed.json b/packages/security-evals/cases/wrdn-code-execution/dotagents-git-argument-injection-fixed.json new file mode 100644 index 00000000..4f75943f --- /dev/null +++ b/packages/security-evals/cases/wrdn-code-execution/dotagents-git-argument-injection-fixed.json @@ -0,0 +1,11 @@ +{ + "kind": "clean", + "summary": "sources/git.ts passes the skill source URL and ref from agents.toml to git clone and git fetch as positional arguments with no -- separator, and the config schema accepts any git: string, so a source or ref starting with --upload-pack= (or another option) turns into a git option that runs an attacker-chosen command when someone installs skills in a repo carrying that agents.toml", + "repository": "getsentry/dotagents", + "ref": "461da846ce520b49b17f66ce00c5859a1a04a644", + "targets": [ + "src/sources/git.ts", + "src/config/schema.ts" + ], + "source": "https://github.com/getsentry/dotagents/commit/461da846ce520b49b17f66ce00c5859a1a04a644" +} diff --git a/packages/security-evals/cases/wrdn-code-execution/dotagents-git-argument-injection.json b/packages/security-evals/cases/wrdn-code-execution/dotagents-git-argument-injection.json new file mode 100644 index 00000000..5d96d96e --- /dev/null +++ b/packages/security-evals/cases/wrdn-code-execution/dotagents-git-argument-injection.json @@ -0,0 +1,11 @@ +{ + "kind": "bug", + "summary": "sources/git.ts passes the skill source URL and ref from agents.toml to git clone and git fetch as positional arguments with no -- separator, and the config schema accepts any git: string, so a source or ref starting with --upload-pack= (or another option) turns into a git option that runs an attacker-chosen command when someone installs skills in a repo carrying that agents.toml", + "repository": "getsentry/dotagents", + "ref": "f17bb9edbe132b67e1987c74d9991d167b16dcf9", + "targets": [ + "src/sources/git.ts", + "src/config/schema.ts" + ], + "source": "https://github.com/getsentry/dotagents/commit/461da846ce520b49b17f66ce00c5859a1a04a644" +} diff --git a/packages/security-evals/cases/wrdn-code-execution/sentry-internal-feature-flags-config-injection.json b/packages/security-evals/cases/wrdn-code-execution/sentry-internal-feature-flags-config-injection.json new file mode 100644 index 00000000..a20a8a13 --- /dev/null +++ b/packages/security-evals/cases/wrdn-code-execution/sentry-internal-feature-flags-config-injection.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "PUT interpolates request.data values verbatim into SENTRY_FEATURES[\"flag\"]= lines in sentry.conf.py and then calls configure(), which executes the file, so a crafted value injects arbitrary Python that runs on the server", + "repository": "getsentry/sentry", + "ref": "101823f59fee6d71221feceb0cbcb28cc6024606", + "targets": [ + "src/sentry/api/endpoints/internal/feature_flags.py" + ], + "source": "https://github.com/getsentry/sentry/commit/2b2dd08cfd33eb13079bfa15b7e27863f157b866" +} diff --git a/packages/security-evals/cases/wrdn-code-execution/sentry-internal-feature-flags-restricted-values.json b/packages/security-evals/cases/wrdn-code-execution/sentry-internal-feature-flags-restricted-values.json new file mode 100644 index 00000000..7db3ff21 --- /dev/null +++ b/packages/security-evals/cases/wrdn-code-execution/sentry-internal-feature-flags-restricted-values.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "PUT interpolates request.data values verbatim into SENTRY_FEATURES[\"flag\"]= lines in sentry.conf.py and then calls configure(), which executes the file, so a crafted value injects arbitrary Python that runs on the server", + "repository": "getsentry/sentry", + "ref": "2b2dd08cfd33eb13079bfa15b7e27863f157b866", + "targets": [ + "src/sentry/api/endpoints/internal/feature_flags.py" + ], + "source": "https://github.com/getsentry/sentry/commit/2b2dd08cfd33eb13079bfa15b7e27863f157b866" +} diff --git a/packages/security-evals/cases/wrdn-code-execution/sentry-wizard-formatter-filename-injection-fixed.json b/packages/security-evals/cases/wrdn-code-execution/sentry-wizard-formatter-filename-injection-fixed.json new file mode 100644 index 00000000..5a10a289 --- /dev/null +++ b/packages/security-evals/cases/wrdn-code-execution/sentry-wizard-formatter-filename-injection-fixed.json @@ -0,0 +1,11 @@ +{ + "kind": "clean", + "summary": "getFormatterTargetFiles joins the changed and untracked file names from git status into one space-separated string, and the formatter step interpolates it into childProcess.exec command lines (npx prettier --write ${targetFiles}, npx @biomejs/biome ...), so a file in the target repo named with shell syntax such as $(touch${IFS}pwned).js runs arbitrary commands on the developer machine when they run the wizard", + "repository": "getsentry/sentry-wizard", + "ref": "9a4f3c9282d4cb47978e01d391b0ecc87f6f437a", + "targets": [ + "src/utils/clack/index.ts", + "src/utils/git.ts" + ], + "source": "https://github.com/getsentry/sentry-wizard/commit/9a4f3c9282d4cb47978e01d391b0ecc87f6f437a" +} diff --git a/packages/security-evals/cases/wrdn-code-execution/sentry-wizard-formatter-filename-injection.json b/packages/security-evals/cases/wrdn-code-execution/sentry-wizard-formatter-filename-injection.json new file mode 100644 index 00000000..f6192a90 --- /dev/null +++ b/packages/security-evals/cases/wrdn-code-execution/sentry-wizard-formatter-filename-injection.json @@ -0,0 +1,11 @@ +{ + "kind": "bug", + "summary": "getFormatterTargetFiles joins the changed and untracked file names from git status into one space-separated string, and the formatter step interpolates it into childProcess.exec command lines (npx prettier --write ${targetFiles}, npx @biomejs/biome ...), so a file in the target repo named with shell syntax such as $(touch${IFS}pwned).js runs arbitrary commands on the developer machine when they run the wizard", + "repository": "getsentry/sentry-wizard", + "ref": "0144d69f365c5812f927a692e8433eb20304ce9c", + "targets": [ + "src/utils/clack/index.ts", + "src/utils/git.ts" + ], + "source": "https://github.com/getsentry/sentry-wizard/commit/9a4f3c9282d4cb47978e01d391b0ecc87f6f437a" +} diff --git a/packages/security-evals/cases/wrdn-data-exfil/sentry-discover-fn-span-count-function-injection-fixed.json b/packages/security-evals/cases/wrdn-data-exfil/sentry-discover-fn-span-count-function-injection-fixed.json new file mode 100644 index 00000000..af88d49a --- /dev/null +++ b/packages/security-evals/cases/wrdn-data-exfil/sentry-discover-fn-span-count-function-injection-fixed.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "DiscoverDatasetConfig's fn_span_count declares its fn argument as an SnQLStringArg with no allowed_strings, so the caller's value passes through unchanged and becomes the name of the generated ClickHouse function, letting an events API caller call arbitrary ClickHouse functions in the query Sentry sends to Snuba", + "repository": "getsentry/sentry", + "ref": "be425544501ac37ccc6730d78d39b830bf69021e", + "targets": [ + "src/sentry/search/events/datasets/discover.py" + ], + "source": "https://github.com/getsentry/sentry/commit/be425544501ac37ccc6730d78d39b830bf69021e" +} diff --git a/packages/security-evals/cases/wrdn-data-exfil/sentry-discover-fn-span-count-function-injection.json b/packages/security-evals/cases/wrdn-data-exfil/sentry-discover-fn-span-count-function-injection.json new file mode 100644 index 00000000..e0286dbe --- /dev/null +++ b/packages/security-evals/cases/wrdn-data-exfil/sentry-discover-fn-span-count-function-injection.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "DiscoverDatasetConfig's fn_span_count declares its fn argument as an SnQLStringArg with no allowed_strings, so the caller's value passes through unchanged and becomes the name of the generated ClickHouse function, letting an events API caller call arbitrary ClickHouse functions in the query Sentry sends to Snuba", + "repository": "getsentry/sentry", + "ref": "d336e8ecfccf6334d52d6b3b0845f979ade9997c", + "targets": [ + "src/sentry/search/events/datasets/discover.py" + ], + "source": "https://github.com/getsentry/sentry/commit/be425544501ac37ccc6730d78d39b830bf69021e" +} diff --git a/packages/security-evals/cases/wrdn-data-exfil/sentry-missing-members-domain-sqli-fixed.json b/packages/security-evals/cases/wrdn-data-exfil/sentry-missing-members-domain-sqli-fixed.json new file mode 100644 index 00000000..42bbffe7 --- /dev/null +++ b/packages/security-evals/cases/wrdn-data-exfil/sentry-missing-members-domain-sqli-fixed.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "_get_missing_organization_members in organization_missing_org_members.py f-string interpolates shared_domain, the email domain shared by the org owners, into the raw CommitAuthor SQL LIKE clause instead of passing it as a query parameter, so an owner email whose domain carries SQL (via import, relocation, or any path that skips Django email validation) injects into a query against the shared database", + "repository": "getsentry/sentry", + "ref": "dd85c7170dd98bbe2e5fc37c2ac7d182e4152bfc", + "targets": [ + "src/sentry/api/endpoints/organization_missing_org_members.py" + ], + "source": "https://github.com/getsentry/sentry/commit/dd85c7170dd98bbe2e5fc37c2ac7d182e4152bfc" +} diff --git a/packages/security-evals/cases/wrdn-data-exfil/sentry-missing-members-domain-sqli.json b/packages/security-evals/cases/wrdn-data-exfil/sentry-missing-members-domain-sqli.json new file mode 100644 index 00000000..e0a5729b --- /dev/null +++ b/packages/security-evals/cases/wrdn-data-exfil/sentry-missing-members-domain-sqli.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "_get_missing_organization_members in organization_missing_org_members.py f-string interpolates shared_domain, the email domain shared by the org owners, into the raw CommitAuthor SQL LIKE clause instead of passing it as a query parameter, so an owner email whose domain carries SQL (via import, relocation, or any path that skips Django email validation) injects into a query against the shared database", + "repository": "getsentry/sentry", + "ref": "b74117a6cd90e4d11bc44dc4ef542cebb480857b", + "targets": [ + "src/sentry/api/endpoints/organization_missing_org_members.py" + ], + "source": "https://github.com/getsentry/sentry/commit/dd85c7170dd98bbe2e5fc37c2ac7d182e4152bfc" +} diff --git a/packages/security-evals/cases/wrdn-data-exfil/sentry-scim-token-plaintext-settings-page-fixed.json b/packages/security-evals/cases/wrdn-data-exfil/sentry-scim-token-plaintext-settings-page-fixed.json new file mode 100644 index 00000000..be09da4b --- /dev/null +++ b/packages/security-evals/cases/wrdn-data-exfil/sentry-scim-token-plaintext-settings-page-fixed.json @@ -0,0 +1,12 @@ +{ + "kind": "clean", + "summary": "organization_auth_settings.handle_existing_provider puts auth_provider.get_scim_token() into the SSO settings page context, and the template renders the full SCIM bearer token in plaintext to every manager-level viewer for as long as SCIM stays enabled, so anyone who can open the page can copy a long-lived credential that provisions and deprovisions org members", + "repository": "getsentry/sentry", + "ref": "d84bf17e962269389a9f0fe6521e2f87c785f31d", + "targets": [ + "src/sentry/web/frontend/organization_auth_settings.py", + "src/sentry/models/authprovider.py", + "src/sentry/auth/services/auth/model.py" + ], + "source": "https://github.com/getsentry/sentry/commit/d84bf17e962269389a9f0fe6521e2f87c785f31d" +} diff --git a/packages/security-evals/cases/wrdn-data-exfil/sentry-scim-token-plaintext-settings-page.json b/packages/security-evals/cases/wrdn-data-exfil/sentry-scim-token-plaintext-settings-page.json new file mode 100644 index 00000000..d9611910 --- /dev/null +++ b/packages/security-evals/cases/wrdn-data-exfil/sentry-scim-token-plaintext-settings-page.json @@ -0,0 +1,12 @@ +{ + "kind": "bug", + "summary": "organization_auth_settings.handle_existing_provider puts auth_provider.get_scim_token() into the SSO settings page context, and the template renders the full SCIM bearer token in plaintext to every manager-level viewer for as long as SCIM stays enabled, so anyone who can open the page can copy a long-lived credential that provisions and deprovisions org members", + "repository": "getsentry/sentry", + "ref": "6b546fd93d8a2dcd533c655b6415b5f233ac818b", + "targets": [ + "src/sentry/web/frontend/organization_auth_settings.py", + "src/sentry/models/authprovider.py", + "src/sentry/auth/services/auth/model.py" + ], + "source": "https://github.com/getsentry/sentry/commit/d84bf17e962269389a9f0fe6521e2f87c785f31d" +} diff --git a/packages/security-evals/cases/wrdn-data-exfil/sentry-system-options-logs-redacted.json b/packages/security-evals/cases/wrdn-data-exfil/sentry-system-options-logs-redacted.json new file mode 100644 index 00000000..3f82b787 --- /dev/null +++ b/packages/security-evals/cases/wrdn-data-exfil/sentry-system-options-logs-redacted.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "The options.update log line records option_value verbatim for every option, so secret options such as mail passwords or API keys are written to application logs in cleartext", + "repository": "getsentry/sentry", + "ref": "36ae5236a01ac174b5d3a432ac350fd469f05441", + "targets": [ + "src/sentry/api/endpoints/system_options.py" + ], + "source": "https://github.com/getsentry/sentry/commit/36ae5236a01ac174b5d3a432ac350fd469f05441" +} diff --git a/packages/security-evals/cases/wrdn-data-exfil/sentry-system-options-logs-secret-values.json b/packages/security-evals/cases/wrdn-data-exfil/sentry-system-options-logs-secret-values.json new file mode 100644 index 00000000..675a2161 --- /dev/null +++ b/packages/security-evals/cases/wrdn-data-exfil/sentry-system-options-logs-secret-values.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "The options.update log line records option_value verbatim for every option, so secret options such as mail passwords or API keys are written to application logs in cleartext", + "repository": "getsentry/sentry", + "ref": "c1a9c001e3410f2087321d9554af59d62a9b6b1a", + "targets": [ + "src/sentry/api/endpoints/system_options.py" + ], + "source": "https://github.com/getsentry/sentry/commit/36ae5236a01ac174b5d3a432ac350fd469f05441" +} diff --git a/packages/security-evals/cases/wrdn-data-exfil/sentry-uptime-robots-txt-safe-fetch.json b/packages/security-evals/cases/wrdn-data-exfil/sentry-uptime-robots-txt-safe-fetch.json new file mode 100644 index 00000000..2b3a9a10 --- /dev/null +++ b/packages/security-evals/cases/wrdn-data-exfil/sentry-uptime-robots-txt-safe-fetch.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "get_robots_txt_parser / check_url_robots_txt fetch robots.txt for event-derived URLs through RobotFileParser.read (raw urllib), bypassing safe_urlopen and its restricted-IP checks, so an attacker-supplied URL makes Sentry request internal or cloud metadata addresses (SSRF)", + "repository": "getsentry/sentry", + "ref": "e84e7a45b001de51cab826593b8a2f5f0f0b323b", + "targets": [ + "src/sentry/uptime/autodetect/tasks.py" + ], + "source": "https://github.com/getsentry/sentry/commit/e84e7a45b001de51cab826593b8a2f5f0f0b323b" +} diff --git a/packages/security-evals/cases/wrdn-data-exfil/sentry-uptime-robots-txt-ssrf.json b/packages/security-evals/cases/wrdn-data-exfil/sentry-uptime-robots-txt-ssrf.json new file mode 100644 index 00000000..dedd5e60 --- /dev/null +++ b/packages/security-evals/cases/wrdn-data-exfil/sentry-uptime-robots-txt-ssrf.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "get_robots_txt_parser / check_url_robots_txt fetch robots.txt for event-derived URLs through RobotFileParser.read (raw urllib), bypassing safe_urlopen and its restricted-IP checks, so an attacker-supplied URL makes Sentry request internal or cloud metadata addresses (SSRF)", + "repository": "getsentry/sentry", + "ref": "86006042764d70705ca51df36e3f215effd3113d", + "targets": [ + "src/sentry/uptime/autodetect/tasks.py" + ], + "source": "https://github.com/getsentry/sentry/commit/e84e7a45b001de51cab826593b8a2f5f0f0b323b" +} diff --git a/packages/security-evals/cases/wrdn-dos-review/sentry-eap-item-encode-depth-limited.json b/packages/security-evals/cases/wrdn-dos-review/sentry-eap-item-encode-depth-limited.json new file mode 100644 index 00000000..811792d9 --- /dev/null +++ b/packages/security-evals/cases/wrdn-dos-review/sentry-eap-item-encode-depth-limited.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "_encode_value recurses into lists, tuples, and dicts from event data without a depth limit, so a deeply nested attribute in an ingested event exceeds protobuf nesting limits or Python recursion limits and fails serialization for that event", + "repository": "getsentry/sentry", + "ref": "57f0df6e319f9de3718e0e9804eb65ffc07aca3e", + "targets": [ + "src/sentry/eventstream/item_helpers.py" + ], + "source": "https://github.com/getsentry/sentry/commit/57f0df6e319f9de3718e0e9804eb65ffc07aca3e" +} diff --git a/packages/security-evals/cases/wrdn-dos-review/sentry-eap-item-encode-unbounded-recursion.json b/packages/security-evals/cases/wrdn-dos-review/sentry-eap-item-encode-unbounded-recursion.json new file mode 100644 index 00000000..7b9f9d2e --- /dev/null +++ b/packages/security-evals/cases/wrdn-dos-review/sentry-eap-item-encode-unbounded-recursion.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "_encode_value recurses into lists, tuples, and dicts from event data without a depth limit, so a deeply nested attribute in an ingested event exceeds protobuf nesting limits or Python recursion limits and fails serialization for that event", + "repository": "getsentry/sentry", + "ref": "ee57ed6ac987a2774415335b98a69a6513e1cdc5", + "targets": [ + "src/sentry/eventstream/item_helpers.py" + ], + "source": "https://github.com/getsentry/sentry/commit/57f0df6e319f9de3718e0e9804eb65ffc07aca3e" +} diff --git a/packages/security-evals/cases/wrdn-dos-review/sentry-eap-search-boolean-nesting-bounded.json b/packages/security-evals/cases/wrdn-dos-review/sentry-eap-search-boolean-nesting-bounded.json new file mode 100644 index 00000000..55da3160 --- /dev/null +++ b/packages/security-evals/cases/wrdn-dos-review/sentry-eap-search-boolean-nesting-bounded.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "_resolve_boolean_conditions recurses once per boolean operator and parenthesis in the user search query with no maximum depth, so a query with many chained OR/AND conditions builds a filter that exceeds protobuf recursion limits and fails the request", + "repository": "getsentry/sentry", + "ref": "93b67135ae9cee140be76f43a8d4ba6bc58bf080", + "targets": [ + "src/sentry/search/eap/resolver.py" + ], + "source": "https://github.com/getsentry/sentry/commit/93b67135ae9cee140be76f43a8d4ba6bc58bf080" +} diff --git a/packages/security-evals/cases/wrdn-dos-review/sentry-eap-search-boolean-nesting-recursion.json b/packages/security-evals/cases/wrdn-dos-review/sentry-eap-search-boolean-nesting-recursion.json new file mode 100644 index 00000000..55e88476 --- /dev/null +++ b/packages/security-evals/cases/wrdn-dos-review/sentry-eap-search-boolean-nesting-recursion.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "_resolve_boolean_conditions recurses once per boolean operator and parenthesis in the user search query with no maximum depth, so a query with many chained OR/AND conditions builds a filter that exceeds protobuf recursion limits and fails the request", + "repository": "getsentry/sentry", + "ref": "975da7ea97f7c9f964d6238229338fc02896661e", + "targets": [ + "src/sentry/search/eap/resolver.py" + ], + "source": "https://github.com/getsentry/sentry/commit/93b67135ae9cee140be76f43a8d4ba6bc58bf080" +} diff --git a/packages/security-evals/cases/wrdn-dos-review/sentry-group-events-full-page-size-fixed.json b/packages/security-evals/cases/wrdn-dos-review/sentry-group-events-full-page-size-fixed.json new file mode 100644 index 00000000..9aee2942 --- /dev/null +++ b/packages/security-evals/cases/wrdn-dos-review/sentry-group-events-full-page-size-fixed.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "GroupEventsEndpoint with full=true binds and serializes the complete nodestore body of every event with EventSerializer at the default per_page of 100, so a single request against native events with many threads and frames builds hundreds of megabytes of JSON and holds a web worker for a long time", + "repository": "getsentry/sentry", + "ref": "1e21009cb0f03dd93714a4ac50356053176e5b6a", + "targets": [ + "src/sentry/issues/endpoints/group_events.py" + ], + "source": "https://github.com/getsentry/sentry/commit/1e21009cb0f03dd93714a4ac50356053176e5b6a" +} diff --git a/packages/security-evals/cases/wrdn-dos-review/sentry-group-events-full-page-size.json b/packages/security-evals/cases/wrdn-dos-review/sentry-group-events-full-page-size.json new file mode 100644 index 00000000..5e6e4ab5 --- /dev/null +++ b/packages/security-evals/cases/wrdn-dos-review/sentry-group-events-full-page-size.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "GroupEventsEndpoint with full=true binds and serializes the complete nodestore body of every event with EventSerializer at the default per_page of 100, so a single request against native events with many threads and frames builds hundreds of megabytes of JSON and holds a web worker for a long time", + "repository": "getsentry/sentry", + "ref": "b597a2926d9b321b1da0ffc1018a41c283f4f781", + "targets": [ + "src/sentry/issues/endpoints/group_events.py" + ], + "source": "https://github.com/getsentry/sentry/commit/1e21009cb0f03dd93714a4ac50356053176e5b6a" +} diff --git a/packages/security-evals/cases/wrdn-dos-review/sentry-grouping-url-regex-backtracking.json b/packages/security-evals/cases/wrdn-dos-review/sentry-grouping-url-regex-backtracking.json new file mode 100644 index 00000000..3702ee7c --- /dev/null +++ b/packages/security-evals/cases/wrdn-dos-review/sentry-grouping-url-regex-backtracking.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "The URL pattern scheme [a-zA-Z][a-zA-Z0-9+\\-.]* is unbounded, so an event message with a long run of scheme-valid characters and no :// makes the regex retry from every position, giving quadratic CPU per message during grouping (ReDoS on event data)", + "repository": "getsentry/sentry", + "ref": "f7c64397f494fe2da5c7775baa3943085998fa5a", + "targets": [ + "src/sentry/grouping/parameterization.py" + ], + "source": "https://github.com/getsentry/sentry/commit/a111b7473cc002875a38a7ffd3792ee1d395f362" +} diff --git a/packages/security-evals/cases/wrdn-dos-review/sentry-grouping-url-regex-bounded-scheme.json b/packages/security-evals/cases/wrdn-dos-review/sentry-grouping-url-regex-bounded-scheme.json new file mode 100644 index 00000000..a31085f7 --- /dev/null +++ b/packages/security-evals/cases/wrdn-dos-review/sentry-grouping-url-regex-bounded-scheme.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "The URL pattern scheme [a-zA-Z][a-zA-Z0-9+\\-.]* is unbounded, so an event message with a long run of scheme-valid characters and no :// makes the regex retry from every position, giving quadratic CPU per message during grouping (ReDoS on event data)", + "repository": "getsentry/sentry", + "ref": "a111b7473cc002875a38a7ffd3792ee1d395f362", + "targets": [ + "src/sentry/grouping/parameterization.py" + ], + "source": "https://github.com/getsentry/sentry/commit/a111b7473cc002875a38a7ffd3792ee1d395f362" +} diff --git a/packages/security-evals/cases/wrdn-dos-review/sentry-logs-regex-filter-compile-cost-fixed.json b/packages/security-evals/cases/wrdn-dos-review/sentry-logs-regex-filter-compile-cost-fixed.json new file mode 100644 index 00000000..7ed380e3 --- /dev/null +++ b/packages/security-evals/cases/wrdn-dos-review/sentry-logs-regex-filter-compile-cost-fixed.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "event_search.validate_regex_pattern compiles every logs key://pattern// filter with RE2 on the web worker with only a 64-character length cap, no RE2 max_mem and no limit on filters per query, so a short pattern like \\p{L}{100} repeated across many filters makes each search request burn CPU and memory compiling huge programs before any Snuba query runs", + "repository": "getsentry/sentry", + "ref": "a00270ad6dfe4beb9637e21582ad163c3284cfe3", + "targets": [ + "src/sentry/api/event_search.py" + ], + "source": "https://github.com/getsentry/sentry/commit/a00270ad6dfe4beb9637e21582ad163c3284cfe3" +} diff --git a/packages/security-evals/cases/wrdn-dos-review/sentry-logs-regex-filter-compile-cost.json b/packages/security-evals/cases/wrdn-dos-review/sentry-logs-regex-filter-compile-cost.json new file mode 100644 index 00000000..26ae18b9 --- /dev/null +++ b/packages/security-evals/cases/wrdn-dos-review/sentry-logs-regex-filter-compile-cost.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "event_search.validate_regex_pattern compiles every logs key://pattern// filter with RE2 on the web worker with only a 64-character length cap, no RE2 max_mem and no limit on filters per query, so a short pattern like \\p{L}{100} repeated across many filters makes each search request burn CPU and memory compiling huge programs before any Snuba query runs", + "repository": "getsentry/sentry", + "ref": "453b3a60dab0cc4c12d1b080a2be39be13acdb37", + "targets": [ + "src/sentry/api/event_search.py" + ], + "source": "https://github.com/getsentry/sentry/commit/a00270ad6dfe4beb9637e21582ad163c3284cfe3" +} diff --git a/packages/security-evals/cases/wrdn-dos-review/sentry-release-set-commits-unbounded-fixed.json b/packages/security-evals/cases/wrdn-dos-review/sentry-release-set-commits-unbounded-fixed.json new file mode 100644 index 00000000..89bf4dc6 --- /dev/null +++ b/packages/security-evals/cases/wrdn-dos-review/sentry-release-set-commits-unbounded-fixed.json @@ -0,0 +1,11 @@ +{ + "kind": "clean", + "summary": "set_commits processes every commit in the release create/update commits list with no size limit, issuing several queries per commit inside the web request while holding the release lock for up to 10 minutes, so one request with a huge commits array ties up a worker, the database, and the lock", + "repository": "getsentry/sentry", + "ref": "b61768dc5582d1018ee2ad8a582591ddae110a53", + "targets": [ + "src/sentry/models/releases/set_commits.py", + "src/sentry/api/serializers/rest_framework/release.py" + ], + "source": "https://github.com/getsentry/sentry/commit/b61768dc5582d1018ee2ad8a582591ddae110a53" +} diff --git a/packages/security-evals/cases/wrdn-dos-review/sentry-release-set-commits-unbounded.json b/packages/security-evals/cases/wrdn-dos-review/sentry-release-set-commits-unbounded.json new file mode 100644 index 00000000..2e1171fb --- /dev/null +++ b/packages/security-evals/cases/wrdn-dos-review/sentry-release-set-commits-unbounded.json @@ -0,0 +1,11 @@ +{ + "kind": "bug", + "summary": "set_commits processes every commit in the release create/update commits list with no size limit, issuing several queries per commit inside the web request while holding the release lock for up to 10 minutes, so one request with a huge commits array ties up a worker, the database, and the lock", + "repository": "getsentry/sentry", + "ref": "bac9728af44bcfd338a4b3e424732a6f5157a451", + "targets": [ + "src/sentry/models/releases/set_commits.py", + "src/sentry/api/serializers/rest_framework/release.py" + ], + "source": "https://github.com/getsentry/sentry/commit/b61768dc5582d1018ee2ad8a582591ddae110a53" +} diff --git a/packages/security-evals/cases/wrdn-pii/sentry-activity-email-ignores-enhanced-privacy-fixed.json b/packages/security-evals/cases/wrdn-pii/sentry-activity-email-ignores-enhanced-privacy-fixed.json new file mode 100644 index 00000000..b56595bc --- /dev/null +++ b/packages/security-evals/cases/wrdn-pii/sentry-activity-email-ignores-enhanced-privacy-fixed.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "ActivityNotification.get_subject and the generic activity email templates put the issue title and group details in the subject, preheader, and text body even when the organization has enhanced_privacy enabled, so regression and other activity emails ship event-derived titles, which often carry user data, to inboxes the org asked to keep them out of", + "repository": "getsentry/sentry", + "ref": "aedeffac61f00a3a7c39b3df8fc62f263c2686ca", + "targets": [ + "src/sentry/notifications/notifications/activity/base.py" + ], + "source": "https://github.com/getsentry/sentry/commit/aedeffac61f00a3a7c39b3df8fc62f263c2686ca" +} diff --git a/packages/security-evals/cases/wrdn-pii/sentry-activity-email-ignores-enhanced-privacy.json b/packages/security-evals/cases/wrdn-pii/sentry-activity-email-ignores-enhanced-privacy.json new file mode 100644 index 00000000..ba456e04 --- /dev/null +++ b/packages/security-evals/cases/wrdn-pii/sentry-activity-email-ignores-enhanced-privacy.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "ActivityNotification.get_subject and the generic activity email templates put the issue title and group details in the subject, preheader, and text body even when the organization has enhanced_privacy enabled, so regression and other activity emails ship event-derived titles, which often carry user data, to inboxes the org asked to keep them out of", + "repository": "getsentry/sentry", + "ref": "660849756331cc9e2a232ee8a2133ea7e2267b29", + "targets": [ + "src/sentry/notifications/notifications/activity/base.py" + ], + "source": "https://github.com/getsentry/sentry/commit/aedeffac61f00a3a7c39b3df8fc62f263c2686ca" +} diff --git a/packages/security-evals/cases/wrdn-pii/sentry-ai-suggested-fix-ignores-org-opt-out-fixed.json b/packages/security-evals/cases/wrdn-pii/sentry-ai-suggested-fix-ignores-org-opt-out-fixed.json new file mode 100644 index 00000000..4d3c21c6 --- /dev/null +++ b/packages/security-evals/cases/wrdn-pii/sentry-ai-suggested-fix-ignores-org-opt-out-fixed.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "EventAiSuggestedFixEndpoint.get decides whether to send an event's message and stacktrace to OpenAI from the OpenAI policy and per-request consent flags only and never checks the organization's sentry:ai_suggested_solution option, so event data, including any user data inside it, goes to a third-party model for orgs that turned the feature off", + "repository": "getsentry/sentry", + "ref": "2204fde6870055ee4fcd7551464559282973c188", + "targets": [ + "src/sentry/api/endpoints/event_ai_suggested_fix.py" + ], + "source": "https://github.com/getsentry/sentry/commit/2204fde6870055ee4fcd7551464559282973c188" +} diff --git a/packages/security-evals/cases/wrdn-pii/sentry-ai-suggested-fix-ignores-org-opt-out.json b/packages/security-evals/cases/wrdn-pii/sentry-ai-suggested-fix-ignores-org-opt-out.json new file mode 100644 index 00000000..977f42cc --- /dev/null +++ b/packages/security-evals/cases/wrdn-pii/sentry-ai-suggested-fix-ignores-org-opt-out.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "EventAiSuggestedFixEndpoint.get decides whether to send an event's message and stacktrace to OpenAI from the OpenAI policy and per-request consent flags only and never checks the organization's sentry:ai_suggested_solution option, so event data, including any user data inside it, goes to a third-party model for orgs that turned the feature off", + "repository": "getsentry/sentry", + "ref": "d8ff0cb911134cf4a8ad24e179247a2e96c06d20", + "targets": [ + "src/sentry/api/endpoints/event_ai_suggested_fix.py" + ], + "source": "https://github.com/getsentry/sentry/commit/2204fde6870055ee4fcd7551464559282973c188" +} diff --git a/packages/security-evals/cases/wrdn-pii/sentry-python-asgi-query-string-fixed.json b/packages/security-evals/cases/wrdn-pii/sentry-python-asgi-query-string-fixed.json new file mode 100644 index 00000000..b8bdfcf6 --- /dev/null +++ b/packages/security-evals/cases/wrdn-pii/sentry-python-asgi-query-string-fixed.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "_asgi_common._get_request_attributes records http.query and a url.full that carries the query string on every ASGI request span without checking should_send_default_pii, so apps that never opted into PII still ship query strings, which routinely hold emails, tokens, and user IDs, to Sentry", + "repository": "getsentry/sentry-python", + "ref": "f17dad95f33f59aa65bf023a4435e059c4145d37", + "targets": [ + "sentry_sdk/integrations/_asgi_common.py" + ], + "source": "https://github.com/getsentry/sentry-python/commit/f17dad95f33f59aa65bf023a4435e059c4145d37" +} diff --git a/packages/security-evals/cases/wrdn-pii/sentry-python-asgi-query-string.json b/packages/security-evals/cases/wrdn-pii/sentry-python-asgi-query-string.json new file mode 100644 index 00000000..019fed29 --- /dev/null +++ b/packages/security-evals/cases/wrdn-pii/sentry-python-asgi-query-string.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "_asgi_common._get_request_attributes records http.query and a url.full that carries the query string on every ASGI request span without checking should_send_default_pii, so apps that never opted into PII still ship query strings, which routinely hold emails, tokens, and user IDs, to Sentry", + "repository": "getsentry/sentry-python", + "ref": "400963132d9c8fb07ab7b4824e5a44c75cb35dbb", + "targets": [ + "sentry_sdk/integrations/_asgi_common.py" + ], + "source": "https://github.com/getsentry/sentry-python/commit/f17dad95f33f59aa65bf023a4435e059c4145d37" +} diff --git a/packages/security-evals/cases/wrdn-pii/sentry-python-langchain-tool-call-output-fixed.json b/packages/security-evals/cases/wrdn-pii/sentry-python-langchain-tool-call-output-fixed.json new file mode 100644 index 00000000..ca73252f --- /dev/null +++ b/packages/security-evals/cases/wrdn-pii/sentry-python-langchain-tool-call-output-fixed.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "SentryLangchainCallback.on_llm_end records the model's tool_calls, with their arguments, as gen_ai.response.tool_calls span data without checking should_send_default_pii or include_prompts, so apps that disabled prompt capture still send tool-call payloads built from user conversations to Sentry", + "repository": "getsentry/sentry-python", + "ref": "e52c3b117a404fb7755feced02680d29e421b0ee", + "targets": [ + "sentry_sdk/integrations/langchain.py" + ], + "source": "https://github.com/getsentry/sentry-python/commit/e52c3b117a404fb7755feced02680d29e421b0ee" +} diff --git a/packages/security-evals/cases/wrdn-pii/sentry-python-langchain-tool-call-output.json b/packages/security-evals/cases/wrdn-pii/sentry-python-langchain-tool-call-output.json new file mode 100644 index 00000000..628161f2 --- /dev/null +++ b/packages/security-evals/cases/wrdn-pii/sentry-python-langchain-tool-call-output.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "SentryLangchainCallback.on_llm_end records the model's tool_calls, with their arguments, as gen_ai.response.tool_calls span data without checking should_send_default_pii or include_prompts, so apps that disabled prompt capture still send tool-call payloads built from user conversations to Sentry", + "repository": "getsentry/sentry-python", + "ref": "3f4e2a8497ae942b380ce4d2459cac7bb364506e", + "targets": [ + "sentry_sdk/integrations/langchain.py" + ], + "source": "https://github.com/getsentry/sentry-python/commit/e52c3b117a404fb7755feced02680d29e421b0ee" +} diff --git a/packages/security-evals/cases/wrdn-pii/sentry-python-logs-metrics-user-attributes-fixed.json b/packages/security-evals/cases/wrdn-pii/sentry-python-logs-metrics-user-attributes-fixed.json new file mode 100644 index 00000000..9e4defad --- /dev/null +++ b/packages/security-evals/cases/wrdn-pii/sentry-python-logs-metrics-user-attributes-fixed.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "Client._capture_log and Client._capture_metric copy the isolation scope user's id, username, and email onto every log and metric as user.* attributes without checking should_send_default_pii, so apps with send_default_pii off still send user identities with each log line and metric", + "repository": "getsentry/sentry-python", + "ref": "5606bb3e58e1c27e2c1e48d6dc69e09e30469897", + "targets": [ + "sentry_sdk/client.py" + ], + "source": "https://github.com/getsentry/sentry-python/commit/5606bb3e58e1c27e2c1e48d6dc69e09e30469897" +} diff --git a/packages/security-evals/cases/wrdn-pii/sentry-python-logs-metrics-user-attributes.json b/packages/security-evals/cases/wrdn-pii/sentry-python-logs-metrics-user-attributes.json new file mode 100644 index 00000000..a6f43238 --- /dev/null +++ b/packages/security-evals/cases/wrdn-pii/sentry-python-logs-metrics-user-attributes.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "Client._capture_log and Client._capture_metric copy the isolation scope user's id, username, and email onto every log and metric as user.* attributes without checking should_send_default_pii, so apps with send_default_pii off still send user identities with each log line and metric", + "repository": "getsentry/sentry-python", + "ref": "6046f2d04632176654415d5ad16ad17d558a81a3", + "targets": [ + "sentry_sdk/client.py" + ], + "source": "https://github.com/getsentry/sentry-python/commit/5606bb3e58e1c27e2c1e48d6dc69e09e30469897" +} diff --git a/packages/security-evals/cases/wrdn-pii/sentry-user-serializer-secondary-emails-fixed.json b/packages/security-evals/cases/wrdn-pii/sentry-user-serializer-secondary-emails-fixed.json new file mode 100644 index 00000000..156a653e --- /dev/null +++ b/packages/security-evals/cases/wrdn-pii/sentry-user-serializer-secondary-emails-fixed.json @@ -0,0 +1,10 @@ +{ + "kind": "clean", + "summary": "UserSerializer.serialize returns every serialized user's full list of secondary email addresses (with verification state) no matter who is asking, so any endpoint that serializes another user without popping emails, such as organization user details, hands org members each other's private secondary emails", + "repository": "getsentry/sentry", + "ref": "58b51b430f0883cd5560eca37dbb7a8427e5a8c1", + "targets": [ + "src/sentry/users/api/serializers/user.py" + ], + "source": "https://github.com/getsentry/sentry/commit/58b51b430f0883cd5560eca37dbb7a8427e5a8c1" +} diff --git a/packages/security-evals/cases/wrdn-pii/sentry-user-serializer-secondary-emails.json b/packages/security-evals/cases/wrdn-pii/sentry-user-serializer-secondary-emails.json new file mode 100644 index 00000000..b9ba7dc3 --- /dev/null +++ b/packages/security-evals/cases/wrdn-pii/sentry-user-serializer-secondary-emails.json @@ -0,0 +1,10 @@ +{ + "kind": "bug", + "summary": "UserSerializer.serialize returns every serialized user's full list of secondary email addresses (with verification state) no matter who is asking, so any endpoint that serializes another user without popping emails, such as organization user details, hands org members each other's private secondary emails", + "repository": "getsentry/sentry", + "ref": "8b34c7c654c24b2b2a5630097d965044b7d71d57", + "targets": [ + "src/sentry/users/api/serializers/user.py" + ], + "source": "https://github.com/getsentry/sentry/commit/58b51b430f0883cd5560eca37dbb7a8427e5a8c1" +} diff --git a/packages/security-evals/package.json b/packages/security-evals/package.json new file mode 100644 index 00000000..b55a6ee2 --- /dev/null +++ b/packages/security-evals/package.json @@ -0,0 +1,24 @@ +{ + "name": "warden-security-evals", + "private": true, + "type": "module", + "scripts": { + "evals:security": "tsx scripts/run.ts", + "add-case": "tsx scripts/add-case.ts", + "baseline:update": "tsx scripts/update-baseline.ts", + "test": "vitest run --config vitest.unit.config.ts", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@sentry/warden": "workspace:*", + "dotenv": "^17.4.2", + "zod": "^4.6.7" + }, + "devDependencies": { + "@sentry/roach": "^0.2.0", + "@types/node": "^25.9.1", + "tsx": "^4.23.12", + "typescript": "^6.0.3", + "vitest": "^4.1.11" + } +} diff --git a/packages/security-evals/roach.json b/packages/security-evals/roach.json new file mode 100644 index 00000000..57f7331d --- /dev/null +++ b/packages/security-evals/roach.json @@ -0,0 +1,84 @@ +{ + "tenant": "getsentry/warden", + "allow": [ + "https://api.anthropic.com", + "https://openrouter.ai", + "https://api.openai.com", + "https://pi.dev" + ], + "rules": [ + { + "name": "model", + "match": { + "method": "POST", + "url": "https://api.anthropic.com/v1/" + }, + "values": { + "worktree-root": "(?:/[A-Za-z0-9_.-]+)+/security-evals-worktrees", + "git-commit": { + "builtin": "gitCommit" + }, + "uuid": { + "builtin": "uuid" + }, + "time": { + "builtin": "isoTime" + }, + "finding-id": "(? --skill [--name ] [--force] + +Writes cases//.json (bug, at the fix's parent) and +cases//-fixed.json (clean, at the fix). Targets are the non-test +.py/.ts files the fix modified.`; + +const { values, positionals } = parseArgs({ + allowPositionals: true, + options: { + skill: { type: 'string' }, + name: { type: 'string' }, + force: { type: 'boolean' }, + help: { type: 'boolean', short: 'h' }, + }, +}); +const [url] = positionals; +if (values.help || !url || !values.skill || positionals.length > 1) { + console.log(USAGE); + process.exit(values.help ? 0 : 2); +} + +const result = await scaffoldCase({ + url, + skill: values.skill, + name: values.name, + force: values.force, + github: createGitHubReader(), +}); +const cwd = process.env['INIT_CWD'] ?? process.cwd(); +console.log(`Bug: ${relative(cwd, result.bugPath)} @ ${result.bugRef}`); +console.log(`Clean: ${relative(cwd, result.cleanPath)} @ ${result.fixRef}`); +console.log(`Targets:\n${result.targets.map((target) => ` ${target}`).join('\n')}`); +if (result.skipped.length > 0) { + console.log(`Skipped:\n${result.skipped.map((file) => ` ${file.path} (${file.reason})`).join('\n')}`); +} +console.log(` +Next: + 1. Replace the TODO summary in both files with the same sentence: the bug, where it lives, what an attacker gets. + 2. pnpm -C packages/security-evals test (free; checks the pair, ids, and summary) + 3. pnpm evals:security --repo ${result.repository}= --skills-repo \\ + --skills security-review,${values.skill} --entries ${result.bugId},${result.cleanId} --repeats 3`); diff --git a/packages/security-evals/scripts/prepare-clone.sh b/packages/security-evals/scripts/prepare-clone.sh new file mode 100755 index 00000000..ffa5073e --- /dev/null +++ b/packages/security-evals/scripts/prepare-clone.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# Usage: prepare-clone.sh +# Ensure holds a blobless clone of the GitHub repository. A restored +# cache is used as is: the suite fetches any pinned SHA the clone lacks. +set -eu + +if [ "$#" -ne 2 ]; then + printf 'usage: %s \n' "$0" >&2 + exit 2 +fi +repository=$1 +dir=$2 + +if [ -d "$dir/.git" ]; then + printf 'Using cached %s clone at %s\n' "$repository" "$dir" + exit 0 +fi + +timeout 1800 git clone --quiet --filter=blob:none --no-checkout \ + "https://github.com/$repository.git" "$dir" diff --git a/packages/security-evals/scripts/run.ts b/packages/security-evals/scripts/run.ts new file mode 100644 index 00000000..e6bb533b --- /dev/null +++ b/packages/security-evals/scripts/run.ts @@ -0,0 +1,88 @@ +#!/usr/bin/env tsx +import { spawnSync } from 'node:child_process'; +import { resolve } from 'node:path'; +import { parseArgs } from 'node:util'; + +// Each flag maps to WARDEN_SECURITY_EVALS_; see src/options.ts for defaults. +const FLAGS = { + 'sentry-repo': 'SENTRY_REPO', + 'skills-repo': 'SKILLS_REPO', + skills: 'SKILLS', + entries: 'ENTRIES', + repeats: 'REPEATS', + runtime: 'RUNTIME', + model: 'MODEL', + effort: 'EFFORT', + 'aux-model': 'AUX_MODEL', + 'aux-effort': 'AUX_EFFORT', + 'judge-model': 'JUDGE_MODEL', + 'max-turns': 'MAX_TURNS', + concurrency: 'CONCURRENCY', + 'cache-dir': 'CACHE_DIR', + output: 'OUTPUT', + baseline: 'BASELINE', +} as const; + +const PATH_FLAGS = new Set(['sentry-repo', 'skills-repo', 'cache-dir', 'output', 'baseline']); + +const USAGE = `usage: pnpm evals:security --repo getsentry/sentry= [--skills-repo ] [options] + + --repo + Local clone of a repository the selected entries scan. + Repeat it per repository (required) + --sentry-repo Same as --repo getsentry/sentry= + --skills-repo Local getsentry/warden-skills clone (required for wrdn-* lanes) + --skills Lanes to run (default: security-review and the five wrdn-* skills) + --entries Entry ids, or prefixes ending in * (default: all) + --repeats Passes per shard (default: 1) + --runtime, --model, --effort, --aux-model, --aux-effort + Model lane (default: pi, openrouter/x-ai/grok-4.5 high, + openrouter/openai/gpt-5.6-luna high) + --judge-model Scoring model (default: openrouter/anthropic/claude-sonnet-4.6) + --max-turns Agent turn cap (default: 100) + --concurrency Concurrent chunks per skill (default: 4) + --cache-dir Worktrees and results (default: packages/security-evals/.cache) + --output Results JSON (default: /results.json) + --baseline Lanes to compare against (default: baseline/results.json) + --remove-worktrees Delete each worktree after its shard (CI disk limits)`; + +const { values } = parseArgs({ + options: { + ...Object.fromEntries(Object.keys(FLAGS).map((flag) => [flag, { type: 'string' as const }])), + repo: { type: 'string', multiple: true }, + 'remove-worktrees': { type: 'boolean' }, + help: { type: 'boolean', short: 'h' }, + }, +}); + +if (values.help) { + console.log(USAGE); + process.exit(0); +} + +// pnpm runs scripts from the package directory; INIT_CWD is where the caller ran it. +const callerPath = (path: string): string => resolve(process.env['INIT_CWD'] ?? process.cwd(), path); + +const env: NodeJS.ProcessEnv = { ...process.env }; +for (const [flag, name] of Object.entries(FLAGS)) { + const value = (values as Record)[flag]; + if (typeof value === 'string') { + env[`WARDEN_SECURITY_EVALS_${name}`] = PATH_FLAGS.has(flag) && value !== 'none' ? callerPath(value) : value; + } +} +if (values.repo) { + env['WARDEN_SECURITY_EVALS_REPOS'] = values.repo.map((pair) => { + const split = pair.indexOf('='); + return split < 0 ? pair : `${pair.slice(0, split)}=${callerPath(pair.slice(split + 1))}`; + }).join(','); +} +if (values['remove-worktrees']) { + env['WARDEN_SECURITY_EVALS_REMOVE_WORKTREES'] = '1'; +} + +const result = spawnSync('pnpm', ['exec', 'vitest', 'run', '--config', 'vitest.config.ts'], { + cwd: resolve(import.meta.dirname, '..'), + env, + stdio: 'inherit', +}); +process.exit(result.status ?? 1); diff --git a/packages/security-evals/scripts/update-baseline.ts b/packages/security-evals/scripts/update-baseline.ts new file mode 100644 index 00000000..2c8c3de0 --- /dev/null +++ b/packages/security-evals/scripts/update-baseline.ts @@ -0,0 +1,19 @@ +#!/usr/bin/env tsx +import { writeFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { loadResultsFile, updateBaseline } from '../src/baseline.js'; +import { DEFAULT_BASELINE } from '../src/options.js'; +import { serializeResults } from '../src/results.js'; + +// usage: pnpm -C packages/security-evals baseline:update +// Folds the lanes a run executed into baseline/results.json. +const [path] = process.argv.slice(2); +if (!path) { + console.error('usage: baseline:update '); + process.exit(2); +} +const run = loadResultsFile(resolve(process.env['INIT_CWD'] ?? process.cwd(), path)); +const baseline = updateBaseline(loadResultsFile(DEFAULT_BASELINE), run); +writeFileSync(DEFAULT_BASELINE, serializeResults(baseline)); +const lanes = Object.entries(run.lanes).filter(([, lane]) => !lane.fromBaseline).map(([skill]) => skill); +console.log(`Updated ${lanes.join(', ')} in ${DEFAULT_BASELINE}`); diff --git a/packages/security-evals/src/baseline.test.ts b/packages/security-evals/src/baseline.test.ts new file mode 100644 index 00000000..bd0f0558 --- /dev/null +++ b/packages/security-evals/src/baseline.test.ts @@ -0,0 +1,118 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { loadResultsFile, mergeBaseline, updateBaseline } from './baseline.js'; +import { DEFAULT_BASELINE } from './options.js'; +import { createResults, recordShardRun, serializeResults } from './results.js'; +import { recordRepeats, testConfig, testEntry, testUsage } from './testing.js'; + +const current = testConfig(); + +describe('mergeBaseline', () => { + it('adds matching lanes the run skipped, limited to the selected entries', () => { + const baseline = createResults([testEntry('authz'), testEntry('dos')]); + recordRepeats(baseline, 'security-review', { authz: 1, dos: 0 }, 1); + + const results = createResults([testEntry('authz')]); + recordRepeats(results, 'wrdn-authz', { authz: 1 }, 1); + const warnings = mergeBaseline(results, baseline, ['wrdn-authz'], () => current); + + expect(warnings).toEqual([]); + expect(results.lanes['security-review']?.fromBaseline).toBe(true); + expect(results.entries['authz']!.lanes).toEqual({ + 'security-review': { hits: 1, runs: 1, errors: 0 }, + 'wrdn-authz': { hits: 1, runs: 1, errors: 0 }, + }); + }); + + it('drops a lane recorded with another model, warden version, or skill ref, with a warning', () => { + const baseline = createResults([testEntry('authz')]); + recordRepeats(baseline, 'security-review', { authz: 1 }, 1, testConfig({ wardenVersion: '0.9.0' })); + recordRepeats(baseline, 'wrdn-pii', { authz: 1 }, 1, testConfig({ skillRef: 'getsentry/warden-skills@old' })); + + const results = createResults([testEntry('authz')]); + const warnings = mergeBaseline(results, baseline, [], () => current); + + expect(warnings).toEqual([ + 'Dropped baseline lane security-review: wardenVersion 0.9.0 != 1.0.0', + 'Dropped baseline lane wrdn-pii: skillRef getsentry/warden-skills@old != builtin', + ]); + expect(results.lanes).toEqual({}); + expect(results.entries['authz']!.lanes).toEqual({}); + }); + + it('ignores tallies recorded at a different ref', () => { + const baseline = createResults([testEntry('authz', { ref: 'b'.repeat(40) })]); + recordRepeats(baseline, 'security-review', { authz: 1 }, 1); + + const results = createResults([testEntry('authz')]); + mergeBaseline(results, baseline, [], () => current); + expect(results.entries['authz']!.lanes).toEqual({}); + }); +}); + +describe('updateBaseline', () => { + it('replaces tallies for entries the run covered and keeps the rest', () => { + const baseline = createResults([testEntry('authz'), testEntry('dos')]); + recordRepeats(baseline, 'security-review', { authz: 0, dos: 1 }, 3); + + const run = createResults([testEntry('authz')]); + recordRepeats(run, 'security-review', { authz: 3 }, 3); + updateBaseline(baseline, run); + + expect(baseline.entries['authz']!.lanes['security-review']).toEqual({ hits: 3, runs: 3, errors: 0 }); + expect(baseline.entries['dos']!.lanes['security-review']).toEqual({ hits: 1, runs: 3, errors: 0 }); + expect(baseline.lanes['security-review']).toMatchObject({ runs: 6 }); + }); + + it('starts a lane over when its config changed', () => { + const baseline = createResults([testEntry('authz'), testEntry('dos')]); + recordRepeats(baseline, 'security-review', { authz: 0, dos: 1 }, 3, testConfig({ model: 'old' })); + + const run = createResults([testEntry('authz')]); + recordRepeats(run, 'security-review', { authz: 3 }, 3); + updateBaseline(baseline, run); + + expect(baseline.lanes['security-review']).toMatchObject({ runs: 3, config: { model: 'm' } }); + expect(baseline.entries['dos']!.lanes).toEqual({}); + }); + + it('skips lanes the run copied from the baseline', () => { + const baseline = createResults([testEntry('authz')]); + const run = createResults([testEntry('authz')]); + recordRepeats(run, 'security-review', { authz: 1 }, 1); + run.lanes['security-review']!.fromBaseline = true; + expect(updateBaseline(baseline, run).lanes).toEqual({}); + }); + + it('refuses results with errored entries', () => { + const run = createResults([testEntry('authz')]); + recordShardRun(run, { + skill: 'wrdn-authz', config: current, usage: testUsage(), error: 'boom', hits: new Map(), errors: ['authz'], + }); + expect(() => updateBaseline(createResults([]), run)).toThrow('1 errored entry runs'); + }); +}); + +describe('loadResultsFile', () => { + it('round-trips a results file, keeping which lanes came from the baseline', () => { + const dir = mkdtempSync(join(tmpdir(), 'security-evals-results-')); + try { + const results = createResults([testEntry('authz')]); + recordRepeats(results, 'security-review', { authz: 1 }, 1); + results.lanes['security-review']!.fromBaseline = true; + writeFileSync(join(dir, 'results.json'), serializeResults(results)); + + const loaded = loadResultsFile(join(dir, 'results.json')); + expect(loaded.lanes['security-review']?.fromBaseline).toBe(true); + expect(loaded.entries['authz']!.lanes['security-review']).toEqual({ hits: 1, runs: 1, errors: 0 }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('loads the committed baseline', () => { + expect(() => loadResultsFile(DEFAULT_BASELINE)).not.toThrow(); + }); +}); diff --git a/packages/security-evals/src/baseline.ts b/packages/security-evals/src/baseline.ts new file mode 100644 index 00000000..4778260b --- /dev/null +++ b/packages/security-evals/src/baseline.ts @@ -0,0 +1,140 @@ +import { readFileSync } from 'node:fs'; +import { z } from 'zod'; +import type { LaneConfig } from './options.js'; +import { summarize, type LaneResult, type LaneUsage, type SuiteResults } from './results.js'; + +const TallySchema = z.object({ hits: z.number().int(), runs: z.number().int(), errors: z.number().int() }); + +const LaneConfigSchema = z.object({ + runtime: z.string(), + model: z.string(), + effort: z.string(), + auxiliaryModel: z.string(), + auxiliaryEffort: z.string(), + maxTurns: z.number(), + judgeModel: z.string(), + wardenVersion: z.string(), + skillRef: z.string(), +}); + +const UsageSchema = z.object({ + inputTokens: z.number(), + outputTokens: z.number(), + cacheReadInputTokens: z.number(), + costUSD: z.number(), + judgeCostUSD: z.number(), +}); + +/** A results file. Summaries are recomputed on load, so they are not checked. */ +const ResultsFileSchema = z.object({ + version: z.literal(2), + lanes: z.record(z.string(), z.object({ + config: LaneConfigSchema, + usage: UsageSchema, + runs: z.number().int(), + failedRuns: z.number().int(), + fromBaseline: z.literal(true).optional(), + })), + entries: z.record(z.string(), z.object({ + source: z.enum(['corpus', 'case']), + kind: z.enum(['bug', 'clean']), + repository: z.string(), + ref: z.string(), + targets: z.array(z.string()), + lanes: z.record(z.string(), TallySchema), + })), +}); + +/** Read and check a results file written by the suite. */ +export function loadResultsFile(path: string): SuiteResults { + const parsed = ResultsFileSchema.safeParse(JSON.parse(readFileSync(path, 'utf8'))); + if (!parsed.success) { + throw new Error(`${path}: ${z.prettifyError(parsed.error)}`); + } + const results: SuiteResults = { ...parsed.data, summary: {} }; + results.summary = summarize(results); + return results; +} + +/** Fields where two lane configs differ. Empty means one can stand in for the other. */ +function configMismatch(recorded: LaneConfig, expected: LaneConfig): string[] { + return (Object.keys(expected) as (keyof LaneConfig)[]) + .filter((key) => recorded[key] !== expected[key]) + .map((key) => `${key} ${String(recorded[key])} != ${String(expected[key])}`); +} + +/** + * Copy lanes this run skipped from a baseline, limited to the selected + * entries, so a candidate lane compares without rerunning the rest. A lane + * recorded with a different config is dropped, never compared. Returns one + * warning per dropped lane. + */ +export function mergeBaseline( + results: SuiteResults, + baseline: SuiteResults, + ranSkills: string[], + expectedConfig: (skill: string) => LaneConfig, +): string[] { + const warnings: string[] = []; + for (const [skill, lane] of Object.entries(baseline.lanes)) { + if (ranSkills.includes(skill)) continue; + const mismatch = configMismatch(lane.config, expectedConfig(skill)); + if (mismatch.length > 0) { + warnings.push(`Dropped baseline lane ${skill}: ${mismatch.join('; ')}`); + continue; + } + results.lanes[skill] = { ...lane, fromBaseline: true }; + for (const [entryId, entry] of Object.entries(results.entries)) { + const recorded = baseline.entries[entryId]; + const tally = recorded?.lanes[skill]; + if (tally && recorded.ref === entry.ref) entry.lanes[skill] = tally; + } + } + return warnings; +} + +/** + * Fold the lanes a run executed into the baseline. Entry tallies are replaced + * per entry, so a filtered run only touches what it ran. A lane whose config + * changed is replaced outright. Lane usage and run counts add up across the + * runs that fed it. Refuses results with errored entries. + */ +export function updateBaseline(baseline: SuiteResults, run: SuiteResults): SuiteResults { + const ran = Object.entries(run.lanes).filter(([, lane]) => !lane.fromBaseline); + for (const [skill] of ran) { + const errors = Object.values(run.entries).reduce((sum, entry) => sum + (entry.lanes[skill]?.errors ?? 0), 0); + if (errors > 0) { + throw new Error(`${skill} has ${errors} errored entry runs; rerun them before updating the baseline`); + } + } + + for (const [skill, lane] of ran) { + let previous: LaneResult | undefined = baseline.lanes[skill]; + if (previous && configMismatch(previous.config, lane.config).length > 0) { + for (const entry of Object.values(baseline.entries)) { + entry.lanes = Object.fromEntries(Object.entries(entry.lanes).filter(([name]) => name !== skill)); + } + previous = undefined; + } + const usage = { ...lane.usage }; + for (const key of Object.keys(usage) as (keyof LaneUsage)[]) usage[key] += previous?.usage[key] ?? 0; + baseline.lanes[skill] = { + config: lane.config, + usage, + runs: lane.runs + (previous?.runs ?? 0), + failedRuns: lane.failedRuns + (previous?.failedRuns ?? 0), + }; + + for (const [entryId, entry] of Object.entries(run.entries)) { + const tally = entry.lanes[skill]; + if (!tally) continue; + const recorded = baseline.entries[entryId]; + // An entry whose ref moved starts over: its old tallies describe other code. + const target = recorded && recorded.ref === entry.ref ? recorded : { ...entry, lanes: {} }; + target.lanes[skill] = tally; + baseline.entries[entryId] = target; + } + } + baseline.summary = summarize(baseline); + return baseline; +} diff --git a/packages/security-evals/src/entries.test.ts b/packages/security-evals/src/entries.test.ts new file mode 100644 index 00000000..db5337dd --- /dev/null +++ b/packages/security-evals/src/entries.test.ts @@ -0,0 +1,69 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { filterEntries, groupShards, loadCaseEntries, loadCaseFiles, loadCorpusEntries, type SuiteEntry } from './entries.js'; + +function entry(id: string, ref: string, targets: string[]): SuiteEntry { + return { id, source: 'corpus', kind: 'bug', repository: 'getsentry/sentry', ref, targets, summary: id }; +} + +describe('loadCorpusEntries', () => { + it('loads one entry per corpus finding at its pinned sha', () => { + const entries = loadCorpusEntries(); + expect(entries).toHaveLength(86); + expect(new Set(entries.map((e) => e.ref)).size).toBe(6); + expect(entries[0]).toMatchObject({ id: 'sentry-vuln-001', kind: 'bug', lines: '265-275' }); + }); +}); + +describe('loadCaseEntries', () => { + it('names cases by owning skill and keeps clean cases separate', () => { + const cases = loadCaseEntries(); + expect(cases.map((c) => c.id)).toContain('wrdn-dos-review/sentry-grouping-url-regex-backtracking'); + expect(cases.filter((c) => c.kind === 'clean').length).toBeGreaterThan(0); + expect(cases.every((c) => /^[0-9a-f]{40}$/.test(c.ref) && c.targets.length > 0)).toBe(true); + }); + + it('rejects a malformed case with its path', () => { + const dir = mkdtempSync(join(tmpdir(), 'security-evals-cases-')); + try { + mkdirSync(join(dir, 'wrdn-pii')); + writeFileSync(join(dir, 'wrdn-pii', 'leak.json'), JSON.stringify({ + kind: 'clean', summary: 's', repository: 'getsentry/sentry', ref: 'a'.repeat(40), + targets: ['x.py'], source: 'https://github.com/getsentry/sentry/pull/1', noClean: 'why', + })); + expect(() => loadCaseFiles(dir)).toThrow(/wrdn-pii\/leak\.json:[\s\S]*noClean only applies to bug cases/); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + +describe('filterEntries', () => { + const entries = [entry('sentry-vuln-001', 'a', ['x.py']), entry('wrdn-authz/scim', 'b', ['y.py'])]; + + it('matches exact ids and trailing-star prefixes', () => { + expect(filterEntries(entries, ['wrdn-authz/*']).map((e) => e.id)).toEqual(['wrdn-authz/scim']); + expect(filterEntries(entries, ['sentry-vuln-001']).map((e) => e.id)).toEqual(['sentry-vuln-001']); + expect(filterEntries(entries, [])).toBe(entries); + }); + + it('fails when nothing matches', () => { + expect(() => filterEntries(entries, ['nope'])).toThrow('No entries match nope'); + }); +}); + +describe('groupShards', () => { + it('makes one shard per ref with deduped targets', () => { + const shards = groupShards([ + entry('one', 'b', ['x.py']), + entry('two', 'a', ['y.py']), + entry('three', 'b', ['x.py', 'z.py']), + ]); + expect(shards.map((s) => [s.ref, s.targets, s.entries.length])).toEqual([ + ['a', ['y.py'], 1], + ['b', ['x.py', 'z.py'], 2], + ]); + }); +}); diff --git a/packages/security-evals/src/entries.ts b/packages/security-evals/src/entries.ts new file mode 100644 index 00000000..311cccd5 --- /dev/null +++ b/packages/security-evals/src/entries.ts @@ -0,0 +1,156 @@ +import { readdirSync, readFileSync } from 'node:fs'; +import { basename, join } from 'node:path'; +import { z } from 'zod'; + +/** Benchmark corpus that seeds most entries. */ +const CORPUS_PATH = join( + import.meta.dirname, + '..', '..', 'docs', 'src', 'data', 'benchmarking', 'sentry-vulnerability-corpus.json', +); + +/** Hand-picked cases for bug classes the corpus does not cover, one directory per owning skill. */ +export const CASES_DIR = join(import.meta.dirname, '..', 'cases'); + +/** Summary the scaffolder writes. The validator rejects any case that still has it. */ +export const SUMMARY_TODO = 'TODO: one sentence naming the bug, where it lives, and what an attacker gets'; + +/** + * One scored expectation. A `bug` entry is a hit when a skill reports it. A + * `clean` entry describes a bug the code at `ref` already fixed, so a hit there + * is a false positive. + */ +export interface SuiteEntry { + id: string; + source: 'corpus' | 'case'; + kind: 'bug' | 'clean'; + repository: string; + ref: string; + /** Files the skills scan for this entry. The rest of the worktree is context. */ + targets: string[]; + /** Line range of the bug in the first target, when known. */ + lines?: string; + summary: string; +} + +/** Entries that share a repository and ref run as one Warden pass over one worktree. */ +export interface SuiteShard { + repository: string; + ref: string; + targets: string[]; + entries: SuiteEntry[]; +} + +const CorpusSchema = z.object({ + findings: z.array(z.object({ + id: z.string(), + repository: z.string(), + sha: z.string(), + summary: z.string(), + code: z.object({ path: z.string(), lines: z.string().optional() }), + })), +}); + +const RelativePath = z.string().min(1).refine( + (path) => !path.startsWith('/') && !path.split('/').includes('..'), + 'must be a repository-relative path', +); + +/** + * One file under `cases//`. A bug case pairs with a clean case + * that shares its source, targets, and summary at the fixed ref, unless + * `noClean` says why it cannot. + */ +export const CaseSchema = z.object({ + kind: z.enum(['bug', 'clean']), + summary: z.string().min(1), + repository: z.string().regex(/^[\w.-]+\/[\w.-]+$/), + ref: z.string().regex(/^[0-9a-f]{40}$/), + targets: z.array(RelativePath).min(1), + source: z.url(), + noClean: z.string().min(1).optional(), +}).strict().refine((data) => data.kind === 'bug' || data.noClean === undefined, { + message: 'noClean only applies to bug cases', + path: ['noClean'], +}); + +export type CaseData = z.infer; + +/** A parsed case file and the skill directory that owns it. */ +export interface CaseFile { + id: string; + owner: string; + data: CaseData; +} + +/** Load one entry per corpus finding. */ +export function loadCorpusEntries(path = CORPUS_PATH): SuiteEntry[] { + const corpus = CorpusSchema.parse(JSON.parse(readFileSync(path, 'utf8'))); + return corpus.findings.map((finding) => ({ + id: finding.id, + source: 'corpus', + kind: 'bug', + repository: finding.repository, + ref: finding.sha, + targets: [finding.code.path], + lines: finding.code.lines, + summary: finding.summary, + })); +} + +/** Parse every case file. Ids read as `/`. A bad file throws with its path. */ +export function loadCaseFiles(dir = CASES_DIR): CaseFile[] { + return readdirSync(dir, { withFileTypes: true }) + .filter((owner) => owner.isDirectory()) + .flatMap((owner) => readdirSync(join(dir, owner.name)) + .filter((file) => file.endsWith('.json')) + .map((file) => { + const path = join(dir, owner.name, file); + const parsed = CaseSchema.safeParse(JSON.parse(readFileSync(path, 'utf8'))); + if (!parsed.success) { + throw new Error(`${path}: ${z.prettifyError(parsed.error)}`); + } + return { id: `${owner.name}/${basename(file, '.json')}`, owner: owner.name, data: parsed.data }; + })) + .sort((a, b) => a.id.localeCompare(b.id)); +} + +/** Load hand-picked cases as suite entries. */ +export function loadCaseEntries(dir = CASES_DIR): SuiteEntry[] { + return loadCaseFiles(dir).map(({ id, data }) => ({ + id, + source: 'case', + kind: data.kind, + repository: data.repository, + ref: data.ref, + targets: data.targets, + summary: data.summary, + })); +} + +/** + * Keep entries matching any filter. A filter is an exact id, or a prefix + * ending in `*` (`wrdn-dos-review/*`, `sentry-vuln-0*`). No filters keeps all. + */ +export function filterEntries(entries: SuiteEntry[], filters: string[]): SuiteEntry[] { + if (filters.length === 0) return entries; + const selected = entries.filter((entry) => filters.some((filter) => filter.endsWith('*') + ? entry.id.startsWith(filter.slice(0, -1)) + : entry.id === filter)); + if (selected.length === 0) { + throw new Error(`No entries match ${filters.join(', ')}`); + } + return selected; +} + +/** Group entries into one shard per repository and ref. */ +export function groupShards(entries: SuiteEntry[]): SuiteShard[] { + const shards = new Map(); + for (const entry of entries) { + const key = `${entry.repository}@${entry.ref}`; + const shard = shards.get(key) ?? { repository: entry.repository, ref: entry.ref, targets: [], entries: [] }; + shard.entries.push(entry); + shard.targets = [...new Set([...shard.targets, ...entry.targets])].sort(); + shards.set(key, shard); + } + return [...shards.values()].sort((a, b) => a.ref.localeCompare(b.ref)); +} diff --git a/packages/security-evals/src/options.test.ts b/packages/security-evals/src/options.test.ts new file mode 100644 index 00000000..e7f33c85 --- /dev/null +++ b/packages/security-evals/src/options.test.ts @@ -0,0 +1,70 @@ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { MATRIX_SKILLS, WARDEN_SKILLS_REF, laneConfig, resolveSuiteOptions } from './options.js'; + +describe('resolveSuiteOptions', () => { + it('defaults to the production lane over every matrix skill', () => { + const options = resolveSuiteOptions({}); + expect(options).toMatchObject({ + skills: [...MATRIX_SKILLS], + entries: [], + repeats: 1, + runtime: 'pi', + model: 'openrouter/x-ai/grok-4.5', + effort: 'high', + auxiliaryModel: 'openrouter/openai/gpt-5.6-luna', + auxiliaryEffort: 'high', + maxTurns: 100, + removeWorktrees: false, + }); + expect(options.baseline).toMatch(/baseline\/results\.json$/); + }); + + it('reads filters and overrides from env', () => { + const options = resolveSuiteOptions({ + WARDEN_SECURITY_EVALS_SKILLS: 'wrdn-pii, security-review', + WARDEN_SECURITY_EVALS_ENTRIES: 'wrdn-pii/*', + WARDEN_SECURITY_EVALS_REPEATS: '3', + WARDEN_SECURITY_EVALS_BASELINE: 'none', + WARDEN_SECURITY_EVALS_SKILLS_REPO: '/src/warden-skills', + }); + expect(options.skillsRepo).toBe('/src/warden-skills'); + expect(options.skills).toEqual(['wrdn-pii', 'security-review']); + expect(options.entries).toEqual(['wrdn-pii/*']); + expect(options.repeats).toBe(3); + expect(options.baseline).toBeUndefined(); + }); + + it('maps --repo pairs and the --sentry-repo alias onto one clone per repository', () => { + expect(resolveSuiteOptions({ + WARDEN_SECURITY_EVALS_REPOS: 'getsentry/getsentry=/src/getsentry', + WARDEN_SECURITY_EVALS_SENTRY_REPO: '/src/sentry', + }).repos).toEqual({ 'getsentry/getsentry': '/src/getsentry', 'getsentry/sentry': '/src/sentry' }); + expect(() => resolveSuiteOptions({ WARDEN_SECURITY_EVALS_REPOS: '/src/sentry' })).toThrow('owner/name='); + expect(() => resolveSuiteOptions({ + WARDEN_SECURITY_EVALS_REPOS: 'getsentry/sentry=/a', + WARDEN_SECURITY_EVALS_SENTRY_REPO: '/b', + })).toThrow('set by both'); + }); + + it('records provenance a baseline lane must match', () => { + const options = resolveSuiteOptions({}); + expect(laneConfig('security-review', options)).toMatchObject({ skillRef: 'builtin', model: 'openrouter/x-ai/grok-4.5' }); + expect(laneConfig('wrdn-pii', options).skillRef).toBe(`getsentry/warden-skills@${WARDEN_SKILLS_REF}`); + expect(laneConfig('wrdn-pii', options).wardenVersion).toMatch(/^\d+\.\d+\.\d+/); + }); + + it('pins the same warden-skills ref the CI workflow checks out', () => { + const workflow = readFileSync( + join(import.meta.dirname, '..', '..', '..', '.github', 'workflows', 'security-evals.yml'), + 'utf8', + ); + expect(workflow).toContain(`WARDEN_SKILLS_REF: ${WARDEN_SKILLS_REF}`); + }); + + it('rejects unknown skills and bad counts', () => { + expect(() => resolveSuiteOptions({ WARDEN_SECURITY_EVALS_SKILLS: 'wrdn-nope' })).toThrow('Unknown skills'); + expect(() => resolveSuiteOptions({ WARDEN_SECURITY_EVALS_REPEATS: '0' })).toThrow('positive integer'); + }); +}); diff --git a/packages/security-evals/src/options.ts b/packages/security-evals/src/options.ts new file mode 100644 index 00000000..cf687b7a --- /dev/null +++ b/packages/security-evals/src/options.ts @@ -0,0 +1,160 @@ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { EffortSchema, RuntimeNameSchema, type Effort, type RuntimeName } from '@sentry/warden'; + +/** Skills the matrix compares: builtin security-review plus the remote security skills production runs. */ +export const MATRIX_SKILLS = [ + 'security-review', + 'wrdn-authz', + 'wrdn-pii', + 'wrdn-data-exfil', + 'wrdn-code-execution', + 'wrdn-dos-review', +] as const; + +/** + * getsentry/warden-skills commit the wrdn-* lanes load from. Pinned so a + * warden-skills edit cannot silently move results. Bump on purpose, update + * .github/workflows/security-evals.yml to match, and rerun the affected lanes. + */ +export const WARDEN_SKILLS_REF = 'daf01f970a28450627a8d2d61d9915a742a735e2'; + +const PACKAGE_DIR = join(import.meta.dirname, '..'); + +/** Committed results used for lanes a run does not execute. */ +export const DEFAULT_BASELINE = join(PACKAGE_DIR, 'baseline', 'results.json'); + +export interface SuiteOptions { + /** Local clone per repository, keyed `owner/name`. Every entry's repository needs one. */ + repos: Record; + /** Local getsentry/warden-skills checkout. Required when a wrdn-* lane is selected. */ + skillsRepo?: string; + skills: string[]; + entries: string[]; + repeats: number; + runtime: RuntimeName; + model: string; + effort: Effort; + auxiliaryModel: string; + auxiliaryEffort: Effort; + judgeModel: string; + maxTurns: number; + /** Concurrent chunk analyses per skill. */ + concurrency: number; + cacheDir: string; + output: string; + /** Results file whose lanes fill in skills this run skips. Undefined disables it. */ + baseline?: string; + /** Delete each worktree after its shard finishes, for disk-limited CI runners. */ + removeWorktrees: boolean; +} + +const PREFIX = 'WARDEN_SECURITY_EVALS_'; + +function list(value: string | undefined): string[] { + return (value ?? '').split(',').map((item) => item.trim()).filter(Boolean); +} + +/** Parse `owner/name=` pairs. `--sentry-repo` is an alias for the getsentry/sentry pair. */ +function repoPaths(value: string | undefined, sentryRepo: string | undefined): Record { + const repos: Record = {}; + for (const pair of list(value)) { + const split = pair.indexOf('='); + const repository = pair.slice(0, split); + const path = pair.slice(split + 1); + if (split < 0 || !/^[\w.-]+\/[\w.-]+$/.test(repository) || !path) { + throw new Error(`${PREFIX}REPOS entries look like owner/name=, got ${pair}`); + } + repos[repository] = path; + } + if (sentryRepo) { + if (repos['getsentry/sentry'] && repos['getsentry/sentry'] !== sentryRepo) { + throw new Error('getsentry/sentry is set by both --repo and --sentry-repo; pass one'); + } + repos['getsentry/sentry'] = sentryRepo; + } + return repos; +} + +function positiveInt(name: string, value: string | undefined, fallback: number): number { + if (!value) return fallback; + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed < 1) { + throw new Error(`${PREFIX}${name} must be a positive integer, got ${value}`); + } + return parsed; +} + +/** + * Read suite options from `WARDEN_SECURITY_EVALS_*`. `scripts/run.ts` maps CLI + * flags onto the same variables. Defaults match the production lane and the + * corpus runbook (maxTurns 100, so a turn limit does not read as a miss). + */ +export function resolveSuiteOptions(env: NodeJS.ProcessEnv = process.env): SuiteOptions { + const read = (name: string): string | undefined => env[`${PREFIX}${name}`]?.trim() || undefined; + + const skills = list(read('SKILLS')); + const unknown = skills.filter((skill) => !(MATRIX_SKILLS as readonly string[]).includes(skill)); + if (unknown.length > 0) { + throw new Error(`Unknown skills: ${unknown.join(', ')}. Choose from ${MATRIX_SKILLS.join(', ')}`); + } + + const cacheDir = read('CACHE_DIR') ?? join(PACKAGE_DIR, '.cache'); + const baseline = read('BASELINE') ?? DEFAULT_BASELINE; + + return { + repos: repoPaths(read('REPOS'), read('SENTRY_REPO')), + skillsRepo: read('SKILLS_REPO'), + skills: skills.length > 0 ? skills : [...MATRIX_SKILLS], + entries: list(read('ENTRIES')), + repeats: positiveInt('REPEATS', read('REPEATS'), 1), + runtime: RuntimeNameSchema.parse(read('RUNTIME') ?? 'pi'), + model: read('MODEL') ?? 'openrouter/x-ai/grok-4.5', + effort: EffortSchema.parse(read('EFFORT') ?? 'high'), + auxiliaryModel: read('AUX_MODEL') ?? 'openrouter/openai/gpt-5.6-luna', + auxiliaryEffort: EffortSchema.parse(read('AUX_EFFORT') ?? 'high'), + judgeModel: read('JUDGE_MODEL') ?? 'openrouter/anthropic/claude-sonnet-4.6', + maxTurns: positiveInt('MAX_TURNS', read('MAX_TURNS'), 100), + concurrency: positiveInt('CONCURRENCY', read('CONCURRENCY'), 4), + cacheDir, + output: read('OUTPUT') ?? join(cacheDir, 'results.json'), + baseline: baseline === 'none' ? undefined : baseline, + removeWorktrees: read('REMOVE_WORKTREES') === '1', + }; +} + +const WARDEN_VERSION = (JSON.parse( + readFileSync(join(PACKAGE_DIR, '..', 'warden', 'package.json'), 'utf8'), +) as { version: string }).version; + +/** + * Everything that changes what a lane finds or how it is scored. A baseline + * lane only stands in for a lane whose config matches field for field. + */ +export interface LaneConfig { + runtime: string; + model: string; + effort: string; + auxiliaryModel: string; + auxiliaryEffort: string; + maxTurns: number; + judgeModel: string; + wardenVersion: string; + /** `builtin` for security-review, `getsentry/warden-skills@` for wrdn-* skills. */ + skillRef: string; +} + +/** The config a lane runs with under these options. */ +export function laneConfig(skill: string, options: SuiteOptions): LaneConfig { + return { + runtime: options.runtime, + model: options.model, + effort: options.effort, + auxiliaryModel: options.auxiliaryModel, + auxiliaryEffort: options.auxiliaryEffort, + maxTurns: options.maxTurns, + judgeModel: options.judgeModel, + wardenVersion: WARDEN_VERSION, + skillRef: skill === 'security-review' ? 'builtin' : `getsentry/warden-skills@${WARDEN_SKILLS_REF}`, + }; +} diff --git a/packages/security-evals/src/results.test.ts b/packages/security-evals/src/results.test.ts new file mode 100644 index 00000000..c63cc0fa --- /dev/null +++ b/packages/security-evals/src/results.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from 'vitest'; +import { createResults, recordShardRun, renderMarkdown, serializeResults, summarize } from './results.js'; +import { recordRepeats, testConfig, testEntry, testUsage } from './testing.js'; + +describe('summarize', () => { + it('reports recall, unique catches, and clean flags', () => { + const results = createResults([ + testEntry('authz'), testEntry('dos'), testEntry('both'), testEntry('fixed', { kind: 'clean' }), + ]); + recordRepeats(results, 'security-review', { authz: 0, dos: 0, both: 2, fixed: 0 }, 2); + recordRepeats(results, 'wrdn-authz', { authz: 1, dos: 0, both: 1, fixed: 1 }, 1); + recordRepeats(results, 'wrdn-dos-review', { authz: 1, dos: 1, both: 0, fixed: 0 }, 1); + + const summary = summarize(results); + + expect(summary['security-review']).toMatchObject({ found: 1, recall: 1 / 3, onlyThisLane: [], errors: 0 }); + expect(summary['wrdn-authz']).toMatchObject({ found: 2, onlyThisLane: [], cleanFlagged: { hits: 1, runs: 1 } }); + expect(summary['wrdn-dos-review']).toMatchObject({ onlyThisLane: ['dos'] }); + expect(results.lanes['security-review']).toMatchObject({ runs: 2, usage: { costUSD: 2, judgeCostUSD: 0.02 } }); + }); + + it('records a failed pass as errors on its entries, not misses', () => { + const results = createResults([testEntry('authz')]); + recordShardRun(results, { + skill: 'wrdn-authz', config: testConfig(), usage: testUsage(3), error: 'boom', hits: new Map(), errors: ['authz'], + }); + expect(results.lanes['wrdn-authz']).toMatchObject({ runs: 0, failedRuns: 1, usage: { costUSD: 3 } }); + expect(results.entries['authz']!.lanes['wrdn-authz']).toEqual({ hits: 0, runs: 0, errors: 1 }); + expect(summarize(results)['wrdn-authz']).toMatchObject({ bugEntries: 0, recall: null, errors: 1 }); + }); +}); + +describe('output', () => { + it('leads with verdicts and never prints a bare unique-catch count', () => { + const results = createResults([testEntry('authz')]); + recordRepeats(results, 'security-review', { authz: 0 }, 1); + recordRepeats(results, 'wrdn-authz', { authz: 1 }, 1); + results.summary = summarize(results); + + const markdown = renderMarkdown(results); + expect(markdown).toContain('- **wrdn-authz: insufficient-evidence**.'); + expect(markdown).toContain('| wrdn-authz | 1.00 | 1/1 | 1 | 0/0 | 0 | 1.00 | 10 | 1 |'); + expect(markdown).toContain('| authz | bug | 0/1 | 1/1 |'); + expect(markdown).not.toContain('missed by security-review'); + }); + + it('marks errored cells', () => { + const results = createResults([testEntry('authz')]); + recordShardRun(results, { + skill: 'security-review', config: testConfig(), usage: testUsage(), error: 'boom', hits: new Map(), errors: ['authz'], + }); + results.summary = summarize(results); + expect(renderMarkdown(results)).toContain('| authz | bug | 0/0 (1 err) |'); + }); + + it('serializes with sorted keys', () => { + const results = createResults([testEntry('b'), testEntry('a')]); + expect(Object.keys(JSON.parse(serializeResults(results)).entries)).toEqual(['a', 'b']); + }); +}); diff --git a/packages/security-evals/src/results.ts b/packages/security-evals/src/results.ts new file mode 100644 index 00000000..d35eb654 --- /dev/null +++ b/packages/security-evals/src/results.ts @@ -0,0 +1,207 @@ +import type { SuiteEntry } from './entries.js'; +import type { LaneConfig } from './options.js'; +import { REFERENCE_SKILL, judgeSkill, type SkillVerdict } from './verdict.js'; + +export interface LaneUsage { + inputTokens: number; + outputTokens: number; + cacheReadInputTokens: number; + costUSD: number; + judgeCostUSD: number; +} + +export interface LaneResult { + config: LaneConfig; + usage: LaneUsage; + /** Passes completed: one per shard and repeat. */ + runs: number; + /** Shard passes that errored. Their entries count as errors, not misses. */ + failedRuns: number; + /** Set when the lane was copied from the baseline instead of run. */ + fromBaseline?: true; +} + +/** One entry in one lane: hits over scored repeats, plus repeats that errored. */ +export interface Tally { + hits: number; + runs: number; + errors: number; +} + +export interface EntryResult { + source: SuiteEntry['source']; + kind: SuiteEntry['kind']; + repository: string; + ref: string; + targets: string[]; + lanes: Record; +} + +export interface LaneSummary { + bugEntries: number; + /** Mean hit rate over bug entries this lane scored. */ + recall: number | null; + found: number; + /** Bug entries this lane found and every other lane that ran them missed. */ + onlyThisLane: string[]; + cleanFlagged: { hits: number; runs: number }; + errors: number; + /** Whether a wrdn-* skill catches what security-review misses. Absent for security-review itself. */ + verdict?: SkillVerdict; +} + +export interface SuiteResults { + version: 2; + lanes: Record; + entries: Record; + summary: Record; +} + +/** One skill's pass over one shard, scored per entry. */ +export interface ShardRun { + skill: string; + config: LaneConfig; + usage: LaneUsage; + /** Set when the whole pass failed. */ + error?: string; + hits: Map; + /** Entries this pass could not score. */ + errors: string[]; +} + +/** Start an empty result set for the selected entries. */ +export function createResults(entries: SuiteEntry[]): SuiteResults { + return { + version: 2, + lanes: {}, + entries: Object.fromEntries(entries.map((entry) => [entry.id, { + source: entry.source, + kind: entry.kind, + repository: entry.repository, + ref: entry.ref, + targets: entry.targets, + lanes: {}, + }])), + summary: {}, + }; +} + +/** Fold one shard pass into the results. */ +export function recordShardRun(results: SuiteResults, run: ShardRun): void { + const lane = results.lanes[run.skill] ?? { + config: run.config, + usage: { inputTokens: 0, outputTokens: 0, cacheReadInputTokens: 0, costUSD: 0, judgeCostUSD: 0 }, + runs: 0, + failedRuns: 0, + }; + for (const key of Object.keys(lane.usage) as (keyof LaneUsage)[]) { + lane.usage[key] += run.usage[key]; + } + if (run.error) { + lane.failedRuns += 1; + } else { + lane.runs += 1; + } + const tally = (entryId: string): Tally | undefined => { + const entry = results.entries[entryId]; + if (!entry) return undefined; + entry.lanes[run.skill] ??= { hits: 0, runs: 0, errors: 0 }; + return entry.lanes[run.skill]; + }; + for (const [entryId, hit] of run.hits) { + const t = tally(entryId); + if (!t) continue; + t.runs += 1; + t.hits += hit ? 1 : 0; + } + for (const entryId of run.errors) { + const t = tally(entryId); + if (t) t.errors += 1; + } + results.lanes[run.skill] = lane; +} + +function hitRate(tally: Tally | undefined): number | undefined { + return tally && tally.runs > 0 ? tally.hits / tally.runs : undefined; +} + +/** Compute per-lane recall, unique catches, clean-case false positives, and each wrdn-* verdict. */ +export function summarize(results: SuiteResults): Record { + const entries = Object.entries(results.entries); + const summary: Record = {}; + + for (const skill of Object.keys(results.lanes).sort()) { + // Entries this lane scored, with its hit rate on each. + const ran = entries.flatMap(([id, entry]) => { + const rate = hitRate(entry.lanes[skill]); + return rate === undefined ? [] : [{ id, entry, rate }]; + }); + const bugs = ran.filter(({ entry }) => entry.kind === 'bug'); + const clean = ran.filter(({ entry }) => entry.kind === 'clean'); + const found = bugs.filter(({ rate }) => rate > 0); + const othersMissed = (entry: EntryResult) => Object.entries(entry.lanes) + .every(([other, tally]) => other === skill || tally.hits === 0); + + summary[skill] = { + bugEntries: bugs.length, + recall: bugs.length > 0 ? bugs.reduce((sum, { rate }) => sum + rate, 0) / bugs.length : null, + found: found.length, + onlyThisLane: found.filter(({ entry }) => othersMissed(entry)).map(({ id }) => id), + cleanFlagged: { hits: clean.filter(({ rate }) => rate > 0).length, runs: clean.length }, + errors: entries.reduce((sum, [, entry]) => sum + (entry.lanes[skill]?.errors ?? 0), 0), + ...(skill === REFERENCE_SKILL ? {} : { verdict: judgeSkill(results, skill) }), + }; + } + return summary; +} + +function cell(tally: Tally | undefined): string { + if (!tally) return '-'; + return `${tally.hits}/${tally.runs}${tally.errors > 0 ? ` (${tally.errors} err)` : ''}`; +} + +/** Render verdicts, the lane table, and the entry x lane hit table as Markdown. */ +export function renderMarkdown(results: SuiteResults): string { + const skills = Object.keys(results.lanes).sort(); + const lines = ['## Security evals', '', '### Verdicts', '']; + for (const skill of skills) { + const verdict = results.summary[skill]?.verdict; + if (!verdict) continue; + const ids = verdict.uniqueCatches.length > 0 ? ` Unique: ${verdict.uniqueCatches.join(', ')}.` : ''; + lines.push(`- **${skill}: ${verdict.verdict}**. ${verdict.reason}.${ids}`); + } + + lines.push( + '', + '| lane | recall | found | only this lane | clean flagged | errors | cost USD | input tokens | output tokens |', + '|---|---|---|---|---|---|---|---|---|', + ); + for (const skill of skills) { + const lane = results.lanes[skill]; + const s = results.summary[skill]; + if (!lane || !s) continue; + const recall = s.recall === null ? '-' : s.recall.toFixed(2); + lines.push( + `| ${skill}${lane.fromBaseline ? ' (baseline)' : ''} | ${recall} | ${s.found}/${s.bugEntries} ` + + `| ${s.onlyThisLane.length} | ${s.cleanFlagged.hits}/${s.cleanFlagged.runs} | ${s.errors} ` + + `| ${lane.usage.costUSD.toFixed(2)} | ${lane.usage.inputTokens} | ${lane.usage.outputTokens} |`, + ); + } + + lines.push('', `| entry | kind | ${skills.join(' | ')} |`, `|---|---|${skills.map(() => '---|').join('')}`); + for (const [id, entry] of Object.entries(results.entries).sort(([a], [b]) => a.localeCompare(b))) { + lines.push(`| ${id} | ${entry.kind} | ${skills.map((skill) => cell(entry.lanes[skill])).join(' | ')} |`); + } + return lines.join('\n'); +} + +/** Serialize with sorted keys so two result files diff cleanly. */ +export function serializeResults(results: SuiteResults): string { + const sortKeys = (_key: string, value: unknown): unknown => { + if (value && typeof value === 'object' && !Array.isArray(value)) { + return Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b))); + } + return value; + }; + return `${JSON.stringify(results, sortKeys, 2)}\n`; +} diff --git a/packages/security-evals/src/scaffold.test.ts b/packages/security-evals/src/scaffold.test.ts new file mode 100644 index 00000000..a4d6858a --- /dev/null +++ b/packages/security-evals/src/scaffold.test.ts @@ -0,0 +1,138 @@ +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { SUMMARY_TODO, loadCaseFiles } from './entries.js'; +import { MATRIX_SKILLS } from './options.js'; +import { createResults } from './results.js'; +import { inferTargets, parseChangeUrl, scaffoldCase, type ChangedFile, type GitHubReader } from './scaffold.js'; +import { validateSuite } from './validate.js'; + +const FIX = '57f0df6e319f9de3718e0e9804eb65ffc07aca3e'; +const PARENT = 'ee57ed6a'.padEnd(40, '0'); + +// Shaped like getsentry/sentry#108555, which bounded recursion in _encode_value. +const FILES: ChangedFile[] = [ + { filename: 'src/sentry/eventstream/item_helpers.py', status: 'modified' }, + { filename: 'tests/sentry/eventstream/test_item_helpers.py', status: 'modified' }, +]; + +function github(overrides: Partial = {}): GitHubReader { + return { + pull: async () => ({ title: 'fix(eap): Limit nesting depth in _encode_value', mergeCommit: FIX }), + commit: async () => ({ sha: FIX, subject: 'fix(eap): Limit nesting depth (#108555)', parents: [PARENT] }), + compare: async () => FILES, + ...overrides, + }; +} + +describe('parseChangeUrl', () => { + it('reads pull request and commit URLs', () => { + expect(parseChangeUrl('https://github.com/getsentry/sentry/pull/108555/files')) + .toEqual({ owner: 'getsentry', repo: 'sentry', pull: 108555 }); + expect(parseChangeUrl(`https://github.com/getsentry/sentry/commit/${FIX}`)) + .toEqual({ owner: 'getsentry', repo: 'sentry', commit: FIX }); + }); + + it('rejects anything else', () => { + expect(() => parseChangeUrl('https://github.com/getsentry/sentry/issues/1')).toThrow('pull request or commit URL'); + }); +}); + +describe('inferTargets', () => { + it('keeps modified non-test .py and .ts files and says why it skipped the rest', () => { + const { targets, skipped } = inferTargets([ + { filename: 'src/sentry/api/endpoints/system_options.py', status: 'modified' }, + { filename: 'static/app/utils/request.ts', status: 'modified' }, + { filename: 'src/sentry/new_helper.py', status: 'added' }, + { filename: 'src/sentry/old_helper.py', status: 'removed' }, + { filename: 'src/sentry/moved.py', status: 'renamed' }, + { filename: 'tests/sentry/api/test_system_options.py', status: 'modified' }, + { filename: 'src/sentry/testutils/factories.py', status: 'modified' }, + { filename: 'src/sentry/conftest.py', status: 'modified' }, + { filename: 'static/app/utils/request.spec.ts', status: 'modified' }, + { filename: 'static/app/views/page.test.ts', status: 'modified' }, + { filename: 'static/app/types/index.d.ts', status: 'modified' }, + { filename: 'CHANGES', status: 'modified' }, + ]); + expect(targets).toEqual(['src/sentry/api/endpoints/system_options.py', 'static/app/utils/request.ts']); + expect(Object.fromEntries(skipped.map(({ path, reason }) => [path, reason]))).toEqual({ + 'src/sentry/new_helper.py': 'added by the fix, so it is missing at one of the two refs', + 'src/sentry/old_helper.py': 'removed by the fix, so it is missing at one of the two refs', + 'src/sentry/moved.py': 'renamed by the fix, so it is missing at one of the two refs', + 'tests/sentry/api/test_system_options.py': 'test file', + 'src/sentry/testutils/factories.py': 'test file', + 'src/sentry/conftest.py': 'test file', + 'static/app/utils/request.spec.ts': 'test file', + 'static/app/views/page.test.ts': 'test file', + 'static/app/types/index.d.ts': 'not a .py or .ts file', + CHANGES: 'not a .py or .ts file', + }); + }); +}); + +describe('scaffoldCase', () => { + let casesDir: string; + + beforeEach(() => { + casesDir = mkdtempSync(join(tmpdir(), 'security-evals-cases-')); + }); + + afterEach(() => { + rmSync(casesDir, { recursive: true, force: true }); + }); + + it('writes the bug at the fix parent and the clean twin at the fix', async () => { + const url = 'https://github.com/getsentry/sentry/pull/108555'; + const result = await scaffoldCase({ url, skill: 'wrdn-dos-review', github: github(), casesDir }); + + expect(result).toMatchObject({ + bugId: 'wrdn-dos-review/fix-eap-limit-nesting-depth-in-encode-value', + cleanId: 'wrdn-dos-review/fix-eap-limit-nesting-depth-in-encode-value-fixed', + bugRef: PARENT, + fixRef: FIX, + targets: ['src/sentry/eventstream/item_helpers.py'], + }); + expect(JSON.parse(readFileSync(result.bugPath, 'utf8'))).toEqual({ + kind: 'bug', + summary: SUMMARY_TODO, + repository: 'getsentry/sentry', + targets: ['src/sentry/eventstream/item_helpers.py'], + source: url, + ref: PARENT, + }); + expect(JSON.parse(readFileSync(result.cleanPath, 'utf8'))).toMatchObject({ kind: 'clean', ref: FIX, source: url }); + }); + + it('writes a pair the validator rejects only for its TODO summary', async () => { + await scaffoldCase({ + url: `https://github.com/getsentry/sentry/commit/${FIX}`, + skill: 'wrdn-dos-review', + name: 'eap-item-encode', + github: github(), + casesDir, + }); + const problems = validateSuite({ + cases: loadCaseFiles(casesDir), corpus: [], skills: MATRIX_SKILLS, baseline: createResults([]), + }); + expect(problems).toEqual([ + 'wrdn-dos-review/eap-item-encode: summary is still the TODO placeholder', + 'wrdn-dos-review/eap-item-encode-fixed: summary is still the TODO placeholder', + ]); + }); + + it('refuses unmerged pull requests, unknown skills, fixes with no target, and overwrites', async () => { + const url = 'https://github.com/getsentry/sentry/pull/1'; + await expect(scaffoldCase({ + url, skill: 'wrdn-dos-review', casesDir, github: github({ pull: async () => ({ title: 't', mergeCommit: null }) }), + })).rejects.toThrow('is not merged'); + await expect(scaffoldCase({ url, skill: 'wrdn-nope', casesDir, github: github() })).rejects.toThrow('Unknown skill'); + await expect(scaffoldCase({ + url, skill: 'wrdn-dos-review', casesDir, github: github({ compare: async () => [FILES[1]!] }), + })).rejects.toThrow('modified no non-test .py or .ts file'); + + await scaffoldCase({ url, skill: 'wrdn-dos-review', name: 'same', casesDir, github: github() }); + await expect(scaffoldCase({ url, skill: 'wrdn-dos-review', name: 'same', casesDir, github: github() })) + .rejects.toThrow('Case already exists'); + }); +}); diff --git a/packages/security-evals/src/scaffold.ts b/packages/security-evals/src/scaffold.ts new file mode 100644 index 00000000..e8751172 --- /dev/null +++ b/packages/security-evals/src/scaffold.ts @@ -0,0 +1,183 @@ +import { existsSync, mkdirSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { parseGitHubPullRequestUrl, slugifyEvalName } from '../../evals/src/scaffold.js'; +import { CASES_DIR, CaseSchema, SUMMARY_TODO, type CaseData } from './entries.js'; +import { MATRIX_SKILLS } from './options.js'; + +/** A file the fix touched, as GitHub's compare API reports it. */ +export interface ChangedFile { + filename: string; + status: string; +} + +/** The three GitHub reads the scaffolder needs. Injected so tests run offline. */ +export interface GitHubReader { + pull(owner: string, repo: string, number: number): Promise<{ title: string; mergeCommit: string | null }>; + commit(owner: string, repo: string, ref: string): Promise<{ sha: string; subject: string; parents: string[] }>; + compare(owner: string, repo: string, base: string, head: string): Promise; +} + +export type ChangeRef = + | { owner: string; repo: string; pull: number; commit?: never } + | { owner: string; repo: string; commit: string; pull?: never }; + +/** Parse a GitHub pull request URL or commit URL. */ +export function parseChangeUrl(url: string): ChangeRef { + const commit = /^https:\/\/github\.com\/([\w.-]+)\/([\w.-]+)\/commit\/([0-9a-f]{7,40})\/?$/.exec(url); + if (commit?.[1] && commit[2] && commit[3]) { + return { owner: commit[1], repo: commit[2], commit: commit[3] }; + } + try { + const pull = parseGitHubPullRequestUrl(url); + return { owner: pull.owner, repo: pull.repo, pull: pull.pullNumber }; + } catch { + throw new Error(`Expected a GitHub pull request or commit URL, got ${url}`); + } +} + +const TEST_DIR = /(^|\/)(tests?|__tests__|testutils|fixtures)\//; +const TEST_FILE = /(^|\/)(test_[^/]*\.py|[^/]*_test\.py|conftest\.py|[^/]*\.(test|spec)\.ts)$/; + +/** + * Pick targets from the files a fix touched: non-test `.py` and `.ts` files + * that exist at both the bug ref and the fixed ref. Everything else is + * skipped with a reason, so the author can add a path by hand if needed. + */ +export function inferTargets(files: ChangedFile[]): { targets: string[]; skipped: { path: string; reason: string }[] } { + const targets: string[] = []; + const skipped: { path: string; reason: string }[] = []; + for (const { filename: path, status } of files) { + if (!/\.(py|ts)$/.test(path) || path.endsWith('.d.ts')) { + skipped.push({ path, reason: 'not a .py or .ts file' }); + } else if (TEST_DIR.test(path) || TEST_FILE.test(path)) { + skipped.push({ path, reason: 'test file' }); + } else if (status !== 'modified' && status !== 'changed') { + skipped.push({ path, reason: `${status} by the fix, so it is missing at one of the two refs` }); + } else { + targets.push(path); + } + } + return { targets: targets.sort(), skipped }; +} + +export interface ScaffoldedCase { + repository: string; + bugId: string; + cleanId: string; + bugPath: string; + cleanPath: string; + bugRef: string; + fixRef: string; + targets: string[]; + skipped: { path: string; reason: string }[]; +} + +/** + * Write a bug case at the fix's parent and its clean twin at the fix, both + * with the TODO summary the validator rejects until an author replaces it. + */ +export async function scaffoldCase(args: { + url: string; + skill: string; + github: GitHubReader; + name?: string; + force?: boolean; + casesDir?: string; +}): Promise { + const { url, skill, github, casesDir = CASES_DIR } = args; + if (!(MATRIX_SKILLS as readonly string[]).includes(skill)) { + throw new Error(`Unknown skill ${skill}. Choose from ${MATRIX_SKILLS.join(', ')}`); + } + const change = parseChangeUrl(url); + const { owner, repo } = change; + + const pull = change.pull === undefined ? undefined : await github.pull(owner, repo, change.pull); + const fixCommit = pull ? pull.mergeCommit : change.commit; + if (!fixCommit) { + throw new Error(`${url} is not merged. Pass the commit that fixed the bug instead.`); + } + const fix = await github.commit(owner, repo, fixCommit); + const fixRef = fix.sha; + const bugRef = fix.parents[0]; + if (!bugRef) { + throw new Error(`${fixRef} has no parent commit to hold the bug`); + } + + const { targets, skipped } = inferTargets(await github.compare(owner, repo, bugRef, fixRef)); + if (targets.length === 0) { + throw new Error(`The fix modified no non-test .py or .ts file. Write the case by hand.\n${ + skipped.map((file) => ` ${file.path}: ${file.reason}`).join('\n')}`); + } + + const name = args.name ?? slugifyEvalName(pull?.title ?? fix.subject); + if (!/^[a-z0-9][a-z0-9-]*$/.test(name)) { + throw new Error(`Invalid case name ${name}. Use lowercase letters, numbers, and dashes.`); + } + const repository = `${owner}/${repo}`; + const base = { summary: SUMMARY_TODO, repository, targets, source: url }; + const dir = join(casesDir, skill); + const bugPath = join(dir, `${name}.json`); + const cleanPath = join(dir, `${name}-fixed.json`); + const cases: [string, CaseData][] = [ + [bugPath, CaseSchema.parse({ kind: 'bug', ...base, ref: bugRef })], + [cleanPath, CaseSchema.parse({ kind: 'clean', ...base, ref: fixRef })], + ]; + const existing = cases.map(([path]) => path).filter((path) => existsSync(path)); + if (existing.length > 0 && !args.force) { + throw new Error(`Case already exists: ${existing.join(', ')}. Pass --force to overwrite.`); + } + mkdirSync(dir, { recursive: true }); + for (const [path, data] of cases) { + writeFileSync(path, `${JSON.stringify(data, null, 2)}\n`); + } + + return { + repository, + bugId: `${skill}/${name}`, + cleanId: `${skill}/${name}-fixed`, + bugPath, + cleanPath, + bugRef, + fixRef, + targets, + skipped, + }; +} + +/** Read GitHub's REST API with `GITHUB_TOKEN`/`GH_TOKEN` when set. */ +export function createGitHubReader(token = process.env['GITHUB_TOKEN'] ?? process.env['GH_TOKEN']): GitHubReader { + const get = async (path: string): Promise => { + const response = await fetch(`https://api.github.com${path}`, { + headers: { + accept: 'application/vnd.github+json', + ...(token ? { authorization: `Bearer ${token}` } : {}), + }, + }); + if (!response.ok) { + throw new Error(`GitHub ${path}: ${response.status} ${await response.text()}`); + } + return response.json() as Promise; + }; + return { + async pull(owner, repo, number) { + const pull = await get<{ title: string; merged: boolean; merge_commit_sha: string | null }>( + `/repos/${owner}/${repo}/pulls/${number}`, + ); + return { title: pull.title, mergeCommit: pull.merged ? pull.merge_commit_sha : null }; + }, + async commit(owner, repo, ref) { + const commit = await get<{ sha: string; commit: { message: string }; parents: { sha: string }[] }>( + `/repos/${owner}/${repo}/commits/${ref}`, + ); + return { + sha: commit.sha, + subject: commit.commit.message.split('\n')[0] ?? '', + parents: commit.parents.map((parent) => parent.sha), + }; + }, + async compare(owner, repo, base, head) { + const comparison = await get<{ files?: ChangedFile[] }>(`/repos/${owner}/${repo}/compare/${base}...${head}`); + return comparison.files ?? []; + }, + }; +} diff --git a/packages/security-evals/src/score.test.ts b/packages/security-evals/src/score.test.ts new file mode 100644 index 00000000..c98a1a63 --- /dev/null +++ b/packages/security-evals/src/score.test.ts @@ -0,0 +1,78 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { getRuntime, type Finding } from '@sentry/warden'; +import type * as WardenPublic from '@sentry/warden'; +import type { SuiteEntry } from './entries.js'; +import { buildScorePrompt, scoreEntry } from './score.js'; + +vi.mock('@sentry/warden', async (importOriginal) => ({ + ...(await importOriginal()), + getRuntime: vi.fn(), +})); + +const runAuxiliary = vi.fn(); +const usage = { inputTokens: 1, outputTokens: 1, costUSD: 0.002 }; + +const entry: SuiteEntry = { + id: 'sentry-vuln-001', + source: 'corpus', + kind: 'bug', + repository: 'getsentry/sentry', + ref: 'a', + targets: ['src/sentry/api/accept.py'], + lines: '265-275', + summary: 'Invite acceptance deletes arbitrary members.', +}; + +function finding(path: string, title: string): Finding { + return { + id: 'ABC-DEF', + severity: 'high', + title, + description: `${title} details`, + location: { path, startLine: 270, endLine: 272 }, + } as Finding; +} + +describe('scoreEntry', () => { + beforeEach(() => { + runAuxiliary.mockReset(); + vi.mocked(getRuntime).mockReturnValue({ runAuxiliary } as unknown as ReturnType); + }); + + it('skips the judge when no finding is in a target file', async () => { + const verdict = await scoreEntry(entry, [finding('src/other.py', 'Other bug')], { model: 'judge' }); + expect(verdict.hit).toBe(false); + expect(runAuxiliary).not.toHaveBeenCalled(); + }); + + it('asks the judge about target-file findings only', async () => { + runAuxiliary.mockResolvedValue({ + success: true, + data: { matched: true, reasoning: 'same bug' }, + usage, + }); + + const verdict = await scoreEntry(entry, [ + finding('src/other.py', 'Other bug'), + finding('src/sentry/api/accept.py', 'Member deletion without token check'), + ], { model: 'judge' }); + + expect(verdict).toEqual({ hit: true, reasoning: 'same bug', usage }); + const prompt = runAuxiliary.mock.calls[0]![0].prompt as string; + expect(prompt).toContain('Location: src/sentry/api/accept.py lines 265-275'); + expect(prompt).toContain('Member deletion without token check'); + expect(prompt).not.toContain('Other bug'); + }); + + it('throws instead of scoring a miss when the judge fails', async () => { + runAuxiliary.mockResolvedValue({ success: false, error: 'timeout', usage }); + await expect(scoreEntry(entry, [finding('src/sentry/api/accept.py', 'x')], { model: 'judge' })) + .rejects.toThrow('Judge failed for sentry-vuln-001: timeout'); + }); +}); + +describe('buildScorePrompt', () => { + it('states the corpus match rule', () => { + expect(buildScorePrompt(entry, [])).toContain('different bug in the same file does not match'); + }); +}); diff --git a/packages/security-evals/src/score.ts b/packages/security-evals/src/score.ts new file mode 100644 index 00000000..f3358e5f --- /dev/null +++ b/packages/security-evals/src/score.ts @@ -0,0 +1,79 @@ +import { z } from 'zod'; +import { getRuntime, type Finding, type UsageStats } from '@sentry/warden'; +import type { SuiteEntry } from './entries.js'; + +const JUDGE_MAX_TOKENS = 2048; +const JUDGE_TIMEOUT_MS = 60_000; + +const VerdictSchema = z.object({ + matched: z.boolean(), + reasoning: z.string(), +}); + +export interface EntryVerdict { + hit: boolean; + reasoning: string; + usage?: UsageStats; +} + +/** Findings a skill reported in any of the entry's target files. */ +export function candidateFindings(entry: SuiteEntry, findings: Finding[]): Finding[] { + return findings.filter((finding) => finding.location && entry.targets.includes(finding.location.path)); +} + +/** Build the corpus semantic-match prompt for one entry. */ +export function buildScorePrompt(entry: SuiteEntry, findings: Finding[]): string { + const location = entry.lines ? `${entry.targets[0]} lines ${entry.lines}` : entry.targets.join(', '); + const block = findings.map((finding, index) => { + const loc = finding.location; + const lines = loc?.endLine ? `${loc.startLine}-${loc.endLine}` : `${loc?.startLine ?? '?'}`; + return `[${index}] ${loc?.path}:${lines}\n title: ${finding.title}\n description: ${finding.description}`; + }).join('\n\n'); + + return `You score a security code reviewer against a known vulnerability. + +## Known vulnerability +Location: ${location} +${entry.summary} + +## Reported findings +${block} + +## Rule +A finding matches when it would lead a reviewer to the same bug in roughly the +same code location. Wording and exact line numbers do not need to match. A +finding about a different bug in the same file does not match, even when it is +real. A broad finding that only names the file or the bug class without this +specific flaw does not match. + +Respond with ONLY a JSON object: +{"matched": true, "reasoning": "one sentence"}`; +} + +/** Decide whether a skill's findings name the entry's bug. No candidates means no hit and no model call. */ +export async function scoreEntry( + entry: SuiteEntry, + findings: Finding[], + options: { model: string; apiKey?: string }, +): Promise { + const candidates = candidateFindings(entry, findings); + if (candidates.length === 0) { + return { hit: false, reasoning: 'no findings in target files' }; + } + + const result = await getRuntime('pi').runAuxiliary({ + task: 'eval_judge', + agentName: 'security-eval-judge', + apiKey: options.apiKey || undefined, + prompt: buildScorePrompt(entry, candidates), + schema: VerdictSchema, + model: options.model, + maxTokens: JUDGE_MAX_TOKENS, + timeout: JUDGE_TIMEOUT_MS, + maxRetries: 2, + }); + if (!result.success) { + throw new Error(`Judge failed for ${entry.id}: ${result.error}`); + } + return { hit: result.data.matched, reasoning: result.data.reasoning, usage: result.usage }; +} diff --git a/packages/security-evals/src/security.eval.ts b/packages/security-evals/src/security.eval.ts new file mode 100644 index 00000000..2760da87 --- /dev/null +++ b/packages/security-evals/src/security.eval.ts @@ -0,0 +1,85 @@ +import { appendFileSync, existsSync, mkdirSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { afterAll, describe, it } from 'vitest'; +import { filterEntries, groupShards, loadCaseEntries, loadCorpusEntries } from './entries.js'; +import { mergeBaseline, loadResultsFile } from './baseline.js'; +import { WARDEN_SKILLS_REF, laneConfig, resolveSuiteOptions } from './options.js'; +import { createResults, recordShardRun, renderMarkdown, serializeResults, summarize } from './results.js'; +import { loadSkills, runShardPass } from './shard.js'; +import { SKILLS_CHECKOUT, ensureWorktree, removeWorktree, repoCheckout, resolveCheckout } from './worktrees.js'; + +const options = resolveSuiteOptions(); +const entries = filterEntries([...loadCorpusEntries(), ...loadCaseEntries()], options.entries); +const shards = groupShards(entries); + +// Both worktree directory names are Roach values in roach.json; keep them in sync. +const worktreesDir = join(options.cacheDir, 'security-evals-worktrees'); +const skillsWorktreesDir = join(options.cacheDir, 'skills-worktrees'); +const apiKey = process.env['WARDEN_ANTHROPIC_API_KEY'] || process.env['ANTHROPIC_API_KEY'] || ''; +const results = createResults(entries); + +// Setup, once per run: a skills problem fails collection here, so no pass runs and +// no entry is scored as a miss. No network unless the clone lacks the pinned ref. +const needsSkillsRepo = options.skills.some((skill) => skill !== 'security-review'); +const skillsWorktree = needsSkillsRepo + ? ensureWorktree( + resolveCheckout(SKILLS_CHECKOUT, options.skillsRepo), + skillsWorktreesDir, + WARDEN_SKILLS_REF, + ) + : undefined; +const skills = await loadSkills(options.skills, skillsWorktree); + +// A pass runs every selected skill over a whole shard, which can take an hour on large shards. +const PASS_TIMEOUT_MS = 3 * 60 * 60 * 1000; + +describe('security evals', () => { + for (const shard of shards) { + // Resolved while vitest collects tests, so a missing clone fails before any pass runs. + const clone = resolveCheckout(repoCheckout(shard.repository), options.repos[shard.repository]); + for (let repeat = 1; repeat <= options.repeats; repeat++) { + const name = `${shard.ref.slice(0, 8)} pass ${repeat}/${options.repeats} ` + + `(${shard.targets.length} files, ${shard.entries.length} entries)`; + it(name, { timeout: PASS_TIMEOUT_MS }, async () => { + const worktree = ensureWorktree(clone, worktreesDir, shard.ref); + try { + const runs = await runShardPass({ + shard, + worktree, + skills, + options, + apiKey, + log: (message) => console.log(` ${message}`), + }); + for (const run of runs) recordShardRun(results, run); + const failed = runs.filter((run) => run.errors.length > 0) + .map((run) => `${run.skill} (${run.errors.length} entries)`); + if (failed.length > 0) { + throw new Error(`Entries errored, recorded as errors: ${failed.join(', ')}`); + } + } finally { + if (options.removeWorktrees && repeat === options.repeats) { + removeWorktree(clone, worktree); + } + } + }); + } + } +}); + +afterAll(() => { + if (Object.keys(results.lanes).length === 0) return; + if (options.baseline && existsSync(options.baseline)) { + const warnings = mergeBaseline(results, loadResultsFile(options.baseline), options.skills, + (skill) => laneConfig(skill, options)); + for (const warning of warnings) console.warn(warning); + } + results.summary = summarize(results); + + mkdirSync(dirname(options.output), { recursive: true }); + writeFileSync(options.output, serializeResults(results)); + const markdown = renderMarkdown(results); + console.log(`\n${markdown}\n\nResults: ${options.output}`); + const stepSummary = process.env['GITHUB_STEP_SUMMARY']; + if (stepSummary) appendFileSync(stepSummary, `${markdown}\n`); +}); diff --git a/packages/security-evals/src/setup.ts b/packages/security-evals/src/setup.ts new file mode 100644 index 00000000..c9a08e9e --- /dev/null +++ b/packages/security-evals/src/setup.ts @@ -0,0 +1,14 @@ +import { existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { config as dotenvConfig } from 'dotenv'; +import { bridgeWardenProviderApiKeyEnv } from '../../warden/src/utils/index.js'; + +// Same env files as packages/evals, later files win. +const root = join(import.meta.dirname, '..', '..', '..'); +for (const file of ['.env', '.env.local', '.env.test']) { + const path = join(root, file); + if (existsSync(path)) { + dotenvConfig({ path, override: true }); + } +} +bridgeWardenProviderApiKeyEnv(); diff --git a/packages/security-evals/src/shard.test.ts b/packages/security-evals/src/shard.test.ts new file mode 100644 index 00000000..0d44aead --- /dev/null +++ b/packages/security-evals/src/shard.test.ts @@ -0,0 +1,139 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import type * as Warden from '@sentry/warden'; +import { runSkill, type SkillReport } from '@sentry/warden'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { buildFileEventContext } from '../../warden/src/cli/context.js'; +import { resolveSuiteOptions } from './options.js'; +import { scoreEntry } from './score.js'; +import { loadSkills, runShardPass } from './shard.js'; +import { testEntry } from './testing.js'; + +vi.mock('@sentry/warden', async (importOriginal) => ({ + ...await importOriginal(), + runSkill: vi.fn(), +})); +vi.mock('../../warden/src/cli/context.js', () => ({ buildFileEventContext: vi.fn() })); +vi.mock('./score.js', () => ({ scoreEntry: vi.fn() })); + +function writeSkill(root: string, dir: string, name: string): void { + mkdirSync(join(root, 'skills', dir), { recursive: true }); + writeFileSync( + join(root, 'skills', dir, 'SKILL.md'), + `---\nname: ${name}\ndescription: Finds ${name} bugs.\n---\n\nLook for ${name} bugs.\n`, + ); +} + +describe('loadSkills', () => { + let worktree: string; + + beforeEach(() => { + worktree = mkdtempSync(join(tmpdir(), 'warden-skills-')); + }); + + afterEach(() => { + rmSync(worktree, { recursive: true, force: true }); + }); + + it('loads builtin security-review and wrdn skills from the skills worktree', async () => { + writeSkill(worktree, 'wrdn-authz', 'wrdn-authz'); + + const skills = await loadSkills(['security-review', 'wrdn-authz'], worktree); + + expect([...skills.keys()]).toEqual(['security-review', 'wrdn-authz']); + expect(skills.get('security-review')?.rootDir).toMatch(/builtin-skills\/security-review$/); + expect(skills.get('wrdn-authz')?.rootDir).toBe(join(worktree, 'skills', 'wrdn-authz')); + }); + + it('needs no skills checkout for security-review alone', async () => { + const skills = await loadSkills(['security-review'], undefined); + expect([...skills.keys()]).toEqual(['security-review']); + }); + + it('fails once with the skill and path when a wrdn skill cannot load', async () => { + await expect(loadSkills(['wrdn-authz'], undefined)).rejects.toThrow('wrdn-authz needs a getsentry/warden-skills checkout'); + await expect(loadSkills(['wrdn-pii'], worktree)).rejects.toThrow( + `Could not load wrdn-pii from ${join(worktree, 'skills', 'wrdn-pii')}`, + ); + }); + + it('refuses a directory whose skill has another name', async () => { + writeSkill(worktree, 'wrdn-pii', 'wrdn-other'); + await expect(loadSkills(['wrdn-pii'], worktree)).rejects.toThrow('loaded wrdn-other'); + }); +}); + +describe('runShardPass', () => { + const usage = { inputTokens: 100, outputTokens: 10, costUSD: 0.5 }; + const shard = { + repository: 'getsentry/sentry', + ref: 'a'.repeat(40), + targets: ['a.py', 'b.py'], + entries: [testEntry('on-a', { targets: ['a.py'] }), testEntry('on-b', { targets: ['b.py'] })], + }; + const skill = { name: 'wrdn-authz' } as never; + + function pass(skills: string[] = ['wrdn-authz']) { + return runShardPass({ + shard, + worktree: '/worktree', + skills: new Map(skills.map((name) => [name, skill])), + options: resolveSuiteOptions({}), + apiKey: 'key', + log: () => undefined, + }); + } + + beforeEach(() => { + vi.mocked(buildFileEventContext).mockResolvedValue({ + pullRequest: { files: [{ filename: 'a.py' }, { filename: 'b.py' }] }, + } as never); + vi.mocked(scoreEntry).mockResolvedValue({ hit: true, usage: { costUSD: 0.01 } } as never); + }); + + afterEach(() => { + vi.resetAllMocks(); + }); + + it('scores every entry when the pass succeeds', async () => { + vi.mocked(runSkill).mockResolvedValue({ findings: [], usage } as unknown as SkillReport); + const [run] = await pass(); + expect(run).toMatchObject({ skill: 'wrdn-authz', errors: [], usage: { costUSD: 0.5, judgeCostUSD: 0.02 } }); + expect([...run!.hits]).toEqual([['on-a', true], ['on-b', true]]); + }); + + it('records entries on a failed chunk as errors and still scores the rest', async () => { + vi.mocked(runSkill).mockResolvedValue({ + findings: [], usage, hunkFailures: [{ filename: 'b.py' }], + } as unknown as SkillReport); + const [run] = await pass(); + expect(run).toMatchObject({ errors: ['on-b'] }); + expect([...run!.hits.keys()]).toEqual(['on-a']); + }); + + it('records every entry as an error when the skill reports or throws one, keeping its spend', async () => { + vi.mocked(runSkill).mockResolvedValueOnce({ + findings: [], usage, error: { message: 'provider 429' }, + } as unknown as SkillReport); + vi.mocked(runSkill).mockRejectedValueOnce(new Error('GnuTLS recv error')); + const runs = await pass(['wrdn-authz', 'security-review']); + expect(runs.map(({ skill, error, errors, hits }) => ({ skill, error, errors, hits: hits.size }))).toEqual([ + { skill: 'wrdn-authz', error: 'provider 429', errors: ['on-a', 'on-b'], hits: 0 }, + { skill: 'security-review', error: 'GnuTLS recv error', errors: ['on-a', 'on-b'], hits: 0 }, + ]); + expect(runs[0]!.usage.costUSD).toBe(0.5); + }); + + it('records a judge failure as errors, not misses', async () => { + vi.mocked(runSkill).mockResolvedValue({ findings: [], usage } as unknown as SkillReport); + vi.mocked(scoreEntry).mockRejectedValue(new Error('judge timed out')); + const [run] = await pass(); + expect(run).toMatchObject({ error: 'judge timed out', errors: ['on-a', 'on-b'] }); + }); + + it('refuses a worktree that is missing a target', async () => { + vi.mocked(buildFileEventContext).mockResolvedValue({ pullRequest: { files: [{ filename: 'a.py' }] } } as never); + await expect(pass()).rejects.toThrow('missing targets: b.py'); + }); +}); diff --git a/packages/security-evals/src/shard.ts b/packages/security-evals/src/shard.ts new file mode 100644 index 00000000..55fc289d --- /dev/null +++ b/packages/security-evals/src/shard.ts @@ -0,0 +1,127 @@ +import { join } from 'node:path'; +import { resolveSkillAsync, runSkill, type SkillReport, type UsageStats } from '@sentry/warden'; +// Not on the public surface; the CLI's file mode builds its context the same way. +import { buildFileEventContext } from '../../warden/src/cli/context.js'; +import type { SuiteShard } from './entries.js'; +import { laneConfig, type SuiteOptions } from './options.js'; +import type { LaneUsage, ShardRun } from './results.js'; +import { scoreEntry } from './score.js'; + +type SkillDefinition = Awaited>; + +const BUILTIN_SECURITY_REVIEW = join(import.meta.dirname, '..', '..', 'warden', 'src', 'builtin-skills', 'security-review'); + +/** Corpus runbook scan limits, so explicit targets are never dropped by PR-sized budgets. */ +const SCAN = { maxFiles: 200, maxChangedLines: 50_000 }; + +/** + * Load each skill once, from disk only. security-review comes from this + * checkout by path, so a same-named skill in the scanned repo cannot shadow + * it. wrdn-* skills come from a worktree of warden-skills at the pinned ref. + * Any failure throws once, before a pass runs, instead of reading as misses. + */ +export async function loadSkills(names: string[], skillsWorktree: string | undefined): Promise> { + const skills = new Map(); + for (const name of names) { + let dir = BUILTIN_SECURITY_REVIEW; + if (name !== 'security-review') { + if (!skillsWorktree) { + throw new Error(`${name} needs a getsentry/warden-skills checkout; set --skills-repo`); + } + dir = join(skillsWorktree, 'skills', name); + } + try { + const skill = await resolveSkillAsync(dir); + if (skill.name !== name) { + throw new Error(`loaded ${skill.name}`); + } + skills.set(name, skill); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Could not load ${name} from ${dir}: ${message}`); + } + } + return skills; +} + +function reportUsage(report: SkillReport): LaneUsage { + const parts: UsageStats[] = [report.usage, ...Object.values(report.auxiliaryUsage ?? {})] + .filter((usage): usage is UsageStats => usage !== undefined); + return { + inputTokens: parts.reduce((sum, usage) => sum + usage.inputTokens, 0), + outputTokens: parts.reduce((sum, usage) => sum + usage.outputTokens, 0), + cacheReadInputTokens: parts.reduce((sum, usage) => sum + (usage.cacheReadInputTokens ?? 0), 0), + costUSD: parts.reduce((sum, usage) => sum + usage.costUSD, 0), + judgeCostUSD: 0, + }; +} + +/** + * One Warden pass over a shard's worktree: build the file context once, run + * every selected skill against it, then score each entry per skill. An entry + * the pass could not score (failed chunk, failed skill, failed judge) is + * recorded as an error for that skill, never as a miss. + */ +export async function runShardPass(args: { + shard: SuiteShard; + worktree: string; + skills: Map; + options: SuiteOptions; + apiKey: string; + log: (message: string) => void; +}): Promise { + const { shard, worktree, skills, options, apiKey, log } = args; + const context = await buildFileEventContext({ patterns: shard.targets, cwd: worktree, scan: SCAN }); + const scanned = new Set(context.pullRequest?.files.map((file) => file.filename)); + const missing = shard.targets.filter((target) => !scanned.has(target)); + if (missing.length > 0) { + throw new Error(`Shard ${shard.ref} is missing targets: ${missing.join(', ')}`); + } + + return Promise.all([...skills].map(async ([name, skill]): Promise => { + const config = laneConfig(name, options); + const allEntries = shard.entries.map((entry) => entry.id); + let usage: LaneUsage = { inputTokens: 0, outputTokens: 0, cacheReadInputTokens: 0, costUSD: 0, judgeCostUSD: 0 }; + const failed = (error: unknown): ShardRun => { + const message = error instanceof Error ? error.message : String(error); + log(`${name} @ ${shard.ref.slice(0, 8)} failed: ${message}`); + return { skill: name, config, usage, error: message, hits: new Map(), errors: allEntries }; + }; + + try { + const report = await runSkill(skill, context, { + apiKey: apiKey || undefined, + runtime: options.runtime, + model: options.model, + effort: options.effort, + auxiliaryModel: options.auxiliaryModel, + auxiliaryEffort: options.auxiliaryEffort, + synthesisModel: options.auxiliaryModel, + maxTurns: options.maxTurns, + concurrency: options.concurrency, + verifyFindings: true, + }); + usage = reportUsage(report); + if (report.error) return failed(report.error.message); + + const failedFiles = new Set(report.hunkFailures?.map((failure) => failure.filename)); + const hits = new Map(); + const errors: string[] = []; + for (const entry of shard.entries) { + if (entry.targets.some((target) => failedFiles.has(target))) { + log(`${name} @ ${shard.ref.slice(0, 8)}: ${entry.id} errored, a target chunk failed`); + errors.push(entry.id); + continue; + } + const verdict = await scoreEntry(entry, report.findings, { model: options.judgeModel, apiKey }); + usage.judgeCostUSD += verdict.usage?.costUSD ?? 0; + hits.set(entry.id, verdict.hit); + } + log(`${name} @ ${shard.ref.slice(0, 8)}: ${report.findings.length} findings, ` + + `${[...hits.values()].filter(Boolean).length}/${hits.size} entries hit, ${errors.length} errored`); + return { skill: name, config, usage, hits, errors }; + } catch (error) { + return failed(error); + } + })); +} diff --git a/packages/security-evals/src/testing.ts b/packages/security-evals/src/testing.ts new file mode 100644 index 00000000..3c95a36e --- /dev/null +++ b/packages/security-evals/src/testing.ts @@ -0,0 +1,53 @@ +import type { SuiteEntry } from './entries.js'; +import type { LaneConfig } from './options.js'; +import { recordShardRun, type LaneUsage, type SuiteResults } from './results.js'; + +/** Test fixtures shared by the results, verdict, and baseline tests. */ +export const testConfig = (overrides: Partial = {}): LaneConfig => ({ + runtime: 'pi', + model: 'm', + effort: 'high', + auxiliaryModel: 'a', + auxiliaryEffort: 'high', + maxTurns: 100, + judgeModel: 'j', + wardenVersion: '1.0.0', + skillRef: 'builtin', + ...overrides, +}); + +export const testUsage = (costUSD = 1): LaneUsage => ({ + inputTokens: 10, outputTokens: 1, cacheReadInputTokens: 5, costUSD, judgeCostUSD: 0.01, +}); + +export function testEntry(id: string, overrides: Partial = {}): SuiteEntry { + return { + id, + source: id.includes('/') ? 'case' : 'corpus', + kind: 'bug', + repository: 'getsentry/sentry', + ref: 'a'.repeat(40), + targets: ['x.py'], + summary: id, + ...overrides, + }; +} + +/** Record `repeats` passes of one lane, hitting each entry in the first `hits[id]` passes. */ +export function recordRepeats( + results: SuiteResults, + skill: string, + hits: Record, + repeats: number, + config: LaneConfig = testConfig(), +): void { + for (let pass = 0; pass < repeats; pass++) { + recordShardRun(results, { + skill, + config, + usage: testUsage(), + hits: new Map(Object.entries(hits).map(([id, count]) => [id, pass < count])), + errors: [], + }); + } +} diff --git a/packages/security-evals/src/validate.test.ts b/packages/security-evals/src/validate.test.ts new file mode 100644 index 00000000..0ed79413 --- /dev/null +++ b/packages/security-evals/src/validate.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, it } from 'vitest'; +import { loadResultsFile } from './baseline.js'; +import { SUMMARY_TODO, loadCaseFiles, loadCorpusEntries, type CaseData, type CaseFile } from './entries.js'; +import { DEFAULT_BASELINE, MATRIX_SKILLS } from './options.js'; +import { createResults, recordShardRun } from './results.js'; +import { testConfig, testEntry, testUsage } from './testing.js'; +import { validateSuite } from './validate.js'; + +// The committed suite. This is the check that keeps it honest as cases are added. +describe('the committed suite', () => { + it('has no problems', () => { + expect(validateSuite({ + cases: loadCaseFiles(), + corpus: loadCorpusEntries(), + skills: MATRIX_SKILLS, + baseline: loadResultsFile(DEFAULT_BASELINE), + })).toEqual([]); + }); +}); + +const BUG = 'b'.repeat(40); +const FIX = 'f'.repeat(40); + +function caseFile(id: string, data: Partial = {}): CaseFile { + return { + id, + owner: id.split('/')[0]!, + data: { + kind: 'bug', + summary: 'Unbounded recursion on nested input', + repository: 'getsentry/sentry', + ref: BUG, + targets: ['src/a.py'], + source: 'https://github.com/getsentry/sentry/commit/ffff', + ...data, + }, + }; +} + +const pair = [caseFile('wrdn-dos-review/bug'), caseFile('wrdn-dos-review/bug-fixed', { kind: 'clean', ref: FIX })]; + +function check(cases: CaseFile[], extra: Partial[0]> = {}): string[] { + return validateSuite({ cases, corpus: [], skills: MATRIX_SKILLS, baseline: createResults([]), ...extra }); +} + +describe('validateSuite', () => { + it('accepts a bug case with its clean twin', () => { + expect(check(pair)).toEqual([]); + }); + + it('accepts a bug case with a noClean reason and no twin', () => { + expect(check([caseFile('wrdn-authz/catch', { noClean: 'PR catch, fix not pinned' })])).toEqual([]); + }); + + it('rejects a bug case with neither a twin nor a noClean reason', () => { + expect(check([pair[0]!])).toEqual([ + 'wrdn-dos-review/bug: no clean twin (same source and targets at the fixed ref) and no noClean reason', + ]); + }); + + it('rejects a twin whose targets or source drifted', () => { + const moved = caseFile('wrdn-dos-review/bug-fixed', { kind: 'clean', ref: FIX, targets: ['src/b.py'] }); + expect(check([pair[0]!, moved])).toHaveLength(1); + }); + + it('rejects a twin with a different summary, and noClean next to a twin', () => { + expect(check([ + caseFile('wrdn-dos-review/bug', { noClean: 'stale' }), + caseFile('wrdn-dos-review/bug-fixed', { kind: 'clean', ref: FIX, summary: 'Something else' }), + ])).toEqual([ + 'wrdn-dos-review/bug: has clean twin wrdn-dos-review/bug-fixed and a noClean reason; drop noClean', + 'wrdn-dos-review/bug: clean twin wrdn-dos-review/bug-fixed has a different summary; they describe the same bug', + ]); + }); + + it('rejects a clean case pinned at the bug ref', () => { + const problems = check([pair[0]!, caseFile('wrdn-dos-review/bug-fixed', { kind: 'clean' })]); + expect(problems).toContain('wrdn-dos-review/bug: clean case wrdn-dos-review/bug-fixed pins the same code'); + }); + + it('rejects the TODO summary the scaffolder writes', () => { + const problems = check([ + caseFile('wrdn-dos-review/bug', { summary: SUMMARY_TODO }), + caseFile('wrdn-dos-review/bug-fixed', { kind: 'clean', ref: FIX, summary: SUMMARY_TODO }), + ]); + expect(problems).toEqual([ + 'wrdn-dos-review/bug: summary is still the TODO placeholder', + 'wrdn-dos-review/bug-fixed: summary is still the TODO placeholder', + ]); + }); + + it('rejects a case directory that is not a matrix skill', () => { + expect(check([caseFile('wrdn-nope/bug', { noClean: 'x' })])[0]).toContain('cases/wrdn-nope is not a matrix skill'); + }); + + it('rejects a case that duplicates a corpus entry and a duplicate id', () => { + const corpus = [testEntry('sentry-vuln-001', { ref: BUG, targets: ['src/a.py'] }), testEntry('sentry-vuln-001')]; + expect(check(pair, { corpus })).toEqual([ + 'sentry-vuln-001: duplicate entry id', + 'wrdn-dos-review/bug: duplicates corpus entry sentry-vuln-001 (src/a.py at bbbbbbbb)', + ]); + }); + + it('rejects baseline entries that no longer exist or moved, and unknown lanes', () => { + const baseline = createResults([ + testEntry('wrdn-dos-review/gone'), + testEntry('wrdn-dos-review/bug', { ref: 'c'.repeat(40) }), + ]); + recordShardRun(baseline, { + skill: 'wrdn-retired', config: testConfig(), usage: testUsage(), hits: new Map(), errors: [], + }); + expect(check(pair, { baseline })).toEqual([ + 'baseline: lane wrdn-retired is not a matrix skill', + 'baseline: wrdn-dos-review/gone is not a corpus entry or case', + 'baseline: wrdn-dos-review/bug was recorded at getsentry/sentry@cccccccc, the entry now pins getsentry/sentry@bbbbbbbb', + ]); + }); +}); diff --git a/packages/security-evals/src/validate.ts b/packages/security-evals/src/validate.ts new file mode 100644 index 00000000..18f6ff42 --- /dev/null +++ b/packages/security-evals/src/validate.ts @@ -0,0 +1,80 @@ +import { SUMMARY_TODO, type CaseFile, type SuiteEntry } from './entries.js'; +import type { SuiteResults } from './results.js'; + +const sameTargets = (a: string[], b: string[]): boolean => + a.length === b.length && [...a].sort().every((target, index) => target === [...b].sort()[index]); + +/** + * Check everything about the suite that needs no model and no clone. Returns + * one message per problem; an empty list means cases, corpus, and baseline + * agree. + */ +export function validateSuite(args: { + cases: CaseFile[]; + corpus: SuiteEntry[]; + skills: readonly string[]; + baseline: SuiteResults; +}): string[] { + const { cases, corpus, skills, baseline } = args; + const problems: string[] = []; + + const ids = [...corpus.map((entry) => entry.id), ...cases.map((file) => file.id)]; + for (const id of new Set(ids.filter((id, index) => ids.indexOf(id) !== index))) { + problems.push(`${id}: duplicate entry id`); + } + + for (const { id, owner, data } of cases) { + if (!skills.includes(owner)) { + problems.push(`${id}: cases/${owner} is not a matrix skill (${skills.join(', ')})`); + } + if (data.summary.trim() === SUMMARY_TODO || data.summary.trim().startsWith('TODO')) { + problems.push(`${id}: summary is still the TODO placeholder`); + } + for (const entry of corpus) { + const shared = data.targets.filter((target) => entry.targets.includes(target)); + if (entry.repository === data.repository && entry.ref === data.ref && shared.length > 0) { + problems.push(`${id}: duplicates corpus entry ${entry.id} (${shared.join(', ')} at ${data.ref.slice(0, 8)})`); + } + } + } + + const clean = cases.filter((file) => file.data.kind === 'clean'); + for (const { id, owner, data } of cases.filter((file) => file.data.kind === 'bug')) { + for (const other of clean) { + if (other.data.repository === data.repository && other.data.ref === data.ref + && other.data.targets.some((target) => data.targets.includes(target))) { + problems.push(`${id}: clean case ${other.id} pins the same code`); + } + } + const twins = clean.filter((other) => other.owner === owner && other.data.source === data.source + && other.data.repository === data.repository && other.data.ref !== data.ref + && sameTargets(other.data.targets, data.targets)); + const [twin] = twins; + if (!twin && !data.noClean) { + problems.push(`${id}: no clean twin (same source and targets at the fixed ref) and no noClean reason`); + } + if (twin && data.noClean) { + problems.push(`${id}: has clean twin ${twin.id} and a noClean reason; drop noClean`); + } + for (const twin of twins.filter((other) => other.data.summary !== data.summary)) { + problems.push(`${id}: clean twin ${twin.id} has a different summary; they describe the same bug`); + } + } + + const pinned = new Map(); + for (const entry of corpus) pinned.set(entry.id, entry); + for (const file of cases) pinned.set(file.id, file.data); + for (const skill of Object.keys(baseline.lanes)) { + if (!skills.includes(skill)) problems.push(`baseline: lane ${skill} is not a matrix skill`); + } + for (const [id, recorded] of Object.entries(baseline.entries)) { + const entry = pinned.get(id); + if (!entry) { + problems.push(`baseline: ${id} is not a corpus entry or case`); + } else if (entry.repository !== recorded.repository || entry.ref !== recorded.ref) { + problems.push(`baseline: ${id} was recorded at ${recorded.repository}@${recorded.ref.slice(0, 8)}, ` + + `the entry now pins ${entry.repository}@${entry.ref.slice(0, 8)}`); + } + } + return problems; +} diff --git a/packages/security-evals/src/verdict.test.ts b/packages/security-evals/src/verdict.test.ts new file mode 100644 index 00000000..dcf8910f --- /dev/null +++ b/packages/security-evals/src/verdict.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, it } from 'vitest'; +import { createResults, recordShardRun, summarize } from './results.js'; +import { recordRepeats, testConfig, testEntry, testUsage } from './testing.js'; +import { VERDICT_RULES, judgeSkill } from './verdict.js'; + +const SKILL = 'wrdn-dos-review'; +const owned = (count: number) => Array.from({ length: count }, (_, index) => testEntry(`${SKILL}/case-${index}`)); + +/** Five owned bug cases, both lanes at `repeats`, with the given hit counts (out of `repeats`). */ +function matrix(args: { repeats: number; mine?: Record; reference?: Record; cases?: number }) { + const entries = owned(args.cases ?? VERDICT_RULES.minOwnedBugCases); + const results = createResults(entries); + const none = Object.fromEntries(entries.map((entry) => [entry.id, 0])); + recordRepeats(results, 'security-review', { ...none, ...args.reference }, args.repeats); + recordRepeats(results, SKILL, { ...none, ...args.mine }, args.repeats); + return results; +} + +describe('judgeSkill', () => { + it('earns-keep for a bug caught in 2 of 3 repeats that security-review never caught', () => { + const verdict = judgeSkill(matrix({ repeats: 3, mine: { [`${SKILL}/case-0`]: 2 } }), SKILL); + expect(verdict).toMatchObject({ verdict: 'earns-keep', uniqueCatches: [`${SKILL}/case-0`] }); + }); + + it('earns-keep even with few cases, since one real unique catch is enough', () => { + const verdict = judgeSkill(matrix({ repeats: 3, cases: 1, mine: { [`${SKILL}/case-0`]: 3 } }), SKILL); + expect(verdict.verdict).toBe('earns-keep'); + }); + + it('no-unique-catches only with 5 owned bug cases at 3 repeats and nothing it beat security-review on', () => { + const verdict = judgeSkill(matrix({ + repeats: 3, + mine: { [`${SKILL}/case-0`]: 3 }, + reference: { [`${SKILL}/case-0`]: 3 }, + }), SKILL); + expect(verdict).toMatchObject({ verdict: 'no-unique-catches', uniqueCatches: [], ownedBugCases: { evidence: 5 } }); + }); + + it('insufficient-evidence with fewer than 5 owned bug cases', () => { + const verdict = judgeSkill(matrix({ repeats: 3, cases: 4 }), SKILL); + expect(verdict.verdict).toBe('insufficient-evidence'); + expect(verdict.reason).toContain('needs 5'); + }); + + it('insufficient-evidence below 3 repeats, even with 5 cases and no catches', () => { + expect(judgeSkill(matrix({ repeats: 2 }), SKILL).verdict).toBe('insufficient-evidence'); + }); + + it('a single-repeat catch is not a unique catch', () => { + const verdict = judgeSkill(matrix({ repeats: 1, mine: { [`${SKILL}/case-0`]: 1 } }), SKILL); + expect(verdict).toMatchObject({ verdict: 'insufficient-evidence', uniqueCatches: [], unsettled: [`${SKILL}/case-0`] }); + }); + + it('a bug it caught once that security-review missed blocks no-unique-catches', () => { + const verdict = judgeSkill(matrix({ repeats: 3, mine: { [`${SKILL}/case-0`]: 1 } }), SKILL); + expect(verdict).toMatchObject({ verdict: 'insufficient-evidence', unsettled: [`${SKILL}/case-0`] }); + }); + + it('a bug security-review caught once is unsettled, not unique', () => { + const verdict = judgeSkill(matrix({ + repeats: 3, + mine: { [`${SKILL}/case-0`]: 3 }, + reference: { [`${SKILL}/case-0`]: 1 }, + }), SKILL); + expect(verdict).toMatchObject({ verdict: 'insufficient-evidence', uniqueCatches: [], unsettled: [`${SKILL}/case-0`] }); + }); + + it('an errored entry blocks every verdict, including earns-keep', () => { + const results = matrix({ repeats: 3, mine: { [`${SKILL}/case-0`]: 3 } }); + recordShardRun(results, { + skill: SKILL, config: testConfig(), usage: testUsage(), error: 'boom', hits: new Map(), errors: [`${SKILL}/case-1`], + }); + expect(judgeSkill(results, SKILL)).toMatchObject({ verdict: 'insufficient-evidence', errors: 1 }); + }); + + it('security-review errors also block the verdict', () => { + const results = matrix({ repeats: 3 }); + recordShardRun(results, { + skill: 'security-review', config: testConfig(), usage: testUsage(), hits: new Map(), errors: [`${SKILL}/case-2`], + }); + expect(judgeSkill(results, SKILL).verdict).toBe('insufficient-evidence'); + }); + + it('needs a security-review lane to compare against', () => { + const results = createResults(owned(5)); + recordRepeats(results, SKILL, { [`${SKILL}/case-0`]: 3 }, 3); + expect(judgeSkill(results, SKILL).reason).toContain('no security-review lane'); + }); + + it('counts only bug cases the skill owns toward the case minimum', () => { + const entries = [...owned(4), testEntry('wrdn-authz/other'), testEntry('sentry-vuln-001')]; + const results = createResults(entries); + const none = Object.fromEntries(entries.map((entry) => [entry.id, 0])); + recordRepeats(results, 'security-review', none, 3); + recordRepeats(results, SKILL, none, 3); + expect(judgeSkill(results, SKILL)).toMatchObject({ + verdict: 'insufficient-evidence', + ownedBugCases: { evidence: 4, selected: 4 }, + }); + }); + + it('is attached to every wrdn-* lane summary and never to security-review', () => { + const summary = summarize(matrix({ repeats: 3 })); + expect(summary['security-review']?.verdict).toBeUndefined(); + expect(summary[SKILL]?.verdict?.verdict).toBe('no-unique-catches'); + }); +}); diff --git a/packages/security-evals/src/verdict.ts b/packages/security-evals/src/verdict.ts new file mode 100644 index 00000000..60a70ce9 --- /dev/null +++ b/packages/security-evals/src/verdict.ts @@ -0,0 +1,93 @@ +import type { SuiteResults, Tally } from './results.js'; + +/** The lane every other skill has to beat. */ +export const REFERENCE_SKILL = 'security-review'; + +/** Evidence a skill needs before the suite says anything about keeping it. */ +export const VERDICT_RULES = { + /** Repeats both lanes need on an entry before it counts as evidence. */ + minRepeats: 3, + /** Share of repeats that makes a catch reliable: 2 of 3. */ + reliableHitRate: 2 / 3, + /** Bug cases under `cases//` with full evidence before zero unique catches means anything. */ + minOwnedBugCases: 5, +} as const; + +export type VerdictName = 'earns-keep' | 'no-unique-catches' | 'insufficient-evidence'; + +export interface SkillVerdict { + verdict: VerdictName; + reason: string; + /** Bugs this skill caught reliably that security-review never caught. */ + uniqueCatches: string[]; + /** Bugs this skill caught at least once that security-review did not catch reliably. More repeats settle them. */ + unsettled: string[]; + /** Bug cases the skill owns: how many have full evidence, out of how many were selected. */ + ownedBugCases: { evidence: number; selected: number }; + /** Entry runs that errored in this lane or in security-review. Any error blocks a verdict. */ + errors: number; +} + +const rate = (tally: Tally): number => tally.hits / tally.runs; +const plural = (count: number, word: string): string => `${count} ${word}${count === 1 ? '' : 's'}`; + +/** + * Decide whether a wrdn-* skill catches bugs security-review misses. Only two + * outcomes are conclusive, and both need repeats; anything short of that is + * insufficient evidence, never a quiet zero. + */ +export function judgeSkill(results: SuiteResults, skill: string): SkillVerdict { + const { minRepeats, reliableHitRate, minOwnedBugCases } = VERDICT_RULES; + const entries = Object.entries(results.entries); + const errors = entries.reduce((sum, [, entry]) => sum + + (entry.lanes[skill]?.errors ?? 0) + (entry.lanes[REFERENCE_SKILL]?.errors ?? 0), 0); + + // Bug entries both lanes scored at least once. + const compared = entries.flatMap(([id, entry]) => { + const mine = entry.lanes[skill]; + const reference = entry.lanes[REFERENCE_SKILL]; + return entry.kind === 'bug' && mine && reference && mine.runs > 0 && reference.runs > 0 + ? [{ id, mine, reference, full: mine.runs >= minRepeats && reference.runs >= minRepeats }] + : []; + }); + const uniqueCatches = compared + .filter(({ mine, reference, full }) => full && rate(mine) >= reliableHitRate && reference.hits === 0) + .map(({ id }) => id); + const unsettled = compared + .filter(({ id, mine, reference }) => !uniqueCatches.includes(id) + && mine.hits > 0 && rate(reference) < reliableHitRate) + .map(({ id }) => id); + const owned = entries.filter(([id, entry]) => entry.source === 'case' && entry.kind === 'bug' + && id.startsWith(`${skill}/`)); + const ownedEvidence = compared.filter(({ id, full }) => full && id.startsWith(`${skill}/`)).length; + + const verdict = (name: VerdictName, reason: string): SkillVerdict => ({ + verdict: name, + reason, + uniqueCatches, + unsettled, + ownedBugCases: { evidence: ownedEvidence, selected: owned.length }, + errors, + }); + + if (!results.lanes[REFERENCE_SKILL]) { + return verdict('insufficient-evidence', `no ${REFERENCE_SKILL} lane to compare against`); + } + if (errors > 0) { + return verdict('insufficient-evidence', `${plural(errors, 'entry run')} errored; rerun before reading anything into it`); + } + if (uniqueCatches.length > 0) { + return verdict('earns-keep', `${plural(uniqueCatches.length, 'bug')} caught in at least 2 of 3 repeats ` + + `that ${REFERENCE_SKILL} never caught`); + } + if (unsettled.length > 0) { + return verdict('insufficient-evidence', `${plural(unsettled.length, 'bug')} it caught that ${REFERENCE_SKILL} ` + + `did not catch reliably; more repeats settle ${unsettled.length === 1 ? 'it' : 'them'}`); + } + if (ownedEvidence < minOwnedBugCases) { + return verdict('insufficient-evidence', `${ownedEvidence} owned bug cases with ${minRepeats}+ repeats in both lanes, ` + + `needs ${minOwnedBugCases} (${owned.length} selected)`); + } + return verdict('no-unique-catches', `0 unique catches across ${ownedEvidence} owned bug cases and ` + + `${plural(compared.length, 'bug entry')} overall at ${minRepeats}+ repeats`); +} diff --git a/packages/security-evals/src/worktrees.test.ts b/packages/security-evals/src/worktrees.test.ts new file mode 100644 index 00000000..9eab66b9 --- /dev/null +++ b/packages/security-evals/src/worktrees.test.ts @@ -0,0 +1,97 @@ +import { execFileSync } from 'node:child_process'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { SKILLS_CHECKOUT, ensureWorktree, removeWorktree, repoCheckout, resolveCheckout } from './worktrees.js'; + +function configuredEmail(): string { + try { + return execFileSync('git', ['config', '--get', 'user.email'], { encoding: 'utf8' }).trim(); + } catch { + return ''; + } +} + +// CI runners have no git identity; keep a configured one so commits stay attributable locally. +const IDENTITY = configuredEmail() ? [] : ['-c', 'user.name=Warden Test', '-c', 'user.email=test@warden.dev']; + +function git(args: string[], cwd: string): string { + return execFileSync('git', [...IDENTITY, '-c', 'commit.gpgsign=false', ...args], { cwd, encoding: 'utf8' }).trim(); +} + +function commit(repo: string, content: string): string { + writeFileSync(join(repo, 'app.py'), content); + git(['add', 'app.py'], repo); + git(['commit', '-q', '-m', content], repo); + return git(['rev-parse', 'HEAD'], repo); +} + +describe('ensureWorktree', () => { + let root: string; + let upstream: string; + let clone: string; + + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'security-evals-')); + upstream = join(root, 'upstream'); + clone = join(root, 'clone'); + git(['init', '-q', '--initial-branch=main', upstream], root); + }); + + afterEach(() => { + rmSync(root, { recursive: true, force: true }); + }); + + it('creates a detached worktree once and reuses it', () => { + const ref = commit(upstream, 'v1'); + git(['clone', '-q', upstream, clone], root); + const worktrees = join(root, 'worktrees'); + + const path = ensureWorktree(clone, worktrees, ref); + expect(path).toBe(join(worktrees, ref)); + expect(readFileSync(join(path, 'app.py'), 'utf8')).toBe('v1'); + expect(ensureWorktree(clone, worktrees, ref)).toBe(path); + + removeWorktree(clone, path); + expect(existsSync(path)).toBe(false); + }); + + it('fetches a ref the clone does not have yet', () => { + commit(upstream, 'v1'); + git(['clone', '-q', upstream, clone], root); + const later = commit(upstream, 'v2'); + + const path = ensureWorktree(clone, join(root, 'worktrees'), later); + expect(readFileSync(join(path, 'app.py'), 'utf8')).toBe('v2'); + }); + + it('checks out a blobless clone, fetching blobs on demand', () => { + const ref = commit(upstream, 'v1'); + git(['config', 'uploadpack.allowFilter', 'true'], upstream); + git(['clone', '-q', '--filter=blob:none', '--no-checkout', `file://${upstream}`, clone], root); + + const path = ensureWorktree(clone, join(root, 'worktrees'), ref); + expect(readFileSync(join(path, 'app.py'), 'utf8')).toBe('v1'); + }); + + it('refuses a worktree that moved off its ref', () => { + const first = commit(upstream, 'v1'); + const second = commit(upstream, 'v2'); + git(['clone', '-q', upstream, clone], root); + const worktrees = join(root, 'worktrees'); + const path = ensureWorktree(clone, worktrees, first); + git(['checkout', '-q', '--detach', second], path); + + expect(() => ensureWorktree(clone, worktrees, first)).toThrow(`expected ${first}`); + }); +}); + +describe('resolveCheckout', () => { + it('fails fast without a clone instead of cloning', () => { + expect(() => resolveCheckout(repoCheckout('getsentry/getsentry'), undefined)) + .toThrow('WARDEN_SECURITY_EVALS_REPOS (or --repo getsentry/getsentry=) must point at a local getsentry/getsentry clone'); + expect(() => resolveCheckout(SKILLS_CHECKOUT, undefined)).toThrow('getsentry/warden-skills clone'); + expect(() => resolveCheckout(repoCheckout('getsentry/sentry'), tmpdir())).toThrow('is not a git checkout'); + }); +}); diff --git a/packages/security-evals/src/worktrees.ts b/packages/security-evals/src/worktrees.ts new file mode 100644 index 00000000..876a51d8 --- /dev/null +++ b/packages/security-evals/src/worktrees.ts @@ -0,0 +1,85 @@ +import { execFileSync } from 'node:child_process'; +import { existsSync, mkdirSync } from 'node:fs'; +import { join, resolve } from 'node:path'; + +/** A repository the suite reads from a local clone, never clones itself. */ +export interface CheckoutSpec { + repository: string; + /** Env var that names the clone; the CLI flag maps onto it. */ + env: string; + flag: string; +} + +/** A scanned repository. Entries name it; `--repo owner/name=` supplies the clone. */ +export function repoCheckout(repository: string): CheckoutSpec { + return { repository, env: 'WARDEN_SECURITY_EVALS_REPOS', flag: `--repo ${repository}=` }; +} + +export const SKILLS_CHECKOUT: CheckoutSpec = { + repository: 'getsentry/warden-skills', + env: 'WARDEN_SECURITY_EVALS_SKILLS_REPO', + flag: '--skills-repo', +}; + +// Roach hands its proxy variables to test workers and refuses origins other +// than the model APIs, so git goes direct. +const PROXY_VARS = new Set(['HTTPS_PROXY', 'https_proxy', 'HTTP_PROXY', 'http_proxy', 'ALL_PROXY', 'all_proxy']); + +function git(args: string[], cwd: string, extraEnv: NodeJS.ProcessEnv = {}): string { + const env = Object.fromEntries(Object.entries({ ...process.env, ...extraEnv }) + .filter(([name]) => !PROXY_VARS.has(name))); + return execFileSync('git', args, { cwd, encoding: 'utf8', env, stdio: ['ignore', 'pipe', 'pipe'] }).trim(); +} + +function hasCommit(repoPath: string, ref: string): boolean { + try { + // Without this, a blobless clone would fetch a missing commit during the check. + git(['cat-file', '-e', `${ref}^{commit}`], repoPath, { GIT_NO_LAZY_FETCH: '1' }); + return true; + } catch { + return false; + } +} + +/** Resolve a local clone the suite builds worktrees from. Never clones on its own. */ +export function resolveCheckout(spec: CheckoutSpec, path: string | undefined): string { + if (!path) { + throw new Error( + `${spec.env} (or ${spec.flag}) must point at a local ${spec.repository} clone. ` + + `A blobless clone is enough: git clone --filter=blob:none https://github.com/${spec.repository}.git`, + ); + } + const repoPath = resolve(path); + try { + return git(['rev-parse', '--show-toplevel'], repoPath); + } catch { + throw new Error(`${repoPath} is not a git checkout. Point ${spec.env} at a ${spec.repository} clone.`); + } +} + +/** + * Return a detached worktree of `ref`, creating it once and reusing it after. + * Fetches `ref` from origin only when the clone does not already have it. + */ +export function ensureWorktree(repoPath: string, worktreesDir: string, ref: string): string { + const path = join(worktreesDir, ref); + if (existsSync(path)) { + const head = git(['rev-parse', 'HEAD'], path); + if (head !== ref) { + throw new Error(`Worktree ${path} is at ${head}, expected ${ref}. Remove it and rerun.`); + } + return path; + } + + if (!hasCommit(repoPath, ref)) { + git(['fetch', '--quiet', 'origin', ref], repoPath); + } + mkdirSync(worktreesDir, { recursive: true }); + git(['worktree', 'add', '--quiet', '--detach', path, ref], repoPath); + return path; +} + +/** Remove a worktree created by `ensureWorktree`. CI uses this to cap disk use. */ +export function removeWorktree(repoPath: string, path: string): void { + git(['worktree', 'remove', '--force', path], repoPath); +} diff --git a/packages/security-evals/tsconfig.json b/packages/security-evals/tsconfig.json new file mode 100644 index 00000000..d1eb8b1d --- /dev/null +++ b/packages/security-evals/tsconfig.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022"], + "strict": true, + "jsx": "react-jsx", + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "isolatedModules": true, + "noUncheckedIndexedAccess": true, + "noImplicitOverride": true, + "noPropertyAccessFromIndexSignature": true, + "paths": { + "@sentry/warden": ["../warden/src/index.ts"] + }, + "types": ["node", "vitest"] + }, + "include": ["src/**/*", "scripts/**/*"] +} diff --git a/packages/security-evals/vitest.config.ts b/packages/security-evals/vitest.config.ts new file mode 100644 index 00000000..bdd25284 --- /dev/null +++ b/packages/security-evals/vitest.config.ts @@ -0,0 +1,21 @@ +import { defineConfig } from 'vitest/config'; +import { fileURLToPath } from 'node:url'; + +// Run through `pnpm evals:security`, which maps CLI flags onto WARDEN_SECURITY_EVALS_* env vars. +export default defineConfig({ + resolve: { + alias: { + '@sentry/warden': fileURLToPath(new URL('../warden/src/index.ts', import.meta.url)), + '@sentry/warden-service-api': fileURLToPath(new URL('../warden-service-api/src/index.ts', import.meta.url)), + }, + }, + test: { + include: ['src/**/*.eval.ts'], + exclude: ['**/node_modules/**', '**/dist/**'], + // Roach records model calls and replays them while the worktree content stays pinned. + globalSetup: ['@sentry/roach/vitest'], + setupFiles: ['./src/setup.ts', '@sentry/roach/vitest/setup'], + // Passes share one results object and one worktree per shard. + fileParallelism: false, + }, +}); diff --git a/packages/security-evals/vitest.unit.config.ts b/packages/security-evals/vitest.unit.config.ts new file mode 100644 index 00000000..a971045f --- /dev/null +++ b/packages/security-evals/vitest.unit.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from 'vitest/config'; +import { fileURLToPath } from 'node:url'; + +export default defineConfig({ + resolve: { + alias: { + '@sentry/warden': fileURLToPath(new URL('../warden/src/index.ts', import.meta.url)), + '@sentry/warden-service-api': fileURLToPath(new URL('../warden-service-api/src/index.ts', import.meta.url)), + }, + }, + test: { + include: ['src/**/*.test.ts'], + exclude: ['**/node_modules/**', '**/dist/**'], + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 09437490..b29f3593 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -212,6 +212,34 @@ importers: specifier: ^4.1.11 version: 4.1.11(@edge-runtime/vm@3.2.0)(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(@vitest/coverage-v8@4.1.11)(happy-dom@21.0.4)(vite@8.3.5(@types/node@25.9.1)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.12)(yaml@2.9.0)) + packages/security-evals: + dependencies: + '@sentry/warden': + specifier: workspace:* + version: link:../warden + dotenv: + specifier: ^17.4.2 + version: 17.4.2 + zod: + specifier: ^4.6.7 + version: 4.6.7 + devDependencies: + '@sentry/roach': + specifier: ^0.2.0 + version: 0.2.0(rollup@4.60.4)(vitest@4.1.11) + '@types/node': + specifier: ^25.9.1 + version: 25.9.1 + tsx: + specifier: ^4.23.12 + version: 4.23.12 + typescript: + specifier: ^6.0.3 + version: 6.0.3 + vitest: + specifier: ^4.1.11 + version: 4.1.11(@edge-runtime/vm@3.2.0)(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(@vitest/coverage-v8@4.1.11)(happy-dom@21.0.4)(vite@8.3.5(@types/node@25.9.1)(esbuild@0.28.2)(jiti@2.7.0)(tsx@4.23.12)(yaml@2.9.0)) + packages/warden: dependencies: '@anthropic-ai/claude-agent-sdk':