chore(evals): Switch Sonnet defaults to GPT 6 Luna

.github/workflows/evals.yml

@@ -86,7 +86,7 @@ jobs:
     env:
       # Pi provider/model selector for every eval and the judge. Override with the
       # WARDEN_EVAL_MODEL repo variable; add the matching WARDEN_<PROVIDER>_API_KEY below.
-      WARDEN_EVAL_MODEL: ${{ vars.WARDEN_EVAL_MODEL || 'openrouter/anthropic/claude-sonnet-4.6' }}
+      WARDEN_EVAL_MODEL: ${{ vars.WARDEN_EVAL_MODEL || 'openrouter/openai/gpt-6-luna' }}
       WARDEN_EVAL_JUDGE_MODEL: ${{ vars.WARDEN_EVAL_JUDGE_MODEL }}
       WARDEN_OPENROUTER_API_KEY: ${{ secrets.WARDEN_OPENROUTER_API_KEY }}
       WARDEN_ANTHROPIC_API_KEY: ${{ secrets.WARDEN_ANTHROPIC_API_KEY }}

packages/evals/README.md

@@ -32,7 +32,7 @@ name, such as `packages/evals/code-review/` or `packages/evals/security-review/`
 ```yaml
 skill: skills/bug-detection.md
 runtime: pi
-model: anthropic/claude-sonnet-4-6
+model: openai/gpt-6-luna
 
 evals:
   - name: null-property-access
@@ -121,7 +121,7 @@ Eval test names are formatted as:
 ```
 
 The suite chooses runtime and model. The current full-pipeline suites run Pi
-with `anthropic/claude-sonnet-4-6`; future suites can matrix the same fixtures
+with `openai/gpt-6-luna`; future suites can matrix the same fixtures
 over multiple runtimes or models without changing case identity.
 
 The Vitest entrypoints are intentionally split by eval layer:
@@ -139,8 +139,8 @@ The Vitest entrypoints are intentionally split by eval layer:
 | Field | Required | Description |
 |-------|----------|-------------|
 | `skill` | Yes | Path to test skill, relative to `packages/evals/` |
-| `runtime` | No | Default runtime for all evals: `claude` or `pi` (default: `claude`) |
-| `model` | No | Default model for all evals (default: `claude-sonnet-4-6`; Pi models must use provider/model format, e.g. `anthropic/claude-sonnet-4-6`) |
+| `runtime` | No | Default runtime for all evals: `claude` or `pi` (default: `pi`) |
+| `model` | No | Default model for all evals (default: `openai/gpt-6-luna`; Pi models must use provider/model format) |
 | `evals` | Yes | List of eval scenarios (at least one) |
 
 ### Per-eval fields
@@ -198,11 +198,11 @@ positive.
 # through its dist/ entry point
 pnpm build
 
-# Run all evals with the checked-in Anthropic models
-WARDEN_ANTHROPIC_API_KEY=sk-ant-... pnpm evals
+# Run all evals with the checked-in OpenAI models
+WARDEN_OPENAI_API_KEY=sk-... pnpm evals
 
 # Run all evals through another provider
-WARDEN_EVAL_MODEL=openrouter/anthropic/claude-sonnet-4.6 \
+WARDEN_EVAL_MODEL=openrouter/openai/gpt-6-luna \
   WARDEN_OPENROUTER_API_KEY=sk-or-... pnpm evals
 
 # Run evals for a specific skill
@@ -228,9 +228,9 @@ pnpm evals:scaffold https://github.com/getsentry/sentry/pull/12345
 ```
 
 Evals make real API calls. Suites choose the runtime and model; the checked-in
-suites run Pi with `anthropic/claude-sonnet-4-6`, and the judge uses the same
-model. With only that default, evals are skipped unless `ANTHROPIC_API_KEY` or
-`WARDEN_ANTHROPIC_API_KEY` is set.
+suites run Pi with `openai/gpt-6-luna`, and the judge uses the same
+model. With only that default, evals are skipped unless `OPENAI_API_KEY` or
+`WARDEN_OPENAI_API_KEY` is set.
 
 To run against any other provider, set:
 
@@ -249,7 +249,7 @@ CI runs evals when eval files or harness code change on a PR, when changes land
 on `main`, or when a maintainer adds the `run-evals` label to a same-repository
 PR. Fork PRs do not receive eval secrets.
 
-CI defaults `WARDEN_EVAL_MODEL` to `openrouter/anthropic/claude-sonnet-4.6`
+CI defaults `WARDEN_EVAL_MODEL` to `openrouter/openai/gpt-6-luna`
 with `WARDEN_OPENROUTER_API_KEY`. Set the `WARDEN_EVAL_MODEL` (and optionally
 `WARDEN_EVAL_JUDGE_MODEL`) repository variable to switch providers, and add the
 matching `WARDEN_<PROVIDER>_API_KEY` secret to the workflow env.
@@ -350,6 +350,6 @@ them in CLI output. Review that list before committing the eval.
 4. **Context**: Build `EventContext` from real `git diff main...eval`
 5. **Execution**: Run the skill via `runSkill()` with the real SDK pipeline;
    the agent operates in the temp repo with Read/Grep tools
-6. **Judgment**: An LLM judge (Sonnet) evaluates findings against assertions
+6. **Judgment**: An LLM judge (GPT 6 Luna by default) evaluates findings against assertions
 7. **Verdict**: Pass if all required `should_find` are met and no
    `should_not_find` are violated

packages/evals/eval-bug-detection.yaml

@@ -1,6 +1,6 @@
 skill: skills/bug-detection.md
 runtime: pi
-model: anthropic/claude-sonnet-4-6
+model: openai/gpt-6-luna
 
 evals:
   - name: null-property-access

packages/evals/eval-precision.yaml

@@ -1,6 +1,6 @@
 skill: skills/precision.md
 runtime: pi
-model: anthropic/claude-sonnet-4-6
+model: openai/gpt-6-luna
 
 evals:
   - name: ignores-style-issues

packages/evals/eval-security-scanning.yaml

@@ -1,6 +1,6 @@
 skill: skills/security-scanning.md
 runtime: pi
-model: anthropic/claude-sonnet-4-6
+model: openai/gpt-6-luna
 
 evals:
   - name: sql-injection

packages/evals/src/code-review.eval.ts

@@ -14,7 +14,7 @@ const evals = discoverEvalScenarios({
   category: 'code-review',
   skill: '../warden/src/builtin-skills/code-review/SKILL.md',
   runtime: 'pi',
-  model: 'anthropic/claude-sonnet-4-6',
+  model: 'openai/gpt-6-luna',
 });
 const CODE_REVIEW_RUN_TIMEOUT_MS = 120_000;
 const CODE_REVIEW_EVAL_TIMEOUT_MS = CODE_REVIEW_RUN_TIMEOUT_MS + 60_000;

packages/evals/src/harness.test.ts

@@ -101,7 +101,7 @@ describe('createWardenEvalJudge', () => {
       usage: emptyUsage(),
     });
 
-    const judge = createWardenEvalJudge({ apiKey: 'test-key', model: 'anthropic/claude-sonnet-4-6' });
+    const judge = createWardenEvalJudge({ apiKey: 'test-key', model: 'openai/gpt-6-luna' });
     const result = await judge.assess(makeJudgeContext(meta, findings));
 
     expect(result.score).toBe(1);
@@ -122,7 +122,7 @@ describe('createWardenEvalJudge', () => {
       usage: emptyUsage(),
     });
 
-    const judge = createWardenEvalJudge({ apiKey: 'test-key', model: 'anthropic/claude-sonnet-4-6' });
+    const judge = createWardenEvalJudge({ apiKey: 'test-key', model: 'openai/gpt-6-luna' });
     const result = await judge.assess(makeJudgeContext(meta, [makeFinding()]));
 
     expect(result.score).toBe(0);
@@ -149,7 +149,7 @@ describe('createWardenEvalJudge', () => {
       error: 'No JSON found in judge response',
     });
 
-    const judge = createWardenEvalJudge({ apiKey: 'test-key', model: 'anthropic/claude-sonnet-4-6' });
+    const judge = createWardenEvalJudge({ apiKey: 'test-key', model: 'openai/gpt-6-luna' });
     const result = await judge.assess(makeJudgeContext(meta, [makeFinding()]));
 
     expect(result.score).toBe(0);

packages/evals/src/index.test.ts

@@ -180,7 +180,7 @@ describe('standalone scenario files', () => {
       category: 'security-review',
       skill: '../warden/src/builtin-skills/security-review/SKILL.md',
       runtime: 'pi',
-      model: 'anthropic/claude-sonnet-4-6',
+      model: 'openai/gpt-6-luna',
       baseDir: evalsDir,
     });
 
@@ -189,7 +189,7 @@ describe('standalone scenario files', () => {
     expect(meta.skillName).toBe('security-review');
     expect(meta.skillPath).toContain('src/builtin-skills/security-review/SKILL.md');
     expect(meta.runtime).toBe('pi');
-    expect(meta.model).toBe('anthropic/claude-sonnet-4-6');
+    expect(meta.model).toBe('openai/gpt-6-luna');
     expect(meta.filePaths[0]).toContain('evals/fixtures/sentry-replay-delete-read-scope/');
   });
 
@@ -198,7 +198,7 @@ describe('standalone scenario files', () => {
       category: 'security-review',
       skill: '../warden/src/builtin-skills/security-review/SKILL.md',
       runtime: 'pi',
-      model: 'anthropic/claude-sonnet-4-6',
+      model: 'openai/gpt-6-luna',
       baseDir: evalsDir,
     });
 
@@ -211,7 +211,7 @@ describe('standalone scenario files', () => {
       category: 'code-review',
       skill: '../warden/src/builtin-skills/code-review/SKILL.md',
       runtime: 'pi',
-      model: 'anthropic/claude-sonnet-4-6',
+      model: 'openai/gpt-6-luna',
       baseDir: evalsDir,
     });
 
@@ -391,7 +391,7 @@ describe('EvalFileSchema', () => {
     const valid = {
       skill: '../warden/src/builtin-skills/security-review/SKILL.md',
       runtime: 'pi',
-      model: 'anthropic/claude-sonnet-4-6',
+      model: 'openai/gpt-6-luna',
       evals: [{
         name: 'sentry-miss',
         given: 'Sentry endpoint with a known authorization miss',

packages/evals/src/integrity-review.eval.ts

@@ -14,7 +14,7 @@ const evals = discoverEvalScenarios({
   category: 'integrity-review',
   skill: '../warden/src/builtin-skills/integrity-review/SKILL.md',
   runtime: 'pi',
-  model: 'anthropic/claude-sonnet-4-6',
+  model: 'openai/gpt-6-luna',
 });
 // Multi-file fixtures run several hunks plus verification and merge passes,
 // which can exceed the 120s the single-file suites use.

packages/evals/src/judge.test.ts

@@ -30,7 +30,7 @@ const meta: EvalMeta = {
   given: 'code that dereferences a possibly undefined value',
   skillPath: '/skills/bug-detection.md',
   filePaths: ['/fixtures/handler.ts'],
-  model: 'openrouter/anthropic/claude-sonnet-4.6',
+  model: 'openrouter/openai/gpt-6-luna',
   runtime: 'pi',
   should_find: [{ finding: 'null access on user', required: true }],
   should_not_find: ['style issues'],
@@ -49,12 +49,12 @@ describe('runJudge', () => {
     };
     runAuxiliary.mockResolvedValue({ success: true, data: response, usage });
 
-    const result = await runJudge(meta, [], { model: 'openrouter/anthropic/claude-sonnet-4.6' });
+    const result = await runJudge(meta, [], { model: 'openrouter/openai/gpt-6-luna' });
 
     expect(mockedGetRuntime).toHaveBeenCalledWith('pi');
     expect(runAuxiliary).toHaveBeenCalledWith(expect.objectContaining({
       task: 'eval_judge',
-      model: 'openrouter/anthropic/claude-sonnet-4.6',
+      model: 'openrouter/openai/gpt-6-luna',
       apiKey: undefined,
       maxRetries: 0,
     }));
@@ -64,15 +64,15 @@ describe('runJudge', () => {
   it('passes an Anthropic key through when one is configured', async () => {
     runAuxiliary.mockResolvedValue({ success: false, error: 'boom', usage });
 
-    await runJudge(meta, [], { model: 'anthropic/claude-sonnet-4-6', apiKey: 'sk-ant' });
+    await runJudge(meta, [], { model: 'openai/gpt-6-luna', apiKey: 'sk-ant' });
 
     expect(runAuxiliary).toHaveBeenCalledWith(expect.objectContaining({ apiKey: 'sk-ant' }));
   });
 
   it('marks every assertion failed when the judge call fails', async () => {
     runAuxiliary.mockResolvedValue({ success: false, error: 'No API key for provider: openrouter', usage });
 
-    const result = await runJudge(meta, [], { model: 'openrouter/anthropic/claude-sonnet-4.6' });
+    const result = await runJudge(meta, [], { model: 'openrouter/openai/gpt-6-luna' });
 
     expect(result.error).toBe('Judge call failed: No API key for provider: openrouter');
     expect(result.usage).toBe(usage);
@@ -91,7 +91,7 @@ describe('runJudge', () => {
       usage,
     });
 
-    const result = await runJudge(meta, [], { model: 'openrouter/anthropic/claude-sonnet-4.6' });
+    const result = await runJudge(meta, [], { model: 'openrouter/openai/gpt-6-luna' });
 
     expect(result.error).toBe('Judge returned 0 verdicts, expected 1');
   });

packages/evals/src/judge.ts

@@ -87,7 +87,7 @@ Requirements:
 }
 
 export interface JudgeOptions {
-  /** Pi model selector for the judge, e.g. `openrouter/anthropic/claude-sonnet-4.6`. */
+  /** Pi model selector for the judge, e.g. `openrouter/openai/gpt-6-luna`. */
   model: string;
   /** Anthropic key for Warden's legacy `apiKey` option. Other providers read their env key. */
   apiKey?: string;

packages/evals/src/provider.test.ts

@@ -2,7 +2,7 @@ import { describe, expect, it } from 'vitest';
 import { DEFAULT_JUDGE_MODEL, resolveEvalProvider } from './provider.js';
 
 describe('resolveEvalProvider', () => {
-  it('is disabled without a model override or Anthropic key', () => {
+  it('is disabled without a model override or OpenAI key', () => {
     expect(resolveEvalProvider({})).toEqual({
       apiKey: '',
       model: undefined,
@@ -13,46 +13,52 @@ describe('resolveEvalProvider', () => {
     });
   });
 
-  it('keeps scenario models when only an Anthropic key is set', () => {
-    const config = resolveEvalProvider({ WARDEN_ANTHROPIC_API_KEY: 'sk-ant' });
+  it('keeps scenario models when only an OpenAI key is set', () => {
+    const config = resolveEvalProvider({ WARDEN_OPENAI_API_KEY: 'sk-openai' });
 
     expect(config).toMatchObject({
-      apiKey: 'sk-ant',
       model: undefined,
       runtime: undefined,
       judgeModel: DEFAULT_JUDGE_MODEL,
       enabled: true,
     });
   });
 
-  it('accepts the unprefixed Anthropic key', () => {
-    expect(resolveEvalProvider({ ANTHROPIC_API_KEY: 'sk-ant' }).apiKey).toBe('sk-ant');
+  it('accepts the unprefixed OpenAI key', () => {
+    expect(resolveEvalProvider({ OPENAI_API_KEY: 'sk-openai' }).enabled).toBe(true);
+  });
+
+  it('stays disabled with only an Anthropic key, since the default models are OpenAI', () => {
+    const config = resolveEvalProvider({ WARDEN_ANTHROPIC_API_KEY: 'sk-ant' });
+
+    expect(config.apiKey).toBe('sk-ant');
+    expect(config.enabled).toBe(false);
   });
 
   it('routes evals, auxiliary calls, and the judge through WARDEN_EVAL_MODEL on Pi', () => {
     const config = resolveEvalProvider({
-      WARDEN_EVAL_MODEL: ' openrouter/anthropic/claude-sonnet-4.6 ',
+      WARDEN_EVAL_MODEL: ' openrouter/openai/gpt-6-luna ',
       WARDEN_OPENROUTER_API_KEY: 'sk-or',
     });
 
     expect(config).toEqual({
       apiKey: '',
-      model: 'openrouter/anthropic/claude-sonnet-4.6',
+      model: 'openrouter/openai/gpt-6-luna',
       runtime: 'pi',
-      auxiliaryModel: 'openrouter/anthropic/claude-sonnet-4.6',
-      judgeModel: 'openrouter/anthropic/claude-sonnet-4.6',
+      auxiliaryModel: 'openrouter/openai/gpt-6-luna',
+      judgeModel: 'openrouter/openai/gpt-6-luna',
       enabled: true,
     });
   });
 
   it('uses WARDEN_EVAL_JUDGE_MODEL for the judge when set', () => {
     const config = resolveEvalProvider({
       WARDEN_EVAL_MODEL: 'openrouter/x-ai/grok-4.3',
-      WARDEN_EVAL_JUDGE_MODEL: 'openrouter/anthropic/claude-sonnet-4.6',
+      WARDEN_EVAL_JUDGE_MODEL: 'openrouter/openai/gpt-6-luna',
     });
 
     expect(config.model).toBe('openrouter/x-ai/grok-4.3');
-    expect(config.judgeModel).toBe('openrouter/anthropic/claude-sonnet-4.6');
+    expect(config.judgeModel).toBe('openrouter/openai/gpt-6-luna');
   });
 
   it('treats empty overrides as unset', () => {
@@ -64,7 +70,7 @@ describe('resolveEvalProvider', () => {
   });
 
   it('rejects a model override without a provider prefix', () => {
-    expect(() => resolveEvalProvider({ WARDEN_EVAL_MODEL: 'claude-sonnet-4-6' })).toThrow(
+    expect(() => resolveEvalProvider({ WARDEN_EVAL_MODEL: 'gpt-6-luna' })).toThrow(
       'WARDEN_EVAL_MODEL must be a provider/model selector'
     );
   });

packages/evals/src/provider.ts

@@ -1,13 +1,13 @@
 import type { RuntimeName } from '@sentry/warden';
 
-/** Env var that points every live eval at one Pi model selector, e.g. `openrouter/anthropic/claude-sonnet-4.6`. */
+/** Env var that points every live eval at one Pi model selector, e.g. `openrouter/openai/gpt-6-luna`. */
 const EVAL_MODEL_ENV = 'WARDEN_EVAL_MODEL';
 
 /** Env var that overrides the judge model. Defaults to the eval model. */
 const EVAL_JUDGE_MODEL_ENV = 'WARDEN_EVAL_JUDGE_MODEL';
 
 /** Judge model used when no eval model is configured. */
-export const DEFAULT_JUDGE_MODEL = 'anthropic/claude-sonnet-4-6';
+export const DEFAULT_JUDGE_MODEL = 'openai/gpt-6-luna';
 
 export interface EvalProviderConfig {
   /** Anthropic key for Warden's legacy `apiKey` option. Empty when unset. */
@@ -23,7 +23,7 @@ export interface EvalProviderConfig {
   auxiliaryModel?: string;
   /** Pi model selector for the LLM judge. */
   judgeModel: string;
-  /** Whether live evals have a configured model or Anthropic key to run with. */
+  /** Whether live evals have a configured model or an OpenAI key for the default models. */
   enabled: boolean;
 }
 
@@ -33,21 +33,22 @@ function readModelSelector(env: NodeJS.ProcessEnv, name: string): string | undef
     return undefined;
   }
   if (!value.includes('/')) {
-    throw new Error(`${name} must be a provider/model selector, for example openrouter/anthropic/claude-sonnet-4.6`);
+    throw new Error(`${name} must be a provider/model selector, for example openrouter/openai/gpt-6-luna`);
   }
   return value;
 }
 
 /**
  * Resolve which model and credentials live evals use.
  *
- * Without `WARDEN_EVAL_MODEL`, scenarios keep their YAML models and evals run only
- * when an Anthropic key is present. With it, every scenario, Warden's auxiliary
- * calls, and the judge use that selector, and credentials come from the
- * provider's `WARDEN_<PROVIDER>_API_KEY`.
+ * Without `WARDEN_EVAL_MODEL`, scenarios keep their checked-in OpenAI models and
+ * evals run only when an OpenAI key is present. With it, every scenario,
+ * Warden's auxiliary calls, and the judge use that selector, and credentials
+ * come from the provider's `WARDEN_<PROVIDER>_API_KEY`.
  */
 export function resolveEvalProvider(env: NodeJS.ProcessEnv = process.env): EvalProviderConfig {
   const apiKey = env['WARDEN_ANTHROPIC_API_KEY'] || env['ANTHROPIC_API_KEY'] || '';
+  const openaiKey = env['WARDEN_OPENAI_API_KEY'] || env['OPENAI_API_KEY'] || '';
   const model = readModelSelector(env, EVAL_MODEL_ENV);
   const judgeModel = readModelSelector(env, EVAL_JUDGE_MODEL_ENV) ?? model ?? DEFAULT_JUDGE_MODEL;
 
@@ -57,6 +58,6 @@ export function resolveEvalProvider(env: NodeJS.ProcessEnv = process.env): EvalP
     runtime: model ? 'pi' : undefined,
     auxiliaryModel: model,
     judgeModel,
-    enabled: Boolean(model || apiKey),
+    enabled: Boolean(model || openaiKey),
   };
 }