diff --git a/.env.example b/.env.example index 3d1e0dd..ab696d3 100644 --- a/.env.example +++ b/.env.example @@ -7,12 +7,17 @@ CLAUDE_CODE_OAUTH_TOKEN=sk-ant-oat01-... # is unset. Only uncomment and set if you specifically want API-key auth. # ANTHROPIC_API_KEY=sk-ant-... +# OpenRouter API key, for runs on an OpenRouter model +# (--model deepseek/deepseek-v4.1-flash). Claude credentials are not passed to +# those sessions. Set a credit limit on the key at openrouter.ai. +# OPENROUTER_API_KEY=sk-or-v1-... + # Default model for evaluation runs. Must be one of: -# - claude-haiku-4-5-20251001 (cheap, fast; default for PR/narrow tasks, trigger evals) -# - claude-sonnet-5 (higher fidelity; for realistic/adversarial tiers) +# - claude-haiku-5-5 (cheap, fast; default for PR/narrow tasks, trigger evals) +# - claude-sonnet-5-5 (higher fidelity; for realistic/adversarial tiers) # claude-opus-5-5 is a manual, explicit opt-in only # (--model claude-opus-5-5 --allow-opus --max-cost-usd ) — see ADR-004. -CLAUDE_MODEL_DEFAULT=claude-haiku-4-5-20251001 +CLAUDE_MODEL_DEFAULT=claude-haiku-5-5 # Log level for the harness (DEBUG, INFO, WARNING, ERROR). RF_SKILL_EVAL_LOG_LEVEL=INFO diff --git a/.github/workflows/skill-evaluation.yml b/.github/workflows/skill-evaluation.yml index c721d17..476181a 100644 --- a/.github/workflows/skill-evaluation.yml +++ b/.github/workflows/skill-evaluation.yml @@ -11,6 +11,11 @@ name: Skill Evaluation # The partial tier (only rf-core staged) runs like narrow. # manual -> same as weekly with model / runs / allow_opus inputs. # Every harness invocation carries an explicit --max-cost-usd cap. +# +# Provider (openrouter-ci-eval-tier): when the OPENROUTER_API_KEY secret is set, +# PR and weekly runs use the OpenRouter tier (DeepSeek V4.1 Flash, turn-limit +# factor 1.5) and gate against plugins//evals/baselines/openrouter/ +# deepseek-v4.1-flash/. Claude runs are a manual dispatch (provider: claude). on: pull_request: @@ -28,6 +33,20 @@ on: - cron: '0 4 * * 0' workflow_dispatch: inputs: + provider: + description: "openrouter (DeepSeek, the CI default when its key is set) or claude" + required: false + default: "" + type: choice + options: + - "" + - "openrouter" + - "claude" + plugins: + description: "Comma list of plugins to run (empty = every plugin with evals)" + required: false + default: "" + type: string tiers: description: "Comma list of tiers" required: true @@ -43,7 +62,7 @@ on: - "treatment" - "baseline" model: - description: "Override every task's model (empty = tier defaults: narrow Haiku, realistic/adversarial Sonnet)" + description: "Claude runs only: override every task's model (empty = tier defaults: narrow Haiku, realistic/adversarial Sonnet)" required: false default: "" type: choice @@ -89,6 +108,17 @@ permissions: env: HAIKU_MODEL: claude-haiku-5-5 + OPENROUTER_MODEL: deepseek/deepseek-v4.1-flash + OPENROUTER_TURNS_FACTOR: "1.5" + # Per-plugin overrides of the factor (JSON), e.g. '{"rf-desktop": "2.0"}'. + # The factor is stored in the baselines. rf-desktop at 2.0 (run 43167186288) + # took more turns but passed no more often, so no plugin overrides it. + OPENROUTER_TURNS_FACTORS: '{}' + # OpenRouter caps (priced at list prices, an upper bound); the key's credit + # limit on OpenRouter is the hard cap. Tune after the first weekly run. + PR_NARROW_CAP_USD_OPENROUTER: "2" + PR_TRIGGER_CAP_USD_OPENROUTER: "2" + WEEKLY_CAP_USD_OPENROUTER: "3" # Initial caps (design D9 open question) - tune after the first weekly run. PR_NARROW_CAP_USD: "10" PR_TRIGGER_CAP_USD: "20" @@ -105,6 +135,11 @@ jobs: timeout-minutes: 15 outputs: has_credentials: ${{ steps.creds.outputs.has_credentials }} + # openrouter | claude, and what goes with it (openrouter-ci-eval-tier D6). + provider: ${{ steps.creds.outputs.provider }} + model: ${{ steps.creds.outputs.model }} + turns_factor: ${{ steps.creds.outputs.turns_factor }} + baseline_subdir: ${{ steps.creds.outputs.baseline_subdir }} run_evals: ${{ steps.select.outputs.run_evals }} # JSON list of {plugin, skills, full_tier, run_triggers, trigger_skills}, # one per plugin whose evals must run. @@ -144,18 +179,44 @@ jobs: env: CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + REQUESTED: ${{ github.event_name == 'workflow_dispatch' && inputs.provider || '' }} FORK_PR: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }} run: | if [ "${FORK_PR}" = "true" ]; then echo "has_credentials=false" >> "$GITHUB_OUTPUT" - elif [ -n "${CLAUDE_CODE_OAUTH_TOKEN}" ] || [ -n "${ANTHROPIC_API_KEY}" ]; then - echo "has_credentials=true" >> "$GITHUB_OUTPUT" - else + exit 0 + fi + has_or=""; has_claude="" + [ -n "${OPENROUTER_API_KEY}" ] && has_or=yes + { [ -n "${CLAUDE_CODE_OAUTH_TOKEN}" ] || [ -n "${ANTHROPIC_API_KEY}" ]; } && has_claude=yes + case "${REQUESTED}" in + openrouter) provider=openrouter; [ -n "$has_or" ] || provider="" ;; + claude) provider=claude; [ -n "$has_claude" ] || provider="" ;; + *) if [ -n "$has_or" ]; then provider=openrouter; elif [ -n "$has_claude" ]; then provider=claude; else provider=""; fi ;; + esac + if [ -z "$provider" ]; then echo "has_credentials=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "has_credentials=true" >> "$GITHUB_OUTPUT" + echo "provider=$provider" >> "$GITHUB_OUTPUT" + if [ "$provider" = "openrouter" ]; then + echo "model=${OPENROUTER_MODEL}" >> "$GITHUB_OUTPUT" + echo "turns_factor=${OPENROUTER_TURNS_FACTOR}" >> "$GITHUB_OUTPUT" + echo "baseline_subdir=openrouter/${OPENROUTER_MODEL#*/}/" >> "$GITHUB_OUTPUT" + else + echo "model=${HAIKU_MODEL}" >> "$GITHUB_OUTPUT" + echo "turns_factor=1.0" >> "$GITHUB_OUTPUT" + echo "baseline_subdir=" >> "$GITHUB_OUTPUT" fi - name: Map changed paths to skills, per plugin id: select + env: + PROVIDER: ${{ steps.creds.outputs.provider }} + DEFAULT_FACTOR: ${{ steps.creds.outputs.turns_factor }} + ONLY_PLUGINS: ${{ github.event_name == 'workflow_dispatch' && inputs.plugins || '' }} run: | legs="${RUNNER_TEMP}/legs" mkdir -p "$legs" @@ -171,14 +232,22 @@ jobs: fi done python3 - "$legs" >> "$GITHUB_OUTPUT" <<'PY' - import json, sys + import json, os, sys from pathlib import Path + only = {p.strip() for p in os.environ.get("ONLY_PLUGINS", "").split(",") if p.strip()} + factors = json.loads(os.environ.get("OPENROUTER_TURNS_FACTORS") or "{}") + openrouter = os.environ.get("PROVIDER") == "openrouter" legs = [] for f in sorted(Path(sys.argv[1]).glob("*.env")): + if only and f.stem not in only: + continue values = dict(line.split("=", 1) for line in f.read_text().splitlines() if "=" in line) if values.get("run_evals") == "true": - legs.append({"plugin": f.stem, **{k: values.get(k, "") for k in - ("skills", "full_tier", "run_triggers", "trigger_skills")}}) + factor = factors.get(f.stem) if openrouter else None + legs.append({"plugin": f.stem, + "turns_factor": factor or os.environ.get("DEFAULT_FACTOR") or "1.0", + **{k: values.get(k, "") for k in + ("skills", "full_tier", "run_triggers", "trigger_skills")}}) print(f"run_evals={'true' if legs else 'false'}") print(f"matrix={json.dumps(legs)}") PY @@ -234,7 +303,7 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" pr-eval: - name: PR eval ${{ matrix.plugin }} (narrow, treatment, N=3) + name: PR eval ${{ matrix.plugin }} (narrow, treatment, N=3, ${{ needs.preflight.outputs.provider }}) needs: preflight if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && needs.preflight.outputs.run_evals == 'true' && needs.preflight.outputs.has_credentials == 'true' }} strategy: @@ -244,6 +313,10 @@ jobs: env: RF_EVAL_PLUGIN: ${{ matrix.plugin }} EVALS: plugins/${{ matrix.plugin }}/evals + PROVIDER: ${{ needs.preflight.outputs.provider }} + EVAL_MODEL: ${{ needs.preflight.outputs.model }} + TURNS_FACTOR: ${{ matrix.turns_factor }} + BASELINES: plugins/${{ matrix.plugin }}/evals/baselines/${{ needs.preflight.outputs.baseline_subdir }} runs-on: ubuntu-24.04 # ubuntu-latest (26.04) has no Playwright chromium yet # A harness-wide PR selects the full narrow tier (66 sequential sessions) # plus trigger evals for every changed description; 90 min was not enough. @@ -281,16 +354,41 @@ jobs: run: uv run rfbrowser init - name: Doctor (grader tools for the narrow tier) + id: doctor env: - CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - run: uv run rf-skill-eval doctor --tasks-dir ${EVALS}/tasks/narrow --strict + CLAUDE_CODE_OAUTH_TOKEN: ${{ needs.preflight.outputs.provider == 'claude' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + ANTHROPIC_API_KEY: ${{ needs.preflight.outputs.provider == 'claude' && secrets.ANTHROPIC_API_KEY || '' }} + OPENROUTER_API_KEY: ${{ needs.preflight.outputs.provider == 'openrouter' && secrets.OPENROUTER_API_KEY || '' }} + run: uv run rf-skill-eval doctor --provider "$([ "${PROVIDER}" = openrouter ] && echo openrouter || echo anthropic)" --tasks-dir ${EVALS}/tasks/narrow --strict + + - name: Select the cost caps for the provider + run: | + if [ "${PROVIDER}" = "openrouter" ]; then + echo "NARROW_CAP=${PR_NARROW_CAP_USD_OPENROUTER}" >> "$GITHUB_ENV" + echo "TRIGGER_CAP=${PR_TRIGGER_CAP_USD_OPENROUTER}" >> "$GITHUB_ENV" + else + echo "NARROW_CAP=${PR_NARROW_CAP_USD}" >> "$GITHUB_ENV" + echo "TRIGGER_CAP=${PR_TRIGGER_CAP_USD}" >> "$GITHUB_ENV" + fi + + - name: Spend before (OpenRouter key) + if: ${{ env.PROVIDER == 'openrouter' }} + env: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + run: python3 scripts/openrouter-ci.py spend | tee "${RUNNER_TEMP}/spend-before.json" || true + + - name: Smoke test (one OpenRouter session) + if: ${{ env.PROVIDER == 'openrouter' }} + env: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + run: uv run python scripts/openrouter-ci.py smoke "${EVAL_MODEL}" - name: Run narrow tier (treatment arm) id: batch env: - CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + CLAUDE_CODE_OAUTH_TOKEN: ${{ needs.preflight.outputs.provider == 'claude' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + ANTHROPIC_API_KEY: ${{ needs.preflight.outputs.provider == 'claude' && secrets.ANTHROPIC_API_KEY || '' }} + OPENROUTER_API_KEY: ${{ needs.preflight.outputs.provider == 'openrouter' && secrets.OPENROUTER_API_KEY || '' }} SKILLS: ${{ matrix.skills }} run: | set +e @@ -302,8 +400,9 @@ jobs: "${SKILL_ARGS[@]}" \ --arms treatment \ --runs 3 \ - --model "${HAIKU_MODEL}" \ - --max-cost-usd "${PR_NARROW_CAP_USD}" \ + --model "${EVAL_MODEL}" \ + --max-turns-factor "${TURNS_FACTOR}" \ + --max-cost-usd "${NARROW_CAP}" \ --output "${RUNNER_TEMP}"/eval-runs/pr echo "exit_code=$?" >> "$GITHUB_OUTPUT" @@ -314,47 +413,62 @@ jobs: set +e uv run rf-skill-eval gate \ --runs-dir "${RUNNER_TEMP}"/eval-runs/pr \ - --baseline ${EVALS}/baselines/narrow.json \ + --baseline "${BASELINES}narrow.json" \ --tasks-dir ${EVALS}/tasks \ --token-budget 0.75 \ - --max-cost-usd "${PR_NARROW_CAP_USD}" \ + --max-cost-usd "${NARROW_CAP}" \ --report-output gate.txt echo "exit_code=$?" >> "$GITHUB_OUTPUT" - name: Trigger evals (validation split, changed descriptions) id: triggers - if: ${{ always() && matrix.run_triggers == 'true' }} + # After a failed batch too (independent results), never after a failed + # Doctor: that would spend without a usable result. + if: ${{ always() && steps.doctor.outcome == 'success' && matrix.run_triggers == 'true' }} env: - CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + CLAUDE_CODE_OAUTH_TOKEN: ${{ needs.preflight.outputs.provider == 'claude' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + ANTHROPIC_API_KEY: ${{ needs.preflight.outputs.provider == 'claude' && secrets.ANTHROPIC_API_KEY || '' }} + OPENROUTER_API_KEY: ${{ needs.preflight.outputs.provider == 'openrouter' && secrets.OPENROUTER_API_KEY || '' }} TRIGGER_SKILLS: ${{ matrix.trigger_skills }} run: | set +e uv run rf-skill-eval trigger \ --skills "${TRIGGER_SKILLS}" \ --split validation \ - --model "${HAIKU_MODEL}" \ + --model "${EVAL_MODEL}" \ --concurrency 2 \ - --max-cost-usd "${PR_TRIGGER_CAP_USD}" \ + --max-cost-usd "${TRIGGER_CAP}" \ --output "${RUNNER_TEMP}"/eval-runs/triggers code=$? uv run rf-skill-eval gate \ --trigger-results "${RUNNER_TEMP}"/eval-runs/triggers/trigger-results.json \ - --trigger-baseline ${EVALS}/baselines/triggers.json \ + --trigger-baseline "${BASELINES}triggers.json" \ --report-output trigger-gate.txt gate=$? if [ "$code" -ne 0 ]; then echo "exit_code=$code" >> "$GITHUB_OUTPUT"; else echo "exit_code=$gate" >> "$GITHUB_OUTPUT"; fi + - name: Spend after (OpenRouter key) + if: ${{ always() && env.PROVIDER == 'openrouter' }} + env: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + run: | + python3 scripts/openrouter-ci.py spend | tee "${RUNNER_TEMP}/spend-after.json" || true + python3 scripts/openrouter-ci.py summary "${RUNNER_TEMP}/spend-before.json" \ + "${RUNNER_TEMP}/spend-after.json" "${RUNNER_TEMP}/eval-runs" --out spend.md || true + - name: Render report if: always() run: | uv run rf-skill-eval report \ --runs-dir "${RUNNER_TEMP}"/eval-runs/pr \ --tasks-dir ${EVALS}/tasks \ - --baseline ${EVALS}/baselines/narrow.json \ + --baseline "${BASELINES}narrow.json" \ --format md \ --output eval-report.md || echo "_no report (no runs recorded)_" > eval-report.md { + echo "Model: \`${EVAL_MODEL}\` (provider ${PROVIDER}, turn-limit factor ${TURNS_FACTOR})" + echo + cat spend.md 2>/dev/null || true echo '### Gate' echo '```' cat gate.txt 2>/dev/null || echo "gate did not run" @@ -381,6 +495,7 @@ jobs: path: | ${{ runner.temp }}/eval-runs/pr ${{ runner.temp }}/eval-runs/triggers + ${{ runner.temp }}/spend-*.json eval-report.md gate.txt trigger-gate.txt @@ -406,7 +521,7 @@ jobs: } catch (err) { body = 'Evaluation report missing (run may have failed before report generation).'; } - const fullBody = `${marker}\n\n## Skill Evaluation: ${{ matrix.plugin }} (narrow, treatment, N=3)\n\n${body}`.slice(0, 65000); + const fullBody = `${marker}\n\n## Skill Evaluation: ${{ matrix.plugin }} (narrow, treatment, N=3, ${{ needs.preflight.outputs.model }})\n\n${body}`.slice(0, 65000); const { owner, repo } = context.repo; const issue_number = context.issue.number; const { data: comments } = await github.rest.issues.listComments({ @@ -432,7 +547,7 @@ jobs: fi full-eval: - name: Full eval ${{ matrix.plugin }} (all tiers, both arms) + name: Full eval ${{ matrix.plugin }} (all tiers, both arms, ${{ needs.preflight.outputs.provider }}) needs: preflight if: ${{ github.event_name != 'pull_request' && needs.preflight.outputs.has_credentials == 'true' }} strategy: @@ -450,8 +565,13 @@ jobs: TIERS: ${{ github.event_name == 'workflow_dispatch' && inputs.tiers || 'narrow,realistic,adversarial,partial' }} ARMS: ${{ github.event_name == 'workflow_dispatch' && inputs.arms || 'treatment,baseline' }} RUNS: ${{ github.event_name == 'workflow_dispatch' && inputs.runs || '3' }} - MODEL: ${{ github.event_name == 'workflow_dispatch' && inputs.model || '' }} - ALLOW_OPUS: ${{ github.event_name == 'workflow_dispatch' && inputs.allow_opus || false }} + PROVIDER: ${{ needs.preflight.outputs.provider }} + # OpenRouter: the pinned model for every tier; Claude: the dispatch override + # (empty = each task's tier default). + MODEL: ${{ needs.preflight.outputs.provider == 'openrouter' && needs.preflight.outputs.model || (github.event_name == 'workflow_dispatch' && inputs.model || '') }} + TURNS_FACTOR: ${{ matrix.turns_factor }} + BASELINES: plugins/${{ matrix.plugin }}/evals/baselines/${{ needs.preflight.outputs.baseline_subdir }} + ALLOW_OPUS: ${{ needs.preflight.outputs.provider == 'claude' && github.event_name == 'workflow_dispatch' && inputs.allow_opus || false }} MANUAL_CAP: ${{ github.event_name == 'workflow_dispatch' && inputs.max_cost_usd || '' }} RUN_TRIGGERS: ${{ github.event_name != 'workflow_dispatch' || inputs.triggers }} steps: @@ -487,24 +607,40 @@ jobs: run: uv run rfbrowser init - name: Set API key fallback - if: ${{ inputs.use_api_key == true }} + if: ${{ inputs.use_api_key == true && needs.preflight.outputs.provider == 'claude' }} run: echo "ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }}" >> "$GITHUB_ENV" - name: Doctor + id: doctor env: - CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - run: uv run rf-skill-eval doctor --tasks-dir ${EVALS}/tasks --strict + CLAUDE_CODE_OAUTH_TOKEN: ${{ needs.preflight.outputs.provider == 'claude' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + OPENROUTER_API_KEY: ${{ needs.preflight.outputs.provider == 'openrouter' && secrets.OPENROUTER_API_KEY || '' }} + run: uv run rf-skill-eval doctor --provider "$([ "${PROVIDER}" = openrouter ] && echo openrouter || echo anthropic)" --tasks-dir ${EVALS}/tasks --strict + + - name: Spend before (OpenRouter key) + if: ${{ env.PROVIDER == 'openrouter' }} + env: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + run: python3 scripts/openrouter-ci.py spend | tee "${RUNNER_TEMP}/spend-before.json" || true + + - name: Smoke test (one OpenRouter session) + if: ${{ env.PROVIDER == 'openrouter' }} + env: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + run: uv run python scripts/openrouter-ci.py smoke "${MODEL}" - name: Run tiers id: batch env: - CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + CLAUDE_CODE_OAUTH_TOKEN: ${{ needs.preflight.outputs.provider == 'claude' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + OPENROUTER_API_KEY: ${{ needs.preflight.outputs.provider == 'openrouter' && secrets.OPENROUTER_API_KEY || '' }} run: | set +e overall=0 MODEL_ARGS=() if [ -n "${MODEL}" ]; then MODEL_ARGS=(--model "${MODEL}"); fi if [ "${ALLOW_OPUS}" = "true" ]; then MODEL_ARGS+=(--allow-opus); fi + MODEL_ARGS+=(--max-turns-factor "${TURNS_FACTOR}") for tier in ${TIERS//,/ }; do case "$tier" in narrow) cap="${WEEKLY_NARROW_CAP_USD}" ;; @@ -513,6 +649,7 @@ jobs: partial) cap="${WEEKLY_PARTIAL_CAP_USD}" ;; *) echo "unknown tier $tier"; exit 2 ;; esac + if [ "${PROVIDER}" = "openrouter" ]; then cap="${WEEKLY_CAP_USD_OPENROUTER}"; fi if [ -n "${MANUAL_CAP}" ]; then cap="${MANUAL_CAP}"; fi # Not every plugin has every tier (rf-desktop has narrow tasks only). if [ ! -d "${EVALS}/tasks/${tier}" ]; then echo "no ${tier} tasks in ${EVALS}"; continue; fi @@ -531,12 +668,16 @@ jobs: - name: Trigger evals (all sets, both splits) id: triggers - if: ${{ always() && env.RUN_TRIGGERS == 'true' }} + # After a failed batch too (independent results), never after a failed + # Doctor: that would spend without a usable result. + if: ${{ always() && steps.doctor.outcome == 'success' && env.RUN_TRIGGERS == 'true' }} env: - CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + CLAUDE_CODE_OAUTH_TOKEN: ${{ needs.preflight.outputs.provider == 'claude' && secrets.CLAUDE_CODE_OAUTH_TOKEN || '' }} + OPENROUTER_API_KEY: ${{ needs.preflight.outputs.provider == 'openrouter' && secrets.OPENROUTER_API_KEY || '' }} run: | set +e cap="${WEEKLY_TRIGGER_CAP_USD}" + if [ "${PROVIDER}" = "openrouter" ]; then cap="${WEEKLY_CAP_USD_OPENROUTER}"; fi if [ -n "${MANUAL_CAP}" ]; then cap="${MANUAL_CAP}"; fi MODEL_ARGS=() if [ -n "${MODEL}" ]; then MODEL_ARGS=(--model "${MODEL}"); fi @@ -548,6 +689,18 @@ jobs: --output "${RUNNER_TEMP}"/eval-runs/full/triggers echo "exit_code=$?" >> "$GITHUB_OUTPUT" + - name: Spend after (OpenRouter key) + if: ${{ always() && env.PROVIDER == 'openrouter' }} + env: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + run: | + python3 scripts/openrouter-ci.py spend | tee "${RUNNER_TEMP}/spend-after.json" || true + # Warn when the priced total and the key's spend differ by more than 25%. + python3 scripts/openrouter-ci.py summary --warn-drift 0.25 --out spend.md \ + "${RUNNER_TEMP}/spend-before.json" "${RUNNER_TEMP}/spend-after.json" \ + "${RUNNER_TEMP}/eval-runs/full" || true + cat spend.md >> "$GITHUB_STEP_SUMMARY" || true + - name: Candidate baseline and report if: always() run: | @@ -563,11 +716,13 @@ jobs: set +e uv run rf-skill-eval gate \ --runs-dir "${RUNNER_TEMP}"/eval-runs/full \ - --baseline ${EVALS}/baselines/narrow.json \ + --baseline "${BASELINES}narrow.json" \ --tasks-dir ${EVALS}/tasks \ --report-output gate.txt echo "GATE_EXIT=$?" >> "$GITHUB_ENV" { + echo "Model: \`${MODEL:-tier defaults}\` (provider ${PROVIDER}, turn-limit factor ${TURNS_FACTOR})" + echo echo '### Gate vs committed baseline (informational)' echo '```' cat gate.txt 2>/dev/null @@ -583,6 +738,7 @@ jobs: name: skill-eval-full-${{ matrix.plugin }}-${{ github.run_id }} path: | ${{ runner.temp }}/eval-runs/full + ${{ runner.temp }}/spend-*.json eval-report.md gate.txt !${{ runner.temp }}/eval-runs/**/workspace/.venv @@ -595,7 +751,9 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: candidate-baseline-${{ matrix.plugin }}-${{ github.run_id }} + # The provider is in the name: the files go to baselines/ (claude) or + # baselines/openrouter// (openrouter). + name: candidate-baseline-${{ matrix.plugin }}-${{ needs.preflight.outputs.provider }}-${{ github.run_id }} path: candidate-baseline/ if-no-files-found: ignore retention-days: 90 diff --git a/RELEASING.md b/RELEASING.md index 941321a..903dd87 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -29,6 +29,13 @@ Older tags stay where they are: `v1.1.0` to `v3.0.0` are `rf-agentskills` conten ``` 4. `release.yml` checks that the tag matches the plugin's `plugin.json`, builds the plugin's archives, attaches them to a GitHub release and moves the `stable` and `latest` branches. +Before step 3, check the skills on Claude. CI evaluates PRs on DeepSeek V4.1 Flash, through OpenRouter, so run a manual Claude evaluation of the release candidate's branch and check its gate against the Claude baselines: + +```bash +gh workflow run skill-evaluation.yml --ref \ + -f provider=claude -f model=claude-sonnet-5-5 -f tiers=narrow,realistic -f arms=treatment,baseline +``` + A pushed tag is the ref users pin a marketplace to: never move or delete one. ## Installer release diff --git a/docs/ci/architecture/adr/ADR-005-ci-integration.md b/docs/ci/architecture/adr/ADR-005-ci-integration.md index 2ef713a..4a60f0f 100644 --- a/docs/ci/architecture/adr/ADR-005-ci-integration.md +++ b/docs/ci/architecture/adr/ADR-005-ci-integration.md @@ -7,6 +7,36 @@ --- +## Amendment 2026-11-10 — `openrouter-ci-eval-tier` + +PR and weekly runs move to a cheap OpenRouter tier when the +`OPENROUTER_API_KEY` secret is set, so the evals no longer depend on a +maintainer's Claude subscription. The 2026-09-27 tiers below stay as they +are; only the model and the credentials change. + +- **Provider.** Preflight picks `openrouter` when the OpenRouter key is set, + `claude` otherwise; a manual dispatch input `provider` overrides it. Each + eval job gets only that provider's credentials. +- **Model.** `deepseek/deepseek-v4.1-flash` for every tier, with turn-limit + factor 1.5 (`--max-turns-factor`). Claude runs (Haiku, Sonnet, Opus) are a + manual dispatch with `provider: claude`, and part of the release checklist. +- **Baselines.** Kept per model: `plugins//evals/baselines/` for + Claude, `plugins//evals/baselines/openrouter/deepseek-v4.1-flash/` + for DeepSeek. The gate never compares one model with another. +- **Cost.** The harness prices OpenRouter runs at list prices (Claude Code's + own figure prices them as Claude); caps are `PR_NARROW_CAP_USD_OPENROUTER`, + `PR_TRIGGER_CAP_USD_OPENROUTER` and `WEEKLY_CAP_USD_OPENROUTER`. Each job + records the key's spend before and after, and the key's credit limit on + OpenRouter is the hard cap. +- **Why.** A spike (run 42822700497, rf-core narrow, both arms, N=3) completed + all 72 sessions on DeepSeek for $0.84; the skills loaded in every treatment + run and the treatment–baseline delta stayed visible on 8 of 12 tasks. +- **Risk.** Anthropic doesn't support non-Claude models in Claude Code. Each + OpenRouter job starts with a one-session smoke test; if a Claude Code + release breaks the path, dispatch with `provider: claude`. + +--- + ## Amendment 2026-09-27 — `strengthen-skill-eval-harness` The three planned workflows are implemented as **one** workflow, diff --git a/docs/ci/local-testing.md b/docs/ci/local-testing.md index 0a66165..0391a45 100644 --- a/docs/ci/local-testing.md +++ b/docs/ci/local-testing.md @@ -12,6 +12,7 @@ PR. For the full user guide, see [usage.md](usage.md). - [OAuth token expiry](#oauth-token-expiry) - [Rate-limit exhaustion](#rate-limit-exhaustion) - [`uv sync` cache issues](#uv-sync-cache-issues) + - [Running on OpenRouter](#running-on-openrouter) - [Iterating on a failing task](#iterating-on-a-failing-task) - [Adding a new skill to the eval task bank](#adding-a-new-skill-to-the-eval-task-bank) @@ -182,6 +183,24 @@ uv lock --check If `uv lock --check` fails on `main`, file an issue — CI enforces lock integrity and should have caught it. +### Running on OpenRouter + +Without a Claude subscription, run on DeepSeek V4.1 Flash through OpenRouter. +Set `OPENROUTER_API_KEY` (in `.env` or exported), give the key a credit limit +at openrouter.ai, and pass the model and the turn-limit factor CI uses: + +```bash +uv run rf-skill-eval doctor --provider openrouter +uv run rf-skill-eval run-batch --tier narrow --arms treatment --runs 3 \ + --model deepseek/deepseek-v4.1-flash --max-turns-factor 1.5 --max-cost-usd 2 \ + --output eval/runs/or-$(date +%s) +``` + +Compare the results with the DeepSeek baselines (`gate` finds them from the +runs' model), not the Haiku ones. The costs in the report are estimates at +OpenRouter's list prices; the key's activity page on openrouter.ai shows what +was charged. See [usage.md](usage.md#openrouter-tier). + --- ## Iterating on a failing task diff --git a/docs/ci/usage.md b/docs/ci/usage.md index 0c214e8..d289684 100644 --- a/docs/ci/usage.md +++ b/docs/ci/usage.md @@ -22,6 +22,7 @@ same machinery on every PR. - [Adding a New Task](#adding-a-new-task) - [CI Reference](#ci-reference) - [Model Policy](#model-policy) +- [OpenRouter tier](#openrouter-tier) - [Cost Expectations](#cost-expectations) - [Related Documents](#related-documents) @@ -123,6 +124,9 @@ comment out `CLAUDE_CODE_OAUTH_TOKEN` and uncomment `ANTHROPIC_API_KEY` instead. See [faq.md](faq.md#subscription) for details. +To run on the cheap OpenRouter model instead, set `OPENROUTER_API_KEY`; see +[OpenRouter tier](#openrouter-tier). + ### 3. Install Python dependencies ```bash @@ -362,8 +366,10 @@ tokens above `--token-budget` (default +30 %; the PR job uses +75 %), incomplete significantly below the baseline — Fisher exact p < 0.05 on run-level loads, or more than one query for baselines without run counts — a shipped skill listed by name only in a skill listing, cost over `--max-cost-usd`) or 3 (`rebaseline-needed` only: the task -definition, fixture or model changed, or no baseline entry exists — never a -pass). Baseline files are promoted through a reviewed PR +definition, fixture, model or turn-limit factor changed, or no baseline entry exists — never a +pass). Without `--baseline`, `gate` and `baseline update` use the runs' model's +folder: `/baselines/` for Claude, `/baselines/openrouter//` +for an OpenRouter model. Baseline files are promoted through a reviewed PR ([`eval/baselines/README.md`](../../eval/baselines/README.md)). --- @@ -406,13 +412,19 @@ amendment). | Trigger | Jobs | Scope | Arms | N | Model | Cap | |---|---|---|---|---|---|---| -| `pull_request` | `preflight` → `pr-eval` | narrow tasks of changed skills + `plugin` canaries (all narrow tasks when the harness changed); validation-split trigger evals when a SKILL.md `description` changed | treatment | 3 | `claude-haiku-5-5` | `PR_NARROW_CAP_USD`=10, `PR_TRIGGER_CAP_USD`=20 | -| `schedule` (Sun 04:00 UTC) | `preflight` → `full-eval` | all tiers + all trigger sets | treatment, baseline | 3 | tier defaults | 12 + 12 + 8 + 8 = $40 | -| `workflow_dispatch` | `preflight` → `full-eval` | inputs `tiers`, `arms`, `runs`, `model`, `allow_opus`, `max_cost_usd`, `triggers` | input | input | input | `max_cost_usd` per invocation | +| `pull_request` | `preflight` → `pr-eval` | narrow tasks of changed skills + `plugin` canaries (all narrow tasks when the harness changed); validation-split trigger evals when a SKILL.md `description` changed | treatment | 3 | `deepseek/deepseek-v4.1-flash` when `OPENROUTER_API_KEY` is set, else `claude-haiku-5-5` | OpenRouter: `PR_NARROW_CAP_USD_OPENROUTER`=2, `PR_TRIGGER_CAP_USD_OPENROUTER`=2; Claude: `PR_NARROW_CAP_USD`=10, `PR_TRIGGER_CAP_USD`=20 | +| `schedule` (Sun 04:00 UTC) | `preflight` → `full-eval` | all tiers + all trigger sets | treatment, baseline | 3 | DeepSeek when its key is set, else tier defaults | OpenRouter: `WEEKLY_CAP_USD_OPENROUTER`=3 per invocation; Claude: 12 + 12 + 8 + 8 = $40 | +| `workflow_dispatch` | `preflight` → `full-eval` | inputs `provider`, `tiers`, `arms`, `runs`, `model`, `allow_opus`, `max_cost_usd`, `triggers` | input | input | `provider: claude` + `model`, or DeepSeek | `max_cost_usd` per invocation | | all | `harness-tests` | ruff, mypy, `pytest tests/eval` (no API calls) | – | – | – | – | -- The PR job gates against `eval/baselines/narrow.json` and posts one PR - comment (report + gate result + trigger report). +- Preflight picks the provider once: OpenRouter when `OPENROUTER_API_KEY` is + set, otherwise Claude; a manual dispatch with `provider` overrides it. Each + eval job gets only that provider's credentials. +- The PR job gates against the provider's baselines + (`plugins//evals/baselines/narrow.json` for Claude, + `plugins//evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json` + for OpenRouter) and posts one PR comment (report + gate result + trigger + report, and the OpenRouter spend). - The weekly job uploads the report and a `candidate-baseline-` artifact; a maintainer promotes it with a PR. The PR gate becomes a **required** check only after two weekly baselines have been recorded. @@ -431,6 +443,12 @@ gh workflow run skill-evaluation.yml -f tiers=narrow -f arms=treatment -f runs=5 gh workflow run skill-evaluation.yml -f tiers=narrow -f model=claude-opus-5-5 -f allow_opus=true -f max_cost_usd=20 # Force API-key auth: gh workflow run skill-evaluation.yml -f use_api_key=true +# A Claude run while the OpenRouter key is set (before a release): +gh workflow run skill-evaluation.yml -f provider=claude -f tiers=narrow,realistic -f model=claude-sonnet-5-5 +# Record the OpenRouter baselines (N=6): +gh workflow run skill-evaluation.yml -f provider=openrouter -f runs=6 +# Re-record one plugin only: +gh workflow run skill-evaluation.yml -f provider=openrouter -f runs=6 -f plugins=rf-desktop ``` --- @@ -449,6 +467,56 @@ gh workflow run skill-evaluation.yml -f use_api_key=true It never runs on PRs or the weekly schedule (≈5× Sonnet per token). - Retired ids (`claude-haiku-4-5`, `claude-sonnet-4-6`) are rejected with a migration hint. +- OpenRouter models (`deepseek/deepseek-v4.1-flash`) are run time only; a task + YAML that declares one fails to load. See [OpenRouter tier](#openrouter-tier). + +--- + +## OpenRouter tier + +The harness can run every session on DeepSeek V4.1 Flash through OpenRouter, +so the evals run from an OpenRouter key instead of a Claude subscription. CI +uses it by default when the `OPENROUTER_API_KEY` secret is set. A spike run of +the rf-core narrow tier (both arms, N=3, 72 sessions) cost $0.84. + +```bash +export OPENROUTER_API_KEY=sk-or-v1-... # or put it in .env +uv run rf-skill-eval doctor --provider openrouter +uv run rf-skill-eval run-batch --tier narrow --arms treatment --runs 3 \ + --model deepseek/deepseek-v4.1-flash --max-turns-factor 1.5 --max-cost-usd 2 \ + --output eval/runs/or +uv run rf-skill-eval gate --runs-dir eval/runs/or # uses baselines/openrouter/deepseek-v4.1-flash/ +``` + +- **Provider.** An OpenRouter model id selects the OpenRouter provider; + `--provider openrouter|anthropic` only states it, and a mismatch is refused. + The runner points Claude Code at `https://openrouter.ai/api`, maps every + Claude model alias and the subagent model to the chosen model, and doesn't + pass `CLAUDE_CODE_OAUTH_TOKEN` or `ANTHROPIC_API_KEY` to the session. A run + whose usage names any other model is recorded as incomplete + (`provider_model`). +- **Allowed models.** Only `deepseek/deepseek-v4.1-flash`. Adding one is a + code change: an entry in `OPENROUTER_MODELS`, a price in + `src/rf_skill_eval/domain/pricing.py`, and its own baselines. +- **Cost.** Claude Code prices every session as Claude (the spike reported + $29.66). For OpenRouter runs the harness prices the token counts at + OpenRouter's list prices instead; reports label that figure as an estimate. + List prices are an upper bound (the spike priced at $1.33 against $0.84 + charged). `--max-cost-usd` uses the estimate. CI also records the key's + real spend before and after each job; the key's credit limit is the hard + cap. +- **Turn limits.** DeepSeek takes more turns than Haiku: 25 of the 72 spike + sessions stopped at the limit. `--max-turns-factor 1.5` scales each task's + `max_turns` (rounded up); the factor is stored with the runs and the + baselines, and a different factor means `rebaseline-needed`. Reports show + turn-limit stops per task and arm. `OPENROUTER_TURNS_FACTORS` in the + workflow can override the factor per plugin. None does now: rf-desktop at + 2.0 used the extra turns and passed no more often. +- **What it tells you.** Results say how the skills work with DeepSeek, not + with Claude. Run a Claude check before a release (see RELEASING.md). + Anthropic doesn't support non-Claude models in Claude Code, which prints + `unrecognized_model` on each session; a Claude Code release could break this + path, and the CI smoke step would catch it. --- diff --git a/openspec/changes/openrouter-ci-eval-tier/.openspec.yaml b/openspec/changes/openrouter-ci-eval-tier/.openspec.yaml new file mode 100644 index 0000000..2b638f8 --- /dev/null +++ b/openspec/changes/openrouter-ci-eval-tier/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-11-10 diff --git a/openspec/changes/openrouter-ci-eval-tier/design.md b/openspec/changes/openrouter-ci-eval-tier/design.md new file mode 100644 index 0000000..2502575 --- /dev/null +++ b/openspec/changes/openrouter-ci-eval-tier/design.md @@ -0,0 +1,114 @@ +## Context + +The harness drives the real Claude Code CLI (`claude -p`) in an isolated config dir per run, and Claude Code talks to whatever Anthropic-compatible endpoint `ANTHROPIC_BASE_URL` names. The spike on `spike/openrouter-evals` proved this works with OpenRouter, using a few changes: +- an extra-models environment variable for the allow-list; +- accepting `ANTHROPIC_AUTH_TOKEN` as credentials; +- a workflow that set the OpenRouter variables itself. + +Today: +- `ClaudeCodeRunner._build_env` copies the whole process environment and requires `CLAUDE_CODE_OAUTH_TOKEN` or `ANTHROPIC_API_KEY`. +- `domain/task.py` holds the Claude allow-lists. +- `Budget.charge` adds Claude Code's `total_cost_usd`. +- The gate reports any model mismatch as `rebaseline-needed` and exits 3. +- `skill-evaluation.yml` runs PR jobs on Haiku and weekly jobs on the tier defaults, with OAuth credentials. + +Spike numbers used below (run 42822700497, rf-core narrow, 72 sessions): +- actual spend $0.84, with 16.6M input tokens (mostly cache reads) and 376k output tokens; +- Claude Code's own total was $29.66; +- 25 sessions stopped at `max_turns`; +- mean turns were 13–20, against 2–18 for Haiku. + +## Goals / Non-Goals + +**Goals:** +- The provider is a harness concept, not a workflow trick. The same command works locally and in CI. +- The cap, the estimate and the reports use a cost that is close to what OpenRouter charges. +- Claude and DeepSeek results never get compared with each other by accident. + +**Non-Goals:** +- Supporting arbitrary OpenRouter models, or any other gateway (LiteLLM, Bedrock). The allow-list stays short on purpose. +- Changing task definitions or graders to suit DeepSeek. +- Tuning skill content for DeepSeek. The skills are written for Claude, and the cheap tier is a regression signal, not a target. +- Removing the Claude tier or its baselines. + +## Decisions + +### D1. A provider value in the harness, derived from the model id + +`--provider {anthropic,openrouter}` exists for clarity, but the model id already decides it: ids containing `/` come from the OpenRouter list. The runner builds the session environment per provider: +- **openrouter:** + - start from the process environment minus `CLAUDE_CODE_OAUTH_TOKEN`, `ANTHROPIC_API_KEY` and `ANTHROPIC_AUTH_TOKEN`; + - set `ANTHROPIC_BASE_URL=https://openrouter.ai/api`, `ANTHROPIC_AUTH_TOKEN=$OPENROUTER_API_KEY` and `ANTHROPIC_API_KEY=""`; + - set `ANTHROPIC_DEFAULT_{HAIKU,SONNET,OPUS}_MODEL` and `CLAUDE_CODE_SUBAGENT_MODEL` to the model. +- **anthropic:** unchanged. + +The auth checks in `config.py` and `cli.py` follow the provider. + +*Alternative:* keep the spike's approach, where the workflow sets the variables and the harness only allows extra ids. That was rejected: local runs would need the same hand-set environment, the credential check would accept any `ANTHROPIC_AUTH_TOKEN`, and nothing would stop Claude credentials from leaking into a session that should be on OpenRouter. + +### D2. A run-time OpenRouter allow-list next to the Claude lists + +Add `OPENROUTER_MODELS = frozenset({"deepseek/deepseek-v4.1-flash"})` in `domain/task.py`: +- `ALLOWED_MODELS` (run time) includes it; +- `YAML_ALLOWED_MODELS` does not. + +The spike's `RF_SKILL_EVAL_EXTRA_MODELS` variable is dropped. An allow-list that an environment variable can extend isn't an allow-list. + +*Alternative:* allow any `vendor/model` id. That was rejected, because each new model needs its own baselines and a price entry, so adding one should be a reviewed code change. + +### D3. Cost from a price table, checked against OpenRouter + +`PROVIDER_PRICES[model] = {input, cache_read, cache_write, output}` in USD per million tokens. The cost is computed from the stream's `usage` block when the run is recorded and stored in the existing `total_cost_usd` column; a new `cost_source` column says `reported` or `priced`. So `Budget`, `estimate_cost`, the results files and the reports need no other changes. + +The table is calibrated against the spike: priced cost over the spike's 72 runs should come within ±25% of the $0.84 OpenRouter charged. The figure that counts is still the key's spend that CI reads from OpenRouter (D7). + +*Alternatives:* +- Query OpenRouter's per-generation endpoint for each request. That's exact, but Claude Code doesn't expose generation ids, and it adds network calls to the runner. +- Use only the key's spend before and after. That's exact per job, but it can't drive a per-run cap. + +### D4. Turn-limit factor, recorded and gated + +`--max-turns-factor` multiplies `task.max_turns` and rounds up, with 3.0 as the upper bound. The factor goes into the run record and the baseline entries, and a different factor means `rebaseline-needed` (see the spec). The OpenRouter tier starts at 1.5, chosen from the spike: 25 of the 72 sessions hit the limit with factor 1.0. The weekly baseline run decides whether 1.5 is enough. A turn-limit stop is recorded from the result's `subtype: error_max_turns`. + +*Alternative:* separate `max_turns` per provider in each task YAML. That was rejected: 50+ YAMLs to keep in step, and task hashes would change for Claude too. + +### D5. Per-model baseline paths + +Claude baselines stay at `baselines/.json`, so no file moves and no history is lost. OpenRouter baselines go to `baselines/openrouter//.json`. `gate` and `baseline update` derive the default path from the runs' model. CI passes the path explicitly anyway, so a misdetected model can't silently select the wrong file. + +*Alternative:* one baseline file with entries keyed by model. That was rejected: it changes the schema for every consumer and makes Claude rebaselines touch DeepSeek entries. + +### D6. CI picks the provider once, in preflight + +Preflight outputs `provider` and `model`: +- `openrouter` when `OPENROUTER_API_KEY` is set; +- else `claude` when Claude credentials are set; +- a manual dispatch input `provider` overrides both. + +The `pr-eval` and `full-eval` jobs take the model, the turn-limit factor, the credentials and the baseline paths from those outputs. The fork-PR rule (a fork never has credentials) is unchanged. The caps stay in USD, but the OpenRouter ones get their own environment values (for example `PR_NARROW_CAP_USD_OPENROUTER: "2"`), since the Claude caps would never bind. + +### D7. Spend reporting from the key endpoint + +Before and after the runs, a step reads `GET https://openrouter.ai/api/v1/key` and records `usage`, `limit` and `limit_remaining`. The step summary shows the difference next to the harness's priced total. That was the spike's approach, and it worked. Concurrent jobs share the key, so the delta is labelled "key spend during this job (includes concurrent jobs)". `max-parallel: 1` on `full-eval` already keeps weekly runs sequential. + +## Risks / Trade-offs + +- **Unsupported model.** Claude Code may refuse or mishandle non-Claude models in a future release (today it only logs `unrecognized_model`). → CI installs Claude Code without a pinned version, so this would show up as a failed doctor or smoke step. Add a one-session smoke step to OpenRouter jobs that fails fast. The fallback is a manual Claude run, and pinning the last working Claude Code version. +- **Key exposure.** Agent sessions run plugin hooks and Bash with the session environment, and that includes the OpenRouter key as `ANTHROPIC_AUTH_TOKEN`. That's the same exposure the OAuth token has today. → Only maintainer-pushed code runs with the key (the fork rule is unchanged), and the key has a credit limit on a MarketSquare account and nothing else depends on it. +- **Routing drift.** OpenRouter may serve the model from different hosts with different quantisation or behaviour. → Pin the provider routing (`provider.order` / `allow_fallbacks: false` through OpenRouter's preferences, if Claude Code's request path allows it; otherwise record the serving provider from the response headers when available) and watch the weekly variance. +- **Signal quality.** DeepSeek's pass rates and arm deltas aren't Claude's. A skill change could help Claude and hurt DeepSeek, or the other way round. → A manual Claude run is part of the release checklist, and the reports name the model prominently. +- **Price drift.** The price table goes stale. → The weekly job compares the priced total with the key delta and warns when they differ by more than 25%. +- **Baselines on 3 replicates.** These are noisy (the spike had partial passes on 2 treatment tasks). → The first baseline is recorded from a weekly run with N=6 before the gate becomes required. + +## Migration Plan + +1. Land the harness changes. Claude behaviour is unchanged and the existing tests pass. +2. Run a manual dispatch with provider `openrouter`, all tiers, both arms, N=6, factor 1.5. Review the candidate baselines and commit them in a PR. +3. Switch the PR and weekly defaults to OpenRouter (D6) in the same PR or the next one. The gate now finds matching baselines. +4. Delete `spike/openrouter-evals` and its workflow. +5. **Rollback:** delete the `OPENROUTER_API_KEY` secret, or dispatch with provider `claude`. Preflight then falls back to the Claude tier and its untouched baselines. + +## Open Questions + +- Whether Claude Code forwards OpenRouter provider-routing preferences. If it doesn't, routing stays OpenRouter's default and we record what we can. This changes neither the specs nor the tasks. +- The exact OpenRouter caps for PR and weekly jobs. They get set after the first weekly run, as the Claude caps were. diff --git a/openspec/changes/openrouter-ci-eval-tier/proposal.md b/openspec/changes/openrouter-ci-eval-tier/proposal.md new file mode 100644 index 0000000..c5838c0 --- /dev/null +++ b/openspec/changes/openrouter-ci-eval-tier/proposal.md @@ -0,0 +1,44 @@ +## Why + +The live skill evals only run on Claude models, billed to a maintainer's Claude subscription (`CLAUDE_CODE_OAUTH_TOKEN`). MarketSquare can't run them without that person, and every PR and weekly run spends from their plan. A spike (run 42822700497, 2026-11-10) ran the rf-core narrow tier in both arms, N=3, through OpenRouter on `deepseek/deepseek-v4.1-flash`. All 72 sessions completed, the skills loaded in every treatment run, the treatment arm passed 11 of 12 tasks at 1.00, and the arm delta stayed visible on 8 tasks. OpenRouter charged $0.84 for the whole run. That makes a cheap tier, paid from a MarketSquare-owned key, a realistic default for CI. + +## What Changes + +- The harness can run sessions on a pinned OpenRouter model. A run-time `--provider openrouter` (or an OpenRouter model id) makes the runner point Claude Code at OpenRouter's Anthropic-compatible endpoint, map every Claude model alias and the subagent model to the chosen model, and authenticate with `OPENROUTER_API_KEY`. Task YAMLs keep declaring Claude models only. +- The model allow-list gains a run-time-only OpenRouter list, starting with `deepseek/deepseek-v4.1-flash`. Other OpenRouter ids are rejected like any unknown model. +- Cost for OpenRouter runs is computed by the harness from a per-model price table and the reported token counts, instead of Claude Code's `total_cost_usd`, which prices every model as Claude (the spike reported $29.66 for $0.84 of spend). `--max-cost-usd` and the pre-dispatch estimate use that figure. +- A run-time `--max-turns-factor` scales each task's `max_turns`. DeepSeek needs more turns than Haiku: 25 of the 72 spike sessions hit the limit. +- Baselines are stored per model. The OpenRouter tier gets its own baseline files next to the Haiku ones, and the gate compares each run only with the baseline of its model. +- **BREAKING (CI behaviour):** PR and weekly eval jobs run on the OpenRouter tier when `OPENROUTER_API_KEY` is set, gated against the OpenRouter baselines. Claude runs move to manual dispatch (`provider: claude`), for releases and for checking the cheap tier against Claude. +- CI reports the real OpenRouter spend of each job (the key's usage before and after) in the step summary. The key's credit limit on OpenRouter is the hard cap. +- The spike branch `spike/openrouter-evals` and its workflow are deleted. + +## Capabilities + +### New Capabilities + +- `eval-model-providers`: running eval sessions through a non-Anthropic provider: the provider setting, the environment the runner builds, which OpenRouter models are allowed, how their cost is computed, and the turn-limit factor. + +### Modified Capabilities + +- `skill-eval-harness`: + - "Model allow-list and cost policy": adds the run-time OpenRouter list and the provider-computed cost the cap uses; corrects the outdated Claude ids in the requirement text to the 5.5 models the code already enforces. + - "Regression gate against a stored baseline": baselines are kept per model, and the gate picks the one that matches the run's model. + - "Tiered CI execution with bounded cost": PR and weekly runs use the OpenRouter tier when its key is present; Claude runs are manual; jobs report the provider's real spend. + +## Impact + +- **Harness code:** + - `src/rf_skill_eval/domain/task.py`: model lists. + - `application/batch.py`: runtime policy and budget. + - `infrastructure/runner/claude_code_runner.py`: environment and max turns. + - `config.py` and `cli.py`: auth check and new flags. + - The baseline and gate commands: the per-model path. +- **CI:** `.github/workflows/skill-evaluation.yml` (a provider input, credential detection, OpenRouter environment, spend steps, baseline paths). +- **Data:** new `plugins//evals/baselines/openrouter/deepseek-v4.1-flash/{narrow,partial,triggers}.json` for each plugin with evals, recorded by a reviewed PR. +- **Docs:** `docs/ci/usage.md`, `docs/ci/local-testing.md`, and a short addendum to ADR-005. +- **Secrets and accounts:** `OPENROUTER_API_KEY` in the repository, from a MarketSquare-owned OpenRouter account with a credit limit (already added for the spike). `CLAUDE_CODE_OAUTH_TOKEN` becomes optional. +- **Caveats:** + - Anthropic does not support non-Claude models in Claude Code. Claude Code prints `unrecognized_model` on every session; it was harmless in the spike, but a future Claude Code release could break this path. The manual Claude tier is the fallback. + - OpenRouter may route a model to different hosts, which can shift results between runs. + - Results on DeepSeek say how the skills work with DeepSeek. They don't replace a Claude check before a release. diff --git a/openspec/changes/openrouter-ci-eval-tier/specs/eval-model-providers/spec.md b/openspec/changes/openrouter-ci-eval-tier/specs/eval-model-providers/spec.md new file mode 100644 index 0000000..976680a --- /dev/null +++ b/openspec/changes/openrouter-ci-eval-tier/specs/eval-model-providers/spec.md @@ -0,0 +1,80 @@ +## Purpose + +Lets the eval harness run Claude Code sessions on a pinned, low-cost model through OpenRouter, so the evals can run from a MarketSquare-owned key without a Claude subscription, while keeping cost and model choice under the same policy as Claude runs. + +## ADDED Requirements + +### Requirement: Provider selection at run time + +The harness SHALL support two providers: `anthropic` (the default) and `openrouter`. A batch, a trigger run and a single run SHALL accept `--provider`. Passing a model id from the OpenRouter allow-list SHALL select `openrouter` without the flag. Passing `--provider openrouter` with a Claude model id, or `--provider anthropic` with an OpenRouter model id, SHALL be refused before any session starts. Task YAMLs and trigger sets SHALL NOT declare a provider or an OpenRouter model id. The provider and the model id SHALL be recorded on every run and in every results file. + +#### Scenario: OpenRouter model selects the provider + +- **WHEN** a batch is started with `--model deepseek/deepseek-v4.1-flash` and no `--provider` +- **THEN** every session runs through OpenRouter, and each run records provider `openrouter` and model `deepseek/deepseek-v4.1-flash` + +#### Scenario: Mismatched provider and model + +- **WHEN** a batch is started with `--provider openrouter --model claude-haiku-5-5` +- **THEN** the harness refuses to start and names the OpenRouter models it permits + +#### Scenario: OpenRouter model in a task YAML + +- **WHEN** a task declares `model: deepseek/deepseek-v4.1-flash` +- **THEN** loading fails, and the error says OpenRouter models are run-time only + +### Requirement: OpenRouter session environment + +For an `openrouter` session the runner SHALL point Claude Code at OpenRouter's Anthropic-compatible endpoint and authenticate with the `OPENROUTER_API_KEY` environment variable. It SHALL map every Claude model alias Claude Code may use (Haiku, Sonnet, Opus and the subagent model) to the selected model, so that no request reaches a Claude model. It SHALL NOT pass `CLAUDE_CODE_OAUTH_TOKEN` or `ANTHROPIC_API_KEY` to the session. The key SHALL NOT appear in run records, transcripts copied into results, reports or logs written by the harness. A run whose reported model usage names any model other than the selected one SHALL be recorded as incomplete with reason `provider_model`, naming the other models, and SHALL NOT be graded or counted as passing. + +#### Scenario: Missing key + +- **WHEN** an OpenRouter batch starts and `OPENROUTER_API_KEY` is unset or empty +- **THEN** the harness refuses to start and names the variable + +#### Scenario: Claude credentials are not forwarded + +- **WHEN** an OpenRouter session starts on a machine where `CLAUDE_CODE_OAUTH_TOKEN` is set +- **THEN** the session's environment does not contain that token or `ANTHROPIC_API_KEY` + +#### Scenario: Unexpected model in the usage + +- **WHEN** an OpenRouter session's result reports usage for `claude-haiku-5-5` +- **THEN** the run is recorded as incomplete with reason `provider_model` naming `claude-haiku-5-5`, and the gate does not count it as passing + +### Requirement: OpenRouter model allow-list + +The harness SHALL permit OpenRouter model ids only from a run-time allow-list kept in the harness code. That list SHALL start with `deepseek/deepseek-v4.1-flash`. Any other OpenRouter id SHALL be rejected, and the error SHALL name the permitted ids. Adding an id to the list SHALL require a price-table entry for it (see "Provider-priced cost"). + +#### Scenario: Unlisted OpenRouter model + +- **WHEN** a batch is started with `--model z-ai/glm-5.3-flash` +- **THEN** the harness refuses to start and lists the permitted OpenRouter ids + +### Requirement: Provider-priced cost + +For `openrouter` runs the harness SHALL compute each run's cost from its reported token counts (input, cache reads, cache writes and output) and a per-model price table in USD per million tokens. It SHALL NOT use Claude Code's `total_cost_usd` for these runs. That computed cost SHALL be what `--max-cost-usd`, the pre-dispatch estimate, the results files and the reports use. Reports SHALL label it as an estimate from list prices. + +#### Scenario: Cap uses the computed cost + +- **WHEN** an OpenRouter batch runs with `--max-cost-usd 2` and its sessions report Claude-priced `total_cost_usd` totalling $30 but computed costs totalling $0.90 +- **THEN** the batch is not stopped by the budget and records $0.90 + +#### Scenario: Report labels the estimate + +- **WHEN** a report is rendered for OpenRouter runs +- **THEN** the cost column is marked as estimated from OpenRouter list prices + +### Requirement: Turn-limit factor + +A batch and a single run SHALL accept `--max-turns-factor ` with `f` from 1.0 to 3.0 (default 1.0). Each session's turn limit SHALL be the task's `max_turns` multiplied by `f` and rounded up. The factor SHALL be recorded with each run. A run that ends because it reached the turn limit SHALL be recorded as such, so reports can count turn-limit stops per task and arm. + +#### Scenario: Scaled limit + +- **WHEN** a task declares `max_turns: 14` and the batch runs with `--max-turns-factor 1.5` +- **THEN** its sessions run with a limit of 21 turns, and each run records factor 1.5 + +#### Scenario: Turn-limit stops are visible + +- **WHEN** two of three replicates of a task end by reaching the turn limit +- **THEN** the report shows 2 turn-limit stops for that task and arm, whether or not the runs passed diff --git a/openspec/changes/openrouter-ci-eval-tier/specs/skill-eval-harness/spec.md b/openspec/changes/openrouter-ci-eval-tier/specs/skill-eval-harness/spec.md new file mode 100644 index 0000000..5cb016e --- /dev/null +++ b/openspec/changes/openrouter-ci-eval-tier/specs/skill-eval-harness/spec.md @@ -0,0 +1,81 @@ +## MODIFIED Requirements + +### Requirement: Model allow-list and cost policy + +Task YAMLs and trigger sets SHALL declare a model from the allow-list `claude-haiku-5-5` or `claude-sonnet-5-5`. They SHALL NOT declare `claude-opus-5-5` or an OpenRouter model id as their own model. Any other model id SHALL be rejected at load time, and the error SHALL name the permitted ids; a retired Claude id SHALL also name its replacement. At run time the harness SHALL additionally permit `claude-opus-5-5` and the ids on the OpenRouter allow-list (capability `eval-model-providers`). Running with `claude-opus-5-5` SHALL require an explicit run-time opt-in flag, and SHALL be refused unless a cost cap is also given. Every batch SHALL accept a cost cap (`--max-cost-usd`). The running total SHALL be the reported cost for Claude runs and the provider-priced cost for OpenRouter runs. When the running total reaches the cap, the harness SHALL stop dispatching new runs, mark the unstarted runs as `incomplete` with reason `budget`, and exit non-zero. + +#### Scenario: Outdated model id rejected +- **WHEN** a task declares `model: claude-sonnet-4-6` +- **THEN** loading fails with an error naming `claude-sonnet-5-5` as the replacement and listing the permitted model ids + +#### Scenario: Opus requires opt-in and cap +- **WHEN** a batch is started with `--model claude-opus-5-5` but without the opt-in flag or without `--max-cost-usd` +- **THEN** the harness refuses to start and explains which flag is missing + +#### Scenario: OpenRouter model permitted at run time only +- **WHEN** a batch is started with `--model deepseek/deepseek-v4.1-flash` +- **THEN** the harness accepts the model, while a task YAML declaring the same id fails to load + +#### Scenario: Budget exhausted +- **WHEN** the cumulative cost (reported for Claude runs, provider-priced for OpenRouter runs) reaches `--max-cost-usd` during a batch +- **THEN** no further sessions start, the unstarted runs are recorded as `incomplete` (reason `budget`), and the command exits non-zero + +### Requirement: Regression gate against a stored baseline + +The repository SHALL contain a baseline results file for each gated tier and each model the CI gates on. Baselines for the Claude models SHALL stay at `plugins//evals/baselines/.json`; baselines for an OpenRouter model SHALL be stored at `plugins//evals/baselines/openrouter//.json`, where `` is the part of the id after the `/`. For every task×arm a baseline SHALL record pass rate, replicate count, mean tokens, turns, duration and cost, along with the model id, the provider, the turn-limit factor, the harness version and a content hash of the task definition. A gate command SHALL compare a new treatment result with the stored baseline. When no baseline path is given, the gate SHALL use the default path for the model of the runs being gated. It SHALL fail when a gating task's treatment pass rate is below the baseline: by default, when the baseline entry has at least 6 replicates, a one-sided Fisher exact test on pass/fail counts with p < 0.05; with fewer baseline replicates or an explicit tolerance, a drop of more than the tolerance (default: more than one replicate's worth, i.e. > 1/N). It SHALL also fail when mean input tokens rise by more than the configured budget (default 30%; the PR job MAY pass a wider budget calibrated from measured run-to-run variance). For trigger results, the gate SHALL compare each skill's validation split with the stored trigger baseline of the same model: when the baseline records run-level counts, skill loads on should-trigger runs and non-loads on should-not-trigger runs SHALL each be compared with a one-sided Fisher exact test (p < 0.05); otherwise accuracy may not drop by more than one query. The gate SHALL also fail when any shipped skill's description was listed by name only in a recorded skill listing. Tasks whose definition hash, model or turn-limit factor differs from the baseline entry SHALL be reported as `rebaseline-needed` and excluded from the comparison. They SHALL NOT be treated as passing. Updating the baseline file SHALL be an explicit command whose output is committed via a reviewed pull request; that command SHALL write to the default path for the runs' model unless an output directory is given. + +#### Scenario: Pass-rate regression +- **WHEN** a gating task had baseline treatment pass rate 1.00 (N=3) and the new run has 0.33 +- **THEN** the gate fails and names the task and both pass rates + +#### Scenario: Flaky task against a 9-run baseline +- **WHEN** a gating task had baseline treatment pass rate 5/9 and the new run has 1/3 +- **THEN** the gate passes (Fisher p = 0.24); against a 9/9 baseline, 1/3 fails (p = 0.045) + +#### Scenario: Token budget regression +- **WHEN** a task's mean input tokens rise by 45% against the baseline with the default 30% budget +- **THEN** the gate fails and names the task and the increase + +#### Scenario: Borderline trigger queries are not a regression +- **WHEN** a skill's baseline has two should-trigger queries that each loaded the skill in 3 of 6 runs, and a PR sample of 3 runs loads them once each +- **THEN** the trigger gate passes, because run-level loads are not significantly lower + +#### Scenario: Description listed by name only +- **WHEN** a trigger session's skill listing shows `rf-setup` without its description +- **THEN** the gate fails with a `listing` finding for rf-setup that points to the listing-room test + +#### Scenario: Changed task needs rebaseline +- **WHEN** a task's definition changed since the baseline was recorded +- **THEN** the gate reports the task as `rebaseline-needed` and does not count it as passing + +#### Scenario: Baseline chosen by model +- **WHEN** the gate runs without `--baseline` on narrow-tier runs of `deepseek/deepseek-v4.1-flash` for rf-core +- **THEN** it compares them with `plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json`, not with the Haiku baseline + +#### Scenario: Different turn-limit factor needs rebaseline +- **WHEN** the baseline entry was recorded with turn-limit factor 1.5 and the new runs used 1.0 +- **THEN** the gate reports the task as `rebaseline-needed` + +### Requirement: Tiered CI execution with bounded cost + +CI SHALL run the harness in tiers, on a CI provider chosen per run: the OpenRouter tier (`deepseek/deepseek-v4.1-flash` with the turn-limit factor recorded in its baselines) whenever the `OPENROUTER_API_KEY` secret is available, and the Claude tier otherwise or when a manual run asks for it. On pull requests that touch skills, the plugin, eval tasks or the harness, CI SHALL run the narrow tier in the treatment arm with N=3 on the CI provider's model. That run SHALL be scoped to tasks of changed skills plus bundle canaries and SHALL be gated against the stored baseline of that model. On pull requests that change a skill's frontmatter `description`, CI SHALL run that skill's trigger set on the validation split on the same model. On a weekly schedule and on manual dispatch, CI SHALL run all tiers in both arms plus all trigger sets, and SHALL publish the report and a candidate baseline for the model it ran as artifacts. A manual dispatch SHALL accept the provider (`openrouter` or `claude`) and, for `claude`, the Claude model; Claude runs SHALL NOT start on pull requests or the schedule while the OpenRouter key is available. Every CI invocation SHALL pass an explicit cost cap. Every OpenRouter job SHALL report the key's actual spend during the job, read from OpenRouter before and after the runs, in its summary. When credentials are not available (for example fork PRs), the job SHALL report that evaluation was not run. It SHALL NOT report success. + +#### Scenario: PR scoped to changed skill +- **WHEN** a pull request only changes `skills/rf-selenium/` and the OpenRouter key is available +- **THEN** CI runs the selenium narrow tasks and the bundle canaries in the treatment arm with N=3 on `deepseek/deepseek-v4.1-flash`, and gates them against that model's stored baseline + +#### Scenario: Description change triggers trigger eval +- **WHEN** a pull request changes the `description` in a skill's SKILL.md frontmatter +- **THEN** CI runs that skill's validation trigger queries on the CI provider's model and reports precision and recall against that model's stored trigger baseline + +#### Scenario: Manual Claude run +- **WHEN** a maintainer dispatches the evaluation with provider `claude` and model `claude-sonnet-5-5` +- **THEN** the job runs on the Claude credentials, gates against the Claude baselines and publishes a Claude candidate baseline + +#### Scenario: Spend reported +- **WHEN** an OpenRouter PR job finishes +- **THEN** its summary shows the key's spend during the job in USD next to the harness's estimated cost + +#### Scenario: Fork PR without secrets +- **WHEN** CI runs on a pull request where no OpenRouter key or Claude credentials are available +- **THEN** the evaluation job reports "not run: no credentials" and does not post a passing result diff --git a/openspec/changes/openrouter-ci-eval-tier/tasks.md b/openspec/changes/openrouter-ci-eval-tier/tasks.md new file mode 100644 index 0000000..cf77fcf --- /dev/null +++ b/openspec/changes/openrouter-ci-eval-tier/tasks.md @@ -0,0 +1,62 @@ +## 1. Model policy and provider + +- [x] 1.1 Add `OPENROUTER_MODELS` (`deepseek/deepseek-v4.1-flash`) to `domain/task.py`, permitted at run time and rejected in task YAML and trigger sets with a "run-time only" message. Verify with new cases in `tests/eval/test_domain_task.py` and `test_task_yaml_loading.py` (YAML rejected, run time accepted, unlisted `z-ai/glm-5.3-flash` rejected with the permitted ids named). +- [x] 1.2 Add a `Provider` value (`anthropic`, `openrouter`), derived from the model id, plus a `--provider` option on `run`, `run-batch` and `trigger` that refuses mismatched pairs before any session starts. Verify with CLI tests for the mismatch error and the derived default. +- [x] 1.3 Record the provider, the model and the turn-limit factor on every run (a new column in `runs`, migrated like the earlier columns) and in results and baseline entries. Verify with `test_persistence.py` (an old database opens, new runs store the provider) and `test_baseline_gate.py` (entries carry the provider and the factor). + +## 2. Session environment and auth + +- [x] 2.1 Make `ClaudeCodeRunner._build_env` provider-aware (design D1). For OpenRouter: + - drop `CLAUDE_CODE_OAUTH_TOKEN`, `ANTHROPIC_API_KEY` and any inherited `ANTHROPIC_AUTH_TOKEN`; + - set the base URL, `ANTHROPIC_AUTH_TOKEN` from `OPENROUTER_API_KEY`, and the four model mappings; + - raise `AuthConfigError` naming `OPENROUTER_API_KEY` when it's missing. + + Verify with runner unit tests that inspect the built environment for both providers. +- [x] 2.2 Make `config.has_auth` and `cli._require_auth` provider-aware, and make `doctor` report which credential the selected provider needs. Verify with `test_cli_doctor.py` cases for both providers. +- [x] 2.3 Record a run as incomplete with reason `provider_model` when its `modelUsage` names any model other than the selected one (the same way arm leaks are recorded), so it is not graded or counted as passing. Verify with a recorded-transcript test in `test_infrastructure_session_parser.py` and a gate test. +- [x] 2.4 Check that the key never reaches stored output: grep a fake key through a full fake batch's database, reports and copied config. Verify with a test that fails if the fake key string appears anywhere under the output dir. + +## 3. Cost and turn limits + +- [x] 3.1 Add `PROVIDER_PRICES` and compute the cost of OpenRouter runs from the usage block (input, cache read, cache write, output). Store it in `total_cost_usd` and record `cost_source` as `priced`. Verify with unit tests that check the arithmetic and that the budget stops on priced cost, not on Claude Code's reported cost. +- [ ] 3.2 Calibrate the DeepSeek prices against the spike: re-score the 72 spike transcripts with the price table. They're in the artifact `spike-openrouter-narrow` of run 42822700497 (`gh run download 42822700497 -n spike-openrouter-narrow`; it expires 90 days after 2026-11-10). Verify that the priced total is within ±25% of the $0.84 OpenRouter charged, and record both numbers in the PR. +- [x] 3.3 Label priced costs as estimates in the md/json reports. Verify with a `test_reporting.py` snapshot. +- [x] 3.4 Add `--max-turns-factor` (1.0–3.0, rounded up) to `run` and `run-batch`, record turn-limit stops from `subtype: error_max_turns`, and show the stop counts per task and arm in reports. Verify with a runner test asserting `--max-turns 21` for `max_turns: 14` at factor 1.5, and a report test for the stop count. + +## 4. Per-model baselines and gate + +- [x] 4.1 Derive the default baseline path from the runs' model (`baselines/openrouter//.json` for OpenRouter, unchanged for Claude) in `gate`, `report` and `baseline update`. Verify with `test_baseline_gate.py` cases for both defaults and an explicit `--baseline` override. +- [x] 4.2 Make a turn-limit factor mismatch a `rebaseline-needed` finding. Verify with a gate test for factor 1.5 against 1.0. +- [x] 4.3 Remove the spike's `RF_SKILL_EVAL_EXTRA_MODELS` and `ANTHROPIC_AUTH_TOKEN` auth shortcuts, so they don't come back through a merge from the spike branch. Verify with `grep -r RF_SKILL_EVAL_EXTRA_MODELS src tests` returning nothing, and with `uv run ruff check src tests/eval`, `uv run mypy` and `uv run pytest tests/eval -q` all green. + +## 5. CI workflow + +- [x] 5.1 Make preflight in `skill-evaluation.yml` output `provider`, `model`, `turns_factor` and the baseline subdirectory: OpenRouter when `OPENROUTER_API_KEY` is set, otherwise Claude, with a `provider` dispatch input overriding both. Keep the fork-PR rule. Verify with `tests/eval/test_ci_preflight.py` cases and by checking that the workflow still parses (`uv run python -c "import yaml; yaml.safe_load(open('.github/workflows/skill-evaluation.yml'))"`). +- [ ] 5.2 Wire `pr-eval` and `full-eval` to the preflight outputs: + - the model and the turn-limit factor; + - the credentials for the provider only; + - separate OpenRouter caps (`*_CAP_USD_OPENROUTER`); + - the baseline paths for the model; + - the candidate-baseline artifact names, which include the model. + + Verify on a PR from a branch in this repository: the eval job runs on DeepSeek and gates against the OpenRouter baselines. +- [ ] 5.3 Add the steps for OpenRouter jobs: + - a one-session smoke step that fails fast; + - "Spend before" and "Spend after" steps that read `/api/v1/key`; + - a summary line with the key's spend during the job next to the priced total; + - on the weekly job, a warning when they differ by more than 25%. + + Verify in the job summary of the PR run from 5.2. +- [ ] 5.4 Check the no-credentials path: dispatch with provider `openrouter` from a fork, or with the secret temporarily unavailable through an environment that lacks it, and see "not run: no credentials". Verify from the run summary. If testing this needs a secret removed, ask the maintainer first. + +## 6. Baselines + +- [ ] 6.1 Dispatch the evaluation with provider `openrouter`, all tiers, both arms, N=6 and factor 1.5, for every plugin with evals. Verify that the run completes within the OpenRouter caps and produces one candidate baseline per plugin. +- [ ] 6.2 Review the candidate baselines (flaky tasks, turn-limit stops, and whether 1.5 is enough), commit them under `plugins//evals/baselines/openrouter/deepseek-v4.1-flash/` in a reviewed PR, and record the reviewed spend. Verify that the next PR eval run reports the gate as `pass`, not `rebaseline-needed`. + +## 7. Docs and clean-up + +- [x] 7.1 Update `docs/ci/usage.md` and `docs/ci/local-testing.md` (running on OpenRouter locally, the key, `--provider`, `--max-turns-factor`, priced cost, where the baselines live) and add a dated addendum to ADR-005 for the default-provider change. Verify with `uv run pytest tests -q -k "docs or stale or links"` and by reading the rendered files. +- [x] 7.2 Add "manual Claude eval (provider `claude`, Sonnet 5.5, narrow and realistic)" to the release checklist in `RELEASING.md`. Verify that the checklist renders and names the dispatch inputs. +- [ ] 7.3 Delete the branch `spike/openrouter-evals` locally and on origin (it holds the spike workflow), after asking the maintainer. Verify that `git ls-remote origin spike/openrouter-evals` prints nothing. +- [ ] 7.4 Run `openspec validate openrouter-ci-eval-tier --strict` and confirm it passes before the archive. diff --git a/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json b/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json new file mode 100644 index 0000000..c1947c8 --- /dev/null +++ b/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json @@ -0,0 +1,51 @@ +{ + "claude_code_version": "2.1.322 (Claude Code)", + "entries": { + "adv-restinstance-expectation-leak|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.026965, + "mean_duration_s": 77.08, + "mean_input_tokens": 489836.8, + "mean_output_tokens": 10493.2, + "mean_turns": 26.33, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-restinstance", + "task_hash": "aecee21f71999774880b80b2509398574df8bf8917c053dcc3858b04ad0b0971", + "task_id": "adv-restinstance-expectation-leak", + "turn_limit_stops": 1, + "turns_factor": 1.5 + }, + "adv-restinstance-expectation-leak|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.030018, + "mean_duration_s": 86.32, + "mean_input_tokens": 587180.8, + "mean_output_tokens": 8671.8, + "mean_turns": 26.0, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-restinstance", + "task_hash": "aecee21f71999774880b80b2509398574df8bf8917c053dcc3858b04ad0b0971", + "task_id": "adv-restinstance-expectation-leak", + "turn_limit_stops": 4, + "turns_factor": 1.5 + } + }, + "harness_version": "0.1.0", + "kind": "tier-baseline", + "schema_version": 1, + "tier": "adversarial" +} diff --git a/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json b/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json new file mode 100644 index 0000000..b47abe8 --- /dev/null +++ b/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json @@ -0,0 +1,93 @@ +{ + "claude_code_version": "2.1.322 (Claude Code)", + "entries": { + "narrow-requests-users-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.009859, + "mean_duration_s": 28.32, + "mean_input_tokens": 132557.3, + "mean_output_tokens": 2866.7, + "mean_turns": 11.0, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-requests", + "task_hash": "a649e0f83496832c1f42e1c1aee15005d24ecc595133d05d8fa28b1354754a4c", + "task_id": "narrow-requests-users-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-requests-users-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.032796, + "mean_duration_s": 125.93, + "mean_input_tokens": 499791.2, + "mean_output_tokens": 13815.7, + "mean_turns": 21.67, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-requests", + "task_hash": "a649e0f83496832c1f42e1c1aee15005d24ecc595133d05d8fa28b1354754a4c", + "task_id": "narrow-requests-users-01", + "turn_limit_stops": 1, + "turns_factor": 1.5 + }, + "narrow-restinstance-schema-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.011716, + "mean_duration_s": 38.74, + "mean_input_tokens": 232008.8, + "mean_output_tokens": 3515.3, + "mean_turns": 17.0, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-restinstance", + "task_hash": "bdd4fdeaac254014d77f0d3177a71c9e57315df5bdca765c63cc512a8849df64", + "task_id": "narrow-restinstance-schema-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-restinstance-schema-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.032096, + "mean_duration_s": 92.27, + "mean_input_tokens": 527758.7, + "mean_output_tokens": 9892.5, + "mean_turns": 22.83, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-restinstance", + "task_hash": "bdd4fdeaac254014d77f0d3177a71c9e57315df5bdca765c63cc512a8849df64", + "task_id": "narrow-restinstance-schema-01", + "turn_limit_stops": 3, + "turns_factor": 1.5 + } + }, + "harness_version": "0.1.0", + "kind": "tier-baseline", + "schema_version": 1, + "tier": "narrow" +} diff --git a/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json b/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json new file mode 100644 index 0000000..97797d5 --- /dev/null +++ b/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json @@ -0,0 +1,51 @@ +{ + "claude_code_version": "2.1.322 (Claude Code)", + "entries": { + "partial-requests-users-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.010817, + "mean_duration_s": 18.67, + "mean_input_tokens": 118719.3, + "mean_output_tokens": 1887.5, + "mean_turns": 11.5, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-requests", + "task_hash": "61a51a99892fbbe4bb9bfb7a9505341483d3ba9960c49ec1ab50a9eb39854550", + "task_id": "partial-requests-users-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "partial-requests-users-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.035372, + "mean_duration_s": 107.71, + "mean_input_tokens": 560272.5, + "mean_output_tokens": 12909.3, + "mean_turns": 24.17, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-requests", + "task_hash": "61a51a99892fbbe4bb9bfb7a9505341483d3ba9960c49ec1ab50a9eb39854550", + "task_id": "partial-requests-users-01", + "turn_limit_stops": 2, + "turns_factor": 1.5 + } + }, + "harness_version": "0.1.0", + "kind": "tier-baseline", + "schema_version": 1, + "tier": "partial" +} diff --git a/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/realistic.json b/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/realistic.json new file mode 100644 index 0000000..2ce0191 --- /dev/null +++ b/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/realistic.json @@ -0,0 +1,51 @@ +{ + "claude_code_version": "2.1.322 (Claude Code)", + "entries": { + "realistic-api-testing-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.016401, + "mean_duration_s": 41.32, + "mean_input_tokens": 242541.3, + "mean_output_tokens": 7374.3, + "mean_turns": 16.83, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-requests", + "task_hash": "ef138fe68471123497708f07e44d03e641d2c75374da7c04248239ed3c83e5dd", + "task_id": "realistic-api-testing-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "realistic-api-testing-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.037013, + "mean_duration_s": 149.54, + "mean_input_tokens": 574239.0, + "mean_output_tokens": 16853.2, + "mean_turns": 23.5, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-requests", + "task_hash": "ef138fe68471123497708f07e44d03e641d2c75374da7c04248239ed3c83e5dd", + "task_id": "realistic-api-testing-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + } + }, + "harness_version": "0.1.0", + "kind": "tier-baseline", + "schema_version": 1, + "tier": "realistic" +} diff --git a/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json b/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json new file mode 100644 index 0000000..27fb678 --- /dev/null +++ b/plugins/rf-api/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json @@ -0,0 +1,69 @@ +{ + "harness_version": "0.1.0", + "kind": "trigger-baseline", + "model": "deepseek/deepseek-v4.1-flash", + "provider": "openrouter", + "schema_version": 1, + "skills": { + "rf-requests": { + "train": { + "accuracy": 0.7391, + "fn": 0, + "fp": 6, + "negative_loads": 17, + "negative_runs": 33, + "positive_loads": 36, + "positive_runs": 36, + "precision": 0.6667, + "queries": 23, + "recall": 1.0, + "tn": 5, + "tp": 12 + }, + "validation": { + "accuracy": 0.75, + "fn": 0, + "fp": 2, + "negative_loads": 7, + "negative_runs": 12, + "positive_loads": 11, + "positive_runs": 12, + "precision": 0.6667, + "queries": 8, + "recall": 1.0, + "tn": 2, + "tp": 4 + } + }, + "rf-restinstance": { + "train": { + "accuracy": 0.6957, + "fn": 0, + "fp": 7, + "negative_loads": 21, + "negative_runs": 33, + "positive_loads": 36, + "positive_runs": 36, + "precision": 0.6316, + "queries": 23, + "recall": 1.0, + "tn": 4, + "tp": 12 + }, + "validation": { + "accuracy": 0.625, + "fn": 0, + "fp": 3, + "negative_loads": 9, + "negative_runs": 12, + "positive_loads": 12, + "positive_runs": 12, + "precision": 0.5714, + "queries": 8, + "recall": 1.0, + "tn": 1, + "tp": 4 + } + } + } +} diff --git a/plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json b/plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json new file mode 100644 index 0000000..edac1d9 --- /dev/null +++ b/plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json @@ -0,0 +1,219 @@ +{ + "claude_code_version": "2.1.322 (Claude Code)", + "entries": { + "adv-language-force-tags-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.0117, + "mean_duration_s": 21.96, + "mean_input_tokens": 144276.2, + "mean_output_tokens": 2417.2, + "mean_turns": 13.5, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.0, + "pass_rate": 0.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "222d98c6bad70c6e939c0b4d8074331c5a0ccf2f49e195ff4c4448e7d830849f", + "task_id": "adv-language-force-tags-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "adv-language-force-tags-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.018232, + "mean_duration_s": 51.36, + "mean_input_tokens": 335080.7, + "mean_output_tokens": 5981.5, + "mean_turns": 17.67, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "222d98c6bad70c6e939c0b4d8074331c5a0ccf2f49e195ff4c4448e7d830849f", + "task_id": "adv-language-force-tags-01", + "turn_limit_stops": 1, + "turns_factor": 1.5 + }, + "adv-language-rf71-typed-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.033495, + "mean_duration_s": 85.76, + "mean_input_tokens": 467768.5, + "mean_output_tokens": 13220.0, + "mean_turns": 23.33, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.1667, + "pass_rate": 0.1667, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "b6e23c49b5f4ef38d0b2e4d7ecebdd6db099714678c10dee53e2c2be06d339fd", + "task_id": "adv-language-rf71-typed-01", + "turn_limit_stops": 4, + "turns_factor": 1.5 + }, + "adv-language-rf71-typed-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.029322, + "mean_duration_s": 72.38, + "mean_input_tokens": 554805.0, + "mean_output_tokens": 10562.3, + "mean_turns": 23.17, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.8333, + "pass_rate": 0.8333, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "b6e23c49b5f4ef38d0b2e4d7ecebdd6db099714678c10dee53e2c2be06d339fd", + "task_id": "adv-language-rf71-typed-01", + "turn_limit_stops": 1, + "turns_factor": 1.5 + }, + "adv-legacy-syntax-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.014381, + "mean_duration_s": 43.62, + "mean_input_tokens": 182624.2, + "mean_output_tokens": 6198.5, + "mean_turns": 17.5, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "b5f1ac31a1abc6c1c3d5d63399c0b614ff80631b82f3765f70cf8e01f29f4649", + "task_id": "adv-legacy-syntax-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "adv-legacy-syntax-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.023623, + "mean_duration_s": 66.41, + "mean_input_tokens": 534293.8, + "mean_output_tokens": 8264.5, + "mean_turns": 24.0, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.8333, + "pass_rate": 0.8333, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "b5f1ac31a1abc6c1c3d5d63399c0b614ff80631b82f3765f70cf8e01f29f4649", + "task_id": "adv-legacy-syntax-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "adv-make-it-pass|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.010574, + "mean_duration_s": 14.79, + "mean_input_tokens": 110324.8, + "mean_output_tokens": 1886.0, + "mean_turns": 11.5, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "plugin", + "task_hash": "0b6f5a8701a2b2b45402a18311b278972f494da4110363d5bef3d4a6afddff85", + "task_id": "adv-make-it-pass", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "adv-make-it-pass|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.01714, + "mean_duration_s": 31.47, + "mean_input_tokens": 323550.2, + "mean_output_tokens": 4352.8, + "mean_turns": 16.33, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "plugin", + "task_hash": "0b6f5a8701a2b2b45402a18311b278972f494da4110363d5bef3d4a6afddff85", + "task_id": "adv-make-it-pass", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "adv-setup-system-pip|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.024477, + "mean_duration_s": 75.38, + "mean_input_tokens": 349557.5, + "mean_output_tokens": 9714.8, + "mean_turns": 18.17, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.3333, + "pass_rate": 0.3333, + "provider": "openrouter", + "runs": 6, + "skill": "rf-setup", + "task_hash": "a6ded3df3dd52efdb00a8e9404f9e8c9dd84d29a9a6964a98fcc68cea57f3ab6", + "task_id": "adv-setup-system-pip", + "turn_limit_stops": 4, + "turns_factor": 1.5 + }, + "adv-setup-system-pip|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.026218, + "mean_duration_s": 78.19, + "mean_input_tokens": 439165.3, + "mean_output_tokens": 7062.0, + "mean_turns": 20.33, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.8333, + "pass_rate": 0.8333, + "provider": "openrouter", + "runs": 6, + "skill": "rf-setup", + "task_hash": "a6ded3df3dd52efdb00a8e9404f9e8c9dd84d29a9a6964a98fcc68cea57f3ab6", + "task_id": "adv-setup-system-pip", + "turn_limit_stops": 4, + "turns_factor": 1.5 + } + }, + "harness_version": "0.1.0", + "kind": "tier-baseline", + "schema_version": 1, + "tier": "adversarial" +} diff --git a/plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json b/plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json new file mode 100644 index 0000000..a4e8c31 --- /dev/null +++ b/plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json @@ -0,0 +1,513 @@ +{ + "claude_code_version": "2.1.322 (Claude Code)", + "entries": { + "narrow-language-bdd-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.009036, + "mean_duration_s": 18.5, + "mean_input_tokens": 134236.8, + "mean_output_tokens": 2628.7, + "mean_turns": 12.33, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "6b41db2fc2d7cdfaa5bb9b5892eebd6602f54843717d463b3578fdbd816ccb8e", + "task_id": "narrow-language-bdd-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-language-bdd-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.017746, + "mean_duration_s": 53.61, + "mean_input_tokens": 303940.5, + "mean_output_tokens": 5407.5, + "mean_turns": 17.0, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "6b41db2fc2d7cdfaa5bb9b5892eebd6602f54843717d463b3578fdbd816ccb8e", + "task_id": "narrow-language-bdd-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-language-data-driven-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.01335, + "mean_duration_s": 40.27, + "mean_input_tokens": 206358.7, + "mean_output_tokens": 5540.7, + "mean_turns": 19.17, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.8333, + "pass_rate": 0.8333, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "2d87d5bb26e71319ae941b09099c0b4ff6f7ef0feae45da4ec2746a1503d6374", + "task_id": "narrow-language-data-driven-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-language-data-driven-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.021354, + "mean_duration_s": 51.63, + "mean_input_tokens": 478180.5, + "mean_output_tokens": 6614.3, + "mean_turns": 23.5, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "2d87d5bb26e71319ae941b09099c0b4ff6f7ef0feae45da4ec2746a1503d6374", + "task_id": "narrow-language-data-driven-01", + "turn_limit_stops": 2, + "turns_factor": 1.5 + }, + "narrow-language-embedded-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.013019, + "mean_duration_s": 21.91, + "mean_input_tokens": 127580.3, + "mean_output_tokens": 3524.5, + "mean_turns": 12.17, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "aba5e4e437b213e99475bf9af9a77974cab7b81f06bfea64efe77c4eef1cbb10", + "task_id": "narrow-language-embedded-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-language-embedded-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.015626, + "mean_duration_s": 36.75, + "mean_input_tokens": 333621.7, + "mean_output_tokens": 4828.0, + "mean_turns": 18.5, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "aba5e4e437b213e99475bf9af9a77974cab7b81f06bfea64efe77c4eef1cbb10", + "task_id": "narrow-language-embedded-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-language-env-varfiles-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.019272, + "mean_duration_s": 42.19, + "mean_input_tokens": 274618.5, + "mean_output_tokens": 6895.3, + "mean_turns": 21.67, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "30ab6d585897d70c99f46dae04d8a0e0bbc9af17c997ee844c008be1f4376614", + "task_id": "narrow-language-env-varfiles-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-language-env-varfiles-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.034865, + "mean_duration_s": 121.15, + "mean_input_tokens": 703081.5, + "mean_output_tokens": 16674.7, + "mean_turns": 22.0, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "30ab6d585897d70c99f46dae04d8a0e0bbc9af17c997ee844c008be1f4376614", + "task_id": "narrow-language-env-varfiles-01", + "turn_limit_stops": 6, + "turns_factor": 1.5 + }, + "narrow-language-suite-setup-tags-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.014611, + "mean_duration_s": 33.13, + "mean_input_tokens": 221498.5, + "mean_output_tokens": 4473.5, + "mean_turns": 18.33, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.8333, + "pass_rate": 0.8333, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "837a5e31e38470fe69079510467cdb40e84cc72343c8797455b71cadc9b8caa8", + "task_id": "narrow-language-suite-setup-tags-01", + "turn_limit_stops": 1, + "turns_factor": 1.5 + }, + "narrow-language-suite-setup-tags-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.023001, + "mean_duration_s": 73.82, + "mean_input_tokens": 516114.8, + "mean_output_tokens": 7825.2, + "mean_turns": 25.83, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-language", + "task_hash": "837a5e31e38470fe69079510467cdb40e84cc72343c8797455b71cadc9b8caa8", + "task_id": "narrow-language-suite-setup-tags-01", + "turn_limit_stops": 1, + "turns_factor": 1.5 + }, + "narrow-non-rf-control-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.004628, + "mean_duration_s": 2.7, + "mean_input_tokens": 28024.2, + "mean_output_tokens": 195.3, + "mean_turns": 2.0, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "plugin", + "task_hash": "5a96ab353665be0efda0528d87c3af328348936e6e9e509c112220be336327a1", + "task_id": "narrow-non-rf-control-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-non-rf-control-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.004683, + "mean_duration_s": 3.16, + "mean_input_tokens": 35121.0, + "mean_output_tokens": 255.2, + "mean_turns": 2.33, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "plugin", + "task_hash": "5a96ab353665be0efda0528d87c3af328348936e6e9e509c112220be336327a1", + "task_id": "narrow-non-rf-control-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-python-library-flaky-listener-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.026371, + "mean_duration_s": 76.32, + "mean_input_tokens": 433974.5, + "mean_output_tokens": 10412.3, + "mean_turns": 20.83, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.8333, + "pass_rate": 0.8333, + "provider": "openrouter", + "runs": 6, + "skill": "rf-python-library", + "task_hash": "802cedc727783fbf79d46d7fa4fbb827d8f5c2ddeb2956829a21510f1b620d74", + "task_id": "narrow-python-library-flaky-listener-01", + "turn_limit_stops": 3, + "turns_factor": 1.5 + }, + "narrow-python-library-flaky-listener-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.025662, + "mean_duration_s": 74.47, + "mean_input_tokens": 517570.8, + "mean_output_tokens": 11552.0, + "mean_turns": 20.0, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-python-library", + "task_hash": "802cedc727783fbf79d46d7fa4fbb827d8f5c2ddeb2956829a21510f1b620d74", + "task_id": "narrow-python-library-flaky-listener-01", + "turn_limit_stops": 3, + "turns_factor": 1.5 + }, + "narrow-python-library-inventory-scope-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.021482, + "mean_duration_s": 62.53, + "mean_input_tokens": 334541.8, + "mean_output_tokens": 9598.3, + "mean_turns": 19.5, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-python-library", + "task_hash": "b36a43d46c9dfe9494d15efc6524a401e17bc144b914aa0ead66f89c9fb85706", + "task_id": "narrow-python-library-inventory-scope-01", + "turn_limit_stops": 3, + "turns_factor": 1.5 + }, + "narrow-python-library-inventory-scope-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.019928, + "mean_duration_s": 80.19, + "mean_input_tokens": 377418.8, + "mean_output_tokens": 7540.5, + "mean_turns": 18.5, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-python-library", + "task_hash": "b36a43d46c9dfe9494d15efc6524a401e17bc144b914aa0ead66f89c9fb85706", + "task_id": "narrow-python-library-inventory-scope-01", + "turn_limit_stops": 2, + "turns_factor": 1.5 + }, + "narrow-python-library-mode-literal-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.024724, + "mean_duration_s": 74.76, + "mean_input_tokens": 330956.2, + "mean_output_tokens": 10259.2, + "mean_turns": 19.0, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-python-library", + "task_hash": "972eac7a963f65bc18d59424c64d6a7136e4fc143195edf237998bf667f68caf", + "task_id": "narrow-python-library-mode-literal-01", + "turn_limit_stops": 1, + "turns_factor": 1.5 + }, + "narrow-python-library-mode-literal-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.026375, + "mean_duration_s": 61.44, + "mean_input_tokens": 441844.7, + "mean_output_tokens": 10191.8, + "mean_turns": 20.17, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-python-library", + "task_hash": "972eac7a963f65bc18d59424c64d6a7136e4fc143195edf237998bf667f68caf", + "task_id": "narrow-python-library-mode-literal-01", + "turn_limit_stops": 1, + "turns_factor": 1.5 + }, + "narrow-rf-injection-positive-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.025976, + "mean_duration_s": 46.99, + "mean_input_tokens": 358063.3, + "mean_output_tokens": 6273.8, + "mean_turns": 18.67, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "plugin", + "task_hash": "08a7e9b09596ebf8c77b9d2106844c13471ebfd614770985b80fdd953924fc59", + "task_id": "narrow-rf-injection-positive-01", + "turn_limit_stops": 1, + "turns_factor": 1.5 + }, + "narrow-rf-injection-positive-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.014985, + "mean_duration_s": 51.23, + "mean_input_tokens": 206766.8, + "mean_output_tokens": 4228.2, + "mean_turns": 14.83, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "plugin", + "task_hash": "08a7e9b09596ebf8c77b9d2106844c13471ebfd614770985b80fdd953924fc59", + "task_id": "narrow-rf-injection-positive-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-rf-mcp-execute-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.016829, + "mean_duration_s": 14.07, + "mean_input_tokens": 151802.3, + "mean_output_tokens": 1464.7, + "mean_turns": 8.5, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "plugin", + "task_hash": "9961ea125f066728f1e6be2b5b1246a9f9b58f6f6a0e68a7b6fef5c2e02b2361", + "task_id": "narrow-rf-mcp-execute-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-rf-mcp-execute-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.026522, + "mean_duration_s": 37.89, + "mean_input_tokens": 477369.8, + "mean_output_tokens": 4747.2, + "mean_turns": 18.67, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "plugin", + "task_hash": "9961ea125f066728f1e6be2b5b1246a9f9b58f6f6a0e68a7b6fef5c2e02b2361", + "task_id": "narrow-rf-mcp-execute-01", + "turn_limit_stops": 4, + "turns_factor": 1.5 + }, + "narrow-setup-requests-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.009745, + "mean_duration_s": 20.2, + "mean_input_tokens": 140285.3, + "mean_output_tokens": 2193.5, + "mean_turns": 10.33, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-setup", + "task_hash": "c4c361ae89c9992d4a1b6a3f3f6129a384564a317c115ca894f98fcb19fa4b1a", + "task_id": "narrow-setup-requests-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-setup-requests-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.015408, + "mean_duration_s": 54.64, + "mean_input_tokens": 228424.2, + "mean_output_tokens": 5263.0, + "mean_turns": 16.83, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-setup", + "task_hash": "c4c361ae89c9992d4a1b6a3f3f6129a384564a317c115ca894f98fcb19fa4b1a", + "task_id": "narrow-setup-requests-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + } + }, + "harness_version": "0.1.0", + "kind": "tier-baseline", + "schema_version": 1, + "tier": "narrow" +} diff --git a/plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json b/plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json new file mode 100644 index 0000000..9615e2c --- /dev/null +++ b/plugins/rf-core/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json @@ -0,0 +1,99 @@ +{ + "harness_version": "0.1.0", + "kind": "trigger-baseline", + "model": "deepseek/deepseek-v4.1-flash", + "provider": "openrouter", + "schema_version": 1, + "skills": { + "rf-language": { + "train": { + "accuracy": 0.76, + "fn": 2, + "fp": 4, + "negative_loads": 13, + "negative_runs": 36, + "positive_loads": 35, + "positive_runs": 39, + "precision": 0.7333, + "queries": 25, + "recall": 0.8462, + "tn": 8, + "tp": 11 + }, + "validation": { + "accuracy": 0.8182, + "fn": 1, + "fp": 1, + "negative_loads": 4, + "negative_runs": 15, + "positive_loads": 16, + "positive_runs": 18, + "precision": 0.8333, + "queries": 11, + "recall": 0.8333, + "tn": 4, + "tp": 5 + } + }, + "rf-python-library": { + "train": { + "accuracy": 0.6364, + "fn": 0, + "fp": 8, + "negative_loads": 24, + "negative_runs": 33, + "positive_loads": 32, + "positive_runs": 33, + "precision": 0.5789, + "queries": 22, + "recall": 1.0, + "tn": 3, + "tp": 11 + }, + "validation": { + "accuracy": 0.875, + "fn": 0, + "fp": 1, + "negative_loads": 5, + "negative_runs": 12, + "positive_loads": 12, + "positive_runs": 12, + "precision": 0.8, + "queries": 8, + "recall": 1.0, + "tn": 3, + "tp": 4 + } + }, + "rf-setup": { + "train": { + "accuracy": 0.6957, + "fn": 0, + "fp": 7, + "negative_loads": 23, + "negative_runs": 33, + "positive_loads": 35, + "positive_runs": 36, + "precision": 0.6316, + "queries": 23, + "recall": 1.0, + "tn": 4, + "tp": 12 + }, + "validation": { + "accuracy": 0.5, + "fn": 1, + "fp": 3, + "negative_loads": 8, + "negative_runs": 12, + "positive_loads": 9, + "positive_runs": 12, + "precision": 0.5, + "queries": 8, + "recall": 0.75, + "tn": 1, + "tp": 3 + } + } + } +} diff --git a/plugins/rf-desktop/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json b/plugins/rf-desktop/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json new file mode 100644 index 0000000..18a3342 --- /dev/null +++ b/plugins/rf-desktop/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json @@ -0,0 +1,51 @@ +{ + "claude_code_version": "2.1.322 (Claude Code)", + "entries": { + "narrow-platynui-calculator-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.0226, + "mean_duration_s": 76.46, + "mean_input_tokens": 475577.5, + "mean_output_tokens": 8533.7, + "mean_turns": 20.67, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.5, + "pass_rate": 0.5, + "provider": "openrouter", + "runs": 6, + "skill": "rf-platynui", + "task_hash": "70cdf07c4e3c5f1f9c1f35e2ad0d551ff52937c575c4ec43f5b1c688e8cfbe70", + "task_id": "narrow-platynui-calculator-01", + "turn_limit_stops": 4, + "turns_factor": 1.5 + }, + "narrow-platynui-calculator-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.033992, + "mean_duration_s": 72.57, + "mean_input_tokens": 708943.0, + "mean_output_tokens": 11146.8, + "mean_turns": 23.5, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.3333, + "pass_rate": 0.3333, + "provider": "openrouter", + "runs": 6, + "skill": "rf-platynui", + "task_hash": "70cdf07c4e3c5f1f9c1f35e2ad0d551ff52937c575c4ec43f5b1c688e8cfbe70", + "task_id": "narrow-platynui-calculator-01", + "turn_limit_stops": 3, + "turns_factor": 1.5 + } + }, + "harness_version": "0.1.0", + "kind": "tier-baseline", + "schema_version": 1, + "tier": "narrow" +} diff --git a/plugins/rf-desktop/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json b/plugins/rf-desktop/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json new file mode 100644 index 0000000..2bda0ce --- /dev/null +++ b/plugins/rf-desktop/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json @@ -0,0 +1,51 @@ +{ + "claude_code_version": "2.1.322 (Claude Code)", + "entries": { + "partial-platynui-calculator-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.022808, + "mean_duration_s": 60.19, + "mean_input_tokens": 423533.3, + "mean_output_tokens": 8134.0, + "mean_turns": 20.0, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-platynui", + "task_hash": "8f932ab699f8f3dffedd2dbe6dfb20a541947e643cd789295f986576a102b1bc", + "task_id": "partial-platynui-calculator-01", + "turn_limit_stops": 1, + "turns_factor": 1.5 + }, + "partial-platynui-calculator-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.034988, + "mean_duration_s": 107.86, + "mean_input_tokens": 731280.0, + "mean_output_tokens": 12132.0, + "mean_turns": 22.83, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.8333, + "pass_rate": 0.8333, + "provider": "openrouter", + "runs": 6, + "skill": "rf-platynui", + "task_hash": "8f932ab699f8f3dffedd2dbe6dfb20a541947e643cd789295f986576a102b1bc", + "task_id": "partial-platynui-calculator-01", + "turn_limit_stops": 5, + "turns_factor": 1.5 + } + }, + "harness_version": "0.1.0", + "kind": "tier-baseline", + "schema_version": 1, + "tier": "partial" +} diff --git a/plugins/rf-desktop/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json b/plugins/rf-desktop/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json new file mode 100644 index 0000000..f2f13d6 --- /dev/null +++ b/plugins/rf-desktop/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json @@ -0,0 +1,39 @@ +{ + "harness_version": "0.1.0", + "kind": "trigger-baseline", + "model": "deepseek/deepseek-v4.1-flash", + "provider": "openrouter", + "schema_version": 1, + "skills": { + "rf-platynui": { + "train": { + "accuracy": 0.6522, + "fn": 0, + "fp": 8, + "negative_loads": 25, + "negative_runs": 33, + "positive_loads": 36, + "positive_runs": 36, + "precision": 0.6, + "queries": 23, + "recall": 1.0, + "tn": 3, + "tp": 12 + }, + "validation": { + "accuracy": 0.625, + "fn": 0, + "fp": 3, + "negative_loads": 10, + "negative_runs": 12, + "positive_loads": 12, + "positive_runs": 12, + "precision": 0.5714, + "queries": 8, + "recall": 1.0, + "tn": 1, + "tp": 4 + } + } + } +} diff --git a/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json b/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json new file mode 100644 index 0000000..16c0790 --- /dev/null +++ b/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json @@ -0,0 +1,51 @@ +{ + "claude_code_version": "2.1.322 (Claude Code)", + "entries": { + "adv-appium-deprecated-visibility|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.035684, + "mean_duration_s": 126.59, + "mean_input_tokens": 486789.0, + "mean_output_tokens": 11684.3, + "mean_turns": 24.0, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.5, + "pass_rate": 0.5, + "provider": "openrouter", + "runs": 6, + "skill": "rf-appium", + "task_hash": "d42a7ea8bb1e41559d26e50e33c6618c166557ca92ff7080527b35bafd6ca889", + "task_id": "adv-appium-deprecated-visibility", + "turn_limit_stops": 1, + "turns_factor": 1.5 + }, + "adv-appium-deprecated-visibility|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.042609, + "mean_duration_s": 93.53, + "mean_input_tokens": 764811.7, + "mean_output_tokens": 15331.7, + "mean_turns": 26.67, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.6667, + "pass_rate": 0.6667, + "provider": "openrouter", + "runs": 6, + "skill": "rf-appium", + "task_hash": "d42a7ea8bb1e41559d26e50e33c6618c166557ca92ff7080527b35bafd6ca889", + "task_id": "adv-appium-deprecated-visibility", + "turn_limit_stops": 3, + "turns_factor": 1.5 + } + }, + "harness_version": "0.1.0", + "kind": "tier-baseline", + "schema_version": 1, + "tier": "adversarial" +} diff --git a/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json b/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json new file mode 100644 index 0000000..789da8e --- /dev/null +++ b/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json @@ -0,0 +1,51 @@ +{ + "claude_code_version": "2.1.322 (Claude Code)", + "entries": { + "narrow-appium-login-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.020638, + "mean_duration_s": 69.05, + "mean_input_tokens": 282606.7, + "mean_output_tokens": 6601.3, + "mean_turns": 18.33, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-appium", + "task_hash": "9c13d665a3ed54d242a8b8d6ee5fc30985bfc97f00b1c90d528beb2f022c0d13", + "task_id": "narrow-appium-login-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-appium-login-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.028794, + "mean_duration_s": 82.43, + "mean_input_tokens": 427472.5, + "mean_output_tokens": 9473.0, + "mean_turns": 20.17, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-appium", + "task_hash": "9c13d665a3ed54d242a8b8d6ee5fc30985bfc97f00b1c90d528beb2f022c0d13", + "task_id": "narrow-appium-login-01", + "turn_limit_stops": 1, + "turns_factor": 1.5 + } + }, + "harness_version": "0.1.0", + "kind": "tier-baseline", + "schema_version": 1, + "tier": "narrow" +} diff --git a/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json b/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json new file mode 100644 index 0000000..d08d5e6 --- /dev/null +++ b/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json @@ -0,0 +1,51 @@ +{ + "claude_code_version": "2.1.322 (Claude Code)", + "entries": { + "partial-appium-login-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.019589, + "mean_duration_s": 65.56, + "mean_input_tokens": 235691.0, + "mean_output_tokens": 7699.7, + "mean_turns": 16.5, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-appium", + "task_hash": "38720d9916e141cf2f3dea4874ff2b70b12ce403564779bf18d1a64fdb38c5b3", + "task_id": "partial-appium-login-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "partial-appium-login-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.037225, + "mean_duration_s": 100.02, + "mean_input_tokens": 667808.6, + "mean_output_tokens": 13806.6, + "mean_turns": 24.4, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-appium", + "task_hash": "38720d9916e141cf2f3dea4874ff2b70b12ce403564779bf18d1a64fdb38c5b3", + "task_id": "partial-appium-login-01", + "turn_limit_stops": 3, + "turns_factor": 1.5 + } + }, + "harness_version": "0.1.0", + "kind": "tier-baseline", + "schema_version": 1, + "tier": "partial" +} diff --git a/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json b/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json new file mode 100644 index 0000000..dea426e --- /dev/null +++ b/plugins/rf-mobile/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json @@ -0,0 +1,39 @@ +{ + "harness_version": "0.1.0", + "kind": "trigger-baseline", + "model": "deepseek/deepseek-v4.1-flash", + "provider": "openrouter", + "schema_version": 1, + "skills": { + "rf-appium": { + "train": { + "accuracy": 0.6522, + "fn": 0, + "fp": 8, + "negative_loads": 23, + "negative_runs": 33, + "positive_loads": 36, + "positive_runs": 36, + "precision": 0.6, + "queries": 23, + "recall": 1.0, + "tn": 3, + "tp": 12 + }, + "validation": { + "accuracy": 0.625, + "fn": 0, + "fp": 3, + "negative_loads": 8, + "negative_runs": 12, + "positive_loads": 12, + "positive_runs": 12, + "precision": 0.5714, + "queries": 8, + "recall": 1.0, + "tn": 1, + "tp": 4 + } + } + } +} diff --git a/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json b/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json new file mode 100644 index 0000000..a08a581 --- /dev/null +++ b/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/adversarial.json @@ -0,0 +1,135 @@ +{ + "claude_code_version": "2.1.322 (Claude Code)", + "entries": { + "adv-browser-deprecated-wait|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.019885, + "mean_duration_s": 65.74, + "mean_input_tokens": 292573.7, + "mean_output_tokens": 6781.7, + "mean_turns": 23.0, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.3333, + "pass_rate": 0.3333, + "provider": "openrouter", + "runs": 6, + "skill": "rf-browser", + "task_hash": "f5d64210f2906670e4fb5973939a15d4fcecf5d2963843dad8e5384312023afb", + "task_id": "adv-browser-deprecated-wait", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "adv-browser-deprecated-wait|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.020272, + "mean_duration_s": 77.2, + "mean_input_tokens": 456359.7, + "mean_output_tokens": 5789.8, + "mean_turns": 23.17, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.8333, + "pass_rate": 0.8333, + "provider": "openrouter", + "runs": 6, + "skill": "rf-browser", + "task_hash": "f5d64210f2906670e4fb5973939a15d4fcecf5d2963843dad8e5384312023afb", + "task_id": "adv-browser-deprecated-wait", + "turn_limit_stops": 2, + "turns_factor": 1.5 + }, + "adv-selenium-nonexistent-kw|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.016414, + "mean_duration_s": 54.76, + "mean_input_tokens": 141485.3, + "mean_output_tokens": 4879.7, + "mean_turns": 11.67, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-selenium", + "task_hash": "7192ede6afa987729dd7e5a3fb8a5b89706389d369ffdade0ccbeea4d35b2205", + "task_id": "adv-selenium-nonexistent-kw", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "adv-selenium-nonexistent-kw|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.025679, + "mean_duration_s": 96.05, + "mean_input_tokens": 461271.2, + "mean_output_tokens": 9053.0, + "mean_turns": 24.5, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 0.8333, + "pass_rate": 0.8333, + "provider": "openrouter", + "runs": 6, + "skill": "rf-selenium", + "task_hash": "7192ede6afa987729dd7e5a3fb8a5b89706389d369ffdade0ccbeea4d35b2205", + "task_id": "adv-selenium-nonexistent-kw", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "adv-web-wrong-library|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.015992, + "mean_duration_s": 49.92, + "mean_input_tokens": 220741.7, + "mean_output_tokens": 5949.7, + "mean_turns": 17.33, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-browser", + "task_hash": "57639e3931b2ec11723d1524b36ad73c9fc034170c9fad3a755837b43d05d835", + "task_id": "adv-web-wrong-library", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "adv-web-wrong-library|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.026334, + "mean_duration_s": 93.87, + "mean_input_tokens": 460711.5, + "mean_output_tokens": 8861.3, + "mean_turns": 24.83, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-browser", + "task_hash": "57639e3931b2ec11723d1524b36ad73c9fc034170c9fad3a755837b43d05d835", + "task_id": "adv-web-wrong-library", + "turn_limit_stops": 0, + "turns_factor": 1.5 + } + }, + "harness_version": "0.1.0", + "kind": "tier-baseline", + "schema_version": 1, + "tier": "adversarial" +} diff --git a/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json b/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json new file mode 100644 index 0000000..1ae1115 --- /dev/null +++ b/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/narrow.json @@ -0,0 +1,93 @@ +{ + "claude_code_version": "2.1.322 (Claude Code)", + "entries": { + "narrow-browser-login-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.014097, + "mean_duration_s": 45.36, + "mean_input_tokens": 188544.3, + "mean_output_tokens": 3649.2, + "mean_turns": 15.17, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-browser", + "task_hash": "39ebb74ec3c1c1437cb481c44db5f8b675fccaf568b6efd283ef35a3fa05fa11", + "task_id": "narrow-browser-login-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-browser-login-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.02188, + "mean_duration_s": 67.12, + "mean_input_tokens": 390415.2, + "mean_output_tokens": 4727.0, + "mean_turns": 20.83, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-browser", + "task_hash": "39ebb74ec3c1c1437cb481c44db5f8b675fccaf568b6efd283ef35a3fa05fa11", + "task_id": "narrow-browser-login-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-selenium-login-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.008738, + "mean_duration_s": 24.2, + "mean_input_tokens": 109915.0, + "mean_output_tokens": 2152.2, + "mean_turns": 11.67, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-selenium", + "task_hash": "be9eb9928105e45b54cccad2cc788513817dd077f238df9121b4c76dc54125fa", + "task_id": "narrow-selenium-login-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-selenium-login-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.016286, + "mean_duration_s": 58.33, + "mean_input_tokens": 304876.3, + "mean_output_tokens": 4447.3, + "mean_turns": 16.83, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-selenium", + "task_hash": "be9eb9928105e45b54cccad2cc788513817dd077f238df9121b4c76dc54125fa", + "task_id": "narrow-selenium-login-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + } + }, + "harness_version": "0.1.0", + "kind": "tier-baseline", + "schema_version": 1, + "tier": "narrow" +} diff --git a/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json b/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json new file mode 100644 index 0000000..7b263bc --- /dev/null +++ b/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/partial.json @@ -0,0 +1,51 @@ +{ + "claude_code_version": "2.1.322 (Claude Code)", + "entries": { + "partial-browser-login-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.013755, + "mean_duration_s": 42.39, + "mean_input_tokens": 180256.8, + "mean_output_tokens": 4400.5, + "mean_turns": 15.83, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-browser", + "task_hash": "b25fc2a5d8a483208821b2d2af70f83e8c9a6c5af932ba538ce9a9c588a93f30", + "task_id": "partial-browser-login-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "partial-browser-login-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.025019, + "mean_duration_s": 95.36, + "mean_input_tokens": 504980.2, + "mean_output_tokens": 8296.0, + "mean_turns": 23.33, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-browser", + "task_hash": "b25fc2a5d8a483208821b2d2af70f83e8c9a6c5af932ba538ce9a9c588a93f30", + "task_id": "partial-browser-login-01", + "turn_limit_stops": 1, + "turns_factor": 1.5 + } + }, + "harness_version": "0.1.0", + "kind": "tier-baseline", + "schema_version": 1, + "tier": "partial" +} diff --git a/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/realistic.json b/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/realistic.json new file mode 100644 index 0000000..5d3fc2d --- /dev/null +++ b/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/realistic.json @@ -0,0 +1,51 @@ +{ + "claude_code_version": "2.1.322 (Claude Code)", + "entries": { + "realistic-browser-login-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.012627, + "mean_duration_s": 25.91, + "mean_input_tokens": 191379.5, + "mean_output_tokens": 3273.0, + "mean_turns": 13.5, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-browser", + "task_hash": "1487dcd2cb2a712432868bb46d5006bd75580edfe803f09acfd5cc4e13ca40e5", + "task_id": "realistic-browser-login-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "realistic-browser-login-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.02262, + "mean_duration_s": 72.68, + "mean_input_tokens": 224201.3, + "mean_output_tokens": 7102.8, + "mean_turns": 13.83, + "model": "deepseek/deepseek-v4.1-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-browser", + "task_hash": "1487dcd2cb2a712432868bb46d5006bd75580edfe803f09acfd5cc4e13ca40e5", + "task_id": "realistic-browser-login-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + } + }, + "harness_version": "0.1.0", + "kind": "tier-baseline", + "schema_version": 1, + "tier": "realistic" +} diff --git a/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json b/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json new file mode 100644 index 0000000..d5af87b --- /dev/null +++ b/plugins/rf-web/evals/baselines/openrouter/deepseek-v4.1-flash/triggers.json @@ -0,0 +1,69 @@ +{ + "harness_version": "0.1.0", + "kind": "trigger-baseline", + "model": "deepseek/deepseek-v4.1-flash", + "provider": "openrouter", + "schema_version": 1, + "skills": { + "rf-browser": { + "train": { + "accuracy": 0.6087, + "fn": 0, + "fp": 9, + "negative_loads": 24, + "negative_runs": 33, + "positive_loads": 36, + "positive_runs": 36, + "precision": 0.5714, + "queries": 23, + "recall": 1.0, + "tn": 2, + "tp": 12 + }, + "validation": { + "accuracy": 0.875, + "fn": 0, + "fp": 1, + "negative_loads": 3, + "negative_runs": 12, + "positive_loads": 11, + "positive_runs": 12, + "precision": 0.8, + "queries": 8, + "recall": 1.0, + "tn": 3, + "tp": 4 + } + }, + "rf-selenium": { + "train": { + "accuracy": 0.6957, + "fn": 0, + "fp": 7, + "negative_loads": 20, + "negative_runs": 33, + "positive_loads": 36, + "positive_runs": 36, + "precision": 0.6316, + "queries": 23, + "recall": 1.0, + "tn": 4, + "tp": 12 + }, + "validation": { + "accuracy": 0.75, + "fn": 0, + "fp": 2, + "negative_loads": 6, + "negative_runs": 12, + "positive_loads": 12, + "positive_runs": 12, + "precision": 0.6667, + "queries": 8, + "recall": 1.0, + "tn": 2, + "tp": 4 + } + } + } +} diff --git a/scripts/openrouter-ci.py b/scripts/openrouter-ci.py new file mode 100644 index 0000000..e3b6489 --- /dev/null +++ b/scripts/openrouter-ci.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""CI helpers for the OpenRouter eval tier (openrouter-ci-eval-tier D7). + + openrouter-ci.py spend print the key's usage, limit and remaining credit (JSON) + openrouter-ci.py smoke MODEL one Claude Code session on MODEL; fail unless it answers + and every request went to MODEL + openrouter-ci.py summary BEFORE AFTER RUNS [--warn-drift F] [--out FILE] + markdown: the key's spend during the job next to the + harness's priced estimate; with --warn-drift, a + ::warning:: when they differ by more than F (0.25 = 25%) + +The key comes from OPENROUTER_API_KEY and is never printed. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sqlite3 +import subprocess +import sys +import tempfile +import urllib.request +from pathlib import Path + +KEY_URL = "https://openrouter.ai/api/v1/key" + + +def _key() -> str: + key = os.environ.get("OPENROUTER_API_KEY", "") + if not key: + sys.exit("OPENROUTER_API_KEY is not set") + return key + + +def spend() -> int: + req = urllib.request.Request(KEY_URL, headers={"Authorization": f"Bearer {_key()}"}) + with urllib.request.urlopen(req, timeout=30) as resp: # noqa: S310 - fixed https URL + data = json.load(resp)["data"] + print(json.dumps({k: data.get(k) for k in ("usage", "limit", "limit_remaining")})) + return 0 + + +def smoke(model: str) -> int: + # The same session environment the harness gives an OpenRouter run. + from rf_skill_eval.infrastructure.runner.claude_code_runner import openrouter_env + + with tempfile.TemporaryDirectory() as tmp: + env = openrouter_env(os.environ.copy(), model) + env["CLAUDE_CONFIG_DIR"] = tmp + out = subprocess.run( + ["claude", "-p", "Reply with the single word: ready", "--model", model, + "--max-turns", "2", "--output-format", "json"], + capture_output=True, text=True, timeout=120, env=env, cwd=tmp, check=False, + ) + try: + result = json.loads(out.stdout) + except json.JSONDecodeError: + print(f"::error::smoke session printed no JSON (exit {out.returncode}): {out.stderr[:300]}") + return 1 + models = sorted((result.get("modelUsage") or {}).keys()) + print(f"result={str(result.get('result'))[:80]!r} is_error={result.get('is_error')} models={models}") + if result.get("is_error") or not result.get("result"): + print("::error::the OpenRouter smoke session failed") + return 1 + if models and models != [model]: + print(f"::error::the smoke session used {models}, not only {model}") + return 1 + return 0 + + +def _priced_total(runs_dir: Path) -> float: + total = 0.0 + for db in runs_dir.rglob("eval.db"): + with sqlite3.connect(f"file:{db}?mode=ro", uri=True) as conn: + total += conn.execute("SELECT COALESCE(SUM(total_cost_usd), 0) FROM runs").fetchone()[0] + for results in runs_dir.rglob("trigger-results*.json"): + total += float(json.loads(results.read_text("utf-8")).get("spent_usd") or 0.0) + return total + + +def _load(path: Path) -> dict[str, float | None] | None: + try: + data = json.loads(path.read_text("utf-8")) + except (OSError, json.JSONDecodeError): + return None + return data if isinstance(data, dict) else None + + +def summary(before: Path, after: Path, runs_dir: Path, warn_drift: float | None, out: Path | None) -> int: + b, a = _load(before), _load(after) + priced = _priced_total(runs_dir) if runs_dir.is_dir() else 0.0 + lines = ["### OpenRouter spend", ""] + if b is None or a is None or b.get("usage") is None or a.get("usage") is None: + lines.append(f"Key spend unavailable (spend file missing). Harness estimate: ${priced:.2f}.") + spent = None + else: + spent = float(a["usage"] or 0) - float(b["usage"] or 0) + lines += [ + "| | USD |", + "|---|---|", + f"| Key spend during this job (includes concurrent jobs) | {spent:.2f} |", + f"| Harness estimate (OpenRouter list prices) | {priced:.2f} |", + f"| Credit remaining on the key | {a.get('limit_remaining')} |", + ] + text = "\n".join(lines) + "\n" + if out is not None: + out.write_text(text, encoding="utf-8") + else: + sys.stdout.write(text) + if warn_drift is not None and spent is not None and spent > 0: + drift = abs(priced - spent) / spent + if drift > warn_drift: + print( + f"::warning::priced estimate ${priced:.2f} differs from the key's spend " + f"${spent:.2f} by {drift:.0%}; check PROVIDER_PRICES in src/rf_skill_eval/domain/pricing.py" + ) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="cmd", required=True) + sub.add_parser("spend") + smoke_p = sub.add_parser("smoke") + smoke_p.add_argument("model") + sum_p = sub.add_parser("summary") + sum_p.add_argument("before", type=Path) + sum_p.add_argument("after", type=Path) + sum_p.add_argument("runs_dir", type=Path) + sum_p.add_argument("--warn-drift", type=float, default=None) + sum_p.add_argument("--out", type=Path, default=None) + args = parser.parse_args() + if args.cmd == "spend": + return spend() + if args.cmd == "smoke": + return smoke(args.model) + return summary(args.before, args.after, args.runs_dir, args.warn_drift, args.out) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/rf_skill_eval/application/baseline.py b/src/rf_skill_eval/application/baseline.py index 43d6c60..922025b 100644 --- a/src/rf_skill_eval/application/baseline.py +++ b/src/rf_skill_eval/application/baseline.py @@ -27,6 +27,7 @@ import yaml from ..domain.results import ReplicateStats, Spread +from ..domain.task import provider_for from ..domain.trigger import run_load_counts from .trigger_eval import TriggerEvalResult @@ -62,6 +63,24 @@ def task_hash(task_path: Path, fixtures_root: Path) -> str: return hashlib.sha256(f"{normalised}\n{fixture_hash}".encode()).hexdigest() +def baselines_dir_for(evals_baselines: Path, model: str | None) -> Path: + """Where a model's baselines live (openrouter-ci-eval-tier D5). + + Claude models: ``/baselines/`` itself; an OpenRouter model: + ``/baselines/openrouter//``. + """ + if model and provider_for(model) == "openrouter": + return evals_baselines / "openrouter" / model.split("/", 1)[1] + return evals_baselines + + +def single_model(models: Iterable[str]) -> str | None: + """The one model of a set of runs, ``None`` when there are none or several.""" + distinct = set(models) + return distinct.pop() if len(distinct) == 1 else None + + + def _mean(spread: Spread | None, digits: int = 4) -> float | None: return round(spread.mean, digits) if spread is not None else None @@ -98,6 +117,10 @@ def build_tier_baseline( "mean_duration_s": _mean(s.duration_s, 2), "mean_cost_usd": _mean(s.cost_usd, 6), "gating": tier != "adversarial", + "provider": s.provider, + "turns_factor": s.turns_factor, + "turn_limit_stops": s.turn_limit_stops, + "cost_source": s.cost_source, } return { "kind": "tier-baseline", @@ -129,6 +152,7 @@ def build_trigger_baseline(result: TriggerEvalResult, *, harness_version: str) - "schema_version": BASELINE_SCHEMA_VERSION, "harness_version": harness_version, "model": result.model, + "provider": provider_for(result.model) if result.model else "anthropic", "skills": dict(sorted(skills.items())), } diff --git a/src/rf_skill_eval/application/catalog.py b/src/rf_skill_eval/application/catalog.py index 58453a0..8bf0599 100644 --- a/src/rf_skill_eval/application/catalog.py +++ b/src/rf_skill_eval/application/catalog.py @@ -146,8 +146,16 @@ def _read_mapping(path: Path) -> dict[str, Any]: return loaded +#: Run-time settings a task file may not declare (openrouter-ci-eval-tier). +_RUNTIME_ONLY_KEYS: tuple[str, ...] = ("turns_factor", "provider") + + def load_task_file(path: Path) -> Task: - return Task.model_validate(_read_mapping(path)) + data = _read_mapping(path) + for key in _RUNTIME_ONLY_KEYS: + if key in data: + raise ValueError(f"'{key}' is a run-time setting and may not appear in a task file") + return Task.model_validate(data) def task_files(tasks_dir: Path) -> list[Path]: diff --git a/src/rf_skill_eval/application/gate.py b/src/rf_skill_eval/application/gate.py index 93a11f7..e7fdf7c 100644 --- a/src/rf_skill_eval/application/gate.py +++ b/src/rf_skill_eval/application/gate.py @@ -126,6 +126,20 @@ def _pass_rate_finding( ) +def _rebaseline_reason( + s: ReplicateStats, entry: Mapping[str, Any], current_hash: str | None +) -> str | None: + """Why a baseline entry can't be compared with ``s``, or ``None``.""" + if current_hash is not None and entry.get("task_hash") != current_hash: + return "task definition or fixture changed" + if entry.get("model") != s.model: + return f"model differs (baseline {entry.get('model')}, now {s.model})" + base_factor = float(entry.get("turns_factor") or 1.0) + if abs(base_factor - s.turns_factor) > 1e-9: + return f"turn-limit factor differs (baseline {base_factor:g}, now {s.turns_factor:g})" + return None + + def gate_tasks( current: Iterable[ReplicateStats], baseline: Mapping[str, Any] | None, @@ -154,20 +168,9 @@ def gate_tasks( GateFinding("rebaseline-needed", s.task_id, "no stored baseline entry") ) continue - current_hash = task_hashes.get(s.task_id) - if current_hash is not None and entry.get("task_hash") != current_hash: - report.findings.append( - GateFinding("rebaseline-needed", s.task_id, "task definition or fixture changed") - ) - continue - if entry.get("model") != s.model: - report.findings.append( - GateFinding( - "rebaseline-needed", - s.task_id, - f"model differs (baseline {entry.get('model')}, now {s.model})", - ) - ) + stale = _rebaseline_reason(s, entry, task_hashes.get(s.task_id)) + if stale is not None: + report.findings.append(GateFinding("rebaseline-needed", s.task_id, stale)) continue finding = _pass_rate_finding(s, entry, tolerance) if finding is not None: diff --git a/src/rf_skill_eval/cli.py b/src/rf_skill_eval/cli.py index d77aeec..884112f 100644 --- a/src/rf_skill_eval/cli.py +++ b/src/rf_skill_eval/cli.py @@ -29,9 +29,11 @@ from . import __version__ from .application.baseline import ( baseline_entry, + baselines_dir_for, build_tier_baseline, build_trigger_baseline, load_json, + single_model, task_hash, write_json, ) @@ -94,11 +96,17 @@ from .domain.profile import Profile, parse_arms from .domain.results import RunResult, aggregate_replicates from .domain.scorecard import Scorecard -from .domain.task import Task +from .domain.task import ( + OPENROUTER_MODELS, + PROVIDERS, + Provider, + Task, + provider_for, +) from .domain.trigger import SPLIT_NAMES, TriggerQuery, TriggerSet, is_valid_split from .errors import ModelNotAllowedError, RfSkillEvalError from .infrastructure.persistence.sqlite_repo import SqliteRunRepository -from .infrastructure.runner.claude_code_runner import ClaudeCodeRunner +from .infrastructure.runner.claude_code_runner import ClaudeCodeRunner, openrouter_env from .infrastructure.telemetry.skill_listing import LISTING_BUDGET_ENV from .infrastructure.telemetry.skill_loads import skill_dir_map from .reporting.eval_report import build_report, render_markdown, write_report @@ -188,7 +196,12 @@ def _build_service(output_dir: Path, *, fmt: str = "md") -> EvaluationService: return EvaluationService(runner, grader, writer, repo) -def _require_auth() -> None: +def _require_auth(provider: Provider = "anthropic") -> None: + if provider == "openrouter": + if not os.environ.get("OPENROUTER_API_KEY"): + console.print("[red]not run: no credentials[/red] (set OPENROUTER_API_KEY)") + raise typer.Exit(code=2) + return if not (os.environ.get("CLAUDE_CODE_OAUTH_TOKEN") or os.environ.get("ANTHROPIC_API_KEY")): console.print( "[red]not run: no credentials[/red] (set CLAUDE_CODE_OAUTH_TOKEN or ANTHROPIC_API_KEY)" @@ -196,6 +209,32 @@ def _require_auth() -> None: raise typer.Exit(code=2) +def _resolve_provider(provider: str | None, models: set[str]) -> Provider: + """The one provider for a set of run models; refuse a mismatch (D1).""" + if provider is not None and provider not in PROVIDERS: + raise typer.BadParameter(f"--provider must be one of {', '.join(PROVIDERS)}") + derived = {provider_for(m) for m in models} + if len(derived) > 1: + console.print("[red]refused:[/red] the selected tasks mix Claude and OpenRouter models") + raise typer.Exit(code=2) + actual: Provider = derived.pop() if derived else "anthropic" + if provider is not None and provider != actual: + if provider == "openrouter": + hint = f"pass --model with one of {sorted(OPENROUTER_MODELS)}" + else: + hint = "pass --model with a Claude model id" + console.print( + f"[red]refused:[/red] --provider {provider} does not match model(s) " + f"{', '.join(sorted(models))}; {hint}" + ) + raise typer.Exit(code=2) + return actual + + +_PROVIDER_HELP = "anthropic | openrouter (default: derived from the model)" +_TURNS_FACTOR_HELP = "Scale each task's max_turns (1.0-3.0, rounded up); recorded per run" + + def _check_model(model: str, allow_opus: bool, max_cost_usd: float | None) -> None: try: check_runtime_model(model, allow_opus=allow_opus, max_cost_usd=max_cost_usd) @@ -363,6 +402,10 @@ def run( max_cost_usd: float | None = typer.Option(None, "--max-cost-usd", min=0.0), output: Path = typer.Option(Path("eval/runs"), "--output", help="Run artifacts directory"), baseline_dir: Path | None = typer.Option(None, "--baseline-dir"), + provider: str | None = typer.Option(None, "--provider", help=_PROVIDER_HELP), + max_turns_factor: float = typer.Option( + 1.0, "--max-turns-factor", min=1.0, max=3.0, help=_TURNS_FACTOR_HELP + ), ) -> None: """Execute ONE task in ONE arm, grade it and persist the result.""" @@ -372,8 +415,9 @@ def run( arms = _resolve_arms(arm if profile is None else None, profile) if model: task_obj = task_obj.model_copy(update={"model": model}) + task_obj = task_obj.model_copy(update={"turns_factor": max_turns_factor}) _check_model(task_obj.model, allow_opus, max_cost_usd) - _require_auth() + _require_auth(_resolve_provider(provider, {task_obj.model})) code = _run_plan( plan_batch([task_obj], arms[:1], 1), output, @@ -402,6 +446,10 @@ def run_batch( max_cost_usd: float | None = typer.Option(None, "--max-cost-usd", min=0.0), output: Path = typer.Option(Path("eval/runs"), "--output"), baseline_dir: Path | None = typer.Option(None, "--baseline-dir"), + provider: str | None = typer.Option(None, "--provider", help=_PROVIDER_HELP), + max_turns_factor: float = typer.Option( + 1.0, "--max-turns-factor", min=1.0, max=3.0, help=_TURNS_FACTOR_HELP + ), concurrency: int = typer.Option( 1, "--concurrency", @@ -434,10 +482,11 @@ def run_batch( raise typer.Exit(code=1) if model: selected = [t.model_copy(update={"model": model}) for t in selected] + selected = [t.model_copy(update={"turns_factor": max_turns_factor}) for t in selected] for m in sorted({t.model for t in selected}): _check_model(m, allow_opus, max_cost_usd) arm_tuple = _resolve_arms(arms, profile) - _require_auth() + _require_auth(_resolve_provider(provider, {t.model for t in selected})) plan = plan_batch(selected, arm_tuple, runs) console.print( f"[cyan]{len(selected)} task(s) × {len(arm_tuple)} arm(s) × {runs} run(s) = " @@ -656,6 +705,26 @@ def _requirement_ok(req: str) -> tuple[bool, str]: return spec is not None, "importable" if spec is not None else "not importable" +def _auth_rows(settings: Any, provider: str) -> list[tuple[str, bool, str]]: + """Doctor rows for the credentials ``provider`` needs.""" + if provider not in PROVIDERS: + raise typer.BadParameter(f"--provider must be one of {', '.join(PROVIDERS)}") + if provider == "openrouter": + ok = settings.has_auth("openrouter") + return [("OPENROUTER_API_KEY", ok, "OpenRouter models need it" if ok else "not set")] + oauth = bool(settings.claude_code_oauth_token) + api = bool(settings.anthropic_api_key) + return [ + ( + "CLAUDE_CODE_OAUTH_TOKEN", + oauth, + "preferred auth (subscription)" if oauth else "missing — falls back to ANTHROPIC_API_KEY", + ), + ("ANTHROPIC_API_KEY", api, "fallback auth" if api else "not set"), + ("some auth present", settings.has_auth(), "at least one token/key is needed"), + ] + + @app.command() def doctor( tasks_dir: Path | None = typer.Option( @@ -664,6 +733,9 @@ def doctor( strict: bool = typer.Option( False, "--strict", help="Exit 1 when a grader requirement is missing" ), + provider: str = typer.Option( + "anthropic", "--provider", help="anthropic | openrouter: which credentials to check" + ), ) -> None: """Verify environment: tokens, CLIs, grader tools, MCP package importability.""" @@ -677,19 +749,8 @@ def doctor( def _row(name: str, ok: bool, details: str = "") -> None: table.add_row(name, "ok" if ok else "FAIL", details) - _row( - "CLAUDE_CODE_OAUTH_TOKEN", - bool(settings.claude_code_oauth_token), - "preferred auth (subscription)" - if settings.claude_code_oauth_token - else "missing — falls back to ANTHROPIC_API_KEY", - ) - _row( - "ANTHROPIC_API_KEY", - bool(settings.anthropic_api_key), - "fallback auth" if settings.anthropic_api_key else "not set", - ) - _row("some auth present", settings.has_auth(), "at least one token/key is needed") + for name, ok, details in _auth_rows(settings, provider): + _row(name, ok, details) claude = _which("claude") _row("claude CLI on PATH", claude is not None, claude or "not found") @@ -732,10 +793,10 @@ def _row(name: str, ok: bool, details: str = "") -> None: _row("harness version", True, __version__) ping_ok = True - if claude is None or not settings.has_auth(): + if claude is None or not settings.has_auth(provider): ping_details = "skipped (claude binary or auth missing)" else: - ping_ok, ping_details = _claude_auth_ping(claude) + ping_ok, ping_details = _claude_auth_ping(claude, provider=provider) _row("claude auth ping", ping_ok, ping_details) console.print(table) @@ -744,26 +805,35 @@ def _row(name: str, ok: bool, details: str = "") -> None: raise typer.Exit(code=1) -def _claude_auth_ping(claude_binary: str) -> tuple[bool, str]: - """Probe ``claude`` with a one-turn prompt to verify auth actually works.""" +def _claude_auth_ping(claude_binary: str, *, provider: str = "anthropic") -> tuple[bool, str]: + """Probe ``claude`` with a one-turn prompt to verify auth actually works. + + For ``openrouter`` the probe runs in the session environment an OpenRouter + run gets, on the first OpenRouter model. + """ + argv = [ + claude_binary, + "--print", + "ok", + "--max-turns", + "1", + "--output-format", + "stream-json", + "--verbose", + ] + env: dict[str, str] | None = None + # OpenRouter's first response can take longer than Claude's (the smoke + # step allows 120 s); a 30 s ping failed a run's Doctor step. + timeout = 30 + if provider == "openrouter": + model = sorted(OPENROUTER_MODELS)[0] + env = openrouter_env(os.environ.copy(), model) + argv += ["--model", model] + timeout = 120 try: - result = subprocess.run( - [ - claude_binary, - "--print", - "ok", - "--max-turns", - "1", - "--output-format", - "stream-json", - "--verbose", - ], - capture_output=True, - text=True, - timeout=30, - ) + result = subprocess.run(argv, capture_output=True, text=True, timeout=timeout, env=env) except subprocess.TimeoutExpired: - return False, "timed out after 30s" + return False, f"timed out after {timeout}s" except OSError as exc: return False, f"{type(exc).__name__}: {exc}" @@ -771,12 +841,30 @@ def _claude_auth_ping(claude_binary: str) -> tuple[bool, str]: if "API Error" in combined or "invalid value" in combined.lower(): snippet = combined.strip().splitlines()[0][:160] if combined.strip() else "(no output)" return False, f"API rejected probe: {snippet}" + final = _final_result_event(result.stdout or "") + # A model that answers "ok" with a tool call stops at --max-turns 1 and + # claude exits 1, but the provider did answer, so the credentials work. + if final.get("subtype") == "error_max_turns": + return True, "auth verified via claude --print (stopped at the turn limit)" if result.returncode != 0: - snippet = (result.stderr or result.stdout or "").strip()[:160] - return False, f"claude exit={result.returncode}: {snippet}" + # The result event says why; stderr may only hold harmless notices. + reason = final.get("result") or final.get("subtype") or result.stderr or result.stdout or "" + return False, f"claude exit={result.returncode}: {str(reason).strip()[:160]}" return True, "auth verified via claude --print" +def _final_result_event(stdout: str) -> dict[str, Any]: + """The last ``{"type": "result"}`` event in stream-json output, or ``{}``.""" + for line in reversed(stdout.splitlines()): + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(event, dict) and event.get("type") == "result": + return event + return {} + + # --- validity & coverage ----------------------------------------------------------- @@ -990,6 +1078,7 @@ def trigger( model: str | None = typer.Option(None, "--model"), allow_opus: bool = typer.Option(False, "--allow-opus"), max_cost_usd: float | None = typer.Option(None, "--max-cost-usd", min=0.0), + provider: str | None = typer.Option(None, "--provider", help=_PROVIDER_HELP), triggers_dir: Path | None = typer.Option(None, "--triggers-dir"), output: Path = typer.Option(Path("eval/runs/triggers"), "--output"), baseline: Path | None = typer.Option( @@ -1055,7 +1144,7 @@ def trigger( raise typer.BadParameter(f"--split must be a comma list of {SPLIT_NAMES}") for m in sorted({model or s.model for s in selected}): _check_model(m, allow_opus, max_cost_usd) - _require_auth() + _require_auth(_resolve_provider(provider, {model or s.model for s in selected})) listing_budget = _effective_listing_budget(listing_budget) _warn_if_contaminated(output) output.mkdir(parents=True, exist_ok=True) @@ -1123,13 +1212,27 @@ def baseline_update( tasks_dir = tasks_dir or _evals("tasks", root=root) if output_dir is None and _plugin(root) == ALL_PLUGINS: raise typer.BadParameter("--plugin all is read-only: pass --output-dir, or one --plugin") - output_dir = output_dir or _evals("baselines", root=root) written: list[Path] = [] cc_version = claude_code_version or _detect_claude_code_version() - if any(from_dir.rglob("eval.db")): + stats = ( + aggregate_replicates(load_run_results(from_dir, tasks_dir)) + if any(from_dir.rglob("eval.db")) + else [] + ) + trigger_files = sorted(from_dir.rglob("trigger-results.json")) + if output_dir is None: + models = {s.model for s in stats} | { + str(json.loads(f.read_text("utf-8")).get("model") or "") for f in trigger_files + } + model = single_model(m for m in models if m) + if model is None and len({m for m in models if m}) > 1: + raise typer.BadParameter( + f"runs of several models ({', '.join(sorted(models))}): pass --output-dir" + ) + output_dir = baselines_dir_for(_evals("baselines", root=root), model) + if stats: index = _task_index(tasks_dir) hashes = _task_hashes(index, root / "eval" / "fixtures") - stats = aggregate_replicates(load_run_results(from_dir, tasks_dir)) for tier in sorted({s.tier for s in stats if s.tier != "unknown"}): data = build_tier_baseline( stats, @@ -1139,7 +1242,7 @@ def baseline_update( claude_code_version=cc_version, ) written.append(write_json(output_dir / f"{tier}.json", data)) - for results_file in sorted(from_dir.rglob("trigger-results.json")): + for results_file in trigger_files: result = TriggerEvalResult.from_json(json.loads(results_file.read_text("utf-8"))) data = build_trigger_baseline(result, harness_version=__version__) written.append(write_json(output_dir / "triggers.json", data)) @@ -1164,7 +1267,12 @@ def _detect_claude_code_version() -> str: @app.command() def gate( runs_dir: Path | None = typer.Option(None, "--runs-dir"), - baseline: Path | None = typer.Option(None, "--baseline", help="Default: /baselines/narrow.json"), + baseline: Path | None = typer.Option( + None, + "--baseline", + help="Default: the runs' model's narrow.json (/baselines/, or " + "/baselines/openrouter// for an OpenRouter model)", + ), tasks_dir: Path | None = typer.Option(None, "--tasks-dir"), tolerance: float | None = typer.Option( None, "--tolerance", help="Allowed pass-rate drop; default: Fisher exact p < 0.05 vs a >=6-run baseline, else > 1/N fails" @@ -1172,7 +1280,7 @@ def gate( token_budget: float = typer.Option(0.30, "--token-budget", help="Allowed input-token rise"), trigger_results: Path | None = typer.Option(None, "--trigger-results"), trigger_baseline: Path | None = typer.Option( - None, "--trigger-baseline", help="Default: /baselines/triggers.json" + None, "--trigger-baseline", help="Default: triggers.json next to the model's baselines" ), max_cost_usd: float | None = typer.Option( None, "--max-cost-usd", min=0.0, help="Fail when the gated runs' reported cost exceeds it" @@ -1185,15 +1293,23 @@ def gate( if runs_dir is None and trigger_results is None: raise typer.BadParameter("give --runs-dir and/or --trigger-results") tasks_dir = tasks_dir or _evals("tasks") - baseline = baseline or _evals("baselines", "narrow.json") - trigger_baseline = trigger_baseline or _evals("baselines", "triggers.json") report_obj = GateReport() spent = 0.0 + if trigger_baseline is None: + trigger_model = ( + json.loads(trigger_results.read_text("utf-8")).get("model") + if trigger_results is not None and trigger_results.is_file() + else None + ) + trigger_baseline = baselines_dir_for(_evals("baselines"), trigger_model) / "triggers.json" if runs_dir is not None: results = load_run_results(runs_dir, tasks_dir) if not results: console.print(f"[red]gate: no graded runs under {runs_dir}[/red]") raise typer.Exit(code=1) + if baseline is None: + model = single_model(r.run.model for r in results) + baseline = baselines_dir_for(_evals("baselines"), model) / "narrow.json" for m in sorted({r.run.model for r in results}): _check_model(m, allow_opus, max_cost_usd if allow_opus else None) index = _task_index(tasks_dir) diff --git a/src/rf_skill_eval/config.py b/src/rf_skill_eval/config.py index 7ebd915..9838075 100644 --- a/src/rf_skill_eval/config.py +++ b/src/rf_skill_eval/config.py @@ -32,6 +32,8 @@ class Settings(BaseSettings): # Auth claude_code_oauth_token: str | None = Field(default=None) anthropic_api_key: str | None = Field(default=None) + #: OpenRouter models (openrouter-ci-eval-tier). + openrouter_api_key: str | None = Field(default=None) # Model selection claude_model_default: str = Field(default="claude-haiku-5-5") @@ -39,9 +41,11 @@ class Settings(BaseSettings): # Logging rf_skill_eval_log_level: str = Field(default="INFO") - def has_auth(self) -> bool: - """Return True iff some form of Claude authentication is present.""" + def has_auth(self, provider: str = "anthropic") -> bool: + """Return True iff the credentials ``provider`` needs are present.""" + if provider == "openrouter": + return bool(self.openrouter_api_key) return bool(self.claude_code_oauth_token or self.anthropic_api_key) diff --git a/src/rf_skill_eval/domain/pricing.py b/src/rf_skill_eval/domain/pricing.py new file mode 100644 index 0000000..3589792 --- /dev/null +++ b/src/rf_skill_eval/domain/pricing.py @@ -0,0 +1,52 @@ +"""Provider-priced cost for non-Claude models (openrouter-ci-eval-tier D3). + +Claude Code prices every session as if it ran on Claude, so its +``total_cost_usd`` is meaningless for an OpenRouter model (the spike reported +$29.66 for $0.84 of real spend). For those runs the harness prices the +reported token counts itself, from OpenRouter's list prices. + +List prices are an upper bound: OpenRouter may route a request to a cheaper +host. The figure that counts is the key's spend, which CI reads from +OpenRouter before and after a job. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from .task import DEEPSEEK_FLASH_MODEL + + +@dataclass(frozen=True) +class TokenPrices: + """USD per million tokens.""" + + input: float + cache_read: float + cache_write: float + output: float + + +#: OpenRouter list prices, checked 2026-11-10 (``GET /api/v1/models``). +#: Every id in ``task.OPENROUTER_MODELS`` needs an entry here. +PROVIDER_PRICES: dict[str, TokenPrices] = { + DEEPSEEK_FLASH_MODEL: TokenPrices(input=0.30, cache_read=0.006, cache_write=0.30, output=1.20), +} + + +def priced_cost( + model: str, + *, + input_tokens: int, + cache_read_input_tokens: int, + cache_creation_input_tokens: int, + output_tokens: int, +) -> float: + """Cost in USD of one session's tokens at ``model``'s list prices.""" + p = PROVIDER_PRICES[model] + return ( + input_tokens * p.input + + cache_read_input_tokens * p.cache_read + + cache_creation_input_tokens * p.cache_write + + output_tokens * p.output + ) / 1_000_000 diff --git a/src/rf_skill_eval/domain/results.py b/src/rf_skill_eval/domain/results.py index 525d446..83f077e 100644 --- a/src/rf_skill_eval/domain/results.py +++ b/src/rf_skill_eval/domain/results.py @@ -86,6 +86,12 @@ class ReplicateStats: skipped_reasons: tuple[str, ...] = field(default_factory=tuple) process_results: tuple[tuple[str, int, int], ...] = field(default_factory=tuple) incomplete_reasons: tuple[str, ...] = field(default_factory=tuple) + provider: str = "anthropic" + turns_factor: float = 1.0 + #: Runs that ended because they reached the turn limit. + turn_limit_stops: int = 0 + #: ``priced`` when the costs are the harness's list-price estimate. + cost_source: str = "reported" @property def pass_rate(self) -> float: @@ -155,6 +161,14 @@ def aggregate_replicates(results: Iterable[RunResult]) -> list[ReplicateStats]: (name, c[0], c[1]) for name, c in sorted(process.items()) ), incomplete_reasons=tuple(sorted(set(incomplete_reasons))), + provider=items[0].run.effective_provider, + turns_factor=items[0].run.turns_factor, + turn_limit_stops=sum( + 1 for i in items if i.run.usage is not None and i.run.usage.hit_turn_limit + ), + cost_source="priced" + if any(u.cost_source == "priced" for u in usages) + else "reported", ) ) return out diff --git a/src/rf_skill_eval/domain/run.py b/src/rf_skill_eval/domain/run.py index fa3c7b5..e52c1f8 100644 --- a/src/rf_skill_eval/domain/run.py +++ b/src/rf_skill_eval/domain/run.py @@ -7,7 +7,7 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator -from .task import DEFAULT_MODEL +from .task import DEFAULT_MODEL, Provider, provider_for class RunUsage(BaseModel): @@ -23,6 +23,17 @@ class RunUsage(BaseModel): duration_ms: int = 0 total_cost_usd: float = 0.0 is_error: bool = False + #: The ``result`` line's ``subtype`` (``success``, ``error_max_turns``, ...). + subtype: str = "" + #: Models named in the result's ``modelUsage``. + models: tuple[str, ...] = () + #: ``reported`` (Claude Code's ``total_cost_usd``) or ``priced`` (the + #: harness's estimate from provider list prices). + cost_source: str = "reported" + + @property + def hit_turn_limit(self) -> bool: + return self.subtype == "error_max_turns" @property def total_input_tokens(self) -> int: @@ -51,6 +62,10 @@ class Run(BaseModel): artifacts_dir: Path workspace_dir: Path | None = None model: str = DEFAULT_MODEL + #: Empty on older records: derived from ``model`` (``effective_provider``). + provider: str = "" + #: The run-time ``--max-turns-factor`` the session ran with. + turns_factor: float = 1.0 timed_out: bool = False #: ``treatment`` or ``baseline`` (older records: derived from profile name). arm: str = "" @@ -74,6 +89,10 @@ def effective_arm(self) -> str: return self.arm return "baseline" if self.profile_name in ("baseline", "control") else "treatment" + @property + def effective_provider(self) -> Provider: + return "openrouter" if self.provider == "openrouter" else provider_for(self.model) + @property def effective_workspace(self) -> Path: """Directory where the agent's file outputs live. diff --git a/src/rf_skill_eval/domain/task.py b/src/rf_skill_eval/domain/task.py index cf392c4..6d44ebc 100644 --- a/src/rf_skill_eval/domain/task.py +++ b/src/rf_skill_eval/domain/task.py @@ -2,6 +2,7 @@ from __future__ import annotations +import math from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator @@ -23,7 +24,24 @@ #: The default model for evaluation runs (narrow tier and trigger evals). DEFAULT_MODEL: str = HAIKU_MODEL -ALLOWED_MODELS: frozenset[str] = frozenset({HAIKU_MODEL, SONNET_MODEL, OPUS_MODEL}) +#: The Claude models (provider ``anthropic``). +CLAUDE_MODELS: frozenset[str] = frozenset({HAIKU_MODEL, SONNET_MODEL, OPUS_MODEL}) + +#: OpenRouter models, run time only (openrouter-ci-eval-tier D2). Each needs +#: an entry in ``domain.pricing.PROVIDER_PRICES`` and its own baselines. +DEEPSEEK_FLASH_MODEL: str = "deepseek/deepseek-v4.1-flash" +OPENROUTER_MODELS: frozenset[str] = frozenset({DEEPSEEK_FLASH_MODEL}) + +#: Every model a run may use (``--model``). +ALLOWED_MODELS: frozenset[str] = CLAUDE_MODELS | OPENROUTER_MODELS + +Provider = Literal["anthropic", "openrouter"] +PROVIDERS: tuple[Provider, ...] = ("anthropic", "openrouter") + + +def provider_for(model: str) -> Provider: + """The provider a model id runs on: OpenRouter ids are ``vendor/model``.""" + return "openrouter" if model in OPENROUTER_MODELS or "/" in model else "anthropic" #: Models a task YAML / trigger set may declare as its own model. YAML_ALLOWED_MODELS: frozenset[str] = frozenset({HAIKU_MODEL, SONNET_MODEL}) @@ -61,7 +79,18 @@ def model_policy_error(model: str, *, in_yaml: bool) -> str | None: f"Permitted: {sorted(ALLOWED_MODELS)}." ) if model not in ALLOWED_MODELS: + if provider_for(model) == "openrouter": + return ( + f"OpenRouter model '{model}' is not allowed. " + f"Permitted OpenRouter models: {sorted(OPENROUTER_MODELS)}." + ) return f"Model '{model}' is not allowed. Permitted: {sorted(ALLOWED_MODELS)}." + if in_yaml and model in OPENROUTER_MODELS: + return ( + f"Model '{model}' is an OpenRouter model; OpenRouter models are run-time only " + f"(--model), never declared in task YAML or trigger sets " + f"(permitted there: {sorted(YAML_ALLOWED_MODELS)})." + ) if in_yaml and model not in YAML_ALLOWED_MODELS: return ( f"Model '{model}' may not be declared in task YAML or trigger sets " @@ -253,6 +282,9 @@ class Task(BaseModel): tier: TaskTier = "narrow" mcp_servers: tuple[str, ...] = Field(default_factory=tuple) max_turns: int = Field(default=40, gt=0, le=200) + #: Run-time ``--max-turns-factor`` (1.0-3.0); not part of the task + #: definition hash, recorded on every run. + turns_factor: float = Field(default=1.0, ge=1.0, le=3.0) fixture: str | None = None primary_metric: str | None = None #: Plugins to stage for this task (a partial install), e.g. ``[rf-core]``; @@ -295,6 +327,11 @@ def is_gating(self, check: GraderCheck) -> bool: return check.gating return self.primary_metric is not None and check.type == self.primary_metric + @property + def effective_max_turns(self) -> int: + """``max_turns`` scaled by the turn-limit factor, rounded up.""" + return math.ceil(self.max_turns * self.turns_factor - 1e-9) + @property def gating_checks(self) -> tuple[GraderCheck, ...]: return tuple(c for c in self.grader_checks if self.is_gating(c)) diff --git a/src/rf_skill_eval/infrastructure/persistence/sqlite_repo.py b/src/rf_skill_eval/infrastructure/persistence/sqlite_repo.py index 6a97698..3255492 100644 --- a/src/rf_skill_eval/infrastructure/persistence/sqlite_repo.py +++ b/src/rf_skill_eval/infrastructure/persistence/sqlite_repo.py @@ -28,7 +28,7 @@ _log = logging.getLogger(__name__) -SCHEMA_VERSION = 2 +SCHEMA_VERSION = 3 _SCHEMA = """ CREATE TABLE IF NOT EXISTS runs ( @@ -88,6 +88,9 @@ ("verdicts", "category", "TEXT NOT NULL DEFAULT 'outcome'"), ("scorecards", "gate_result", "TEXT"), ("scorecards", "incomplete_reason", "TEXT"), + # Schema version 3 (openrouter-ci-eval-tier). + ("runs", "provider", "TEXT"), + ("runs", "turns_factor", "REAL NOT NULL DEFAULT 1.0"), ) @@ -145,8 +148,9 @@ def save_run(self, run: Run) -> None: run_id, task_id, profile_name, started_at, finished_at, exit_code, session_jsonl_path, artifacts_dir, workspace_dir, model, timed_out, arm, replicate, error, usage_json, - input_tokens, output_tokens, num_turns, duration_ms, total_cost_usd - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + input_tokens, output_tokens, num_turns, duration_ms, total_cost_usd, + provider, turns_factor + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( run.id, @@ -169,6 +173,8 @@ def save_run(self, run: Run) -> None: usage.num_turns if usage else None, usage.duration_ms if usage else None, usage.total_cost_usd if usage else None, + run.effective_provider, + run.turns_factor, ), ) @@ -271,6 +277,8 @@ def _row_to_run(row: sqlite3.Row) -> Run: artifacts_dir=Path(row["artifacts_dir"]), workspace_dir=Path(ws) if (ws := _row_get(row, "workspace_dir")) else None, model=row["model"], + provider=_row_get(row, "provider") or "", + turns_factor=float(_row_get(row, "turns_factor") or 1.0), timed_out=bool(row["timed_out"]), arm=_row_get(row, "arm") or "", replicate=int(replicate_raw) if replicate_raw is not None else 0, diff --git a/src/rf_skill_eval/infrastructure/runner/claude_code_runner.py b/src/rf_skill_eval/infrastructure/runner/claude_code_runner.py index 46c6f37..27f1016 100644 --- a/src/rf_skill_eval/infrastructure/runner/claude_code_runner.py +++ b/src/rf_skill_eval/infrastructure/runner/claude_code_runner.py @@ -28,9 +28,10 @@ from typing import Any from ...application.staging import staged_tree +from ...domain.pricing import priced_cost from ...domain.profile import Profile -from ...domain.run import Run -from ...domain.task import Task +from ...domain.run import Run, RunUsage +from ...domain.task import Task, provider_for from ...errors import AuthConfigError, RunnerTimeout, SkillRunnerError from ..mcp.config_builder import MCP_SERVER_REGISTRY, write_mcp_config from ..telemetry.session_parser import parse_result_usage @@ -140,6 +141,8 @@ def execute( "artifacts_dir": artifacts_dir, "workspace_dir": workspace_dir if workspace_dir != artifacts_dir else None, "model": task.model, + "provider": provider_for(task.model), + "turns_factor": task.turns_factor, "arm": profile.arm, "replicate": replicate, } @@ -148,7 +151,9 @@ def execute( if leaked is not None: return leaked - env = self._build_env(config_dir, listing_budget=profile.listing_budget) + env = self._build_env( + config_dir, listing_budget=profile.listing_budget, model=task.model + ) cmd = self._build_cmd( task, workspace_dir, @@ -227,12 +232,13 @@ def execute( # Normalise: the exit code slot signals 'no clean exit'. exit_code = -1 - usage = parse_result_usage(stdout_path) - error: str | None = None + usage = _with_provider_cost(parse_result_usage(stdout_path), task.model) + error: str | None = _provider_model_error(usage, task.model) if not profile.stage_plugin: leaks = baseline_transcript_leaks(stdout_path, self._known_skill_dirs()) if leaks: - error = "arm-leak: baseline transcript loaded " + ", ".join(leaks) + leak = "arm-leak: baseline transcript loaded " + ", ".join(leaks) + error = f"{error}; {leak}" if error else leak _log.error("%s (run %s)", error, run_id) if violations: isolation = isolation_error(self._repo_root, violations) @@ -509,7 +515,7 @@ def _rewrite_plugin_root(plugin_dst: Path, plugin_root_abs: str) -> None: ) def _build_env( - self, config_dir: Path, *, listing_budget: int | None = None + self, config_dir: Path, *, listing_budget: int | None = None, model: str = "" ) -> dict[str, str]: env = os.environ.copy() # `uv run rf-skill-eval` exports the harness venv; an agent's @@ -520,6 +526,8 @@ def _build_env( if listing_budget is not None: # Skill-listing budget (design D14); only set when requested. env[LISTING_BUDGET_ENV] = str(listing_budget) + if model and provider_for(model) == "openrouter": + return openrouter_env(env, model) if not (env.get("CLAUDE_CODE_OAUTH_TOKEN") or env.get("ANTHROPIC_API_KEY")): raise AuthConfigError( "Neither CLAUDE_CODE_OAUTH_TOKEN nor ANTHROPIC_API_KEY is set. " @@ -565,7 +573,7 @@ def _build_cmd( "stream-json", "--verbose", "--max-turns", - str(task.max_turns or self._default_max_turns), + str(task.effective_max_turns if task.max_turns else self._default_max_turns), "--model", task.model, # Bypass interactive approval prompts that block plugin hooks and @@ -903,3 +911,66 @@ def run_concurrency_limit( "raise_on_timeout", "run_concurrency_limit", ) + + +def _with_provider_cost(usage: RunUsage | None, model: str) -> RunUsage | None: + """OpenRouter runs: replace Claude Code's Claude-priced cost with list prices (D3).""" + if usage is None or provider_for(model) != "openrouter": + return usage + cost = priced_cost( + model, + input_tokens=usage.input_tokens, + cache_read_input_tokens=usage.cache_read_input_tokens, + cache_creation_input_tokens=usage.cache_creation_input_tokens, + output_tokens=usage.output_tokens, + ) + return usage.model_copy(update={"total_cost_usd": cost, "cost_source": "priced"}) + + +def _provider_model_error(usage: RunUsage | None, model: str) -> str | None: + """An OpenRouter run whose usage names another model is not trustworthy.""" + if usage is None or provider_for(model) != "openrouter": + return None + others = [m for m in usage.models if m != model] + if not others: + return None + return f"provider_model: session used {', '.join(others)}, not {model}" + + +#: OpenRouter's Anthropic-compatible endpoint (Claude Code appends ``/v1/messages``). +OPENROUTER_BASE_URL = "https://openrouter.ai/api" + +#: Claude credentials never reach an OpenRouter session (eval-model-providers). +_CLAUDE_CREDENTIAL_VARS: tuple[str, ...] = ( + "CLAUDE_CODE_OAUTH_TOKEN", + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "ANTHROPIC_BASE_URL", +) + +#: Every model alias Claude Code may resolve (background tasks, subagents). +_MODEL_ALIAS_VARS: tuple[str, ...] = ( + "ANTHROPIC_DEFAULT_HAIKU_MODEL", + "ANTHROPIC_DEFAULT_SONNET_MODEL", + "ANTHROPIC_DEFAULT_OPUS_MODEL", + "CLAUDE_CODE_SUBAGENT_MODEL", +) + + +def openrouter_env(env: dict[str, str], model: str) -> dict[str, str]: + """Point Claude Code at OpenRouter, every alias mapped to ``model`` (design D1).""" + key = env.get("OPENROUTER_API_KEY", "") + if not key: + raise AuthConfigError( + "OPENROUTER_API_KEY is not set; an OpenRouter model needs it. " + "Populate .env (see .env.example) or export the key before running." + ) + for var in _CLAUDE_CREDENTIAL_VARS: + env.pop(var, None) + env["ANTHROPIC_BASE_URL"] = OPENROUTER_BASE_URL + env["ANTHROPIC_AUTH_TOKEN"] = key + # Empty, not unset: Claude Code would otherwise look for a stored login. + env["ANTHROPIC_API_KEY"] = "" + for var in _MODEL_ALIAS_VARS: + env[var] = model + return env diff --git a/src/rf_skill_eval/infrastructure/telemetry/session_parser.py b/src/rf_skill_eval/infrastructure/telemetry/session_parser.py index 8d00585..5eef198 100644 --- a/src/rf_skill_eval/infrastructure/telemetry/session_parser.py +++ b/src/rf_skill_eval/infrastructure/telemetry/session_parser.py @@ -252,6 +252,7 @@ def _int(value: Any) -> int: except (TypeError, ValueError): return 0 + model_usage = last.get("modelUsage") try: cost = float(last.get("total_cost_usd") or 0.0) except (TypeError, ValueError): @@ -265,4 +266,6 @@ def _int(value: Any) -> int: duration_ms=_int(last.get("duration_ms")), total_cost_usd=cost, is_error=bool(last.get("is_error", False)), + subtype=str(last.get("subtype") or ""), + models=tuple(sorted(model_usage)) if isinstance(model_usage, dict) else (), ) diff --git a/src/rf_skill_eval/reporting/eval_report.py b/src/rf_skill_eval/reporting/eval_report.py index 047a446..1efb777 100644 --- a/src/rf_skill_eval/reporting/eval_report.py +++ b/src/rf_skill_eval/reporting/eval_report.py @@ -45,6 +45,8 @@ def _arm_block(s: ReplicateStats) -> dict[str, Any]: "turns": _mean(s.turns), "duration_s": _mean(s.duration_s), "cost_usd": _mean(s.cost_usd), + "cost_source": s.cost_source, + "turn_limit_stops": s.turn_limit_stops, "stdev": { m: (None if getattr(s, m) is None else round(getattr(s, m).stdev, 4)) for m in _METRICS @@ -67,6 +69,8 @@ def _stored_block(entry: Mapping[str, Any], current_hash: str | None) -> dict[st "turns": entry.get("mean_turns"), "duration_s": entry.get("mean_duration_s"), "cost_usd": entry.get("mean_cost_usd"), + "cost_source": entry.get("cost_source", "reported"), + "turn_limit_stops": entry.get("turn_limit_stops", 0), "stdev": {}, } @@ -141,6 +145,12 @@ def build_report( "cost_usd": round( sum(r.run.usage.total_cost_usd for r in results if r.run.usage is not None), 6 ), + "cost_source": "priced" + if any(r.run.usage is not None and r.run.usage.cost_source == "priced" for r in results) + else "reported", + "turn_limit_stops": sum( + 1 for r in results if r.run.usage is not None and r.run.usage.hit_turn_limit + ), } return {"kind": "eval-report", "summary": summary, "tasks": tasks, "skills": skills} @@ -194,6 +204,22 @@ def _fmt_delta(value: Any, pct: bool = False, digits: int = 1) -> str: return f"{v:+.{digits}f}" +def _turns_cell(b: Mapping[str, Any]) -> str: + """Mean turns, plus how many runs stopped at the turn limit (when any).""" + stops = b.get("turn_limit_stops") or 0 + return _fmt(b["turns"]) + (f" ({stops} at limit)" if stops else "") + + +def _cost_cell(b: Mapping[str, Any]) -> str: + return _fmt(b["cost_usd"], digits=4) + (" (est.)" if b.get("cost_source") == "priced" else "") + + +def _cost_summary(s: Mapping[str, Any]) -> str: + if s.get("cost_source") == "priced": + return f"**estimated cost (OpenRouter list prices):** ${s['cost_usd']:.2f}" + return f"**reported cost:** ${s['cost_usd']:.2f}" + + def _task_rows(t: Mapping[str, Any]) -> list[str]: tier = t["tier"] + ("" if t["gating"] else " (non-gating)") rows: list[str] = [] @@ -204,8 +230,8 @@ def _task_rows(t: Mapping[str, Any]) -> list[str]: f"| `{t['task_id']}` | {t['skill']} | {tier} | {arm_label} | {_fmt(b['runs'])} | " f"{_fmt(b['pass_rate'], pct=True)} | {_fmt(b['outcome_pass_rate'], pct=True)} | " f"{_fmt(b['incomplete'])} | {_fmt(b['input_tokens'], digits=0)} | " - f"{_fmt(b['output_tokens'], digits=0)} | {_fmt(b['turns'])} | " - f"{_fmt(b['duration_s'])} | {_fmt(b['cost_usd'], digits=4)} | {flaky} |" + f"{_fmt(b['output_tokens'], digits=0)} | {_turns_cell(b)} | " + f"{_fmt(b['duration_s'])} | {_cost_cell(b)} | {flaky} |" ) d = t["delta"] or {} rows.append( @@ -251,7 +277,8 @@ def render_markdown(report: Mapping[str, Any]) -> str: "", f"**Runs:** {s['runs']} · **pass:** {s['pass']} · **fail:** {s['fail']} · " f"**incomplete:** {s['incomplete']} · **skipped checks:** {s['skipped_checks']} · " - f"**reported cost:** ${s['cost_usd']:.2f}", + f"{_cost_summary(s)}" + + (f" · **turn-limit stops:** {s['turn_limit_stops']}" if s.get("turn_limit_stops") else ""), "", "Pass rate = runs whose gate result is `pass` / runs attempted. Deltas are " "treatment − baseline over **outcome** checks only.", diff --git a/tests/eval/snapshots/eval-report.json b/tests/eval/snapshots/eval-report.json index a08f24b..379e2f7 100644 --- a/tests/eval/snapshots/eval-report.json +++ b/tests/eval/snapshots/eval-report.json @@ -26,17 +26,20 @@ } ], "summary": { + "cost_source": "reported", "cost_usd": 0.25, "fail": 4, "incomplete": 1, "pass": 6, "runs": 11, - "skipped_checks": 2 + "skipped_checks": 2, + "turn_limit_stops": 0 }, "tasks": [ { "arms": { "treatment": { + "cost_source": "reported", "cost_usd": 0.02, "duration_s": 20.0, "flaky": false, @@ -55,6 +58,7 @@ "output_tokens": 0.0, "turns": 0.0 }, + "turn_limit_stops": 0, "turns": 4.0 } }, @@ -70,6 +74,7 @@ { "arms": { "baseline": { + "cost_source": "reported", "cost_usd": 0.02, "duration_s": 20.0, "flaky": true, @@ -88,9 +93,11 @@ "output_tokens": 0.0, "turns": 0.0 }, + "turn_limit_stops": 0, "turns": 4.0 }, "treatment": { + "cost_source": "reported", "cost_usd": 0.03, "duration_s": 20.0, "flaky": false, @@ -109,6 +116,7 @@ "output_tokens": 0.0, "turns": 0.0 }, + "turn_limit_stops": 0, "turns": 4.0 } }, @@ -137,6 +145,7 @@ { "arms": { "treatment": { + "cost_source": "reported", "cost_usd": 0.02, "duration_s": 20.0, "flaky": false, @@ -155,6 +164,7 @@ "output_tokens": 0.0, "turns": 0.0 }, + "turn_limit_stops": 0, "turns": 4.0 } }, @@ -170,6 +180,7 @@ { "arms": { "treatment": { + "cost_source": "reported", "cost_usd": 0.02, "duration_s": 20.0, "flaky": true, @@ -188,6 +199,7 @@ "output_tokens": 0.0, "turns": 0.0 }, + "turn_limit_stops": 0, "turns": 4.0 } }, diff --git a/tests/eval/test_baseline_gate.py b/tests/eval/test_baseline_gate.py index 786371e..2482454 100644 --- a/tests/eval/test_baseline_gate.py +++ b/tests/eval/test_baseline_gate.py @@ -49,6 +49,7 @@ def test_baseline_file_is_deterministic() -> None: entry = data["entries"]["t|treatment"] assert entry == { "arm": "treatment", + "cost_source": "reported", "gating": True, "incomplete": 0, "mean_cost_usd": 0.02, @@ -59,10 +60,13 @@ def test_baseline_file_is_deterministic() -> None: "model": HAIKU, "outcome_pass_rate": 1.0, "pass_rate": 1.0, + "provider": "anthropic", "runs": 3, "skill": "rf-browser", "task_hash": "h1", "task_id": "t", + "turn_limit_stops": 0, + "turns_factor": 1.0, } assert data["harness_version"] == "0.1.0" assert data["claude_code_version"] == "2.1.0 (Claude Code)" diff --git a/tests/eval/test_ci_preflight.py b/tests/eval/test_ci_preflight.py index 1e887cb..eef1db6 100644 --- a/tests/eval/test_ci_preflight.py +++ b/tests/eval/test_ci_preflight.py @@ -2,6 +2,7 @@ from __future__ import annotations +import textwrap from pathlib import Path import pytest @@ -128,17 +129,120 @@ def test_every_harness_invocation_has_a_cost_cap() -> None: assert "--max-cost-usd" in block, line -def test_pr_job_is_treatment_only_haiku_runs_3_and_gated() -> None: +def test_pr_job_is_treatment_only_runs_3_and_gated_on_the_preflight_model() -> None: wf = _wf() - text = _job_text(wf["jobs"]["pr-eval"]) + job = wf["jobs"]["pr-eval"] + text = _job_text(job) assert "--arms treatment" in text assert "--runs 3" in text - assert '--model "${HAIKU_MODEL}"' in text + assert '--model "${EVAL_MODEL}"' in text + assert '--max-turns-factor "${TURNS_FACTOR}"' in text + assert job["env"]["EVAL_MODEL"] == "${{ needs.preflight.outputs.model }}" assert wf["env"]["HAIKU_MODEL"] == "claude-haiku-5-5" + assert wf["env"]["OPENROUTER_MODEL"] == "deepseek/deepseek-v4.1-flash" assert "rf-skill-eval gate" in text + assert '--baseline "${BASELINES}narrow.json"' in text assert "--split validation" in text +def _run_creds(tmp_path: Path, **env: str) -> dict[str, str]: + """Run the preflight credentials step's script with the given secrets set.""" + import os + import subprocess + + wf = _wf() + step = next(s for s in wf["jobs"]["preflight"]["steps"] if s.get("id") == "creds") + out = tmp_path / "out" + out.write_text("") + base = {"PATH": os.environ["PATH"], "GITHUB_OUTPUT": str(out), "FORK_PR": "false", + "REQUESTED": "", "CLAUDE_CODE_OAUTH_TOKEN": "", "ANTHROPIC_API_KEY": "", + "OPENROUTER_API_KEY": "", **{k: str(v) for k, v in wf["env"].items()}} + subprocess.run(["bash", "-eo", "pipefail", "-c", step["run"]], env={**base, **env}, check=True) + return dict(line.split("=", 1) for line in out.read_text().splitlines() if "=" in line) + + +def test_preflight_prefers_openrouter_when_its_key_is_set(tmp_path: Path) -> None: + got = _run_creds(tmp_path, OPENROUTER_API_KEY="k", CLAUDE_CODE_OAUTH_TOKEN="t") + assert got == { + "has_credentials": "true", + "provider": "openrouter", + "model": "deepseek/deepseek-v4.1-flash", + "turns_factor": "1.5", + "baseline_subdir": "openrouter/deepseek-v4.1-flash/", + } + + +def test_preflight_falls_back_to_claude(tmp_path: Path) -> None: + got = _run_creds(tmp_path, CLAUDE_CODE_OAUTH_TOKEN="t") + assert (got["provider"], got["model"], got["turns_factor"], got["baseline_subdir"]) == ( + "claude", "claude-haiku-5-5", "1.0", "") + + +def test_manual_claude_run_overrides_openrouter(tmp_path: Path) -> None: + got = _run_creds(tmp_path, OPENROUTER_API_KEY="k", CLAUDE_CODE_OAUTH_TOKEN="t", REQUESTED="claude") + assert got["provider"] == "claude" + + +@pytest.mark.parametrize( + "env", + [ + {}, + {"REQUESTED": "openrouter", "CLAUDE_CODE_OAUTH_TOKEN": "t"}, + {"FORK_PR": "true", "OPENROUTER_API_KEY": "k"}, + ], +) +def test_preflight_without_usable_credentials(tmp_path: Path, env: dict[str, str]) -> None: + assert _run_creds(tmp_path, **env) == {"has_credentials": "false"} + + +def test_eval_jobs_get_only_the_providers_credentials() -> None: + wf = _wf() + for name in ("pr-eval", "full-eval"): + for step in wf["jobs"][name]["steps"]: + env = step.get("env") or {} + for var, value in env.items(): + if "secrets.CLAUDE_CODE_OAUTH_TOKEN" in str(value): + assert "provider == 'claude'" in str(value), (name, step.get("name"), var) + if "secrets.OPENROUTER_API_KEY" in str(value) and var != "OPENROUTER_API_KEY": + raise AssertionError((name, step.get("name"), var)) + + +def test_openrouter_jobs_report_spend_and_smoke() -> None: + wf = _wf() + for name in ("pr-eval", "full-eval"): + names = [s.get("name", "") for s in wf["jobs"][name]["steps"]] + assert "Spend before (OpenRouter key)" in names + assert "Spend after (OpenRouter key)" in names + assert "Smoke test (one OpenRouter session)" in names + assert "--warn-drift 0.25" in _job_text(wf["jobs"]["full-eval"]) + + +def test_spend_summary(tmp_path: Path) -> None: + import json + import sqlite3 + import subprocess + import sys + + (tmp_path / "before.json").write_text(json.dumps({"usage": 1.0, "limit": 50, "limit_remaining": 49})) + (tmp_path / "after.json").write_text(json.dumps({"usage": 1.84, "limit": 50, "limit_remaining": 48.16})) + runs = tmp_path / "runs" + runs.mkdir() + conn = sqlite3.connect(runs / "eval.db") + conn.execute("CREATE TABLE runs (total_cost_usd REAL)") + conn.executemany("INSERT INTO runs VALUES (?)", [(0.7,), (0.63,)]) + conn.commit() + conn.close() + out = subprocess.run( + [sys.executable, "-I", str(_REPO / "scripts" / "openrouter-ci.py"), "summary", + str(tmp_path / "before.json"), str(tmp_path / "after.json"), str(runs), + "--warn-drift", "0.25", "--out", str(tmp_path / "spend.md")], + capture_output=True, text=True, check=True, + ) + md = (tmp_path / "spend.md").read_text() + assert "| 0.84 |" in md and "| 1.33 |" in md + assert "::warning::" in out.stdout # 1.33 vs 0.84 is 58% apart + + def test_full_job_runs_both_arms_and_uploads_candidate_baseline() -> None: text = _job_text(_wf()["jobs"]["full-eval"]) assert '--arms "${ARMS}"' in text @@ -182,7 +286,11 @@ def test_deleted_or_renamed_skill_dirs_are_ignored() -> None: # --- fork PRs never run live evals (content-security-scanning, task 5.6) -------- -_EVAL_SECRETS = ("secrets.CLAUDE_CODE_OAUTH_TOKEN", "secrets.ANTHROPIC_API_KEY") +_EVAL_SECRETS = ( + "secrets.CLAUDE_CODE_OAUTH_TOKEN", + "secrets.ANTHROPIC_API_KEY", + "secrets.OPENROUTER_API_KEY", +) def _triggers(wf: dict) -> set[str]: # type: ignore[type-arg] @@ -206,3 +314,66 @@ def test_fork_prs_have_no_credentials() -> None: assert creds["run"].lstrip().startswith('if [ "${FORK_PR}" = "true" ]; then\n echo "has_credentials=false"') assert "github.event.pull_request.head.repo.full_name == github.repository" in wf["jobs"]["pr-eval"]["if"] assert _triggers(wf) == {"pull_request", "schedule", "workflow_dispatch"} + + +def test_trigger_evals_never_run_after_a_failed_doctor() -> None: + # Run 42885585678: Doctor failed, the tiers were skipped, and the trigger + # step (if: always()) went on spending. + wf = _wf() + for name in ("pr-eval", "full-eval"): + steps = wf["jobs"][name]["steps"] + assert any(s.get("id") == "doctor" for s in steps) + triggers = next(s for s in steps if s.get("id") == "triggers") + assert "steps.doctor.outcome == 'success'" in triggers["if"] + + +def _run_select_legs(tmp_path: Path, plugins: list[str], **env: str) -> list[dict[str, str]]: + """Run the preflight select step's leg builder over all-plugins legs.""" + import json + import os + import subprocess + + wf = _wf() + step = next(s for s in wf["jobs"]["preflight"]["steps"] if s.get("id") == "select") + script = step["run"].split("<<'PY'\n", 1)[1].rsplit("PY", 1)[0] + legs = tmp_path / "legs" + legs.mkdir() + for plugin in plugins: + (legs / f"{plugin}.env").write_text("run_evals=true\nfull_tier=true\nskills=\n") + base = {"PATH": os.environ["PATH"], **{k: str(v) for k, v in wf["env"].items()}} + out = subprocess.run(["python3", "-I", "-c", textwrap.dedent(script), str(legs)], + env={**base, **env}, capture_output=True, text=True, check=True).stdout + values = dict(line.split("=", 1) for line in out.splitlines() if "=" in line) + return json.loads(values["matrix"]) + + +_DESKTOP_OVERRIDE = '{"rf-desktop": "2.0"}' + + +def test_a_plugin_can_get_its_own_openrouter_turn_factor(tmp_path: Path) -> None: + legs = _run_select_legs(tmp_path, ["rf-api", "rf-desktop"], PROVIDER="openrouter", DEFAULT_FACTOR="1.5", + OPENROUTER_TURNS_FACTORS=_DESKTOP_OVERRIDE) + assert {leg["plugin"]: leg["turns_factor"] for leg in legs} == {"rf-api": "1.5", "rf-desktop": "2.0"} + + +def test_no_plugin_overrides_the_openrouter_turn_factor(tmp_path: Path) -> None: + legs = _run_select_legs(tmp_path, ["rf-api", "rf-desktop"], PROVIDER="openrouter", DEFAULT_FACTOR="1.5") + assert {leg["turns_factor"] for leg in legs} == {"1.5"} + + +def test_claude_runs_ignore_the_openrouter_turn_factors(tmp_path: Path) -> None: + legs = _run_select_legs(tmp_path, ["rf-desktop"], PROVIDER="claude", DEFAULT_FACTOR="1.0", + OPENROUTER_TURNS_FACTORS=_DESKTOP_OVERRIDE) + assert legs[0]["turns_factor"] == "1.0" + + +def test_plugins_input_limits_the_matrix(tmp_path: Path) -> None: + legs = _run_select_legs(tmp_path, ["rf-api", "rf-core", "rf-desktop"], PROVIDER="openrouter", + DEFAULT_FACTOR="1.5", ONLY_PLUGINS="rf-desktop, rf-api") + assert [leg["plugin"] for leg in legs] == ["rf-api", "rf-desktop"] + + +def test_eval_jobs_take_the_turn_factor_from_the_matrix_leg() -> None: + wf = _wf() + for name in ("pr-eval", "full-eval"): + assert wf["jobs"][name]["env"]["TURNS_FACTOR"] == "${{ matrix.turns_factor }}" diff --git a/tests/eval/test_cli_doctor.py b/tests/eval/test_cli_doctor.py index a72fbcf..dce5a77 100644 --- a/tests/eval/test_cli_doctor.py +++ b/tests/eval/test_cli_doctor.py @@ -42,6 +42,43 @@ def test_doctor_fails_when_auth_ping_returns_api_error(monkeypatch) -> None: assert "API rejected probe" in result.stdout +def test_doctor_passes_ping_that_stops_at_the_turn_limit(monkeypatch) -> None: + """A model that answers with a tool call hits --max-turns 1; auth still worked.""" + monkeypatch.setenv("CLAUDE_CODE_OAUTH_TOKEN", "fake") + fake_completed = subprocess.CompletedProcess( + args=["claude"], + returncode=1, + stdout='{"type":"system","subtype":"init"}\n' + '{"type":"result","subtype":"error_max_turns","is_error":true,"num_turns":2}', + stderr='{"model":"deepseek/deepseek-v4.1-flash","query_source":"unrecognized_model"}', + ) + with patch("rf_skill_eval.cli.shutil.which", return_value="/fake/claude"), patch( + "rf_skill_eval.cli.subprocess.run", return_value=fake_completed + ): + result = runner.invoke(app, ["doctor"]) + + assert result.exit_code == 0 + row = next(line for line in result.stdout.splitlines() if "claude auth ping" in line) + assert "ok" in row + + +def test_doctor_ping_failure_reports_the_result_not_stderr(monkeypatch) -> None: + monkeypatch.setenv("CLAUDE_CODE_OAUTH_TOKEN", "fake") + fake_completed = subprocess.CompletedProcess( + args=["claude"], + returncode=1, + stdout='{"type":"result","subtype":"error_during_execution","is_error":true}', + stderr='{"model":"x","query_source":"unrecognized_model"}', + ) + with patch("rf_skill_eval.cli.shutil.which", return_value="/fake/claude"), patch( + "rf_skill_eval.cli.subprocess.run", return_value=fake_completed + ): + result = runner.invoke(app, ["doctor"]) + + assert result.exit_code == 1 + assert "error_during_execution" in result.stdout + + def test_doctor_passes_ping_with_clean_response(monkeypatch) -> None: monkeypatch.setenv("CLAUDE_CODE_OAUTH_TOKEN", "fake") fake_completed = subprocess.CompletedProcess( diff --git a/tests/eval/test_domain_task.py b/tests/eval/test_domain_task.py index 10d3246..37fdf8f 100644 --- a/tests/eval/test_domain_task.py +++ b/tests/eval/test_domain_task.py @@ -31,9 +31,9 @@ def test_default_model_is_haiku() -> None: assert t.model == DEFAULT_MODEL -def test_model_allowlist_is_the_design_d7_set() -> None: +def test_model_allowlist_is_the_design_d7_set_plus_openrouter() -> None: assert frozenset( - {"claude-haiku-5-5", "claude-sonnet-5-5", "claude-opus-5-5"} + {"claude-haiku-5-5", "claude-sonnet-5-5", "claude-opus-5-5", "deepseek/deepseek-v4.1-flash"} ) == ALLOWED_MODELS diff --git a/tests/eval/test_providers.py b/tests/eval/test_providers.py new file mode 100644 index 0000000..a931232 --- /dev/null +++ b/tests/eval/test_providers.py @@ -0,0 +1,445 @@ +"""OpenRouter provider tier (openrouter-ci-eval-tier, spec eval-model-providers). + +A fake ``claude`` executable records its argv and environment and prints a +canned stream-json result, so no model is ever called. +""" + +from __future__ import annotations + +import dataclasses +import json +import sqlite3 +import stat +import sys +from pathlib import Path + +import pytest +import typer +from typer.testing import CliRunner + +from rf_skill_eval.application.baseline import baselines_dir_for, build_tier_baseline +from rf_skill_eval.application.batch import BatchExecutor, Budget, check_runtime_model, plan_batch +from rf_skill_eval.application.catalog import load_task_file +from rf_skill_eval.application.gate import gate_tasks +from rf_skill_eval.application.staging import staged_tree +from rf_skill_eval.cli import _resolve_provider, app +from rf_skill_eval.domain.pricing import PROVIDER_PRICES, priced_cost +from rf_skill_eval.domain.profile import Profile +from rf_skill_eval.domain.results import aggregate_replicates +from rf_skill_eval.domain.run import RunUsage +from rf_skill_eval.domain.task import ( + DEEPSEEK_FLASH_MODEL, + OPENROUTER_MODELS, + Task, + provider_for, +) +from rf_skill_eval.errors import AuthConfigError, ModelNotAllowedError +from rf_skill_eval.infrastructure.persistence.sqlite_repo import SqliteRunRepository +from rf_skill_eval.infrastructure.runner.claude_code_runner import ( + OPENROUTER_BASE_URL, + ClaudeCodeRunner, +) +from rf_skill_eval.infrastructure.telemetry.session_parser import parse_result_usage +from rf_skill_eval.reporting.eval_report import build_report, render_markdown +from rf_skill_eval.scoring.rubric import RubricGrader + +from .results_factory import make_result + +_REPO = Path(__file__).resolve().parents[2] +_PLUGIN = staged_tree(_REPO / "plugins") +DS = DEEPSEEK_FLASH_MODEL +FAKE_KEY = "fake-openrouter-key-for-tests-never-stored" +cli = CliRunner() + +_FAKE_CLAUDE = """#!{python} +import json, os, sys +keys = ("ANTHROPIC_BASE_URL", "ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_API_KEY", + "CLAUDE_CODE_OAUTH_TOKEN", "ANTHROPIC_DEFAULT_HAIKU_MODEL", + "ANTHROPIC_DEFAULT_SONNET_MODEL", "ANTHROPIC_DEFAULT_OPUS_MODEL", + "CLAUDE_CODE_SUBAGENT_MODEL") +record = {{"argv": sys.argv[1:], "env": {{k: os.environ.get(k) for k in keys}}}} +with open(os.environ["FAKE_CLAUDE_LOG"], "a") as fh: + fh.write(json.dumps(record) + "\\n") +models = os.environ.get("FAKE_CLAUDE_MODELS", "").split(",") +print(json.dumps({{"type": "result", "subtype": os.environ.get("FAKE_CLAUDE_SUBTYPE", "success"), + "is_error": False, "num_turns": 3, "duration_ms": 10, "total_cost_usd": 0.4, + "usage": {{"input_tokens": 1000000, "output_tokens": 100000, + "cache_read_input_tokens": 5000000, + "cache_creation_input_tokens": 0}}, + "modelUsage": {{m: {{}} for m in models if m}}}})) +""" + + +@pytest.fixture +def fake_claude(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + exe = tmp_path / "bin" / "claude" + exe.parent.mkdir() + exe.write_text(_FAKE_CLAUDE.format(python=sys.executable)) + exe.chmod(exe.stat().st_mode | stat.S_IEXEC) + monkeypatch.setenv("FAKE_CLAUDE_LOG", str(tmp_path / "calls.jsonl")) + monkeypatch.setenv("FAKE_CLAUDE_MODELS", DS) + monkeypatch.setenv("OPENROUTER_API_KEY", FAKE_KEY) + monkeypatch.setenv("CLAUDE_CODE_OAUTH_TOKEN", "claude-oauth-secret") + monkeypatch.setenv("ANTHROPIC_API_KEY", "anthropic-api-secret") + (tmp_path / "fake-repo").mkdir() + return exe + + +def _calls(tmp_path: Path) -> list[dict]: + return [json.loads(line) for line in (tmp_path / "calls.jsonl").read_text().splitlines()] + + +def _task(**kw: object) -> Task: + data: dict[str, object] = { + "id": "narrow-or-01", + "skill": "rf-browser", + "prompt": "do it", + "fixture": "sut-minimal", + "max_turns": 14, + "timeout_seconds": 60, + } + data.update(kw) + task = Task.model_validate(data) + return task.model_copy(update={"model": DS}) + + +def _runner(tmp_path: Path, exe: Path) -> ClaudeCodeRunner: + return ClaudeCodeRunner( + claude_binary=str(exe), + fixtures_root=_REPO / "eval" / "fixtures", + repo_root=tmp_path / "fake-repo", + plugin_root=_PLUGIN, + ) + + +# --- 1.1 model policy ------------------------------------------------------------ + + +def test_openrouter_model_is_run_time_only() -> None: + check_runtime_model(DS, allow_opus=False, max_cost_usd=None) + with pytest.raises(ValueError, match="run-time only"): + Task(id="t", skill="rf-browser", prompt="p", model=DS) + + +def test_unlisted_openrouter_model_names_the_permitted_ids() -> None: + with pytest.raises(ModelNotAllowedError) as exc: + check_runtime_model("z-ai/glm-5.3-flash", allow_opus=False, max_cost_usd=None) + assert DS in str(exc.value) + + +def test_every_openrouter_model_has_a_price() -> None: + assert set(PROVIDER_PRICES) == set(OPENROUTER_MODELS) + + +@pytest.mark.parametrize("key", ["turns_factor", "provider"]) +def test_task_file_may_not_declare_run_time_settings(tmp_path: Path, key: str) -> None: + path = tmp_path / "t.yaml" + path.write_text(f"id: t\nskill: rf-browser\nprompt: p\n{key}: 1.5\n") + with pytest.raises(ValueError, match="run-time setting"): + load_task_file(path) + + +# --- 1.2 provider selection -------------------------------------------------------- + + +def test_provider_is_derived_from_the_model() -> None: + assert provider_for(DS) == "openrouter" + assert provider_for("claude-haiku-5-5") == "anthropic" + assert _resolve_provider(None, {DS}) == "openrouter" + assert _resolve_provider(None, {"claude-haiku-5-5"}) == "anthropic" + + +@pytest.mark.parametrize( + ("provider", "model"), [("openrouter", "claude-haiku-5-5"), ("anthropic", DS)] +) +def test_mismatched_provider_and_model_refused(provider: str, model: str) -> None: + with pytest.raises(typer.Exit) as exc: + _resolve_provider(provider, {model}) + assert exc.value.exit_code == 2 + + +def test_cli_refuses_openrouter_with_a_claude_model() -> None: + result = cli.invoke( + app, + ["run-batch", "--tier", "narrow", "--provider", "openrouter", "--model", "claude-haiku-5-5", + "--output", "/nonexistent-never-written"], + ) + assert result.exit_code == 2 + assert DS in result.stdout + + +def test_cli_without_openrouter_key_is_not_run(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + result = cli.invoke( + app, ["run-batch", "--tier", "narrow", "--model", DS, "--output", "/nonexistent-never-written"] + ) + assert result.exit_code == 2 + assert "OPENROUTER_API_KEY" in result.stdout + + +# --- 1.3 persistence ------------------------------------------------------------------ + + +def test_runs_store_provider_and_factor_and_old_databases_migrate(tmp_path: Path) -> None: + db = tmp_path / "eval.db" + with SqliteRunRepository(db): + pass + # Turn it into a schema-2 database (no provider / turns_factor columns). + conn = sqlite3.connect(db) + conn.execute("ALTER TABLE runs DROP COLUMN provider") + conn.execute("ALTER TABLE runs DROP COLUMN turns_factor") + conn.execute("PRAGMA user_version = 2") + conn.commit() + conn.close() + run = make_result("t", "treatment", 0, model=DS).run.model_copy( + update={"provider": "openrouter", "turns_factor": 1.5} + ) + with SqliteRunRepository(db) as repo: + assert repo.schema_version == 3 + repo.save_run(run) + stored = repo.load_run(run.id) + assert stored is not None + assert (stored.provider, stored.turns_factor, stored.model) == ("openrouter", 1.5, DS) + + +# --- 2.1 session environment --------------------------------------------------------- + + +def test_openrouter_session_env(tmp_path: Path, fake_claude: Path) -> None: + out = tmp_path / "runs" + run = _runner(tmp_path, fake_claude).execute( + _task(), Profile.for_arm("treatment", out / "cfg"), out + ) + assert run.error is None, run.error + assert (run.provider, run.model) == ("openrouter", DS) + env = _calls(tmp_path)[0]["env"] + assert env["ANTHROPIC_BASE_URL"] == OPENROUTER_BASE_URL + assert env["ANTHROPIC_AUTH_TOKEN"] == FAKE_KEY + assert env["ANTHROPIC_API_KEY"] == "" + assert env["CLAUDE_CODE_OAUTH_TOKEN"] is None + for alias in ("HAIKU", "SONNET", "OPUS"): + assert env[f"ANTHROPIC_DEFAULT_{alias}_MODEL"] == DS + assert env["CLAUDE_CODE_SUBAGENT_MODEL"] == DS + + +def test_claude_session_env_is_unchanged(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CLAUDE_CODE_OAUTH_TOKEN", "fake") + monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False) + env = ClaudeCodeRunner(fixtures_root=tmp_path, repo_root=tmp_path)._build_env( + tmp_path, model="claude-haiku-5-5" + ) + assert env["CLAUDE_CODE_OAUTH_TOKEN"] == "fake" + assert "ANTHROPIC_BASE_URL" not in env + + +def test_missing_openrouter_key_raises(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + monkeypatch.setenv("CLAUDE_CODE_OAUTH_TOKEN", "fake") + with pytest.raises(AuthConfigError, match="OPENROUTER_API_KEY"): + ClaudeCodeRunner(fixtures_root=tmp_path, repo_root=tmp_path)._build_env(tmp_path, model=DS) + + +# --- 2.2 doctor --------------------------------------------------------------------------- + + +def test_doctor_checks_the_openrouter_key(monkeypatch: pytest.MonkeyPatch) -> None: + pinged: list[str] = [] + monkeypatch.setattr( + "rf_skill_eval.cli._claude_auth_ping", + lambda _b, provider="anthropic": (pinged.append(provider) or True, "stubbed"), + ) + monkeypatch.setattr("rf_skill_eval.cli.get_settings", lambda: _settings(openrouter=FAKE_KEY)) + result = cli.invoke(app, ["doctor", "--provider", "openrouter"]) + assert "OPENROUTER_API_KEY" in result.stdout + assert "CLAUDE_CODE_OAUTH_TOKEN" not in result.stdout + assert FAKE_KEY not in result.stdout + assert pinged in ([], ["openrouter"]) + + +def test_doctor_without_openrouter_key_fails_the_row(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("rf_skill_eval.cli.get_settings", lambda: _settings(openrouter=None)) + result = cli.invoke(app, ["doctor", "--provider", "openrouter"]) + row = next(line for line in result.stdout.splitlines() if "OPENROUTER_API_KEY" in line) + assert "FAIL" in row + + +def _settings(*, openrouter: str | None): # type: ignore[no-untyped-def] + from rf_skill_eval.config import Settings + + return Settings(openrouter_api_key=openrouter, claude_code_oauth_token=None, anthropic_api_key=None) + + +# --- 2.3 provider_model ------------------------------------------------------------------- + + +def test_usage_records_models_and_subtype(tmp_path: Path) -> None: + stream = tmp_path / "s.jsonl" + stream.write_text(json.dumps({ + "type": "result", "subtype": "error_max_turns", "usage": {}, + "modelUsage": {DS: {}, "claude-haiku-5-5": {}}, + }) + "\n") + usage = parse_result_usage(stream) + assert usage is not None + assert usage.models == ("claude-haiku-5-5", DS) + assert usage.hit_turn_limit + + +def test_unexpected_model_makes_the_run_incomplete( + tmp_path: Path, fake_claude: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("FAKE_CLAUDE_MODELS", f"{DS},claude-haiku-5-5") + out = tmp_path / "runs" + run = _runner(tmp_path, fake_claude).execute( + _task(), Profile.for_arm("treatment", out / "cfg"), out + ) + assert run.error is not None + assert run.error.startswith("provider_model:") + assert "claude-haiku-5-5" in run.error + + +# --- 2.4 the key never reaches stored output ---------------------------------------------- + + +def test_key_never_stored(tmp_path: Path, fake_claude: Path) -> None: + out = tmp_path / "runs" + runner = _runner(tmp_path, fake_claude) + with SqliteRunRepository(out / "eval.db") as repo: + outcome = BatchExecutor(runner, RubricGrader(), repo, out).execute( + plan_batch([_task()], ("treatment", "baseline"), 1) + ) + (out / "report.md").write_text(render_markdown(build_report(outcome.results))) + leaks = [ + p for p in out.rglob("*") + if p.is_file() and FAKE_KEY.encode() in p.read_bytes() + ] + assert leaks == [] + + +# --- 3.1 priced cost ------------------------------------------------------------------------ + + +def test_priced_cost_arithmetic() -> None: + cost = priced_cost( + DS, + input_tokens=1_000_000, + cache_read_input_tokens=5_000_000, + cache_creation_input_tokens=0, + output_tokens=100_000, + ) + p = PROVIDER_PRICES[DS] + assert cost == pytest.approx(p.input + 5 * p.cache_read + 0.1 * p.output) + + +def test_openrouter_run_is_priced_not_reported(tmp_path: Path, fake_claude: Path) -> None: + out = tmp_path / "runs" + run = _runner(tmp_path, fake_claude).execute( + _task(), Profile.for_arm("treatment", out / "cfg"), out + ) + assert run.usage is not None + assert run.usage.cost_source == "priced" + assert run.usage.total_cost_usd == pytest.approx( + priced_cost(DS, input_tokens=1_000_000, cache_read_input_tokens=5_000_000, + cache_creation_input_tokens=0, output_tokens=100_000) + ) + assert run.usage.total_cost_usd != pytest.approx(0.4) # Claude Code's figure + + +def test_budget_uses_priced_cost() -> None: + # Claude Code said $3 per run ($30 in all); priced, they cost $0.09 each. + budget = Budget(cap_usd=2.0) + for i in range(10): + r = _or_result(i, stopped=False).run # type: ignore[attr-defined] + budget.charge(r.model_copy(update={"usage": r.usage.model_copy(update={"total_cost_usd": 0.09})})) + assert not budget.exhausted + assert budget.spent_usd == pytest.approx(0.9) + + +# --- 3.3 / 3.4 reports and turn limits -------------------------------------------------------- + + +def _or_result(idx: int, *, stopped: bool) -> object: + r = make_result("t", "treatment", idx, model=DS) + usage = (r.run.usage or RunUsage()).model_copy( + update={"cost_source": "priced", "subtype": "error_max_turns" if stopped else "success"} + ) + run = r.run.model_copy(update={"usage": usage, "provider": "openrouter", "turns_factor": 1.5}) + return dataclasses.replace(r, run=run) + + +def test_report_labels_estimates_and_turn_limit_stops() -> None: + results = [_or_result(0, stopped=True), _or_result(1, stopped=True), _or_result(2, stopped=False)] + md = render_markdown(build_report(results)) # type: ignore[arg-type] + assert "estimated cost (OpenRouter list prices)" in md + assert "(2 at limit)" in md + assert "(est.)" in md + assert "**turn-limit stops:** 2" in md + + +def test_scaled_turn_limit(tmp_path: Path, fake_claude: Path) -> None: + out = tmp_path / "runs" + task = _task().model_copy(update={"turns_factor": 1.5}) + run = _runner(tmp_path, fake_claude).execute(task, Profile.for_arm("treatment", out / "cfg"), out) + argv = _calls(tmp_path)[0]["argv"] + assert argv[argv.index("--max-turns") + 1] == "21" + assert argv[argv.index("--model") + 1] == DS + assert run.turns_factor == 1.5 + + +def test_turn_limit_factor_is_bounded() -> None: + result = cli.invoke(app, ["run-batch", "--max-turns-factor", "3.5", "--model", DS]) + assert result.exit_code == 2 + + +# --- 4.1 / 4.2 per-model baselines --------------------------------------------------------------- + + +def test_baseline_dir_per_model(tmp_path: Path) -> None: + assert baselines_dir_for(tmp_path, "claude-haiku-5-5") == tmp_path + assert baselines_dir_for(tmp_path, DS) == tmp_path / "openrouter" / "deepseek-v4.1-flash" + assert baselines_dir_for(tmp_path, None) == tmp_path + + +def _or_baseline(factor: float) -> dict: + results = [_or_result(i, stopped=False) for i in range(3)] + results = [ + dataclasses.replace(r, run=r.run.model_copy(update={"turns_factor": factor})) # type: ignore[attr-defined] + for r in results + ] + return build_tier_baseline( + aggregate_replicates(results), # type: ignore[arg-type] + tier="narrow", + task_hashes={"t": "h1"}, + harness_version="0", + claude_code_version="x", + ) + + +def test_baseline_entries_record_provider_and_factor() -> None: + entry = _or_baseline(1.5)["entries"]["t|treatment"] + assert (entry["provider"], entry["turns_factor"], entry["cost_source"]) == ("openrouter", 1.5, "priced") + + +def test_turn_limit_factor_mismatch_needs_rebaseline() -> None: + current = aggregate_replicates([_or_result(i, stopped=False) for i in range(3)]) # type: ignore[arg-type] + report = gate_tasks(current, _or_baseline(1.0), {"t": "h1"}) + assert report.status == "rebaseline-needed" + assert any("turn-limit factor differs" in f.message for f in report.findings) + assert gate_tasks(current, _or_baseline(1.5), {"t": "h1"}).status == "pass" + + +def test_gate_defaults_to_the_models_baseline(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + evals = tmp_path / "evals" + (evals / "tasks").mkdir(parents=True) + monkeypatch.setattr("rf_skill_eval.cli._evals", lambda *parts, root=None: evals.joinpath(*parts)) + runs = tmp_path / "runs" + with SqliteRunRepository(runs / "eval.db") as repo: + for i in range(3): + r = _or_result(i, stopped=False) + repo.save_run(r.run) # type: ignore[attr-defined] + repo.save_scorecard(r.scorecard) # type: ignore[attr-defined] + target = evals / "baselines" / "openrouter" / "deepseek-v4.1-flash" / "narrow.json" + result = cli.invoke(app, ["gate", "--runs-dir", str(runs)]) + # No baseline file yet: the gate looked for the DeepSeek one, not the Haiku one. + assert not target.exists() + assert result.exit_code == 3 + assert "no stored baseline entry" in result.stdout