Merge pull request #1411 from corbitsdev/cl-evals-wire

feat(evals): wire @corbits/evals and add dispatch-routing eval

Author: TheGreatAxios

.gitignore

@@ -25,6 +25,7 @@ worktrees
 evals/capability/results/*
 !evals/capability/results/baseline-0286.json
 evals/public/results/*
+evals/prompts/dispatch-routing/last.json
 local-harness-bench/
 subagents/
 

bun.lock

@@ -0,0 +1,14 @@
+# Dispatch routing eval
+
+```sh
+bun scripts/eval-prompts-dispatch.ts
+# or
+bun run eval:prompts-dispatch
+```
+
+Local Ollama by default. Override with `EVAL_OLLAMA_URL` and `EVAL_MODEL`:
+
+```sh
+EVAL_MODEL=tev1:0.8b bun scripts/eval-prompts-dispatch.ts
+EVAL_OLLAMA_URL=http://localhost:11434 EVAL_MODEL=qwen3:0.6b bun scripts/eval-prompts-dispatch.ts
+```

evals/prompts/dispatch-routing/README.md

@@ -0,0 +1,9 @@
+{"name":"map-retry-backoff","input":"Where does the repo handle retry backoff? Find every call site and sketch the flow between them.","expect":{"route":"explorer","decoys":["coder","planner"]}}
+{"name":"plan-offline-mode","input":"We need offline mode. Write a build plan with phases, risks, and acceptance criteria.","expect":{"route":"planner","decoys":["coder","explorer"]}}
+{"name":"fix-token-refresh","input":"There's a bug in the login flow: the token refresh silently fails. Fix it and add a regression test.","expect":{"route":"coder","decoys":["explorer","reviewer"]}}
+{"name":"review-ship-diff","input":"Review the attached diff for correctness and edge cases before we ship it.","expect":{"route":"reviewer","decoys":["coder"]}}
+{"name":"design-settings-cramped","input":"The settings screen looks cramped. Update the design tokens and spacing rules.","expect":{"route":"designer","decoys":["coder","explorer"]}}
+{"name":"doc-storage-layer","input":"Write the architecture doc for the new storage layer, covering the write path and failure modes.","expect":{"route":"shakespeare","decoys":["explorer","planner"]}}
+{"name":"audit-permissions-boundary","input":"Audit the new permissions boundary for trust issues and secret exposure.","expect":{"route":"warden","decoys":["reviewer"]}}
+{"name":"probe-latency-percentiles","input":"Measure latency percentiles for the new model endpoint across 200 requests and report the distribution.","expect":{"route":"prober","decoys":["qa-lead","reviewer"]}}
+{"name":"qa-onboarding-run","input":"Run the CLI through the onboarding flow end to end and show it works with a pass/fail record.","expect":{"route":"qa-lead","decoys":["prober","coder"]}}

evals/prompts/dispatch-routing/cases.jsonl

@@ -41,6 +41,7 @@
     "check": "bun run lint && bun run typecheck && bun run check:dead-exports && bun run build && bun run check:projects-dir-guard",
     "start": "bun run build && bun ./dist/index.js",
     "eval:capability": "bun scripts/eval-capability.ts",
+    "eval:prompts-dispatch": "bun scripts/eval-prompts-dispatch.ts",
     "eval:public-swe-one": "bun scripts/eval-public-swe-one.ts",
     "tui:smoke": "bun ./src/tui/smoke.ts"
   },
@@ -116,6 +117,7 @@
     "@corbits/code-agent-explorer": "workspace:*"
   },
   "devDependencies": {
+    "@corbits/evals": "^0.1.1",
     "@intx/inference-testing": "0.3.0",
     "@types/bun": "1.4.3",
     "knip": "^6.41.0",