fix(evals): fail closed when the route baseline is missing

Author: TheGreatAxios

scripts/eval-prompts-dispatch.test.ts

@@ -1,4 +1,4 @@
-import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { access, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { describe, expect, test } from "bun:test";
@@ -218,4 +218,32 @@ describe("drop baseline", () => {
       await rm(dir, { recursive: true, force: true });
     }
   });
+
+  test("a missing baseline fails closed and is not created", async () => {
+    const dir = await mkdtemp(join(tmpdir(), "dispatch-routing-"));
+    const baselinePath = join(dir, "baseline.json");
+    const lastPath = join(dir, "last.json");
+    const run = async () => "reviewer\nI refuse.";
+    try {
+      const first = await evaluateRouting({
+        cases: [coderCase],
+        run,
+        lastPath,
+        baselinePath,
+      });
+      expect(first).not.toBe(0);
+      await expect(access(baselinePath)).rejects.toThrow();
+
+      const second = await evaluateRouting({
+        cases: [coderCase],
+        run,
+        lastPath,
+        baselinePath,
+      });
+      expect(second).not.toBe(0);
+      await expect(access(baselinePath)).rejects.toThrow();
+    } finally {
+      await rm(dir, { recursive: true, force: true });
+    }
+  });
 });

scripts/eval-prompts-dispatch.ts

@@ -1,5 +1,5 @@
 #!/usr/bin/env bun
-import { copyFile, readFile } from "node:fs/promises";
+import { access, copyFile, readFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 import { systemPrompt } from "@corbits/code-agent-dispatch";
@@ -195,13 +195,23 @@ type DispatchEvaluate = (
   options: EvaluateOptions<string, string, RoutingExpect>,
 ) => Promise<Report<string, string, RoutingExpect>>;
 
+async function baselinePresent(path: string): Promise<boolean> {
+  try {
+    await access(path);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
 export async function evaluateRouting(args: {
   cases: readonly Case<string, RoutingExpect>[];
   run: (input: string, signal: AbortSignal) => Promise<string>;
   lastPath: string;
   baselinePath: string;
   evaluateFn?: DispatchEvaluate;
 }): Promise<number> {
+  const hadBaseline = await baselinePresent(args.baselinePath);
   const evaluateFn: DispatchEvaluate =
     args.evaluateFn ?? ((name, options) => evaluate(name, options));
   try {
@@ -227,8 +237,18 @@ export async function evaluateRouting(args: {
         ollamaUrl: OLLAMA_URL,
       },
     });
-    await copyFile(args.lastPath, args.baselinePath);
     console.log(report.markdown());
+    const routeMisses = report.checks["routes"]?.missed;
+    if (!hadBaseline) {
+      console.error(
+        `eval "${DISPATCH_EVAL}" failed: no baseline at ${args.baselinePath}`,
+      );
+      return 1;
+    }
+    if (routeMisses !== 0) {
+      return 0;
+    }
+    await copyFile(args.lastPath, args.baselinePath);
     return 0;
   } catch (err: unknown) {
     if (err instanceof EvalFailure) {