Skip to content

Commit 6ae164b

Browse files
TheGreatAxiosTheGreatAxios
committed
fix(evals): fail unbaselined route misses and drop noDiy
1 parent af2a425 commit 6ae164b

2 files changed

Lines changed: 205 additions & 95 deletions

File tree

‎scripts/eval-prompts-dispatch.test.ts‎

Lines changed: 79 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,6 @@ import { evaluate, type CheckValue, type Run } from "@corbits/evals";
66
import {
77
ROUTE_BUDGET_MS,
88
evaluateRouting,
9-
noDiyCheck,
10-
routeLine,
119
routesCheck,
1210
} from "./eval-prompts-dispatch.js";
1311

@@ -79,58 +77,6 @@ describe("routes scores the first line", () => {
7977
});
8078
});
8179

82-
describe("noDiy ignores the brief line", () => {
83-
test("DIY verbs in the brief do not fail", () => {
84-
expect(
85-
passed(
86-
noDiyCheck(
87-
routingRun(
88-
"coder\nI will just patch the login flow and add a test.",
89-
{
90-
route: "coder",
91-
decoys: ["explorer"],
92-
},
93-
),
94-
),
95-
),
96-
).toBe(true);
97-
expect(
98-
passed(
99-
noDiyCheck(
100-
routingRun("shakespeare\nI'll write the architecture doc", {
101-
route: "shakespeare",
102-
decoys: ["explorer", "planner"],
103-
}),
104-
),
105-
),
106-
).toBe(true);
107-
});
108-
109-
test("DIY on the route line still fails", () => {
110-
expect(
111-
passed(
112-
noDiyCheck(
113-
routingRun("I'll write the architecture doc\nshakespeare", {
114-
route: "shakespeare",
115-
decoys: ["explorer"],
116-
}),
117-
),
118-
),
119-
).toBe(false);
120-
expect(
121-
passed(
122-
noDiyCheck(
123-
routingRun("I'll fix the login bug", {
124-
route: "coder",
125-
decoys: ["reviewer"],
126-
}),
127-
),
128-
),
129-
).toBe(false);
130-
expect(routeLine("coder\nI'll fix it")).toBe("coder");
131-
});
132-
});
133-
13480
describe("drop baseline", () => {
13581
const coderCase = {
13682
name: "fix-token-refresh",
@@ -146,7 +92,6 @@ describe("drop baseline", () => {
14692
cases: {
14793
"fix-token-refresh": {
14894
routes: { pass: routesPass, value: routesPass ? 1 : 0 },
149-
noDiy: { pass: true, value: 1 },
15095
fast: { pass: true, value: 1 },
15196
},
15297
},
@@ -297,4 +242,83 @@ describe("drop baseline", () => {
297242
await rm(dir, { recursive: true, force: true });
298243
}
299244
});
245+
246+
test("an unbaselined routes miss fails and is not stored as a pass", async () => {
247+
const dir = await mkdtemp(join(tmpdir(), "dispatch-routing-"));
248+
const baselinePath = join(dir, "baseline.json");
249+
const lastPath = join(dir, "last.json");
250+
const before = stored(true);
251+
await writeFile(baselinePath, before);
252+
const unknown = {
253+
name: "map-the-repo",
254+
input: "Where is the login handler?",
255+
expect: { route: "explorer", decoys: ["coder", "reviewer"] },
256+
};
257+
try {
258+
const code = await evaluateRouting({
259+
cases: [coderCase, unknown],
260+
run: async (input) =>
261+
input === unknown.input
262+
? "coder\nwrong specialist"
263+
: "coder\nAdd a regression test for token refresh.",
264+
lastPath,
265+
baselinePath,
266+
});
267+
expect(code).not.toBe(0);
268+
const after = JSON.parse(await readFile(baselinePath, "utf8")) as {
269+
cases: Record<string, { routes?: { pass: boolean } } | undefined>;
270+
};
271+
expect(after.cases["map-the-repo"]?.routes?.pass).not.toBe(true);
272+
expect(after.cases["fix-token-refresh"]?.routes?.pass).toBe(true);
273+
} finally {
274+
await rm(dir, { recursive: true, force: true });
275+
}
276+
});
277+
278+
test("a new pass beside a baselined miss is stored and a later flip drops", async () => {
279+
const dir = await mkdtemp(join(tmpdir(), "dispatch-routing-"));
280+
const baselinePath = join(dir, "baseline.json");
281+
const lastPath = join(dir, "last.json");
282+
await writeFile(baselinePath, stored(false));
283+
const added = {
284+
name: "map-the-repo",
285+
input: "Where is the login handler?",
286+
expect: { route: "explorer", decoys: ["coder", "reviewer"] },
287+
};
288+
try {
289+
const storedPass = await evaluateRouting({
290+
cases: [coderCase, added],
291+
run: async (input) =>
292+
input === added.input
293+
? "explorer\nLook up the handler."
294+
: "reviewer\nI refuse.",
295+
lastPath,
296+
baselinePath,
297+
});
298+
expect(storedPass).toBe(0);
299+
const after = JSON.parse(await readFile(baselinePath, "utf8")) as {
300+
cases: {
301+
"fix-token-refresh": { routes: { pass: boolean } };
302+
"map-the-repo": { routes: { pass: boolean } };
303+
};
304+
};
305+
expect(after.cases["fix-token-refresh"].routes.pass).toBe(false);
306+
expect(after.cases["map-the-repo"].routes.pass).toBe(true);
307+
const passedText = await readFile(baselinePath, "utf8");
308+
309+
const flipped = await evaluateRouting({
310+
cases: [coderCase, added],
311+
run: async (input) =>
312+
input === added.input
313+
? "coder\nwrong specialist"
314+
: "reviewer\nI refuse.",
315+
lastPath,
316+
baselinePath,
317+
});
318+
expect(flipped).not.toBe(0);
319+
expect(await readFile(baselinePath, "utf8")).toBe(passedText);
320+
} finally {
321+
await rm(dir, { recursive: true, force: true });
322+
}
323+
});
300324
});

‎scripts/eval-prompts-dispatch.ts‎

Lines changed: 126 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
#!/usr/bin/env bun
2-
import { access, copyFile, readFile } from "node:fs/promises";
2+
import { access, copyFile, readFile, writeFile } from "node:fs/promises";
33
import { dirname, join } from "node:path";
44
import { fileURLToPath } from "node:url";
55
import { systemPrompt } from "@corbits/code-agent-dispatch";
6-
import { EvalFailure, evaluate, avoids, under } from "@corbits/evals";
6+
import { EvalFailure, evaluate, under } from "@corbits/evals";
77
import type {
88
Case,
99
CheckValue,
@@ -132,7 +132,7 @@ function loadCases(path: string): Promise<Case<string, RoutingExpect>[]> {
132132
});
133133
}
134134

135-
export function routeLine(output: string): string {
135+
function routeLine(output: string): string {
136136
const line = output.split(/\r?\n/, 1)[0] ?? "";
137137
return line.trim();
138138
}
@@ -164,37 +164,78 @@ export function routesCheck(
164164
return true;
165165
}
166166

167-
const DIY_PHRASES = [
168-
"I will fix",
169-
"I'll fix",
170-
"I will implement",
171-
"I'll implement",
172-
"I can fix",
173-
"I can implement",
174-
"I'll write",
175-
"I will write",
176-
"I'll do it",
177-
"I can do it",
178-
"I'll handle",
179-
"I will handle",
180-
"I'm going to implement",
181-
] as const;
182-
183-
const diyOnText = avoids<string, string, RoutingExpect>(DIY_PHRASES);
184-
185-
export function noDiyCheck(
186-
run: Run<string, string, RoutingExpect>,
187-
): CheckValue | Promise<CheckValue> {
188-
return diyOnText({ ...run, output: routeLine(run.output) });
189-
}
190-
191167
const DISPATCH_EVAL = "dispatch-routing";
192168

193169
type DispatchEvaluate = (
194170
name: string,
195171
options: EvaluateOptions<string, string, RoutingExpect>,
196172
) => Promise<Report<string, string, RoutingExpect>>;
197173

174+
interface StoredCheck {
175+
pass: boolean;
176+
value: number;
177+
}
178+
179+
interface StoredRun {
180+
format: 1;
181+
eval: string;
182+
about?: Record<string, unknown>;
183+
cases: Record<string, Record<string, StoredCheck>>;
184+
}
185+
186+
function isRecord(value: unknown): value is Record<string, unknown> {
187+
return typeof value === "object" && value !== null && !Array.isArray(value);
188+
}
189+
190+
function parseStoredRun(text: string, source: string): StoredRun {
191+
let raw: unknown;
192+
try {
193+
raw = JSON.parse(text);
194+
} catch (cause) {
195+
throw new Error(`${source}: invalid JSON (${(cause as Error).message})`);
196+
}
197+
if (!isRecord(raw) || !isRecord(raw["cases"])) {
198+
throw new Error(`${source}: expected a stored run`);
199+
}
200+
const format = raw["format"];
201+
const evalName = raw["eval"];
202+
if (format !== 1 || typeof evalName !== "string" || evalName === "") {
203+
throw new Error(`${source}: expected a format 1 stored run`);
204+
}
205+
const cases: Record<string, Record<string, StoredCheck>> = {};
206+
for (const [name, checks] of Object.entries(raw["cases"])) {
207+
if (!isRecord(checks)) {
208+
throw new Error(`${source}: case "${name}" must be an object`);
209+
}
210+
const parsed: Record<string, StoredCheck> = {};
211+
for (const [check, outcome] of Object.entries(checks)) {
212+
if (
213+
!isRecord(outcome) ||
214+
typeof outcome["pass"] !== "boolean" ||
215+
typeof outcome["value"] !== "number" ||
216+
!Number.isFinite(outcome["value"])
217+
) {
218+
throw new Error(
219+
`${source}: case "${name}" check "${check}" is invalid`,
220+
);
221+
}
222+
parsed[check] = { pass: outcome["pass"], value: outcome["value"] };
223+
}
224+
cases[name] = parsed;
225+
}
226+
const stored: StoredRun = { format: 1, eval: evalName, cases };
227+
if (isRecord(raw["about"])) {
228+
stored.about = raw["about"];
229+
}
230+
return stored;
231+
}
232+
233+
function routesStoredPass(
234+
checks: Record<string, StoredCheck> | undefined,
235+
): boolean {
236+
return checks?.["routes"]?.pass === true;
237+
}
238+
198239
async function baselinePresent(path: string): Promise<boolean> {
199240
try {
200241
await access(path);
@@ -204,6 +245,59 @@ async function baselinePresent(path: string): Promise<boolean> {
204245
}
205246
}
206247

248+
// A miss is a drop only when the stored routes pass is true, so the miss
249+
// budget would keep a case that has never been baselined green. Passes still
250+
// merge: a miss is never written as a pass, and a stored pass is never
251+
// replaced by a miss, so a later flip of a case that passed beside a sibling
252+
// miss is a drop.
253+
async function commitRoutingBaseline(args: {
254+
hadBaseline: boolean;
255+
lastPath: string;
256+
baselinePath: string;
257+
}): Promise<number> {
258+
const last = parseStoredRun(
259+
await readFile(args.lastPath, "utf8"),
260+
args.lastPath,
261+
);
262+
const missed = Object.entries(last.cases).filter(
263+
([, checks]) => !routesStoredPass(checks),
264+
);
265+
if (missed.length === 0) {
266+
await copyFile(args.lastPath, args.baselinePath);
267+
return 0;
268+
}
269+
if (!args.hadBaseline) {
270+
console.error(
271+
`eval "${DISPATCH_EVAL}" failed: no baseline at ${args.baselinePath}`,
272+
);
273+
return 1;
274+
}
275+
const beforeText = await readFile(args.baselinePath, "utf8");
276+
const baseline = parseStoredRun(beforeText, args.baselinePath);
277+
const unbaselined = missed
278+
.filter(([name]) => baseline.cases[name] === undefined)
279+
.map(([name]) => name);
280+
const cases = { ...baseline.cases };
281+
let changed = false;
282+
for (const [name, checks] of Object.entries(last.cases)) {
283+
if (!routesStoredPass(checks)) continue;
284+
if (routesStoredPass(cases[name])) continue;
285+
cases[name] = checks;
286+
changed = true;
287+
}
288+
if (changed) {
289+
const next: StoredRun = { ...baseline, cases };
290+
await writeFile(args.baselinePath, `${JSON.stringify(next, null, 2)}\n`);
291+
}
292+
if (unbaselined.length !== 0) {
293+
console.error(
294+
`eval "${DISPATCH_EVAL}" failed: unbaselined routes miss (${unbaselined.join(", ")})`,
295+
);
296+
return 1;
297+
}
298+
return 0;
299+
}
300+
207301
export async function evaluateRouting(args: {
208302
cases: readonly Case<string, RoutingExpect>[];
209303
run: (input: string, signal: AbortSignal) => Promise<string>;
@@ -220,12 +314,10 @@ export async function evaluateRouting(args: {
220314
cases: args.cases,
221315
checks: {
222316
routes: routesCheck,
223-
noDiy: noDiyCheck,
224317
fast: under({ ms: ROUTE_BUDGET_MS }),
225318
},
226319
allow: {
227320
routes: { miss: 6, drop: 0 },
228-
noDiy: { miss: 2, drop: 0 },
229321
fast: 0,
230322
},
231323
compare: args.baselinePath,
@@ -238,17 +330,11 @@ export async function evaluateRouting(args: {
238330
},
239331
});
240332
console.log(report.markdown());
241-
if (report.checks["routes"]?.missed !== 0) {
242-
if (!hadBaseline) {
243-
console.error(
244-
`eval "${DISPATCH_EVAL}" failed: no baseline at ${args.baselinePath}`,
245-
);
246-
return 1;
247-
}
248-
return 0;
249-
}
250-
await copyFile(args.lastPath, args.baselinePath);
251-
return 0;
333+
return await commitRoutingBaseline({
334+
hadBaseline,
335+
lastPath: args.lastPath,
336+
baselinePath: args.baselinePath,
337+
});
252338
} catch (err: unknown) {
253339
if (err instanceof EvalFailure) {
254340
console.log(err.report.markdown());

0 commit comments

Comments
 (0)