diff --git a/README.md b/README.md index 2ebc8c1..fb55351 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,8 @@ returned. and passes only at 1; return a boolean to pick your own threshold, or `{ pass, value, explanation }` for detail. - **Sentence**: a yes/no question for Jev about the run. Passes at - p(yes) >= 0.7, fails at <= 0.3, fails as `unsure (p=…)` in between. + p(yes) >= `yes`, fails at <= `no` (0.7 / 0.3 by default), fails as + `unsure (p=…)` in between. `{ question, yes, no }` sets its own bars. - **Helpers**: | helper | passes when | @@ -59,7 +60,8 @@ returned. `mentions` and `avoids` also take `(r) => words` for per-case words. `mentions` and `ranSteps` take `{ atLeast }` to pass on partial credit. -`ask` fails as unsure when Jev's top probability is below 0.6. +`ask` fails as unsure when Jev's top probability is below `sure` +(0.6 by default; a check's own `sure` wins). A check that throws, hangs past `timeout`, or returns anything else fails itself closed; its siblings still run. Jev transport failures fail closed. @@ -70,6 +72,13 @@ Jev sees `{ input, output, expect, steps }`, and a case's string and `"official"` (default), `"gateway"` or a URL. Without an `apiKey` the key comes from the environment as `@corbits/system-one` reads it. +Thresholds: a yes/no check passes at p(yes) >= `yes` and fails at +p(yes) <= `no` (0.7 / 0.3); `ask` fails as unsure when its top +probability is under `sure` (0.6). Retune them per judge with +`jev({ thresholds: { yes, no, sure } })` or per check as above. +Thresholds are tuned to a judge version, so pin `model` — e.g. +`jev({ model: "jev-1.13.0" })` — when they must stay stable. + ## Runs - **Function**: `(input, signal) => output`. `signal` aborts on `timeout`. diff --git a/e2e/jev.e2e.test.ts b/e2e/jev.e2e.test.ts index 25cd0d8..993db13 100644 --- a/e2e/jev.e2e.test.ts +++ b/e2e/jev.e2e.test.ts @@ -154,6 +154,88 @@ describe("batching", () => { }); }); +describe("thresholds", () => { + const question = "Is this reply ready to send to the customer?"; + const noul = (p: number) => ({ type: "noul", noul: p }); + + function flatFetch(answer: unknown) { + return (async (_url: string, init: RequestInit) => { + const body = JSON.parse(String(init.body)) as { questions: Record }; + const answers = Object.fromEntries(Object.keys(body.questions).map((id) => [id, answer])); + return new Response(JSON.stringify({ model: "jev-test", answers }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }) as unknown as EvaluateDeps["fetch"]; + } + + test("judge thresholds retune every check's bars", async () => { + const { judge } = stubJev(noul(0.75)); + expect((await judged(question, judge)).outcome?.pass).toBe(true); + const strict = jev({ + apiKey: "test-key", + thresholds: { yes: 0.8 }, + deps: stubDeps(flatFetch(noul(0.75))), + }); + expect((await judged(question, strict)).outcome).toMatchObject({ + pass: false, + label: "unsure", + explanation: "unsure (p=0.75)", + }); + }); + + test("per-check bars win over the judge's", async () => { + const { judge } = stubJev(noul(0.75)); + expect((await judged({ question, yes: 0.6 }, judge)).outcome?.pass).toBe(true); + expect((await judged({ question, yes: 0.8 }, judge)).outcome).toMatchObject({ + pass: false, + label: "unsure", + }); + }); + + test("ask's own sure wins over the judge's", async () => { + const pick = { + type: "choice", + choice: "ready", + probabilities: { ready: 0.45, revise: 0.35, escalate: 0.2 }, + confidence: 0.45, + }; + const lenient = ask("Ready?", { + options: { ready: "send", revise: "work", escalate: "human" }, + pass: "ready", + sure: 0.4, + }); + const strict = ask("Ready?", { + options: { ready: "send", revise: "work", escalate: "human" }, + pass: "ready", + sure: 0.5, + }); + expect((await answered(pick, lenient)).outcome?.pass).toBe(true); + expect((await answered(pick, strict)).outcome).toMatchObject({ pass: false }); + }); + + test("bad thresholds throw at definition", async () => { + expect(() => jev({ thresholds: { yes: 0.3, no: 0.5 } })).toThrow("must not exceed"); + expect(() => jev({ thresholds: { sure: 1.5 } })).toThrow("[0, 1]"); + expect(() => ask("Ready?", { options: { a: "a" }, pass: "a", sure: 0 })).toThrow("(0, 1]"); + const { judge } = stubJev(noul(0.9)); + await expect(judged({ question, yes: 0.2, no: 0.5 }, judge)).rejects.toThrow( + "no must not exceed yes", + ); + // Even a hand-built ask (bypassing ask()) is validated at evaluate time. + await expect( + judged({ question: "Ready?", options: { a: "a" }, pass: ["a"], sure: 0 }, judge), + ).rejects.toThrow("sure must be in (0, 1]"); + // A check-level `no` above the judge's `yes` clamps to the pass band instead + // of overlapping it: p=0.75 stays "yes" under defaults with no: 0.8. + const clamped = stubJev(noul(0.75)).judge; + expect((await judged({ question, no: 0.8 }, clamped)).outcome).toMatchObject({ + pass: true, + label: "yes", + }); + }); +}); + describe("ask", () => { const ready = ask("Ready to send?", { options: { ready: "send it", revise: "needs work", escalate: "needs a human" }, diff --git a/src/evaluate.ts b/src/evaluate.ts index 4d69e7b..7f5b1ba 100644 --- a/src/evaluate.ts +++ b/src/evaluate.ts @@ -1,6 +1,6 @@ import { matches } from "./checks.js"; import { fromJsonl } from "./datasets.js"; -import { jev, type Ask, type Judge } from "./jev.js"; +import { checkThresholds, jev, type Ask, type Judge, type Sentence } from "./jev.js"; import { flips, fromReport, @@ -16,7 +16,11 @@ import { toMs, type Duration } from "./timeout.js"; import type { Case, CaseResult, CheckFn, CheckResult, Report, Run } from "./types.js"; /** Code, a yes/no question for Jev, or `ask(...)`. */ -export type Check = CheckFn | string | Ask; +export type Check = + | CheckFn + | string + | Ask + | Sentence; /** Cases a check may miss, or `{ miss, drop }`; both default to 0. */ export type Allow = number | { miss?: number; drop?: number }; @@ -144,11 +148,14 @@ function named( function toCheckParts( name: string, checks: Record>, -): { code: Record) => unknown>; jev: Record } { +): { + code: Record) => unknown>; + jev: Record; +} { const entries = Object.entries(checks); if (entries.length === 0) throw new Error(`eval "${name}" has no checks`); const code: Record) => unknown> = {}; - const jev: Record = {}; + const jev: Record = {}; for (const [key, check] of entries) { if (typeof check === "function") { code[key] = check; @@ -157,7 +164,8 @@ function toCheckParts( const valid = typeof check === "string" ? check.trim() !== "" : typeof check?.question === "string"; if (!valid) throw new Error(`eval "${name}": check "${key}" is not a check`); - jev[key] = check as Ask | string; + checkThresholds(check as Ask | Sentence); + jev[key] = check as Ask | Sentence | string; } return { code, jev }; } diff --git a/src/index.ts b/src/index.ts index 463a0e0..b123b5a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -24,6 +24,15 @@ export { type Words, } from "./checks.js"; export { EvalFailure, evaluate, type Allow, type Check, type EvaluateOptions } from "./evaluate.js"; -export { ask, jev, type Ask, type JevOptions, type Judge } from "./jev.js"; +export { + ask, + jev, + DEFAULTS, + type Ask, + type JevOptions, + type Judge, + type Sentence, + type Thresholds, +} from "./jev.js"; export type { Runnable } from "./runs.js"; export type { Duration } from "./timeout.js"; diff --git a/src/jev.ts b/src/jev.ts index aaceb77..96b2992 100644 --- a/src/jev.ts +++ b/src/jev.ts @@ -18,11 +18,40 @@ export type JevOptions = { apiKey?: string; /** Bounds one Jev call. `@corbits/system-one` defaults to 1500ms. */ timeout?: Duration; + /** Pass bars for every Jev check through this judge. */ + thresholds?: Thresholds; /** Replaces `fetch` and the scheduler, e.g. to stub Jev in tests. */ deps?: EvaluateDeps; }; -export type Judge = { readonly config: EvaluateConfig; readonly deps?: EvaluateDeps }; +/** Yes/no passes at p >= `yes` and fails at p <= `no`; `ask` is unsure under `sure`. */ +export type Thresholds = { yes?: number; no?: number; sure?: number }; + +export type Resolved = { yes: number; no: number; sure: number }; + +/** The one documented set of numbers, tuned per workload with `calibrate`. */ +export const DEFAULTS: Resolved = { yes: 0.7, no: 0.3, sure: 0.6 }; + +export type Judge = { + readonly config: EvaluateConfig; + readonly thresholds?: Resolved; + readonly deps?: EvaluateDeps; +}; + +function resolve(label: string, raw: Thresholds): Resolved { + const out: Resolved = { + yes: raw.yes ?? DEFAULTS.yes, + no: raw.no ?? DEFAULTS.no, + sure: raw.sure ?? DEFAULTS.sure, + }; + for (const [key, value] of Object.entries(out)) { + if (!Number.isFinite(value) || value < 0 || value > 1) { + throw new Error(`jev: ${label}.${key} must be in [0, 1]`); + } + } + if (out.no > out.yes) throw new Error(`jev: ${label}.no must not exceed ${label}.yes`); + return out; +} function toEndpoint(endpoint?: string, model?: string): EndpointConfig | undefined { const named = model === undefined ? {} : { model }; @@ -38,7 +67,13 @@ export function jev(options: JevOptions = {}): Judge { if (endpoint !== undefined) config.endpoint = endpoint; if (options.apiKey !== undefined) config.apiKey = options.apiKey; if (options.timeout !== undefined) config.timeoutMs = toMs(options.timeout, "jev timeout"); - return options.deps === undefined ? { config } : { config, deps: options.deps }; + const thresholds = + options.thresholds === undefined ? undefined : resolve("thresholds", options.thresholds); + return { + config, + ...(thresholds === undefined ? {} : { thresholds }), + ...(options.deps === undefined ? {} : { deps: options.deps }), + }; } /** A Jev question with named answers. A plain string check is the yes/no case. */ @@ -47,21 +82,38 @@ export type Ask = { readonly options?: Readonly>; readonly levels?: readonly string[]; readonly pass: readonly string[]; + /** Overrides the judge's `sure` bar. */ + readonly sure?: number; +}; + +/** A yes/no question with its own pass bars. A plain string uses the judge's. */ +export type Sentence = { + readonly question: string; + readonly yes?: number; + readonly no?: number; }; /** Pick one option; pass when Jev picks one of `pass`. */ export function ask( question: string, - spec: { options: Readonly>; pass: string | readonly string[] }, + spec: { + options: Readonly>; + pass: string | readonly string[]; + sure?: number; + }, ): Ask; /** Grade on ordered levels, lowest first; pass at `pass` or above. */ -export function ask(question: string, spec: { levels: readonly string[]; pass: string }): Ask; +export function ask( + question: string, + spec: { levels: readonly string[]; pass: string; sure?: number }, +): Ask; export function ask( question: string, spec: { options?: Readonly>; levels?: readonly string[]; pass: string | readonly string[]; + sure?: number; }, ): Ask { if (question.trim() === "") throw new Error("ask needs a question"); @@ -76,20 +128,18 @@ export function ask( for (const name of pass) { if (!names.includes(name)) throw new Error(`ask: pass "${name}" is not an answer`); } + if (spec.sure !== undefined && (!Number.isFinite(spec.sure) || spec.sure <= 0 || spec.sure > 1)) { + throw new Error("ask: sure must be in (0, 1]"); + } + const sure = spec.sure === undefined ? {} : { sure: spec.sure }; return spec.options === undefined - ? { question, levels: names, pass } - : { question, options: spec.options, pass }; + ? { question, levels: names, pass, ...sure } + : { question, options: spec.options, pass, ...sure }; } -/** Yes/no passes at p >= YES, fails at p <= NO; between is unsure and fails. */ -const YES = 0.7; -const NO = 0.3; -/** A choice or level picked with less probability than this is unsure and fails. */ -const SURE = 0.6; - -function toQuestion(id: string, check: Ask | string): Question { +function toQuestion(id: string, check: Ask | Sentence | string): Question { if (typeof check === "string") return { id, type: "boolean", instructions: check }; - if (check.options !== undefined) { + if ("options" in check && check.options !== undefined) { return { id, type: "choice", @@ -97,12 +147,15 @@ function toQuestion(id: string, check: Ask | string): Question { criteria: { ...check.options }, }; } - return { - id, - type: "score", - instructions: check.question, - criteria: [...(check.levels ?? [])], - }; + if ("levels" in check && check.levels !== undefined) { + return { + id, + type: "score", + instructions: check.question, + criteria: [...check.levels], + }; + } + return { id, type: "boolean", instructions: check.question }; } function failed(explanation: string): Outcome { @@ -113,11 +166,48 @@ function p(n: number): string { return n.toFixed(2); } -function toOutcome(check: Ask | string, decision: Decision): Outcome { +/** Per-check overrides win over the judge's, which win over `DEFAULTS`. */ +function thresholdsFor(check: Ask | Sentence | string, judge: Judge): Resolved { + const base = judge.thresholds ?? DEFAULTS; + if (typeof check === "string") return base; + const any = check as { sure?: number; yes?: number; no?: number }; + const yes = any.yes ?? base.yes; + return { + yes, + // A check-level `no` above the merged `yes` would overlap the pass band. + // Clamping it keeps the band shape; the pass branch already wins the + // overlap, so no verdict changes. + no: Math.min(any.no ?? base.no, yes), + sure: any.sure ?? base.sure, + }; +} + +/** Throws when a per-check override is malformed, so bad bars fail at definition. */ +export function checkThresholds(check: Ask | Sentence | string): void { + if (typeof check === "string") return; + const any = check as { sure?: number; yes?: number; no?: number }; + for (const key of ["yes", "no"] as const) { + const value = any[key]; + if (value !== undefined && (!Number.isFinite(value) || value < 0 || value > 1)) { + throw new Error(`jev: ${key} must be in [0, 1]`); + } + } + const sure = any.sure; + if (sure !== undefined && (!Number.isFinite(sure) || sure <= 0 || sure > 1)) { + throw new Error("jev: sure must be in (0, 1]"); + } + if (any.yes !== undefined && any.no !== undefined && any.no > any.yes) { + throw new Error("jev: no must not exceed yes"); + } +} + +function toOutcome(check: Ask | Sentence | string, decision: Decision, judge: Judge): Outcome { + const t = thresholdsFor(check, judge); if (decision.type === "noul") { const yes = decision.noul; - if (yes >= YES) return { pass: true, value: yes, label: "yes" }; - if (yes <= NO) return { pass: false, value: yes, label: "no", explanation: `no (p=${p(yes)})` }; + if (yes >= t.yes) return { pass: true, value: yes, label: "yes" }; + if (yes <= t.no) + return { pass: false, value: yes, label: "no", explanation: `no (p=${p(yes)})` }; return { pass: false, value: yes, label: "unsure", explanation: `unsure (p=${p(yes)})` }; } const ask = check as Ask; @@ -125,9 +215,9 @@ function toOutcome(check: Ask | string, decision: Decision): Outcome { if (decision.type === "choice") { const value = Math.min( 1, - ask.pass.reduce((t, name) => t + (decision.probabilities[name] ?? 0), 0), + ask.pass.reduce((sum, name) => sum + (decision.probabilities[name] ?? 0), 0), ); - if (top < SURE) { + if (top < t.sure) { return { ...failed(`unsure (p=${p(top)} for "${decision.choice}")`), value, @@ -143,7 +233,8 @@ function toOutcome(check: Ask | string, decision: Decision): Outcome { const index = Math.round(decision.score); const level = levels[index] ?? String(decision.score); const value = decision.score / (levels.length - 1); - if (top < SURE) return { ...failed(`unsure (p=${p(top)} for "${level}")`), value, label: level }; + if (top < t.sure) + return { ...failed(`unsure (p=${p(top)} for "${level}")`), value, label: level }; const pass = index >= levels.indexOf(ask.pass[0] ?? ""); const outcome: Outcome = { pass, value, label: level }; if (!pass) outcome.explanation = `graded "${level}"`; @@ -158,14 +249,14 @@ function toOutcome(check: Ask | string, decision: Decision): Outcome { */ export async function askJevBatch( judge: Judge, - checks: Record, + checks: Record, run: Run, ): Promise> { const names = Object.keys(checks); const state: Record = { input: run.input, output: run.output }; if (run.expect !== undefined) state["expect"] = run.expect; if (run.steps.length > 0) state["steps"] = run.steps; - const questions = names.map((id) => toQuestion(id, checks[id] as Ask | string)); + const questions = names.map((id) => toQuestion(id, checks[id] as Ask | Sentence | string)); let out; try { out = await evaluate( @@ -185,7 +276,7 @@ export async function askJevBatch( const outcome = decision === undefined ? failed("jev returned no decision") - : toOutcome(checks[name] as Ask | string, decision); + : toOutcome(checks[name] as Ask | Sentence | string, decision, judge); return [name, outcome]; }), ); diff --git a/src/runner.ts b/src/runner.ts index ce96df5..8ee8d66 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -1,5 +1,5 @@ import { toOutcome } from "./checks.js"; -import { askJevBatch, type Ask, type Judge } from "./jev.js"; +import { askJevBatch, type Ask, type Judge, type Sentence } from "./jev.js"; import type { Trace } from "./runs.js"; import { withTimeout } from "./timeout.js"; import type { CaseResult, Outcome, Run } from "./types.js"; @@ -7,7 +7,7 @@ import type { CaseResult, Outcome, Run } from "./types.js"; export type NamedCase = { name: string; input: I; expect?: E; skip?: readonly string[] }; /** String and `ask` checks Jev answers in one call per attempt. */ -export type JevGroup = { judge: Judge; checks: Record }; +export type JevGroup = { judge: Judge; checks: Record }; export type RunSpec = { cases: readonly NamedCase[]; @@ -82,7 +82,7 @@ async function runJev( timeoutMs: number | undefined, ): Promise> { const selected = Object.fromEntries( - names.map((name) => [name, group.checks[name] as Ask | string]), + names.map((name) => [name, group.checks[name] as Ask | Sentence | string]), ); try { const pending = askJevBatch(group.judge, selected, run);