feat(jev): tunable thresholds and documented model pinning

README.md

@@ -42,7 +42,8 @@ returned.
   and passes only at 1; return a boolean to pick your own threshold, or
   `{ pass, value, explanation }` for detail.
 - **Sentence**: a yes/no question for Jev about the run. Passes at
-  p(yes) >= 0.7, fails at <= 0.3, fails as `unsure (p=…)` in between.
+  p(yes) >= `yes`, fails at <= `no` (0.7 / 0.3 by default), fails as
+  `unsure (p=…)` in between. `{ question, yes, no }` sets its own bars.
 - **Helpers**:
 
 | helper                      | passes when                                                               |
@@ -59,7 +60,8 @@ returned.
 
 `mentions` and `avoids` also take `(r) => words` for per-case words.
 `mentions` and `ranSteps` take `{ atLeast }` to pass on partial credit.
-`ask` fails as unsure when Jev's top probability is below 0.6.
+`ask` fails as unsure when Jev's top probability is below `sure`
+(0.6 by default; a check's own `sure` wins).
 
 A check that throws, hangs past `timeout`, or returns anything else fails
 itself closed; its siblings still run. Jev transport failures fail closed.
@@ -70,6 +72,13 @@ Jev sees `{ input, output, expect, steps }`, and a case's string and
 `"official"` (default), `"gateway"` or a URL. Without an `apiKey` the
 key comes from the environment as `@corbits/system-one` reads it.
 
+Thresholds: a yes/no check passes at p(yes) >= `yes` and fails at
+p(yes) <= `no` (0.7 / 0.3); `ask` fails as unsure when its top
+probability is under `sure` (0.6). Retune them per judge with
+`jev({ thresholds: { yes, no, sure } })` or per check as above.
+Thresholds are tuned to a judge version, so pin `model` — e.g.
+`jev({ model: "jev-1.13.0" })` — when they must stay stable.
+
 ## Runs
 
 - **Function**: `(input, signal) => output`. `signal` aborts on `timeout`.

e2e/jev.e2e.test.ts

@@ -154,6 +154,88 @@ describe("batching", () => {
   });
 });
 
+describe("thresholds", () => {
+  const question = "Is this reply ready to send to the customer?";
+  const noul = (p: number) => ({ type: "noul", noul: p });
+
+  function flatFetch(answer: unknown) {
+    return (async (_url: string, init: RequestInit) => {
+      const body = JSON.parse(String(init.body)) as { questions: Record<string, unknown> };
+      const answers = Object.fromEntries(Object.keys(body.questions).map((id) => [id, answer]));
+      return new Response(JSON.stringify({ model: "jev-test", answers }), {
+        status: 200,
+        headers: { "content-type": "application/json" },
+      });
+    }) as unknown as EvaluateDeps["fetch"];
+  }
+
+  test("judge thresholds retune every check's bars", async () => {
+    const { judge } = stubJev(noul(0.75));
+    expect((await judged(question, judge)).outcome?.pass).toBe(true);
+    const strict = jev({
+      apiKey: "test-key",
+      thresholds: { yes: 0.8 },
+      deps: stubDeps(flatFetch(noul(0.75))),
+    });
+    expect((await judged(question, strict)).outcome).toMatchObject({
+      pass: false,
+      label: "unsure",
+      explanation: "unsure (p=0.75)",
+    });
+  });
+
+  test("per-check bars win over the judge's", async () => {
+    const { judge } = stubJev(noul(0.75));
+    expect((await judged({ question, yes: 0.6 }, judge)).outcome?.pass).toBe(true);
+    expect((await judged({ question, yes: 0.8 }, judge)).outcome).toMatchObject({
+      pass: false,
+      label: "unsure",
+    });
+  });
+
+  test("ask's own sure wins over the judge's", async () => {
+    const pick = {
+      type: "choice",
+      choice: "ready",
+      probabilities: { ready: 0.45, revise: 0.35, escalate: 0.2 },
+      confidence: 0.45,
+    };
+    const lenient = ask("Ready?", {
+      options: { ready: "send", revise: "work", escalate: "human" },
+      pass: "ready",
+      sure: 0.4,
+    });
+    const strict = ask("Ready?", {
+      options: { ready: "send", revise: "work", escalate: "human" },
+      pass: "ready",
+      sure: 0.5,
+    });
+    expect((await answered(pick, lenient)).outcome?.pass).toBe(true);
+    expect((await answered(pick, strict)).outcome).toMatchObject({ pass: false });
+  });
+
+  test("bad thresholds throw at definition", async () => {
+    expect(() => jev({ thresholds: { yes: 0.3, no: 0.5 } })).toThrow("must not exceed");
+    expect(() => jev({ thresholds: { sure: 1.5 } })).toThrow("[0, 1]");
+    expect(() => ask("Ready?", { options: { a: "a" }, pass: "a", sure: 0 })).toThrow("(0, 1]");
+    const { judge } = stubJev(noul(0.9));
+    await expect(judged({ question, yes: 0.2, no: 0.5 }, judge)).rejects.toThrow(
+      "no must not exceed yes",
+    );
+    // Even a hand-built ask (bypassing ask()) is validated at evaluate time.
+    await expect(
+      judged({ question: "Ready?", options: { a: "a" }, pass: ["a"], sure: 0 }, judge),
+    ).rejects.toThrow("sure must be in (0, 1]");
+    // A check-level `no` above the judge's `yes` clamps to the pass band instead
+    // of overlapping it: p=0.75 stays "yes" under defaults with no: 0.8.
+    const clamped = stubJev(noul(0.75)).judge;
+    expect((await judged({ question, no: 0.8 }, clamped)).outcome).toMatchObject({
+      pass: true,
+      label: "yes",
+    });
+  });
+});
+
 describe("ask", () => {
   const ready = ask("Ready to send?", {
     options: { ready: "send it", revise: "needs work", escalate: "needs a human" },

src/evaluate.ts

@@ -1,6 +1,6 @@
 import { matches } from "./checks.js";
 import { fromJsonl } from "./datasets.js";
-import { jev, type Ask, type Judge } from "./jev.js";
+import { checkThresholds, jev, type Ask, type Judge, type Sentence } from "./jev.js";
 import {
   flips,
   fromReport,
@@ -16,7 +16,11 @@ import { toMs, type Duration } from "./timeout.js";
 import type { Case, CaseResult, CheckFn, CheckResult, Report, Run } from "./types.js";
 
 /** Code, a yes/no question for Jev, or `ask(...)`. */
-export type Check<I = unknown, O = unknown, E = unknown> = CheckFn<I, O, E> | string | Ask;
+export type Check<I = unknown, O = unknown, E = unknown> =
+  | CheckFn<I, O, E>
+  | string
+  | Ask
+  | Sentence;
 
 /** Cases a check may miss, or `{ miss, drop }`; both default to 0. */
 export type Allow = number | { miss?: number; drop?: number };
@@ -144,11 +148,14 @@ function named<I, E>(
 function toCheckParts<I, O, E>(
   name: string,
   checks: Record<string, Check<I, O, E>>,
-): { code: Record<string, (run: Run<I, O, E>) => unknown>; jev: Record<string, Ask | string> } {
+): {
+  code: Record<string, (run: Run<I, O, E>) => unknown>;
+  jev: Record<string, Ask | Sentence | string>;
+} {
   const entries = Object.entries(checks);
   if (entries.length === 0) throw new Error(`eval "${name}" has no checks`);
   const code: Record<string, (run: Run<I, O, E>) => unknown> = {};
-  const jev: Record<string, Ask | string> = {};
+  const jev: Record<string, Ask | Sentence | string> = {};
   for (const [key, check] of entries) {
     if (typeof check === "function") {
       code[key] = check;
@@ -157,7 +164,8 @@ function toCheckParts<I, O, E>(
     const valid =
       typeof check === "string" ? check.trim() !== "" : typeof check?.question === "string";
     if (!valid) throw new Error(`eval "${name}": check "${key}" is not a check`);
-    jev[key] = check as Ask | string;
+    checkThresholds(check as Ask | Sentence);
+    jev[key] = check as Ask | Sentence | string;
   }
   return { code, jev };
 }

src/index.ts

@@ -24,6 +24,15 @@ export {
   type Words,
 } from "./checks.js";
 export { EvalFailure, evaluate, type Allow, type Check, type EvaluateOptions } from "./evaluate.js";
-export { ask, jev, type Ask, type JevOptions, type Judge } from "./jev.js";
+export {
+  ask,
+  jev,
+  DEFAULTS,
+  type Ask,
+  type JevOptions,
+  type Judge,
+  type Sentence,
+  type Thresholds,
+} from "./jev.js";
 export type { Runnable } from "./runs.js";
 export type { Duration } from "./timeout.js";