feat(evals): add interactive collaboration evaluation baseline

evals/collaboration/README.md

@@ -0,0 +1,79 @@
+# Collaboration eval baseline (CL-9879)
+
+Interactive-collaboration checks for the **product** agent path, scored from
+outside: tool-call steps and reply text an observer can see — never model
+internals. Measurement lane only: nothing here changes product prompts or
+runtime.
+
+## What this measures
+
+Whether an agent, mid-run, handles the nine collaboration situations below
+the way the scenario intends (verify instead of trusting, follow a steer,
+re-read a changed file, yield to the operator, ...). Each scenario carries
+frozen **paired controls**: a good transcript that must pass every expected
+action, and a bad transcript that must fail (covering `badMisses`).
+
+## The nine scenarios
+
+| Scenario id                     | Kind                     | The behavior under test                                    |
+| ------------------------------- | ------------------------ | ---------------------------------------------------------- |
+| `collab-misleading-summary`     | `misleading-summary`     | Read the source instead of trusting a teammate summary     |
+| `collab-buried-required-action` | `buried-required-action` | Run the required step buried in a long report              |
+| `collab-conflicting-reports`    | `conflicting-reports`    | Resolve disagreeing reports via the source file            |
+| `collab-failed-checks`          | `failed-checks`          | Report failing checks honestly instead of claiming green   |
+| `collab-report-overflow`        | `report-overflow`        | Answer from an over-long report without tool thrash        |
+| `collab-midflight-steering`     | `midflight-steering`     | Follow an operator steer to a new file mid-run             |
+| `collab-mailbox-yield`          | `mailbox-yield`          | Yield to an operator question instead of acting            |
+| `collab-changed-file-reread`    | `changed-file-reread`    | Re-read (with narrowed bounds) a file that changed mid-run |
+| `collab-direct-question`        | `direct-question`        | Answer a direct operator question without tools            |
+
+Expected-action kinds: `toolCall`, `replyIncludes`, `replyExcludes`,
+`noTools`, `maxRepeats`, `rereadAfterChange`, `narrowedReread`,
+`readAfterOperator`. See `lib.ts` (`scoreTranscript`) for exact semantics.
+
+## Deterministic lane (what runs in CI)
+
+```sh
+bun scripts/eval-collaboration.ts --repeats 2 --out evals/collaboration/baseline-<YYYY-MM-DD>.json
+```
+
+The runner replays the frozen good/bad transcripts through the scorer and
+writes a `CollaborationReport` (see `lib.ts`). Repeats replay the same
+exemplars, so they pin the scorer — not model variance. The runner refuses
+to record a baseline whose paired controls mismatch (non-zero exit):
+a scorer that cannot tell good from bad measures nothing.
+
+`bun test ./evals/collaboration` asserts the same properties directly:
+every good transcript passes, every bad transcript fails for its intended
+reasons, single mutations flip good to fail, the empty transcript passes
+nothing, and the deterministic report reaches sensitivity 1.0.
+
+## Tracked report fields
+
+Per trial: scenario, kind, repeat, arm (`good`/`bad`), pass/fail, whether
+the control behaved as intended, and failing action ids with reasons.
+Totals: trial count, controls ok/failed, and **sensitivity** (fraction of
+paired trials behaving as intended).
+
+Metrics with no deterministic value are reported **unknown, never zero**:
+`tokenUsage`, `livePassRate`, `agentDurationMs`, `cost`, `turnsUsed`.
+`live.status` is `not-run` until a live run records them.
+
+## Live-model protocol (separate, authorized runs only)
+
+Live trials need an authorized inference run and are recorded separately —
+never by editing scripted transcripts into passing. A live run records the
+same `CollabTrial` shape with the observed (not scripted) transcript verdict
+plus the real provider, model, effort, and the unknown metrics above filled
+in. `bun scripts/eval-collaboration.ts --live` refuses with this guidance
+instead of laundering scripted results as live data.
+
+## Conventions
+
+The scenario set is frozen (`version: 1`, fixtures
+`collab-fixtures-v1`). Re-measures reuse `scenarios.json` as-is so runs
+stay comparable. Never edit `scenarios.json` or a recorded baseline to hit
+a target number; a scenario-set change needs a version bump plus a new
+baseline file. Passing scripted controls says the scorer discriminates —
+it says nothing about live model quality, and must never be claimed as a
+product improvement.

evals/collaboration/lib.test.ts

@@ -0,0 +1,142 @@
+import { describe, expect, test } from "bun:test";
+import scenariosJson from "./scenarios.json" with { type: "json" };
+import {
+  buildDeterministicReport,
+  formatSummary,
+  parseScenarioSetFile,
+  scoreTranscript,
+  type ObservedTranscript,
+} from "./lib.js";
+
+const set = parseScenarioSetFile(scenariosJson);
+
+const EXPECTED_KINDS = [
+  "misleading-summary",
+  "buried-required-action",
+  "conflicting-reports",
+  "failed-checks",
+  "report-overflow",
+  "midflight-steering",
+  "mailbox-yield",
+  "changed-file-reread",
+  "direct-question",
+] as const;
+
+describe("collaboration scenario set", () => {
+  test("covers all nine collaboration kinds exactly once", () => {
+    expect(set.version).toBe(1);
+    expect(set.scenarios.map((s) => s.kind).sort()).toEqual(
+      [...EXPECTED_KINDS].sort(),
+    );
+  });
+
+  test("fixture versions are frozen and uniform", () => {
+    for (const scenario of set.scenarios) {
+      expect(scenario.fixtureVersion).toBe("collab-fixtures-v1");
+      expect(scenario.prompt.trim().length).toBeGreaterThan(0);
+      expect(scenario.expectedActions.length).toBeGreaterThan(0);
+    }
+  });
+});
+
+describe("positive controls", () => {
+  test("every good transcript passes its scenario", () => {
+    for (const scenario of set.scenarios) {
+      const verdict = scoreTranscript(scenario, scenario.goodTranscript);
+      expect(`${scenario.id}: ${JSON.stringify(verdict.results)}`).toBe(
+        `${scenario.id}: ${JSON.stringify(
+          verdict.results.map((r) => ({ ...r, ok: true })),
+        )}`,
+      );
+      expect(verdict.passed).toBe(true);
+    }
+  });
+});
+
+describe("negative controls", () => {
+  test("every bad transcript fails for its intended reasons", () => {
+    for (const scenario of set.scenarios) {
+      const verdict = scoreTranscript(scenario, scenario.badTranscript);
+      expect(verdict.passed).toBe(false);
+      const failedIds = verdict.results.filter((r) => !r.ok).map((r) => r.id);
+      for (const miss of scenario.badMisses) {
+        expect(failedIds).toContain(miss);
+      }
+    }
+  });
+});
+
+describe("scorer sensitivity", () => {
+  test("dropping a required tool call flips a good transcript to fail", () => {
+    const scenario = set.scenarios.find(
+      (s) => s.id === "collab-misleading-summary",
+    );
+    if (scenario === undefined) throw new Error("scenario missing");
+    const mutated: ObservedTranscript = {
+      steps: scenario.goodTranscript.steps.filter(
+        (step) => !(step.step === "tool" && step.name === "read"),
+      ),
+    };
+    const verdict = scoreTranscript(scenario, mutated);
+    expect(verdict.passed).toBe(false);
+  });
+
+  test("altering the decisive reply value flips a good transcript to fail", () => {
+    const scenario = set.scenarios.find(
+      (s) => s.id === "collab-conflicting-reports",
+    );
+    if (scenario === undefined) throw new Error("scenario missing");
+    const mutated: ObservedTranscript = {
+      steps: scenario.goodTranscript.steps.map((step) =>
+        step.step === "reply"
+          ? { ...step, text: (step.text ?? "").replace("beta", "alpha") }
+          : step,
+      ),
+    };
+    const verdict = scoreTranscript(scenario, mutated);
+    expect(verdict.passed).toBe(false);
+  });
+
+  test("an empty transcript passes nothing", () => {
+    for (const scenario of set.scenarios) {
+      expect(scoreTranscript(scenario, { steps: [] }).passed).toBe(false);
+    }
+  });
+});
+
+describe("deterministic report", () => {
+  test("paired controls are all satisfied and live metrics stay unknown", () => {
+    const report = buildDeterministicReport(set, {
+      commitSha: "test-sha",
+      repeats: 1,
+      seed: 424242,
+    });
+    expect(report.totals.trialsTotal).toBe(set.scenarios.length * 2);
+    expect(report.totals.controlsFailed).toBe(0);
+    expect(report.totals.sensitivity).toBe(1);
+    expect(report.live.status).toBe("not-run");
+    expect(report.live.unknownMetrics).toContain("tokenUsage");
+    expect(report.live.unknownMetrics).toContain("livePassRate");
+    const summary = formatSummary(report);
+    expect(summary).toContain("sensitivity 100.0%");
+    expect(summary).toContain("unknown");
+  });
+
+  test("refuses a set with drifted fixture versions instead of recording a baseline", () => {
+    const tampered = {
+      ...set,
+      scenarios: set.scenarios.map((scenario, index) =>
+        index === 0
+          ? { ...scenario, fixtureVersion: "tampered-v99" }
+          : scenario,
+      ),
+    };
+    expect(() =>
+      buildDeterministicReport(tampered, {
+        commitSha: "test-sha",
+        repeats: 1,
+        seed: 424242,
+      }),
+    ).toThrow(/tampered-v99.*refusing to record an incomparable baseline/);
+  });
+});