feat(desktop): add independent eval verdict gate

desktop/src/features/agents/lib/evalVerdict.test.mjs

@@ -0,0 +1,88 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+  deriveEvalPromotionDecision,
+  deriveEvalVerdict,
+} from "./evalVerdict.ts";
+
+const completeRun = {
+  agentPubkey: "agent-pubkey",
+  evaluatorPubkey: "evaluator-pubkey",
+  criteria: {
+    id: "refund-created",
+    version: 1,
+    ownerPubkey: "owner-pubkey",
+    lockedBeforeRun: true,
+  },
+  evidence: [
+    { id: "database-receipt", kind: "outcome", source: "database readback" },
+    { id: "observer-trace", kind: "process", source: "observer transcript" },
+  ],
+  graderPassed: true,
+};
+
+test("validates only when outcome and process evidence are both present", () => {
+  assert.deepEqual(deriveEvalVerdict(completeRun), {
+    status: "validated",
+    missingCoverage: [],
+    evidenceIds: ["database-receipt", "observer-trace"],
+  });
+
+  const verdict = deriveEvalVerdict({
+    ...completeRun,
+    evidence: completeRun.evidence.filter(({ kind }) => kind !== "outcome"),
+  });
+  assert.equal(verdict.status, "inconclusive");
+  assert.deepEqual(verdict.missingCoverage, ["outcome evidence"]);
+});
+
+test("refuses self-grading and criteria changed after the run began", () => {
+  const verdict = deriveEvalVerdict({
+    ...completeRun,
+    evaluatorPubkey: completeRun.agentPubkey,
+    criteria: { ...completeRun.criteria, lockedBeforeRun: false },
+  });
+
+  assert.equal(verdict.status, "inconclusive");
+  assert.deepEqual(verdict.missingCoverage, [
+    "criteria locked before run",
+    "independent evaluator",
+  ]);
+});
+
+test("records a supported failure as rejected rather than inconclusive", () => {
+  const verdict = deriveEvalVerdict({ ...completeRun, graderPassed: false });
+  assert.equal(verdict.status, "rejected");
+  assert.deepEqual(verdict.missingCoverage, []);
+});
+
+test("promotion requires validation, non-regression, and owner approval", () => {
+  const verdict = deriveEvalVerdict(completeRun);
+
+  assert.deepEqual(
+    deriveEvalPromotionDecision({
+      verdict,
+      baselinePassRate: 0.8,
+      candidatePassRate: 0.9,
+      ownerApproved: true,
+    }),
+    { allowed: true, reasons: [] },
+  );
+
+  assert.deepEqual(
+    deriveEvalPromotionDecision({
+      verdict,
+      baselinePassRate: 0.9,
+      candidatePassRate: 0.8,
+      ownerApproved: false,
+    }),
+    {
+      allowed: false,
+      reasons: [
+        "candidate regresses against baseline",
+        "owner approval is missing",
+      ],
+    },
+  );
+});

desktop/src/features/agents/lib/evalVerdict.ts

@@ -0,0 +1,106 @@
+export type EvalEvidenceKind = "outcome" | "process";
+
+export interface EvalEvidence {
+  id: string;
+  kind: EvalEvidenceKind;
+  source: string;
+}
+
+export interface EvalCriteria {
+  id: string;
+  version: number;
+  ownerPubkey: string;
+  lockedBeforeRun: boolean;
+}
+
+export interface EvalRunResult {
+  agentPubkey: string;
+  evaluatorPubkey: string;
+  criteria: EvalCriteria;
+  evidence: readonly EvalEvidence[];
+  graderPassed: boolean | null;
+}
+
+export type EvalVerdictStatus = "validated" | "rejected" | "inconclusive";
+
+export interface EvalVerdict {
+  status: EvalVerdictStatus;
+  missingCoverage: readonly string[];
+  evidenceIds: readonly string[];
+}
+
+/**
+ * Produces a conservative verdict from immutable criteria and independently
+ * supplied outcome + process evidence. Missing trust inputs are inconclusive,
+ * never silently converted into a failure or a pass.
+ */
+export function deriveEvalVerdict(run: EvalRunResult): EvalVerdict {
+  const missingCoverage: string[] = [];
+  const evidenceIds = new Set<string>();
+  let hasOutcomeEvidence = false;
+  let hasProcessEvidence = false;
+
+  for (const evidence of run.evidence) {
+    if (!evidence.id.trim() || !evidence.source.trim()) continue;
+    evidenceIds.add(evidence.id);
+    if (evidence.kind === "outcome") hasOutcomeEvidence = true;
+    if (evidence.kind === "process") hasProcessEvidence = true;
+  }
+
+  if (!run.criteria.id.trim() || run.criteria.version < 1) {
+    missingCoverage.push("versioned success criteria");
+  }
+  if (!run.criteria.ownerPubkey.trim()) {
+    missingCoverage.push("criteria owner");
+  }
+  if (!run.criteria.lockedBeforeRun) {
+    missingCoverage.push("criteria locked before run");
+  }
+  if (!run.evaluatorPubkey.trim() || run.evaluatorPubkey === run.agentPubkey) {
+    missingCoverage.push("independent evaluator");
+  }
+  if (!hasOutcomeEvidence) missingCoverage.push("outcome evidence");
+  if (!hasProcessEvidence) missingCoverage.push("process evidence");
+  if (run.graderPassed === null) missingCoverage.push("grader verdict");
+
+  if (missingCoverage.length > 0) {
+    return {
+      status: "inconclusive",
+      missingCoverage,
+      evidenceIds: [...evidenceIds],
+    };
+  }
+
+  return {
+    status: run.graderPassed ? "validated" : "rejected",
+    missingCoverage,
+    evidenceIds: [...evidenceIds],
+  };
+}
+
+export interface EvalPromotionDecision {
+  allowed: boolean;
+  reasons: readonly string[];
+}
+
+/** Promotion additionally requires a non-regressing baseline comparison and owner review. */
+export function deriveEvalPromotionDecision(input: {
+  verdict: EvalVerdict;
+  baselinePassRate: number | null;
+  candidatePassRate: number | null;
+  ownerApproved: boolean;
+}): EvalPromotionDecision {
+  const reasons: string[] = [];
+
+  if (input.verdict.status !== "validated") {
+    reasons.push("candidate verdict is not validated");
+  }
+  if (input.baselinePassRate === null || input.candidatePassRate === null) {
+    reasons.push("baseline comparison is missing");
+  } else if (input.candidatePassRate < input.baselinePassRate) {
+    reasons.push("candidate regresses against baseline");
+  }
+  if (!input.ownerApproved) reasons.push("owner approval is missing");
+
+  return { allowed: reasons.length === 0, reasons };
+}