fix(evals): address the judge review findings

- judge evidence now includes each tool's returned output, so grounding
  rubrics can see the status/carrier values they check
- the exact-argument check runs only for scripted runs; a live model may emit
  valid-but-different arguments
- rubricDigest includes minScore, so a threshold-only change is not comparable
- runScenario retains the structured judge verdict on the result, not just a
  formatted check string
- the model-comparison suite passes a judge so judged scenarios are scored
- the three originally-judged scenarios drop their finalContent pattern in live
  mode, so valid paraphrases are not failures
- the live and comparison test timeouts scale with EVAL_TRIALS
- the context checks assert history role and order, not just presence

Author: sudoKrishna

apps/sim/evals/agent-tool-use/agent-tool-use.compare.live.test.ts

@@ -60,6 +60,9 @@ describe.skipIf(!LIVE)('agent tool-use model comparison', () => {
           mode: 'live',
           model: spec.model,
           providerName: spec.provider.label,
+          ...(scenario.judge
+            ? { judge: { completion, model: spec.model, rubric: scenario.judge } }
+            : {}),
         })
         runs.push({
           provider: spec.provider.id,
@@ -69,7 +72,8 @@ describe.skipIf(!LIVE)('agent tool-use model comparison', () => {
         })
       }
     },
-    TIMEOUT_MS
+    /** The test runs TRIALS trials, each with a task call and possibly a judge call. */
+    TRIALS * TIMEOUT_MS
   )
 
   it('meets the per-model pass-rate floor', () => {

apps/sim/evals/agent-tool-use/agent-tool-use.live.test.ts

@@ -119,6 +119,7 @@ describe.skipIf(!LIVE)('agent tool-use eval suite (live DeepSeek)', () => {
         ).toBeGreaterThanOrEqual(MIN_PASS_RATE)
       }
     },
-    TIMEOUT_MS
+    /** The test runs TRIALS trials, each with a task call and possibly a judge call. */
+    TRIALS * TIMEOUT_MS
   )
 })

apps/sim/evals/agent-tool-use/executor-harness.ts

@@ -267,6 +267,28 @@ export async function runExecutorScenario(
           : `missing [${missingHistory.map((message) => message.content).join(', ')}]`,
     })
 
+    /** Each prior turn must appear as a subsequence with its role, in order. */
+    let cursor = 0
+    const inOrder = memory.history.every((message) => {
+      while (cursor < requestMessages.length) {
+        const candidate = requestMessages[cursor]
+        cursor += 1
+        if (
+          candidate.role === message.role &&
+          typeof candidate.content === 'string' &&
+          candidate.content.includes(message.content)
+        ) {
+          return true
+        }
+      }
+      return false
+    })
+    checks.push({
+      name: 'memory-history-order',
+      passed: inOrder,
+      detail: 'each prior turn reached the provider in order with its role',
+    })
+
     const lastHistoryIndex =
       memory.history.length === 0
         ? -1

apps/sim/evals/agent-tool-use/harness.ts

@@ -7,7 +7,7 @@ import { toolsMockFns } from '@sim/testing/mocks/tools.mock'
 import { isRecordLike } from '@sim/utils/object'
 import type { ChatCompletionChunk } from 'openai/resources/chat/completions'
 import type { CompletionUsage } from 'openai/resources/completions'
-import { type JudgeRubric, judgeAnswer } from '@/evals/agent-tool-use/judge'
+import { type JudgeRubric, type JudgeVerdict, judgeAnswer } from '@/evals/agent-tool-use/judge'
 import type {
   AgentToolUseExpectations,
   AgentToolUseResult,
@@ -438,6 +438,7 @@ export async function runScenario(
         success: response.success,
         ...(response.error ? { error: response.error } : {}),
         durationMs: Date.now() - startedAt,
+        result: response.output,
       })
       return response
     }
@@ -481,9 +482,10 @@ export async function runScenario(
   const iterations = completed?.iterations ?? 0
   const checks = scoreExpectations(expected, toolCalls, finalContent, iterations, streamError, mode)
 
+  let judgeVerdict: JudgeVerdict | undefined
   if (options.judge) {
     try {
-      const verdict = await judgeAnswer({
+      judgeVerdict = await judgeAnswer({
         completion: options.judge.completion,
         model: options.judge.model,
         rubric: options.judge.rubric,
@@ -493,15 +495,16 @@ export async function runScenario(
       })
       checks.push({
         name: 'judge',
-        passed: verdict.passed,
-        detail: `weighted ${verdict.weightedScore.toFixed(2)} [judge ${verdict.identity.model} ${verdict.identity.rubricDigest}]; ${verdict.rationale}`,
+        passed: judgeVerdict.passed,
+        detail: `weighted ${judgeVerdict.weightedScore.toFixed(2)} [judge ${judgeVerdict.identity.model} ${judgeVerdict.identity.rubricDigest}]; ${judgeVerdict.rationale}`,
       })
     } catch (error) {
       checks.push({ name: 'judge', passed: false, detail: `judge failed: ${String(error)}` })
     }
   }
 
-  if (invocations.length > 0) {
+  /** Only scripted runs know the exact arguments; a live model may choose valid ones. */
+  if (invocations.length > 0 && mode !== 'live') {
     const expectedCalls = expectedExecutableCalls(scenario)
     const mismatched = invocations.filter(
       (invocation) =>
@@ -533,6 +536,7 @@ export async function runScenario(
     checks,
     finalContent,
     toolInvocations: invocations,
+    ...(judgeVerdict ? { judge: judgeVerdict } : {}),
     metrics: {
       iterations,
       toolCalls: toolCalls.length,

apps/sim/evals/agent-tool-use/judge.test.ts

@@ -128,6 +128,12 @@ describe('judge identity', () => {
     expect(rubricDigest(reordered)).toBe(rubricDigest(rubric))
   })
 
+  it('changes the digest when only the passing threshold changes', () => {
+    expect(rubricDigest({ criteria: rubric.criteria, minScore: 0.9 })).not.toBe(
+      rubricDigest(rubric)
+    )
+  })
+
   it('changes the digest when a criterion changes', () => {
     const changed = {
       criteria: [

apps/sim/evals/agent-tool-use/judge.ts

@@ -1,9 +1,15 @@
 import { createHash } from 'node:crypto'
 import type { ChatCompletionChunk } from 'openai/resources/chat/completions'
-import type { JudgeCriterion, JudgeRubric } from '@/evals/agent-tool-use/types'
+import type {
+  JudgeCriterion,
+  JudgeIdentity,
+  JudgeRubric,
+  JudgeScore,
+  JudgeVerdict,
+} from '@/evals/agent-tool-use/types'
 import type { OpenAICompatCreateCompletion } from '@/providers/openai-compat/streaming-tool-loop'
 
-export type { JudgeCriterion, JudgeRubric }
+export type { JudgeCriterion, JudgeIdentity, JudgeRubric, JudgeScore, JudgeVerdict }
 
 /**
  * LLM-as-judge scoring for open-ended answers.
@@ -29,36 +35,19 @@ export interface JudgeInput {
 export const JUDGE_PARSER_VERSION = '1'
 
 /**
- * Who judged and how. Two scores are only comparable when this matches; a delta
- * across differing identity is not evidence of improvement.
+ * Stable digest of the rubric's grading contract, independent of key order. The
+ * passing threshold is part of the contract: changing only `minScore` changes
+ * which scores pass, so it must change the digest too.
  */
-export interface JudgeIdentity {
-  model: string
-  rubricDigest: string
-  parserVersion: string
-  temperature?: number
-}
-
-export interface JudgeScore {
-  scores: Record<string, number>
-  rationale: string
-  weightedScore: number
-  passed: boolean
-}
-
-export interface JudgeVerdict extends JudgeScore {
-  identity: JudgeIdentity
-}
-
-/** Stable digest of the rubric's grading contract, independent of key order. */
 export function rubricDigest(rubric: JudgeRubric): string {
-  const canonical = JSON.stringify(
-    rubric.criteria.map((criterion) => ({
+  const canonical = JSON.stringify({
+    minScore: rubric.minScore ?? 0.5,
+    criteria: rubric.criteria.map((criterion) => ({
       id: criterion.id,
       description: criterion.description,
       weight: criterion.weight ?? 1,
-    }))
-  )
+    })),
+  })
   return `sha256:${createHash('sha256').update(canonical).digest('hex').slice(0, 16)}`
 }
 

apps/sim/evals/agent-tool-use/scenarios.ts

@@ -311,8 +311,8 @@ export const AGENT_TOOL_USE_SCENARIOS: AgentToolUseScenario[] = [
       finalContent: /A-1937.*shipped/,
       maxIterations: 3,
     },
-    /** A live model may answer with the user-facing order number and the grounded status. */
-    liveExpect: { finalContent: /shipped/i },
+    /** The rubric decides grounding; a content pattern would reject valid paraphrases. */
+    liveExpect: { finalContent: undefined },
     judge: {
       criteria: [
         {
@@ -571,6 +571,7 @@ export const AGENT_TOOL_USE_SCENARIOS: AgentToolUseScenario[] = [
       ],
       minScore: 0.7,
     },
+    liveExpect: { finalContent: undefined },
   },
   {
     id: 'long-chain-dependency',
@@ -634,6 +635,7 @@ export const AGENT_TOOL_USE_SCENARIOS: AgentToolUseScenario[] = [
       toolCallSequence: undefined,
       requiredTools: ['list_orders', 'get_order', 'get_shipping'],
       successfulToolCalls: 3,
+      finalContent: undefined,
     },
     judge: {
       criteria: [

apps/sim/evals/agent-tool-use/types.ts

@@ -83,6 +83,27 @@ export interface JudgeRubric {
   minScore?: number
 }
 
+/** Who judged and how. Two scores are comparable only when this matches. */
+export interface JudgeIdentity {
+  model: string
+  rubricDigest: string
+  parserVersion: string
+  temperature?: number
+}
+
+/** A parsed judge response, before the identity envelope is attached. */
+export interface JudgeScore {
+  scores: Record<string, number>
+  rationale: string
+  weightedScore: number
+  passed: boolean
+}
+
+/** A judge verdict retained in structured form so reports can compare it. */
+export interface JudgeVerdict extends JudgeScore {
+  identity: JudgeIdentity
+}
+
 /** A single agent behavior case. */
 export interface AgentToolUseScenario {
   id: string
@@ -125,6 +146,8 @@ export interface EvalToolInvocation {
   success: boolean
   error?: string
   durationMs: number
+  /** The tool's returned output, so a judge can verify grounding. */
+  result?: unknown
 }
 
 /** Measured properties of one completed run. */
@@ -163,5 +186,7 @@ export interface AgentToolUseResult {
   finalContent: string
   toolInvocations: EvalToolInvocation[]
   metrics: AgentToolUseMetrics
+  /** Structured judge verdict, present when a judge ran. */
+  judge?: JudgeVerdict
   error?: string
 }