fix(server-utils): Keep OpenRouter evaluation confidence and legend on Vercel AI spans (#25644)

Author: RulaKhaled

packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts

@@ -52,7 +52,7 @@ import {
 import type { TracingChannel } from 'node:diagnostics_channel';
 import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../ai/core/gen-ai-attributes';
 import { isEveGenAiRecordingDefault } from './gen-ai-recording-mode';
-import type { GenAiOptions } from '../../ai/core/utils';
+import { getEvaluationOutputMessages, type GenAiOptions } from '../../ai/core/utils';
 import { getProviderMetadataAttributes, LAST_STEP_ONLY_USAGE_KEYS } from '../../ai/vercel-ai';
 import { WORKERS_AI_INTEGRATION_NAME } from '../../ai/workers-ai/constants';
 import { bindTracingChannelToSpan } from '../../tracing-channel';
@@ -689,7 +689,7 @@ function getOutputMessages(
   finishReason: string | undefined,
 ): string | undefined {
   if (type === 'experimental_evaluate' || type === 'experimental_decide') {
-    return stringify([{ type: 'evaluation', answers: withProviderConfidence(result) }]);
+    return getEvaluationOutputMessages(withProviderAnswerFields(result));
   }
   // `languageModelCall` exposes the response as a `content` parts array; top-level results expose
   // `text` + `toolCalls`. Both normalize into the OTel `gen_ai.output.messages` assistant message.
@@ -701,26 +701,36 @@ function getOutputMessages(
 }
 
 /**
- * The AI SDK TypeSafe and OpenAI providers move each answer's `confidence` out of the answers into
- * `providerMetadata.<provider>.confidence` (keyed by question id). Put it back so decide answers keep it.
+ * Providers move answer fields into `providerMetadata`: TypeSafe and OpenAI as `<provider>.confidence[id]`,
+ * OpenRouter as `openrouter.answers[id].{confidence,legend}`. Put them back on the answers.
  */
-function withProviderConfidence(result: Record<string, unknown>): unknown {
+function withProviderAnswerFields(result: Record<string, unknown>): unknown {
   const { answers, providerMetadata } = result;
-  const metadata = isObjectLike(providerMetadata)
-    ? [providerMetadata.typesafe, providerMetadata.openai].find(m => isObjectLike(m) && isObjectLike(m.confidence))
-    : undefined;
-  const confidence = isObjectLike(metadata) && isObjectLike(metadata.confidence) ? metadata.confidence : undefined;
-  if (!confidence || !isObjectLike(answers)) {
+  if (!isObjectLike(answers) || !isObjectLike(providerMetadata)) {
+    return answers;
+  }
+  const confidenceById = [providerMetadata.typesafe, providerMetadata.openai]
+    .map(metadata => (isObjectLike(metadata) ? metadata.confidence : undefined))
+    .find(isObjectLike);
+  const { openrouter } = providerMetadata;
+  const openrouterById = isObjectLike(openrouter) && isObjectLike(openrouter.answers) ? openrouter.answers : undefined;
+  if (!confidenceById && !openrouterById) {
     return answers;
   }
 
   return Object.fromEntries(
-    Object.entries(answers).map(([id, answer]) => [
-      id,
-      isObjectLike(answer) && typeof confidence[id] === 'number' && answer.confidence === undefined
-        ? { ...answer, confidence: confidence[id] }
-        : answer,
-    ]),
+    Object.entries(answers).map(([id, answer]) => {
+      if (!isObjectLike(answer)) {
+        return [id, answer];
+      }
+      const openrouterAnswer = openrouterById?.[id];
+      const fields = isObjectLike(openrouterAnswer) ? openrouterAnswer : {};
+      const {
+        confidence = asNumber(confidenceById?.[id]) ?? asNumber(fields.confidence),
+        legend = isObjectLike(fields.legend) ? fields.legend : undefined,
+      } = answer;
+      return [id, { ...answer, confidence, legend }];
+    }),
   );
 }
 

packages/server-utils/test/integrations/vercel-ai/decide.test.ts

@@ -29,6 +29,27 @@ describe('Vercel AI decide spans', () => {
     getMainCarrier().__SENTRY__ = undefined;
   });
 
+  function recordedAnswers(provider: string, result: Message['result']): unknown {
+    const message = {
+      type: 'experimental_decide',
+      event: {
+        provider,
+        modelId: 'typesafe/jev-1.13',
+        state: 'I was charged twice and the app keeps crashing',
+        questions: {},
+        recordInputs: true,
+        recordOutputs: true,
+      },
+      result,
+    } satisfies Message;
+
+    const span = createSpanFromMessage(message, {} as Parameters<typeof createSpanFromMessage>[1])!;
+    enrichSpanOnEnd(span, message, {} as Parameters<typeof enrichSpanOnEnd>[2]);
+    span.end();
+
+    return JSON.parse(spanToStaticSpanJSON(endedSpans[0]!).data?.[GEN_AI_OUTPUT_MESSAGES] as string)[0].answers;
+  }
+
   // `ai` 7.0.128 renamed the channel event from `experimental_evaluate` to `experimental_decide`.
   it.each(['experimental_evaluate', 'experimental_decide'] as const)('creates a decide span for `%s`', type => {
     const message = {
@@ -58,29 +79,80 @@ describe('Vercel AI decide spans', () => {
   });
 
   it.each(['typesafe', 'openai'])('restores answer confidence from `providerMetadata.%s`', provider => {
-    const message = {
-      type: 'experimental_decide',
-      event: {
-        provider,
-        modelId: 'gpt-6-luna',
-        state: 'I was charged twice',
-        questions: { department: { type: 'choice', criteria: { billing: 'Charges', support: 'Other' } } },
-        recordInputs: true,
-        recordOutputs: true,
-      },
-      result: {
-        answers: { department: { type: 'choice', choice: 'billing' } },
-        providerMetadata: { [provider]: { confidence: { department: 0.91 } } },
-      },
-    } satisfies Message;
+    const answers = recordedAnswers(provider, {
+      answers: { department: { type: 'choice', choice: 'billing' } },
+      providerMetadata: { [provider]: { confidence: { department: 0.91 } } },
+    });
 
-    const span = createSpanFromMessage(message, {} as Parameters<typeof createSpanFromMessage>[1])!;
-    enrichSpanOnEnd(span, message, {} as Parameters<typeof enrichSpanOnEnd>[2]);
-    span.end();
+    expect(answers).toEqual({ department: { type: 'choice', choice: 'billing', confidence: 0.91 } });
+  });
 
-    const json = spanToStaticSpanJSON(endedSpans[0]!);
-    expect(JSON.parse(json.data?.[GEN_AI_OUTPUT_MESSAGES] as string)).toEqual([
-      { type: 'evaluation', answers: { department: { type: 'choice', choice: 'billing', confidence: 0.91 } } },
-    ]);
+  describe('OpenRouter evaluation models', () => {
+    it('restores choice confidence from `providerMetadata.openrouter.answers`', () => {
+      const answers = recordedAnswers('openrouter', {
+        answers: {
+          department: { type: 'choice', choice: 'billing', probabilities: { billing: 0.82, technical: 0.18 } },
+          wantsRefund: { type: 'boolean', probability: 0.97 },
+        },
+        providerMetadata: {
+          openrouter: { provider: 'TypeSafe', answers: { department: { confidence: 0.64 }, wantsRefund: {} } },
+        },
+      });
+
+      expect(answers).toEqual({
+        department: {
+          type: 'choice',
+          choice: 'billing',
+          probabilities: { billing: 0.82, technical: 0.18 },
+          confidence: 0.64,
+        },
+        wantsRefund: { type: 'boolean', probability: 0.97 },
+      });
+    });
+
+    it('restores score confidence and legend from `providerMetadata.openrouter.answers`', () => {
+      const legend = { '0': 'Not urgent', '1': 'Handle today', '2': 'Page on-call' };
+
+      const answers = recordedAnswers('openrouter', {
+        answers: { urgency: { type: 'score', score: 1.8, probabilities: { '0': 0, '1': 0.2, '2': 0.8 } } },
+        providerMetadata: { openrouter: { answers: { urgency: { confidence: 0.31, legend } } } },
+      });
+
+      expect(answers).toEqual({
+        urgency: {
+          type: 'score',
+          score: 1.8,
+          probabilities: { '0': 0, '1': 0.2, '2': 0.8 },
+          confidence: 0.31,
+          legend,
+        },
+      });
+    });
+
+    it.each([
+      ['no provider metadata', undefined],
+      ['no `answers` entry', { openrouter: { provider: 'TypeSafe', usage: { cost: 0.0004 } } }],
+      ['no entry for the question', { openrouter: { answers: { department: { confidence: 0.64 } } } }],
+    ])('leaves answers unchanged with %s', (_, providerMetadata) => {
+      const answers = recordedAnswers('openrouter', {
+        answers: { urgency: { type: 'score', score: 1.8 } },
+        providerMetadata,
+      });
+
+      expect(answers).toEqual({ urgency: { type: 'score', score: 1.8 } });
+    });
+
+    it('keeps confidence and legend already on the answer', () => {
+      const answers = recordedAnswers('openrouter', {
+        answers: { urgency: { type: 'score', score: 1.8, confidence: 0.9, legend: { '2': 'Page on-call' } } },
+        providerMetadata: {
+          openrouter: { answers: { urgency: { confidence: 0.31, legend: { '2': 'Escalate' } } } },
+        },
+      });
+
+      expect(answers).toEqual({
+        urgency: { type: 'score', score: 1.8, confidence: 0.9, legend: { '2': 'Page on-call' } },
+      });
+    });
   });
 });