feat(server-utils): Instrument Vercel AI experimental_evaluate

dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-evaluate.mjs

@@ -0,0 +1,11 @@
+import * as Sentry from '@sentry/node';
+import { loggingTransport } from '@sentry-internal/node-integration-tests';
+
+Sentry.init({
+  dsn: 'https://public@dsn.ingest.sentry.io/1337',
+  release: '1.0',
+  tracesSampleRate: 1.0,
+  // `NO_RECORDING` turns off recording of inputs and outputs for the privacy test.
+  dataCollection: process.env.NO_RECORDING ? { genAI: { inputs: false, outputs: false } } : {},
+  transport: loggingTransport,
+});

dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-evaluate.mjs

@@ -0,0 +1,43 @@
+import * as Sentry from '@sentry/node';
+import { experimental_evaluate } from 'ai';
+import { Experimental_EvaluationMockModelV4 } from 'ai/test';
+
+async function run() {
+  await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
+    await experimental_evaluate({
+      model: new Experimental_EvaluationMockModelV4({
+        provider: 'gateway',
+        modelId: 'typesafe-ai/jev',
+        doEvaluate: async () => ({
+          answers: {
+            authIssue: { type: 'boolean', probability: 0.97 },
+            department: {
+              type: 'choice',
+              choice: 'billing',
+              probabilities: { billing: 0.64, technical: 0.36 },
+            },
+            wantsRefund: { type: 'boolean', probability: 0.99 },
+            urgency: { type: 'score', score: 1.8, probabilities: { 0: 0, 1: 0.2, 2: 0.8 } },
+          },
+          usage: { inputTokens: 275, outputTokens: 20 },
+          // What the AI SDK TypeSafe provider returns: answer confidence moves into provider metadata.
+          providerMetadata: { typesafe: { confidence: { department: 0.28 } } },
+          warnings: [],
+        }),
+      }),
+      state: 'I cannot log in, and I also want a refund for last month.',
+      questions: {
+        authIssue: { type: 'boolean', instructions: 'Is there a login problem?' },
+        department: {
+          type: 'choice',
+          instructions: 'Which team should handle this?',
+          criteria: { billing: 'Charges and refunds', technical: 'Bugs and outages' },
+        },
+        wantsRefund: { type: 'boolean', instructions: 'Is a refund requested?' },
+        urgency: { type: 'score', instructions: 'How urgent is this ticket?', criteria: ['low', 'medium', 'high'] },
+      },
+    });
+  });
+}
+
+run();

dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts

@@ -4,6 +4,7 @@ import {
   GEN_AI_CONVERSATION_ID,
   GEN_AI_EMBEDDINGS_INPUT,
   GEN_AI_INPUT_MESSAGES,
+  GEN_AI_OPERATION_NAME,
   GEN_AI_OUTPUT_MESSAGES,
   GEN_AI_PROVIDER_NAME,
   GEN_AI_REQUEST_MODEL,
@@ -997,3 +998,105 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe
     },
   );
 });
+
+describe('Vercel AI integration experimental_evaluate', () => {
+  afterAll(() => {
+    cleanupChildProcesses();
+  });
+
+  createEsmTests(
+    __dirname,
+    'scenario-evaluate.mjs',
+    'instrument-evaluate.mjs',
+    (createRunner, test) => {
+      test('creates an evaluate span', async () => {
+        await createRunner()
+          .unordered()
+          .expect({
+            span: container => {
+              const evaluateSpan = container.items.find(
+                span => span.attributes['sentry.op']?.value === 'gen_ai.evaluate',
+              )!;
+              expect(evaluateSpan).toBeDefined();
+              expect(evaluateSpan.name).toBe('evaluate typesafe-ai/jev');
+              expect(evaluateSpan.status).toBe('ok');
+              expect(evaluateSpan.attributes['sentry.origin']?.value).toBe('auto.vercelai.channel');
+              expect(evaluateSpan.attributes[GEN_AI_OPERATION_NAME]?.value).toBe('evaluate');
+              expect(evaluateSpan.attributes[GEN_AI_PROVIDER_NAME]?.value).toBe('gateway');
+              expect(evaluateSpan.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('typesafe-ai/jev');
+              expect(evaluateSpan.attributes[GEN_AI_RESPONSE_MODEL]?.value).toBe('typesafe-ai/jev');
+              expect(evaluateSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(275);
+              expect(evaluateSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(20);
+              expect(evaluateSpan.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(295);
+              expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES]?.value as string)).toEqual([
+                {
+                  type: 'evaluation',
+                  state: 'I cannot log in, and I also want a refund for last month.',
+                  questions: {
+                    authIssue: { type: 'boolean', instructions: 'Is there a login problem?' },
+                    department: {
+                      type: 'choice',
+                      instructions: 'Which team should handle this?',
+                      criteria: { billing: 'Charges and refunds', technical: 'Bugs and outages' },
+                    },
+                    wantsRefund: { type: 'boolean', instructions: 'Is a refund requested?' },
+                    urgency: {
+                      type: 'score',
+                      instructions: 'How urgent is this ticket?',
+                      criteria: ['low', 'medium', 'high'],
+                    },
+                  },
+                },
+              ]);
+              expect(JSON.parse(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]?.value as string)).toEqual([
+                {
+                  type: 'evaluation',
+                  answers: {
+                    authIssue: { type: 'boolean', probability: 0.97 },
+                    department: {
+                      type: 'choice',
+                      choice: 'billing',
+                      probabilities: { billing: 0.64, technical: 0.36 },
+                      confidence: 0.28,
+                    },
+                    wantsRefund: { type: 'boolean', probability: 0.99 },
+                    urgency: { type: 'score', score: 1.8, probabilities: { 0: 0, 1: 0.2, 2: 0.8 } },
+                  },
+                },
+              ]);
+            },
+          })
+          .start()
+          .completed();
+      });
+
+      test('does not record inputs or outputs when recording is off', async () => {
+        await createRunner()
+          .withEnv({ NO_RECORDING: 'true' })
+          .unordered()
+          .expect({
+            span: container => {
+              const evaluateSpan = container.items.find(
+                span => span.attributes['sentry.op']?.value === 'gen_ai.evaluate',
+              )!;
+              expect(evaluateSpan).toBeDefined();
+              expect(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined();
+              expect(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined();
+              // State, questions and answers must not come back through another attribute. Only the attributes
+              // are checked (timestamps could match a number), and `probabilities` only occurs in answers.
+              expect(JSON.stringify(evaluateSpan.attributes)).not.toMatch(
+                /cannot log in|Charges and refunds|probabilities/,
+              );
+            },
+          })
+          .start()
+          .completed();
+      });
+    },
+    {
+      additionalDependencies: {
+        ai: '^7.0.111',
+      },
+    },
+  );
+});

packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts

@@ -65,10 +65,14 @@ const AI_SDK_TELEMETRY_TRACING_CHANNEL = 'ai:telemetry';
 
 const ORIGIN = 'auto.vercelai.channel';
 
+// Not yet in `@sentry/conventions`.
+const GEN_AI_EVALUATE = 'gen_ai.evaluate';
+
 // `gen_ai.operation.name` values, keyed to the span op they map to.
 const GEN_AI_OPERATION_SPAN_OPS = {
   embeddings: GEN_AI_EMBEDDINGS,
   rerank: GEN_AI_RERANK,
+  evaluate: GEN_AI_EVALUATE,
   invoke_agent: GEN_AI_INVOKE_AGENT,
   execute_tool: GEN_AI_EXECUTE_TOOL,
   // The model-call op matches the Vercel AI OTel integration (`gen_ai.generate_content`) rather than
@@ -211,7 +215,8 @@ export type ChannelEventType =
   | 'executeTool'
   | 'embed'
   | 'embedMany'
-  | 'rerank';
+  | 'rerank'
+  | 'experimental_evaluate';
 
 /**
  * The context object the AI SDK passes through one tracing-channel call. It is the same object
@@ -444,6 +449,17 @@ export function createSpanFromMessage(
     }
     case 'rerank':
       return startGenAiSpan('rerank', modelId, baseAttributes);
+    case 'experimental_evaluate':
+      return startGenAiSpan('evaluate', modelId, {
+        ...baseAttributes,
+        ...(recordInputs
+          ? {
+              [GEN_AI_INPUT_MESSAGES]: stringify([
+                { type: 'evaluation', state: event.state, questions: event.questions },
+              ]),
+            }
+          : {}),
+      });
     default:
       // Unknown event type: opt out rather than open a span we can't shape correctly.
       return undefined;
@@ -608,19 +624,52 @@ export function enrichSpanOnEnd(
   span.setAttributes(providerAttributes);
 
   if (recordOutputs) {
-    // `languageModelCall` exposes the response as a `content` parts array; top-level results expose
-    // `text` + `toolCalls`. Both normalize into the OTel `gen_ai.output.messages` assistant message.
-    const parts =
-      type === 'languageModelCall' && Array.isArray(result.content)
-        ? partsFromContent(result.content)
-        : partsFromTextAndToolCalls(result.text, result.toolCalls);
-    const outputMessages = buildOutputMessages(parts, finishReason);
+    const outputMessages = getOutputMessages(type, result, finishReason);
     if (outputMessages) {
       span.setAttribute(GEN_AI_OUTPUT_MESSAGES, outputMessages);
     }
   }
 }
 
+function getOutputMessages(
+  type: ChannelEventType,
+  result: Record<string, unknown>,
+  finishReason: string | undefined,
+): string | undefined {
+  if (type === 'experimental_evaluate') {
+    return stringify([{ type: 'evaluation', answers: withProviderConfidence(result) }]);
+  }
+  // `languageModelCall` exposes the response as a `content` parts array; top-level results expose
+  // `text` + `toolCalls`. Both normalize into the OTel `gen_ai.output.messages` assistant message.
+  const parts =
+    type === 'languageModelCall' && Array.isArray(result.content)
+      ? partsFromContent(result.content)
+      : partsFromTextAndToolCalls(result.text, result.toolCalls);
+  return buildOutputMessages(parts, finishReason);
+}
+
+/**
+ * The AI SDK TypeSafe provider moves each answer's `confidence` out of the answers into
+ * `providerMetadata.typesafe.confidence` (keyed by question id). Put it back so evaluate answers keep it.
+ */
+function withProviderConfidence(result: Record<string, unknown>): unknown {
+  const { answers, providerMetadata } = result;
+  const typesafe = isObjectLike(providerMetadata) ? providerMetadata.typesafe : undefined;
+  const confidence = isObjectLike(typesafe) && isObjectLike(typesafe.confidence) ? typesafe.confidence : undefined;
+  if (!confidence || !isObjectLike(answers)) {
+    return answers;
+  }
+
+  return Object.fromEntries(
+    Object.entries(answers).map(([id, answer]) => [
+      id,
+      isObjectLike(answer) && typeof confidence[id] === 'number' && answer.confidence === undefined
+        ? { ...answer, confidence: confidence[id] }
+        : answer,
+    ]),
+  );
+}
+
 /** Maps a Vercel AI finish reason to the OTel `gen_ai.output.messages` form (`tool-calls` → `tool_call`). */
 function normalizeFinishReason(finishReason: string | undefined): string {
   return finishReason === 'tool-calls' ? 'tool_call' : (finishReason ?? 'stop');