From 91d6874b616a4d9c253c219d3eec256ef3b6a419 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Thu, 24 Sep 2026 13:53:44 +0200 Subject: [PATCH 1/5] feat(server-utils): Instrument Vercel AI experimental_evaluate Co-Authored-By: Claude Opus 5.5 --- .../vercelai/v6_v7/instrument-evaluate.mjs | 10 +++ .../vercelai/v6_v7/scenario-evaluate.mjs | 31 +++++++++ .../suites/tracing/vercelai/v6_v7/test.ts | 68 +++++++++++++++++++ .../vercel-ai/vercel-ai-dc-subscriber.ts | 43 +++++++++--- 4 files changed, 144 insertions(+), 8 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-evaluate.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-evaluate.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-evaluate.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-evaluate.mjs new file mode 100644 index 000000000000..852786895514 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-evaluate.mjs @@ -0,0 +1,10 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + dataCollection: {}, + transport: loggingTransport, +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-evaluate.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-evaluate.mjs new file mode 100644 index 000000000000..3016d3fbc71a --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-evaluate.mjs @@ -0,0 +1,31 @@ +import * as Sentry from '@sentry/node'; +import { experimental_evaluate } from 'ai'; +import { Experimental_EvaluationMockModelV4 } from 'ai/test'; + +async function run() { + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + await experimental_evaluate({ + model: new Experimental_EvaluationMockModelV4({ + provider: 'gateway', + modelId: 'typesafe-ai/jev', + doEvaluate: async () => ({ + answers: { + authIssue: { type: 'boolean', probability: 0.97 }, + wantsRefund: { type: 'boolean', probability: 0.99 }, + urgency: { type: 'score', score: 1.8, probabilities: { 0: 0, 1: 0.2, 2: 0.8 } }, + }, + usage: { inputTokens: 275, outputTokens: 20 }, + warnings: [], + }), + }), + state: 'I cannot log in, and I also want a refund for last month.', + questions: { + authIssue: { type: 'boolean', instructions: 'Is there a login problem?' }, + wantsRefund: { type: 'boolean', instructions: 'Is a refund requested?' }, + urgency: { type: 'score', instructions: 'How urgent is this ticket?', criteria: ['low', 'medium', 'high'] }, + }, + }); + }); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts index 2468f974af55..d4c890579a4e 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts @@ -997,3 +997,71 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe }, ); }); + +describe('Vercel AI integration experimental_evaluate', () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + createEsmTests( + __dirname, + 'scenario-evaluate.mjs', + 'instrument-evaluate.mjs', + (createRunner, test) => { + test('creates an evaluate span', async () => { + await createRunner() + .unordered() + .expect({ + span: container => { + const evaluateSpan = container.items.find( + span => span.attributes['sentry.op']?.value === 'gen_ai.evaluate', + )!; + expect(evaluateSpan).toBeDefined(); + expect(evaluateSpan.name).toBe('evaluate typesafe-ai/jev'); + expect(evaluateSpan.status).toBe('ok'); + expect(evaluateSpan.attributes['sentry.origin']?.value).toBe('auto.vercelai.channel'); + expect(evaluateSpan.attributes['gen_ai.operation.name']?.value).toBe('evaluate'); + expect(evaluateSpan.attributes[GEN_AI_PROVIDER_NAME]?.value).toBe('gateway'); + expect(evaluateSpan.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('typesafe-ai/jev'); + expect(evaluateSpan.attributes[GEN_AI_RESPONSE_MODEL]?.value).toBe('typesafe-ai/jev'); + expect(evaluateSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(275); + expect(evaluateSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(20); + expect(evaluateSpan.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(295); + expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES]?.value as string)).toEqual([ + { + type: 'evaluation', + state: 'I cannot log in, and I also want a refund for last month.', + questions: { + authIssue: { type: 'boolean', instructions: 'Is there a login problem?' }, + wantsRefund: { type: 'boolean', instructions: 'Is a refund requested?' }, + urgency: { + type: 'score', + instructions: 'How urgent is this ticket?', + criteria: ['low', 'medium', 'high'], + }, + }, + }, + ]); + expect(JSON.parse(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]?.value as string)).toEqual([ + { + type: 'evaluation', + answers: { + authIssue: { type: 'boolean', probability: 0.97 }, + wantsRefund: { type: 'boolean', probability: 0.99 }, + urgency: { type: 'score', score: 1.8, probabilities: { 0: 0, 1: 0.2, 2: 0.8 } }, + }, + }, + ]); + }, + }) + .start() + .completed(); + }); + }, + { + additionalDependencies: { + ai: '^7.0.111', + }, + }, + ); +}); diff --git a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts index 76b1ddda69fc..fc9f12735b56 100644 --- a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts +++ b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts @@ -65,10 +65,14 @@ const AI_SDK_TELEMETRY_TRACING_CHANNEL = 'ai:telemetry'; const ORIGIN = 'auto.vercelai.channel'; +// Not yet in `@sentry/conventions`. +const GEN_AI_EVALUATE = 'gen_ai.evaluate'; + // `gen_ai.operation.name` values, keyed to the span op they map to. const GEN_AI_OPERATION_SPAN_OPS = { embeddings: GEN_AI_EMBEDDINGS, rerank: GEN_AI_RERANK, + evaluate: GEN_AI_EVALUATE, invoke_agent: GEN_AI_INVOKE_AGENT, execute_tool: GEN_AI_EXECUTE_TOOL, // The model-call op matches the Vercel AI OTel integration (`gen_ai.generate_content`) rather than @@ -211,7 +215,8 @@ export type ChannelEventType = | 'executeTool' | 'embed' | 'embedMany' - | 'rerank'; + | 'rerank' + | 'experimental_evaluate'; /** * The context object the AI SDK passes through one tracing-channel call. It is the same object @@ -444,6 +449,17 @@ export function createSpanFromMessage( } case 'rerank': return startGenAiSpan('rerank', modelId, baseAttributes); + case 'experimental_evaluate': + return startGenAiSpan('evaluate', modelId, { + ...baseAttributes, + ...(recordInputs + ? { + [GEN_AI_INPUT_MESSAGES]: stringify([ + { type: 'evaluation', state: event.state, questions: event.questions }, + ]), + } + : {}), + }); default: // Unknown event type: opt out rather than open a span we can't shape correctly. return undefined; @@ -608,19 +624,30 @@ export function enrichSpanOnEnd( span.setAttributes(providerAttributes); if (recordOutputs) { - // `languageModelCall` exposes the response as a `content` parts array; top-level results expose - // `text` + `toolCalls`. Both normalize into the OTel `gen_ai.output.messages` assistant message. - const parts = - type === 'languageModelCall' && Array.isArray(result.content) - ? partsFromContent(result.content) - : partsFromTextAndToolCalls(result.text, result.toolCalls); - const outputMessages = buildOutputMessages(parts, finishReason); + const outputMessages = getOutputMessages(type, result, finishReason); if (outputMessages) { span.setAttribute(GEN_AI_OUTPUT_MESSAGES, outputMessages); } } } +function getOutputMessages( + type: ChannelEventType, + result: Record, + finishReason: string | undefined, +): string | undefined { + if (type === 'experimental_evaluate') { + return stringify([{ type: 'evaluation', answers: result.answers }]); + } + // `languageModelCall` exposes the response as a `content` parts array; top-level results expose + // `text` + `toolCalls`. Both normalize into the OTel `gen_ai.output.messages` assistant message. + const parts = + type === 'languageModelCall' && Array.isArray(result.content) + ? partsFromContent(result.content) + : partsFromTextAndToolCalls(result.text, result.toolCalls); + return buildOutputMessages(parts, finishReason); +} + /** Maps a Vercel AI finish reason to the OTel `gen_ai.output.messages` form (`tool-calls` → `tool_call`). */ function normalizeFinishReason(finishReason: string | undefined): string { return finishReason === 'tool-calls' ? 'tool_call' : (finishReason ?? 'stop'); From 7086acf39ee1aefdc1b1da4485c0c191bf35fb14 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Thu, 24 Sep 2026 16:05:24 +0200 Subject: [PATCH 2/5] Use GEN_AI_OPERATION_NAME from conventions in evaluate test --- .../suites/tracing/vercelai/v6_v7/test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts index d4c890579a4e..df740785564e 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts @@ -4,6 +4,7 @@ import { GEN_AI_CONVERSATION_ID, GEN_AI_EMBEDDINGS_INPUT, GEN_AI_INPUT_MESSAGES, + GEN_AI_OPERATION_NAME, GEN_AI_OUTPUT_MESSAGES, GEN_AI_PROVIDER_NAME, GEN_AI_REQUEST_MODEL, @@ -1020,7 +1021,7 @@ describe('Vercel AI integration experimental_evaluate', () => { expect(evaluateSpan.name).toBe('evaluate typesafe-ai/jev'); expect(evaluateSpan.status).toBe('ok'); expect(evaluateSpan.attributes['sentry.origin']?.value).toBe('auto.vercelai.channel'); - expect(evaluateSpan.attributes['gen_ai.operation.name']?.value).toBe('evaluate'); + expect(evaluateSpan.attributes[GEN_AI_OPERATION_NAME]?.value).toBe('evaluate'); expect(evaluateSpan.attributes[GEN_AI_PROVIDER_NAME]?.value).toBe('gateway'); expect(evaluateSpan.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('typesafe-ai/jev'); expect(evaluateSpan.attributes[GEN_AI_RESPONSE_MODEL]?.value).toBe('typesafe-ai/jev'); From e0ff82eba8fbbab8ed23727633aed66d5568583c Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Fri, 25 Sep 2026 09:15:30 +0200 Subject: [PATCH 3/5] Cover lossless answers and disabled recording in evaluate tests --- .../vercelai/v6_v7/instrument-evaluate.mjs | 3 +- .../vercelai/v6_v7/scenario-evaluate.mjs | 10 +++++++ .../suites/tracing/vercelai/v6_v7/test.ts | 30 +++++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-evaluate.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-evaluate.mjs index 852786895514..bae1adf4da11 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-evaluate.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-evaluate.mjs @@ -5,6 +5,7 @@ Sentry.init({ dsn: 'https://public@dsn.ingest.sentry.io/1337', release: '1.0', tracesSampleRate: 1.0, - dataCollection: {}, + // `NO_RECORDING` turns off recording of inputs and outputs for the privacy test. + dataCollection: process.env.NO_RECORDING ? { genAI: { inputs: false, outputs: false } } : {}, transport: loggingTransport, }); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-evaluate.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-evaluate.mjs index 3016d3fbc71a..bedce326e758 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-evaluate.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-evaluate.mjs @@ -11,6 +11,11 @@ async function run() { doEvaluate: async () => ({ answers: { authIssue: { type: 'boolean', probability: 0.97 }, + department: { + type: 'choice', + choice: 'billing', + probabilities: { billing: 0.64, technical: 0.36 }, + }, wantsRefund: { type: 'boolean', probability: 0.99 }, urgency: { type: 'score', score: 1.8, probabilities: { 0: 0, 1: 0.2, 2: 0.8 } }, }, @@ -21,6 +26,11 @@ async function run() { state: 'I cannot log in, and I also want a refund for last month.', questions: { authIssue: { type: 'boolean', instructions: 'Is there a login problem?' }, + department: { + type: 'choice', + instructions: 'Which team should handle this?', + criteria: { billing: 'Charges and refunds', technical: 'Bugs and outages' }, + }, wantsRefund: { type: 'boolean', instructions: 'Is a refund requested?' }, urgency: { type: 'score', instructions: 'How urgent is this ticket?', criteria: ['low', 'medium', 'high'] }, }, diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts index df740785564e..54b2067006ce 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts @@ -1034,6 +1034,11 @@ describe('Vercel AI integration experimental_evaluate', () => { state: 'I cannot log in, and I also want a refund for last month.', questions: { authIssue: { type: 'boolean', instructions: 'Is there a login problem?' }, + department: { + type: 'choice', + instructions: 'Which team should handle this?', + criteria: { billing: 'Charges and refunds', technical: 'Bugs and outages' }, + }, wantsRefund: { type: 'boolean', instructions: 'Is a refund requested?' }, urgency: { type: 'score', @@ -1048,6 +1053,11 @@ describe('Vercel AI integration experimental_evaluate', () => { type: 'evaluation', answers: { authIssue: { type: 'boolean', probability: 0.97 }, + department: { + type: 'choice', + choice: 'billing', + probabilities: { billing: 0.64, technical: 0.36 }, + }, wantsRefund: { type: 'boolean', probability: 0.99 }, urgency: { type: 'score', score: 1.8, probabilities: { 0: 0, 1: 0.2, 2: 0.8 } }, }, @@ -1058,6 +1068,26 @@ describe('Vercel AI integration experimental_evaluate', () => { .start() .completed(); }); + + test('does not record inputs or outputs when recording is off', async () => { + await createRunner() + .withEnv({ NO_RECORDING: 'true' }) + .unordered() + .expect({ + span: container => { + const evaluateSpan = container.items.find( + span => span.attributes['sentry.op']?.value === 'gen_ai.evaluate', + )!; + expect(evaluateSpan).toBeDefined(); + expect(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); + expect(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); + // State, questions and answers must not come back through another attribute. + expect(JSON.stringify(evaluateSpan)).not.toMatch(/cannot log in|Charges and refunds|0\.97/); + }, + }) + .start() + .completed(); + }); }, { additionalDependencies: { From e9fa86caedda17bc7ecc282b9def38955e7d7dd0 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Fri, 25 Sep 2026 10:09:48 +0200 Subject: [PATCH 4/5] Restore TypeSafe answer confidence from provider metadata --- .../vercelai/v6_v7/scenario-evaluate.mjs | 2 ++ .../suites/tracing/vercelai/v6_v7/test.ts | 1 + .../vercel-ai/vercel-ai-dc-subscriber.ts | 24 ++++++++++++++++++- 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-evaluate.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-evaluate.mjs index bedce326e758..42fc5ead26c7 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-evaluate.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-evaluate.mjs @@ -20,6 +20,8 @@ async function run() { urgency: { type: 'score', score: 1.8, probabilities: { 0: 0, 1: 0.2, 2: 0.8 } }, }, usage: { inputTokens: 275, outputTokens: 20 }, + // What the AI SDK TypeSafe provider returns: answer confidence moves into provider metadata. + providerMetadata: { typesafe: { confidence: { department: 0.28 } } }, warnings: [], }), }), diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts index 54b2067006ce..c2ae8880e688 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts @@ -1057,6 +1057,7 @@ describe('Vercel AI integration experimental_evaluate', () => { type: 'choice', choice: 'billing', probabilities: { billing: 0.64, technical: 0.36 }, + confidence: 0.28, }, wantsRefund: { type: 'boolean', probability: 0.99 }, urgency: { type: 'score', score: 1.8, probabilities: { 0: 0, 1: 0.2, 2: 0.8 } }, diff --git a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts index fc9f12735b56..78b4763f0240 100644 --- a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts +++ b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts @@ -637,7 +637,7 @@ function getOutputMessages( finishReason: string | undefined, ): string | undefined { if (type === 'experimental_evaluate') { - return stringify([{ type: 'evaluation', answers: result.answers }]); + return stringify([{ type: 'evaluation', answers: withProviderConfidence(result) }]); } // `languageModelCall` exposes the response as a `content` parts array; top-level results expose // `text` + `toolCalls`. Both normalize into the OTel `gen_ai.output.messages` assistant message. @@ -648,6 +648,28 @@ function getOutputMessages( return buildOutputMessages(parts, finishReason); } +/** + * The AI SDK TypeSafe provider moves each answer's `confidence` out of the answers into + * `providerMetadata.typesafe.confidence` (keyed by question id). Put it back so evaluate answers keep it. + */ +function withProviderConfidence(result: Record): unknown { + const { answers, providerMetadata } = result; + const typesafe = isObjectLike(providerMetadata) ? providerMetadata.typesafe : undefined; + const confidence = isObjectLike(typesafe) && isObjectLike(typesafe.confidence) ? typesafe.confidence : undefined; + if (!confidence || !isObjectLike(answers)) { + return answers; + } + + return Object.fromEntries( + Object.entries(answers).map(([id, answer]) => [ + id, + isObjectLike(answer) && typeof confidence[id] === 'number' && answer.confidence === undefined + ? { ...answer, confidence: confidence[id] } + : answer, + ]), + ); +} + /** Maps a Vercel AI finish reason to the OTel `gen_ai.output.messages` form (`tool-calls` → `tool_call`). */ function normalizeFinishReason(finishReason: string | undefined): string { return finishReason === 'tool-calls' ? 'tool_call' : (finishReason ?? 'stop'); From 5c2d6cff53039317e9dfddfdc474a87489026599 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Fri, 25 Sep 2026 10:26:39 +0200 Subject: [PATCH 5/5] Check only span attributes in evaluate privacy tests --- .../suites/tracing/vercelai/v6_v7/test.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts index c2ae8880e688..d0016ec52a19 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts @@ -1082,8 +1082,11 @@ describe('Vercel AI integration experimental_evaluate', () => { expect(evaluateSpan).toBeDefined(); expect(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); expect(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); - // State, questions and answers must not come back through another attribute. - expect(JSON.stringify(evaluateSpan)).not.toMatch(/cannot log in|Charges and refunds|0\.97/); + // State, questions and answers must not come back through another attribute. Only the attributes + // are checked (timestamps could match a number), and `probabilities` only occurs in answers. + expect(JSON.stringify(evaluateSpan.attributes)).not.toMatch( + /cannot log in|Charges and refunds|probabilities/, + ); }, }) .start()