diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-evaluate.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-evaluate.mjs new file mode 100644 index 000000000000..bae1adf4da11 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/instrument-evaluate.mjs @@ -0,0 +1,11 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + // `NO_RECORDING` turns off recording of inputs and outputs for the privacy test. + dataCollection: process.env.NO_RECORDING ? { genAI: { inputs: false, outputs: false } } : {}, + transport: loggingTransport, +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-evaluate.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-evaluate.mjs new file mode 100644 index 000000000000..42fc5ead26c7 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/scenario-evaluate.mjs @@ -0,0 +1,43 @@ +import * as Sentry from '@sentry/node'; +import { experimental_evaluate } from 'ai'; +import { Experimental_EvaluationMockModelV4 } from 'ai/test'; + +async function run() { + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + await experimental_evaluate({ + model: new Experimental_EvaluationMockModelV4({ + provider: 'gateway', + modelId: 'typesafe-ai/jev', + doEvaluate: async () => ({ + answers: { + authIssue: { type: 'boolean', probability: 0.97 }, + department: { + type: 'choice', + choice: 'billing', + probabilities: { billing: 0.64, technical: 0.36 }, + }, + wantsRefund: { type: 'boolean', probability: 0.99 }, + urgency: { type: 'score', score: 1.8, probabilities: { 0: 0, 1: 0.2, 2: 0.8 } }, + }, + usage: { inputTokens: 275, outputTokens: 20 }, + // What the AI SDK TypeSafe provider returns: answer confidence moves into provider metadata. + providerMetadata: { typesafe: { confidence: { department: 0.28 } } }, + warnings: [], + }), + }), + state: 'I cannot log in, and I also want a refund for last month.', + questions: { + authIssue: { type: 'boolean', instructions: 'Is there a login problem?' }, + department: { + type: 'choice', + instructions: 'Which team should handle this?', + criteria: { billing: 'Charges and refunds', technical: 'Bugs and outages' }, + }, + wantsRefund: { type: 'boolean', instructions: 'Is a refund requested?' }, + urgency: { type: 'score', instructions: 'How urgent is this ticket?', criteria: ['low', 'medium', 'high'] }, + }, + }); + }); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts index 2468f974af55..d0016ec52a19 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6_v7/test.ts @@ -4,6 +4,7 @@ import { GEN_AI_CONVERSATION_ID, GEN_AI_EMBEDDINGS_INPUT, GEN_AI_INPUT_MESSAGES, + GEN_AI_OPERATION_NAME, GEN_AI_OUTPUT_MESSAGES, GEN_AI_PROVIDER_NAME, GEN_AI_REQUEST_MODEL, @@ -997,3 +998,105 @@ describe.each(matrix)('Vercel AI integration (version %s)', (version, vercelAiVe }, ); }); + +describe('Vercel AI integration experimental_evaluate', () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + createEsmTests( + __dirname, + 'scenario-evaluate.mjs', + 'instrument-evaluate.mjs', + (createRunner, test) => { + test('creates an evaluate span', async () => { + await createRunner() + .unordered() + .expect({ + span: container => { + const evaluateSpan = container.items.find( + span => span.attributes['sentry.op']?.value === 'gen_ai.evaluate', + )!; + expect(evaluateSpan).toBeDefined(); + expect(evaluateSpan.name).toBe('evaluate typesafe-ai/jev'); + expect(evaluateSpan.status).toBe('ok'); + expect(evaluateSpan.attributes['sentry.origin']?.value).toBe('auto.vercelai.channel'); + expect(evaluateSpan.attributes[GEN_AI_OPERATION_NAME]?.value).toBe('evaluate'); + expect(evaluateSpan.attributes[GEN_AI_PROVIDER_NAME]?.value).toBe('gateway'); + expect(evaluateSpan.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('typesafe-ai/jev'); + expect(evaluateSpan.attributes[GEN_AI_RESPONSE_MODEL]?.value).toBe('typesafe-ai/jev'); + expect(evaluateSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(275); + expect(evaluateSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(20); + expect(evaluateSpan.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(295); + expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES]?.value as string)).toEqual([ + { + type: 'evaluation', + state: 'I cannot log in, and I also want a refund for last month.', + questions: { + authIssue: { type: 'boolean', instructions: 'Is there a login problem?' }, + department: { + type: 'choice', + instructions: 'Which team should handle this?', + criteria: { billing: 'Charges and refunds', technical: 'Bugs and outages' }, + }, + wantsRefund: { type: 'boolean', instructions: 'Is a refund requested?' }, + urgency: { + type: 'score', + instructions: 'How urgent is this ticket?', + criteria: ['low', 'medium', 'high'], + }, + }, + }, + ]); + expect(JSON.parse(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]?.value as string)).toEqual([ + { + type: 'evaluation', + answers: { + authIssue: { type: 'boolean', probability: 0.97 }, + department: { + type: 'choice', + choice: 'billing', + probabilities: { billing: 0.64, technical: 0.36 }, + confidence: 0.28, + }, + wantsRefund: { type: 'boolean', probability: 0.99 }, + urgency: { type: 'score', score: 1.8, probabilities: { 0: 0, 1: 0.2, 2: 0.8 } }, + }, + }, + ]); + }, + }) + .start() + .completed(); + }); + + test('does not record inputs or outputs when recording is off', async () => { + await createRunner() + .withEnv({ NO_RECORDING: 'true' }) + .unordered() + .expect({ + span: container => { + const evaluateSpan = container.items.find( + span => span.attributes['sentry.op']?.value === 'gen_ai.evaluate', + )!; + expect(evaluateSpan).toBeDefined(); + expect(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); + expect(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); + // State, questions and answers must not come back through another attribute. Only the attributes + // are checked (timestamps could match a number), and `probabilities` only occurs in answers. + expect(JSON.stringify(evaluateSpan.attributes)).not.toMatch( + /cannot log in|Charges and refunds|probabilities/, + ); + }, + }) + .start() + .completed(); + }); + }, + { + additionalDependencies: { + ai: '^7.0.111', + }, + }, + ); +}); diff --git a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts index 76b1ddda69fc..78b4763f0240 100644 --- a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts +++ b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts @@ -65,10 +65,14 @@ const AI_SDK_TELEMETRY_TRACING_CHANNEL = 'ai:telemetry'; const ORIGIN = 'auto.vercelai.channel'; +// Not yet in `@sentry/conventions`. +const GEN_AI_EVALUATE = 'gen_ai.evaluate'; + // `gen_ai.operation.name` values, keyed to the span op they map to. const GEN_AI_OPERATION_SPAN_OPS = { embeddings: GEN_AI_EMBEDDINGS, rerank: GEN_AI_RERANK, + evaluate: GEN_AI_EVALUATE, invoke_agent: GEN_AI_INVOKE_AGENT, execute_tool: GEN_AI_EXECUTE_TOOL, // The model-call op matches the Vercel AI OTel integration (`gen_ai.generate_content`) rather than @@ -211,7 +215,8 @@ export type ChannelEventType = | 'executeTool' | 'embed' | 'embedMany' - | 'rerank'; + | 'rerank' + | 'experimental_evaluate'; /** * The context object the AI SDK passes through one tracing-channel call. It is the same object @@ -444,6 +449,17 @@ export function createSpanFromMessage( } case 'rerank': return startGenAiSpan('rerank', modelId, baseAttributes); + case 'experimental_evaluate': + return startGenAiSpan('evaluate', modelId, { + ...baseAttributes, + ...(recordInputs + ? { + [GEN_AI_INPUT_MESSAGES]: stringify([ + { type: 'evaluation', state: event.state, questions: event.questions }, + ]), + } + : {}), + }); default: // Unknown event type: opt out rather than open a span we can't shape correctly. return undefined; @@ -608,19 +624,52 @@ export function enrichSpanOnEnd( span.setAttributes(providerAttributes); if (recordOutputs) { - // `languageModelCall` exposes the response as a `content` parts array; top-level results expose - // `text` + `toolCalls`. Both normalize into the OTel `gen_ai.output.messages` assistant message. - const parts = - type === 'languageModelCall' && Array.isArray(result.content) - ? partsFromContent(result.content) - : partsFromTextAndToolCalls(result.text, result.toolCalls); - const outputMessages = buildOutputMessages(parts, finishReason); + const outputMessages = getOutputMessages(type, result, finishReason); if (outputMessages) { span.setAttribute(GEN_AI_OUTPUT_MESSAGES, outputMessages); } } } +function getOutputMessages( + type: ChannelEventType, + result: Record, + finishReason: string | undefined, +): string | undefined { + if (type === 'experimental_evaluate') { + return stringify([{ type: 'evaluation', answers: withProviderConfidence(result) }]); + } + // `languageModelCall` exposes the response as a `content` parts array; top-level results expose + // `text` + `toolCalls`. Both normalize into the OTel `gen_ai.output.messages` assistant message. + const parts = + type === 'languageModelCall' && Array.isArray(result.content) + ? partsFromContent(result.content) + : partsFromTextAndToolCalls(result.text, result.toolCalls); + return buildOutputMessages(parts, finishReason); +} + +/** + * The AI SDK TypeSafe provider moves each answer's `confidence` out of the answers into + * `providerMetadata.typesafe.confidence` (keyed by question id). Put it back so evaluate answers keep it. + */ +function withProviderConfidence(result: Record): unknown { + const { answers, providerMetadata } = result; + const typesafe = isObjectLike(providerMetadata) ? providerMetadata.typesafe : undefined; + const confidence = isObjectLike(typesafe) && isObjectLike(typesafe.confidence) ? typesafe.confidence : undefined; + if (!confidence || !isObjectLike(answers)) { + return answers; + } + + return Object.fromEntries( + Object.entries(answers).map(([id, answer]) => [ + id, + isObjectLike(answer) && typeof confidence[id] === 'number' && answer.confidence === undefined + ? { ...answer, confidence: confidence[id] } + : answer, + ]), + ); +} + /** Maps a Vercel AI finish reason to the OTel `gen_ai.output.messages` form (`tool-calls` → `tool_call`). */ function normalizeFinishReason(finishReason: string | undefined): string { return finishReason === 'tool-calls' ? 'tool_call' : (finishReason ?? 'stop');