From aea4552b7bb372472e346371cd464d7e0452f347 Mon Sep 17 00:00:00 2001 From: isaacs Date: Thu, 24 Sep 2026 15:10:20 -0700 Subject: [PATCH] fix(server-utils): Record Vercel AI `experimental_telemetry.metadata` on gen_ai spans v10 got `vercel.ai.telemetry.metadata.` attributes from the AI SDK's own OTel spans. In v11, `ai` 4 to 6 go through the orchestrion subscriber, which never read `metadata`, so v11 dropped it. The orchestrion subscriber now passes the metadata to the channel subscriber, which sets it as span attributes when each span starts. The `invoke_agent`, `generate_content` and `embeddings` spans get the attributes, and `execute_tool` spans don't. This matches the AI SDK, which leaves metadata off its `ai.toolCall` spans. fixes JS-3800 fixes #24720 Re: #24706 Co-Authored-By: Claude Opus 5.5 (1M context) --- .../scenario-telemetry-metadata.mjs | 82 +++++++++++++++++++ .../vercelai/span-streaming-v6/test.ts | 38 +++++++++ .../vercel-ai/vercel-ai-dc-subscriber.ts | 21 ++++- .../vercel-ai-orchestrion-subscriber.ts | 8 ++ 4 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-telemetry-metadata.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-telemetry-metadata.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-telemetry-metadata.mjs new file mode 100644 index 000000000000..2b955a0fb972 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/scenario-telemetry-metadata.mjs @@ -0,0 +1,82 @@ +import * as Sentry from '@sentry/node'; +import { embed, embedMany, generateText, streamText } from 'ai'; +import { MockEmbeddingModelV3, MockLanguageModelV3, simulateReadableStream } from 'ai/test'; + +const usage = { + inputTokens: { total: 10, noCache: 10, cached: 0 }, + outputTokens: { total: 20, noCache: 20, cached: 0 }, + totalTokens: { total: 30, noCache: 30, cached: 0 }, +}; + +async function run() { + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + await generateText({ + experimental_telemetry: { + isEnabled: true, + functionId: 'support-chat', + metadata: { requestId: 'req_generate', tenantId: 'acme' }, + }, + model: new MockLanguageModelV3({ + doGenerate: async () => ({ + finishReason: { unified: 'stop', raw: 'stop' }, + usage, + content: [{ type: 'text', text: 'Hello!' }], + warnings: [], + }), + }), + prompt: 'Hi', + }); + + const result = streamText({ + experimental_telemetry: { + isEnabled: true, + functionId: 'support-stream', + metadata: { requestId: 'req_stream', tenantId: 'acme' }, + }, + model: new MockLanguageModelV3({ + doStream: async () => ({ + stream: simulateReadableStream({ + chunks: [ + { type: 'stream-start', warnings: [] }, + { type: 'text-start', id: '0' }, + { type: 'text-delta', id: '0', delta: 'Hello!' }, + { type: 'text-end', id: '0' }, + { type: 'finish', finishReason: { unified: 'stop', raw: 'stop' }, usage }, + ], + }), + }), + }), + prompt: 'Hi', + }); + for await (const _part of result.fullStream) { + void _part; + } + + await embed({ + experimental_telemetry: { isEnabled: true, metadata: { requestId: 'req_embed', tenantId: 'acme' } }, + model: new MockEmbeddingModelV3({ + doEmbed: async () => ({ embeddings: [[0.1, 0.2, 0.3]], usage: { tokens: 10 } }), + }), + value: 'Embed me', + }); + + await embedMany({ + experimental_telemetry: { isEnabled: true, metadata: { requestId: 'req_embed_many', tenantId: 'acme' } }, + model: new MockEmbeddingModelV3({ + maxEmbeddingsPerCall: 5, + doEmbed: async () => ({ + embeddings: [ + [0.1, 0.2, 0.3], + [0.4, 0.5, 0.6], + ], + usage: { tokens: 20 }, + }), + }), + values: ['First', 'Second'], + }); + }); + + await Sentry.flush(2000); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts index 939ba08e9746..205df0e6d6a3 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v6/test.ts @@ -304,6 +304,44 @@ describe('Vercel AI integration (streaming, v6)', () => { }, ); + function spanWithMetadata(name: string, op: string, requestId: string) { + return expect.objectContaining({ + name, + attributes: expect.objectContaining({ + 'vercel.ai.telemetry.metadata.requestId': attr(requestId), + 'vercel.ai.telemetry.metadata.tenantId': attr('acme'), + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: attr(op), + }), + }); + } + + const EXPECTED_SPANS_TELEMETRY_METADATA = { + items: expect.arrayContaining([ + spanWithMetadata('invoke_agent support-chat', 'gen_ai.invoke_agent', 'req_generate'), + spanWithMetadata('generate_content mock-model-id', 'gen_ai.generate_content', 'req_generate'), + spanWithMetadata('invoke_agent support-stream', 'gen_ai.invoke_agent', 'req_stream'), + spanWithMetadata('generate_content mock-model-id', 'gen_ai.generate_content', 'req_stream'), + spanWithMetadata('embeddings mock-model-id', 'gen_ai.embeddings', 'req_embed'), + spanWithMetadata('embeddings mock-model-id', 'gen_ai.embeddings', 'req_embed_many'), + ]), + }; + + createEsmAndCjsTests( + __dirname, + 'scenario-telemetry-metadata.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('records experimental_telemetry.metadata as span attributes', async () => { + await createRunner().expect({ span: EXPECTED_SPANS_TELEMETRY_METADATA }).start().completed(); + }); + }, + { + additionalDependencies: { + ai: '^6.0.0', + }, + }, + ); + createEsmAndCjsTests( __dirname, 'scenario-error-in-tool.mjs', diff --git a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts index 78b4763f0240..bd6dff667a57 100644 --- a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts +++ b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts @@ -33,7 +33,7 @@ import { GEN_AI_INVOKE_AGENT, GEN_AI_RERANK, } from '@sentry/conventions/op'; -import type { Span, SpanAttributes } from '@sentry/core'; +import type { Span, SpanAttributes, SpanAttributeValue } from '@sentry/core'; import { _INTERNAL_skipAiProviderWrapping, captureException, @@ -86,6 +86,7 @@ type GenAiOperation = keyof typeof GEN_AI_OPERATION_SPAN_OPS; const VERCEL_AI_OPERATION_ID_ATTRIBUTE = 'vercel.ai.operationId'; const VERCEL_AI_MODEL_PROVIDER_ATTRIBUTE = 'vercel.ai.model.provider'; const VERCEL_AI_SETTINGS_MAX_RETRIES_ATTRIBUTE = 'vercel.ai.settings.maxRetries'; +const VERCEL_AI_TELEMETRY_METADATA_ATTRIBUTE_PREFIX = 'vercel.ai.telemetry.metadata.'; // Tracks the top-level operationId (and whether it streams) per `callId` so a model-call span can // name its `doGenerate`/`doStream` operation the same way the OTel integration does. `isStream` is @@ -417,8 +418,9 @@ export function createSpanFromMessage( recordToolDescriptions(callId, event.tools); } - const baseAttributes: Record = { + const baseAttributes: SpanAttributes = { [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: ORIGIN, + ...telemetryMetadataAttributes(event.telemetryMetadata), ...(provider ? { [GEN_AI_PROVIDER_NAME]: provider, [VERCEL_AI_MODEL_PROVIDER_ATTRIBUTE]: provider } : {}), ...(modelId ? { [GEN_AI_REQUEST_MODEL]: modelId } : {}), ...(maxRetries !== undefined ? { [VERCEL_AI_SETTINGS_MAX_RETRIES_ATTRIBUTE]: maxRetries } : {}), @@ -466,6 +468,21 @@ export function createSpanFromMessage( } } +/** + * `experimental_telemetry.metadata` (`ai` <= 6) as `vercel.ai.telemetry.metadata.`, the names the OTel + * integration produced from the SDK's `ai.telemetry.metadata.*`. Only the orchestrion adapter sets + * `event.telemetryMetadata`; `ai` 7 has no `telemetry.metadata`. + */ +function telemetryMetadataAttributes(metadata: unknown): SpanAttributes { + const attributes: SpanAttributes = {}; + if (isObjectLike(metadata)) { + for (const [key, value] of Object.entries(metadata)) { + attributes[`${VERCEL_AI_TELEMETRY_METADATA_ATTRIBUTE_PREFIX}${key}`] = value as SpanAttributeValue; + } + } + return attributes; +} + /** Start a `gen_ai.` span named ` ` (or just `` when no suffix). */ function startGenAiSpan(operation: GenAiOperation, suffix: string | undefined, attributes: SpanAttributes): Span { return startInactiveSpan({ diff --git a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-orchestrion-subscriber.ts b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-orchestrion-subscriber.ts index ea64ae6aacd9..40f076f54bec 100644 --- a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-orchestrion-subscriber.ts +++ b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-orchestrion-subscriber.ts @@ -107,6 +107,9 @@ const callIdBySpan = new WeakMap(); // child `generate_content` span (whose event would fall back to the global default). v7's channel forwards // these flags on every event, so this keeps v6 identical. const recordingBySpan = new WeakMap>(); +// The operation's `experimental_telemetry.metadata`, keyed by its span, so the model-call span records it +// too. The OTel integration put it on both, but not on tool-call spans. +const telemetryMetadataBySpan = new WeakMap(); interface OperationErrorInfo { // The span active when the operation was invoked (its call site, e.g. the enclosing request/`main` span). // When an operation's error bubbles out unhandled it reaches the global handler outside any span, so — as @@ -163,6 +166,7 @@ export function subscribeVercelAiOrchestrionChannels( ...modelFields(callOptions.model), maxRetries: callOptions.maxRetries, value: callOptions.value, + telemetryMetadata: telemetry.metadata, ...recording(telemetry), }, }), @@ -180,6 +184,7 @@ export function subscribeVercelAiOrchestrionChannels( ...modelFields(callOptions.model), maxRetries: callOptions.maxRetries, values: callOptions.values, + telemetryMetadata: telemetry.metadata, ...recording(telemetry), }, }), @@ -263,6 +268,7 @@ function bindOperation( callIdBySpan.set(span, callId); } recordingBySpan.set(span, recording(telemetry)); + telemetryMetadataBySpan.set(span, telemetry.metadata); // v5 has no `executeToolCall` channel, so patch each tool's `execute` to emit the tool-call span. // Inert on v6 (guarded inside `patchToolExecute` when the parent is `executeToolCall`'s own span). if (isObjectLike(callOptions.tools)) { @@ -490,6 +496,7 @@ function patchModelMethod( // top-level `tools` array. Reading both keeps `tool.definitions` populated on the model-call span. tools: callArgs.tools ?? (isObjectLike(callArgs.mode) ? callArgs.mode.tools : undefined), messages: callArgs.prompt, + telemetryMetadata: telemetryMetadataBySpan.get(parent), // Inherit the enclosing operation's per-call recording flags so inputs/tools/outputs are recorded on // the model-call span whenever they are on the parent `invoke_agent` span. ...recordingBySpan.get(parent), @@ -669,6 +676,7 @@ function buildTextMessage(type: 'generateText' | 'streamText' | 'generateObject' // Normalize to the message-array shape the shared core (and v7's channel) expects: a bare string // `prompt` becomes a single user message, matching the SDK's own normalization. messages: normalizePromptMessages(options), + telemetryMetadata: telemetry.metadata, ...recording(telemetry), }, });