From 88d1cc0deae8395f3bf05f95870863b87ff0d22e Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 28 Oct 2026 14:35:11 +0100 Subject: [PATCH 1/2] fix(server-utils): Record AI SDK tool spans for tools run outside its tool loop eve 0.84+ runs tools in its own loop instead of the AI SDK's, so the `ai:telemetry` channel never published `executeTool` and no `gen_ai.execute_tool` spans or tool errors were recorded. eve still calls the AI SDK's registered telemetry integrations, so the Vercel AI integration now registers one that publishes those tool executions to the channel. Co-Authored-By: Claude Opus 5.5 --- .../src/integrations/vercel-ai/index.ts | 2 + .../vercel-ai/vercel-ai-dc-subscriber.ts | 2 +- .../vercel-ai/vercel-ai-tool-telemetry.ts | 62 +++++++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 packages/server-utils/src/integrations/vercel-ai/vercel-ai-tool-telemetry.ts diff --git a/packages/server-utils/src/integrations/vercel-ai/index.ts b/packages/server-utils/src/integrations/vercel-ai/index.ts index 6cf24b1e70b3..7d847a76dfe8 100644 --- a/packages/server-utils/src/integrations/vercel-ai/index.ts +++ b/packages/server-utils/src/integrations/vercel-ai/index.ts @@ -1,6 +1,7 @@ import { defineIntegration, waitForTracingChannelBinding, type IntegrationFn } from '@sentry/core'; import type { GenAiOptions } from '../../ai/core/utils'; import { subscribeVercelAiTracingChannel } from './vercel-ai-dc-subscriber'; +import { registerVercelAiToolTelemetry } from './vercel-ai-tool-telemetry'; import * as dc from '../../utils/diagnosticsChannel'; import { invokeOrchestrionInstrumentation } from '../../orchestrion/instrumentation'; import { vercelAiModuleNames } from '../../orchestrion/config/vercel-ai'; @@ -23,6 +24,7 @@ const _vercelAIIntegration = ((options: VercelAiOptions = {}) => { waitForTracingChannelBinding(() => { subscribeVercelAiTracingChannel(dc.tracingChannel, options); }); + registerVercelAiToolTelemetry(); }, setup(client) { invokeOrchestrionInstrumentation(client, vercelAiModuleNames, instrumentVercelAiOrchestrion, [options]); diff --git a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts index 1079886c6762..2943aa7dddb2 100644 --- a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts +++ b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-dc-subscriber.ts @@ -63,7 +63,7 @@ import { asNumber, asString, isReadableStream, type StreamedModelCallResult, sum * via `node:diagnostics_channel`. Events are discriminated by their `type` field. * @see https://github.com/vercel/ai/pull/15660 */ -const AI_SDK_TELEMETRY_TRACING_CHANNEL = 'ai:telemetry'; +export const AI_SDK_TELEMETRY_TRACING_CHANNEL = 'ai:telemetry'; const ORIGIN = 'auto.ai.vercelai'; diff --git a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-tool-telemetry.ts b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-tool-telemetry.ts new file mode 100644 index 000000000000..799d1ff17977 --- /dev/null +++ b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-tool-telemetry.ts @@ -0,0 +1,62 @@ +import { getActiveSpan, spanToJSON } from '@sentry/core'; +import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../ai/core/gen-ai-attributes'; +import * as dc from '../../utils/diagnosticsChannel'; +import { AI_SDK_TELEMETRY_TRACING_CHANNEL, type VercelAiChannelMessage } from './vercel-ai-dc-subscriber'; + +interface ToolOutput { + type: 'tool-result' | 'tool-error'; + toolCallId: string; + output?: unknown; +} + +/** The subset of `ai`'s `Telemetry` integration interface this uses. */ +interface TelemetryIntegration { + executeTool?( + options: Record & { toolCallId: string; execute: () => PromiseLike }, + ): PromiseLike; + onToolExecutionEnd?(event: { toolOutput: ToolOutput }): void; +} + +const openToolCalls = new Map(); + +/** + * Register a global `ai` (>= 7) telemetry integration that publishes tool executions to the + * `ai:telemetry` channel. Frameworks such as eve (>= 0.84) run tools outside `ai`'s tool loop, so `ai` + * publishes nothing for them, but they still call registered telemetry integrations. + */ +export function registerVercelAiToolTelemetry(): void { + const channel = dc.tracingChannel(AI_SDK_TELEMETRY_TRACING_CHANNEL); + const carrier = globalThis as { AI_SDK_TELEMETRY_INTEGRATIONS?: TelemetryIntegration[] }; + + (carrier.AI_SDK_TELEMETRY_INTEGRATIONS ??= []).push({ + executeTool({ execute, ...event }) { + // `ai`'s own tool loop calls this inside its own `executeTool` channel span. + const activeSpan = getActiveSpan(); + if (activeSpan && spanToJSON(activeSpan).attributes?.[GEN_AI_TOOL_CALL_ID_ATTRIBUTE] === event.toolCallId) { + return execute(); + } + + const message: VercelAiChannelMessage = { type: 'executeTool', event }; + openToolCalls.set(event.toolCallId, message); + try { + return channel.start.runStores(message, execute); + } finally { + // Without a `result` this leaves the span open until `onToolExecutionEnd`. + channel.end.publish(message); + } + }, + + // The tool output only arrives here, after `execute` has settled. + onToolExecutionEnd({ toolOutput }) { + const message = openToolCalls.get(toolOutput.toolCallId); + if (!message) { + return; + } + openToolCalls.delete(toolOutput.toolCallId); + + // Same result shape as `ai`'s own `executeTool` events, so the subscriber records the output or captures the error. + message.result = { output: toolOutput.type === 'tool-error' ? toolOutput : toolOutput.output }; + channel.asyncEnd.publish(message); + }, + }); +} From b12e1230e4dca982fb296589cda99d48d7b01038 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 28 Oct 2026 14:42:23 +0100 Subject: [PATCH 2/2] ref: Link AI SDK telemetry docs Co-Authored-By: Claude Opus 5.5 --- .../src/integrations/vercel-ai/vercel-ai-tool-telemetry.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-tool-telemetry.ts b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-tool-telemetry.ts index 799d1ff17977..ebfe044bcf02 100644 --- a/packages/server-utils/src/integrations/vercel-ai/vercel-ai-tool-telemetry.ts +++ b/packages/server-utils/src/integrations/vercel-ai/vercel-ai-tool-telemetry.ts @@ -23,6 +23,7 @@ const openToolCalls = new Map(); * Register a global `ai` (>= 7) telemetry integration that publishes tool executions to the * `ai:telemetry` channel. Frameworks such as eve (>= 0.84) run tools outside `ai`'s tool loop, so `ai` * publishes nothing for them, but they still call registered telemetry integrations. + * @see https://ai-sdk.dev/docs/ai-sdk-core/telemetry */ export function registerVercelAiToolTelemetry(): void { const channel = dc.tracingChannel(AI_SDK_TELEMETRY_TRACING_CHANNEL);