Skip to content

Commit bcf7aea

Browse files
RulaKhaledclaude
andauthored
fix(server-utils): Record Anthropic chat spans for thread and tool-runner streams on 1.2+ (#25931)
On `@anthropic-ai/sdk` 1.2.0+, streamed turns from `beta.messages.toolRunner({ stream: true })` and `beta.messages.thread().stream()` had no `gen_ai.chat` spans. Every tool runner runs over a thread since 1.2.0, including on Bedrock and Vertex. The thread calls `beta.messages.create()`, which opens our span, but parses the response through a new `APIPromise` our hook never sees, so the span never ended. This hooks `BetaMessageStream._start()`, matches each request to its stream by abort signal, and ends the span from the stream's events with usage and output. `instrumentMessageStream` now also ends the span on `end`, so aborted or truncated streams, `messages.stream()` included, no longer leave it open. An abort before the response is not marked as an error. Fixes #25816 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
1 parent 52948fa commit bcf7aea

7 files changed

Lines changed: 491 additions & 5 deletions

File tree

‎dev-packages/node-integration-tests/suites/tracing/anthropic/v1/instrument.mjs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ Sentry.init({
55
dsn: 'https://public@dsn.ingest.sentry.io/1337',
66
release: '1.0',
77
tracesSampleRate: 1.0,
8-
dataCollection: { genAI: { inputs: false, outputs: false } },
8+
dataCollection: { genAI: { inputs: false, outputs: process.env.RECORD_OUTPUTS === 'true' } },
99
transport: loggingTransport,
1010
// Registers Sentry as the global tracer provider, which is what 1.x traces itself through.
1111
enableOpenTelemetrySetup: process.env.OTEL_SETUP === 'true',
Lines changed: 238 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,238 @@
1+
import Anthropic from '@anthropic-ai/sdk';
2+
import * as Sentry from '@sentry/node';
3+
import express from 'express';
4+
5+
const MODEL = 'claude-3-haiku-20240307';
6+
7+
// The first turn calls the weather tool and the second answers. Ids are `msg_<mode>_<turn>`, so each span can be told apart.
8+
function reply(mode, turn) {
9+
const id = `msg_${mode}_${turn}`;
10+
return turn === 'tools'
11+
? {
12+
id,
13+
content: [{ type: 'tool_use', id: `toolu_${mode}`, name: 'weather', input: { city: 'Vienna' } }],
14+
stop_reason: 'tool_use',
15+
usage: { input_tokens: 10, output_tokens: 20 },
16+
}
17+
: {
18+
id,
19+
content: [{ type: 'text', text: 'Sunny.' }],
20+
stop_reason: 'end_turn',
21+
usage: { input_tokens: 30, output_tokens: 5 },
22+
};
23+
}
24+
25+
function streamEvents({ id, content, stop_reason, usage }) {
26+
const events = [
27+
{
28+
type: 'message_start',
29+
message: {
30+
id,
31+
type: 'message',
32+
role: 'assistant',
33+
model: MODEL,
34+
content: [],
35+
usage: { input_tokens: usage.input_tokens },
36+
},
37+
},
38+
];
39+
content.forEach((block, index) => {
40+
if (block.type === 'tool_use') {
41+
events.push(
42+
{ type: 'content_block_start', index, content_block: { ...block, input: {} } },
43+
{
44+
type: 'content_block_delta',
45+
index,
46+
delta: { type: 'input_json_delta', partial_json: JSON.stringify(block.input) },
47+
},
48+
);
49+
} else {
50+
events.push(
51+
{ type: 'content_block_start', index, content_block: { type: 'text', text: '' } },
52+
{ type: 'content_block_delta', index, delta: { type: 'text_delta', text: block.text } },
53+
);
54+
}
55+
events.push({ type: 'content_block_stop', index });
56+
});
57+
events.push(
58+
{
59+
type: 'message_delta',
60+
delta: { stop_reason, stop_sequence: null },
61+
usage: { output_tokens: usage.output_tokens },
62+
},
63+
{ type: 'message_stop' },
64+
);
65+
return events;
66+
}
67+
68+
let recoveredReplies = 0;
69+
70+
function startMockAnthropicServer() {
71+
const app = express();
72+
app.use(express.json());
73+
74+
app.post('/anthropic/v1/messages', (req, res) => {
75+
const mode = req.headers['x-mode'];
76+
if (mode === 'thread_stream_error') {
77+
res.status(400).json({ type: 'error', error: { type: 'invalid_request_error', message: 'Bad thread' } });
78+
return;
79+
}
80+
const answersTool = req.body.messages.some(
81+
message => Array.isArray(message.content) && message.content.some(block => block.type === 'tool_result'),
82+
);
83+
// The server lost the thread, so the SDK replays the turn that continues it.
84+
if (mode === 'thread_stream_recovered' && req.body.thread?.type === 'continue') {
85+
res.status(404).json({
86+
type: 'error',
87+
error: { type: 'not_found_error', message: 'Thread not found', details: { error_code: 'thread_not_found' } },
88+
});
89+
return;
90+
}
91+
if (mode === 'thread_stream_failed_signal_reuse' && !answersTool) {
92+
res.status(400).json({ type: 'error', error: { type: 'invalid_request_error', message: 'Bad thread' } });
93+
return;
94+
}
95+
const message =
96+
mode === 'thread_stream_recovered'
97+
? { ...reply(mode, 'final'), id: `msg_${mode}_${(recoveredReplies += 1)}` }
98+
: reply(mode, answersTool ? 'final' : 'tools');
99+
100+
if (!req.body.stream) {
101+
res.json({ type: 'message', role: 'assistant', model: MODEL, stop_sequence: null, ...message });
102+
return;
103+
}
104+
105+
res.writeHead(200, { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache', Connection: 'keep-alive' });
106+
// The body closes after the first tool input delta, without `message_stop`.
107+
const events = mode === 'thread_stream_truncated' ? streamEvents(message).slice(0, 3) : streamEvents(message);
108+
events.forEach((event, index) => {
109+
setTimeout(() => {
110+
res.write(`event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`);
111+
if (index === events.length - 1) {
112+
res.end();
113+
}
114+
}, index * 5);
115+
});
116+
});
117+
118+
return new Promise(resolve => {
119+
const server = app.listen(0, () => {
120+
resolve(server);
121+
});
122+
});
123+
}
124+
125+
const tools = [
126+
{
127+
name: 'weather',
128+
description: 'Get the weather by city',
129+
input_schema: { type: 'object', properties: { city: { type: 'string' } } },
130+
run: () => 'sunny',
131+
},
132+
];
133+
const params = { model: MODEL, max_tokens: 100, messages: [{ role: 'user', content: 'Weather in Vienna?' }] };
134+
135+
async function streamFollowUp(client, signal) {
136+
const answer = { role: 'user', content: [{ type: 'tool_result', tool_use_id: 'toolu_1', content: 'sunny' }] };
137+
const followUp = await client.beta.messages.create(
138+
{ ...params, messages: [...params.messages, answer], stream: true },
139+
{ signal },
140+
);
141+
for await (const _event of followUp) {
142+
// consume
143+
}
144+
}
145+
146+
const MODES = {
147+
async runner_stream_eager(client) {
148+
for await (const stream of client.beta.messages.toolRunner({
149+
...params,
150+
tools,
151+
stream: true,
152+
runToolsEagerly: true,
153+
})) {
154+
await stream.finalMessage();
155+
}
156+
},
157+
async runner_stream(client) {
158+
for await (const stream of client.beta.messages.toolRunner({ ...params, tools, stream: true })) {
159+
await stream.finalMessage();
160+
}
161+
},
162+
// How Bedrock and Vertex clients, which have no threads, run every tool runner.
163+
async runner_stream_unthreaded(client) {
164+
for await (const stream of client.beta.messages.toolRunner({ ...params, tools, stream: true, thread: false })) {
165+
await stream.finalMessage();
166+
}
167+
},
168+
async runner(client) {
169+
for await (const _message of client.beta.messages.toolRunner({ ...params, tools })) {
170+
// consume
171+
}
172+
},
173+
async thread_send(client) {
174+
await client.beta.messages.thread({ model: MODEL, max_tokens: 100 }).send(params.messages);
175+
},
176+
async thread_stream(client) {
177+
await client.beta.messages.thread({ model: MODEL, max_tokens: 100 }).stream(params.messages).finalMessage();
178+
},
179+
// A later request sent with a finished thread stream's signal is not that stream's request.
180+
async thread_stream_signal_reuse(client) {
181+
const stream = client.beta.messages.thread({ model: MODEL, max_tokens: 100 }).stream(params.messages);
182+
await stream.finalMessage();
183+
await streamFollowUp(client, stream.controller.signal);
184+
},
185+
async thread_stream_failed_signal_reuse(client) {
186+
const stream = client.beta.messages.thread({ model: MODEL, max_tokens: 100 }).stream(params.messages);
187+
await stream.finalMessage().catch(() => {});
188+
await streamFollowUp(client, stream.controller.signal);
189+
},
190+
async thread_stream_abort(client) {
191+
const stream = client.beta.messages.thread({ model: MODEL, max_tokens: 100 }).stream(params.messages);
192+
stream.once('streamEvent', () => stream.abort());
193+
await stream.finalMessage().catch(() => {});
194+
},
195+
async thread_stream_abort_early(client) {
196+
const stream = client.beta.messages.thread({ model: MODEL, max_tokens: 100 }).stream(params.messages);
197+
stream.abort();
198+
await stream.finalMessage().catch(() => {});
199+
},
200+
async thread_stream_truncated(client) {
201+
await client.beta.messages
202+
.thread({ model: MODEL, max_tokens: 100 })
203+
.stream(params.messages)
204+
.finalMessage()
205+
.catch(() => {});
206+
},
207+
async thread_stream_recovered(client) {
208+
const thread = client.beta.messages.thread({ model: MODEL, max_tokens: 100 });
209+
await thread.stream(params.messages).finalMessage();
210+
await thread.stream([{ role: 'user', content: 'And tomorrow?' }]).finalMessage();
211+
},
212+
async thread_stream_error(client) {
213+
await client.beta.messages
214+
.thread({ model: MODEL, max_tokens: 100 })
215+
.stream(params.messages)
216+
.finalMessage()
217+
.catch(() => {});
218+
},
219+
};
220+
221+
async function run() {
222+
const server = await startMockAnthropicServer();
223+
const baseURL = `http://localhost:${server.address().port}/anthropic`;
224+
225+
await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
226+
for (const [mode, call] of Object.entries(MODES)) {
227+
await Sentry.startSpan({ name: mode }, () =>
228+
call(new Anthropic({ apiKey: 'mock-api-key', baseURL, maxRetries: 0, defaultHeaders: { 'x-mode': mode } })),
229+
);
230+
}
231+
});
232+
233+
await Sentry.flush(2000);
234+
235+
server.close();
236+
}
237+
238+
run();

‎dev-packages/node-integration-tests/suites/tracing/anthropic/v1/test.ts‎

Lines changed: 120 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,15 @@ import {
33
GEN_AI_CONVERSATION_ID,
44
GEN_AI_RESPONSE_ID,
55
GEN_AI_RESPONSE_STREAMING,
6+
GEN_AI_RESPONSE_TEXT,
7+
GEN_AI_RESPONSE_TOOL_CALLS,
68
GEN_AI_TOOL_NAME,
9+
GEN_AI_USAGE_INPUT_TOKENS,
10+
GEN_AI_USAGE_OUTPUT_TOKENS,
711
SENTRY_OP,
812
SENTRY_ORIGIN,
913
} from '@sentry/conventions/attributes';
10-
import { GEN_AI_EXECUTE_TOOL, GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op';
14+
import { GEN_AI_CHAT, GEN_AI_EXECUTE_TOOL, GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op';
1115
import { afterAll, describe, expect } from 'vitest';
1216
import { GEN_AI_TOOL_CALL_ID_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes';
1317
import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner';
@@ -116,6 +120,121 @@ describe('Anthropic integration (1.x)', () => {
116120
}
117121
});
118122

123+
// Since 1.2, tool runners and `thread().stream()` send each turn through a thread, which parses the response itself.
124+
createEsmAndCjsTests(__dirname, 'scenario-threads.mjs', 'instrument.mjs', (createRunner, test) => {
125+
const MODES = [
126+
{ mode: 'runner_stream_eager', streaming: true, turns: ['tools', 'final'] },
127+
{ mode: 'runner_stream', streaming: true, turns: ['tools', 'final'] },
128+
{ mode: 'runner_stream_unthreaded', streaming: true, turns: ['tools', 'final'] },
129+
{ mode: 'runner', streaming: false, turns: ['tools', 'final'] },
130+
{ mode: 'thread_send', streaming: false, turns: ['tools'] },
131+
{ mode: 'thread_stream', streaming: true, turns: ['tools'] },
132+
{ mode: 'thread_stream_signal_reuse', streaming: true, turns: ['tools', 'final'] },
133+
];
134+
const FINAL_TURN_MODES = MODES.filter(({ turns }) => turns.includes('final'));
135+
// Only the span's status is known for these. The stream is aborted on its first event or before the response,
136+
// its body closes before `message_stop`, or the request fails.
137+
const STATUS_ONLY_MODES = [
138+
{ mode: 'thread_stream_abort', status: 'ok' },
139+
{ mode: 'thread_stream_abort_early', status: 'ok' },
140+
{ mode: 'thread_stream_truncated', status: 'ok' },
141+
{ mode: 'thread_stream_error', status: 'error' },
142+
];
143+
144+
describe.each([{}, { OTEL_SETUP: 'true' }])('with %j', env => {
145+
// Every scenario run is a Node process, so the cases below share one run.
146+
let run: Promise<SerializedSpan[]> | undefined;
147+
function getSpans(): Promise<SerializedSpan[]> {
148+
run ??= (async () => {
149+
const runner = createRunner().withEnv({ ...env, RECORD_OUTPUTS: 'true' });
150+
const spansPromise = runner.collectStreamedSpansUntilSegment('main');
151+
await runner.start().completed();
152+
return spansPromise;
153+
})();
154+
return run;
155+
}
156+
157+
async function chatSpansUnder(mode: string): Promise<SerializedSpan[]> {
158+
const spans = await getSpans();
159+
const modeSpan = spans.find(span => span.name === mode)!;
160+
return spans.filter(
161+
span => span.parent_span_id === modeSpan.span_id && span.attributes[SENTRY_OP]?.value === GEN_AI_CHAT,
162+
);
163+
}
164+
165+
test("emits one chat span per turn and none of the SDK's own spans", async () => {
166+
const spans = await getSpans();
167+
168+
expect(spans.filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_CHAT)).toHaveLength(
169+
// Plus the failed request and the follow-up of `thread_stream_failed_signal_reuse`, and the three requests
170+
// of `thread_stream_recovered`.
171+
MODES.flatMap(({ turns }) => turns).length + STATUS_ONLY_MODES.length + 2 + 3,
172+
);
173+
expect(spans.filter(span => span.name.startsWith('anthropic.'))).toEqual([]);
174+
});
175+
176+
test.each(MODES)('nests one chat span per $mode turn under the caller', async ({ mode, turns }) => {
177+
const chatSpans = await chatSpansUnder(mode);
178+
179+
expect(chatSpans.map(span => span.attributes[GEN_AI_RESPONSE_ID]?.value).sort()).toEqual(
180+
turns.map(turn => `msg_${mode}_${turn}`).sort(),
181+
);
182+
});
183+
184+
test.each(MODES)('records the tool call of the first $mode turn', async ({ mode, streaming }) => {
185+
const chatSpan = spanForCall(await chatSpansUnder(mode), `msg_${mode}_tools`);
186+
187+
expect(chatSpan.status).toBe('ok');
188+
expect(chatSpan.attributes[GEN_AI_RESPONSE_STREAMING]?.value ?? false).toBe(streaming);
189+
expect(chatSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(10);
190+
expect(chatSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(20);
191+
expect(JSON.parse(chatSpan.attributes[GEN_AI_RESPONSE_TOOL_CALLS]!.value as string)).toEqual([
192+
{ type: 'tool_use', id: `toolu_${mode}`, name: 'weather', input: { city: 'Vienna' } },
193+
]);
194+
});
195+
196+
test.each(FINAL_TURN_MODES)('records the text of the final $mode turn', async ({ mode, streaming }) => {
197+
const chatSpan = spanForCall(await chatSpansUnder(mode), `msg_${mode}_final`);
198+
199+
expect(chatSpan.status).toBe('ok');
200+
expect(chatSpan.attributes[GEN_AI_RESPONSE_STREAMING]?.value ?? false).toBe(streaming);
201+
expect(chatSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(30);
202+
expect(chatSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(5);
203+
expect(chatSpan.attributes[GEN_AI_RESPONSE_TEXT]?.value).toBe('Sunny.');
204+
expect(chatSpan.attributes[GEN_AI_RESPONSE_TOOL_CALLS]).toBeUndefined();
205+
});
206+
207+
test.each(STATUS_ONLY_MODES)('ends the $mode chat span with status $status', async ({ mode, status }) => {
208+
const chatSpans = await chatSpansUnder(mode);
209+
210+
expect(chatSpans).toHaveLength(1);
211+
expect(chatSpans[0]!.status).toBe(status);
212+
});
213+
214+
// Deliberate. The rejected attempt was a real request that failed, like an http.client span that a retry
215+
// recovers from. Hiding it would mean copying the SDK's private recovery rules: they replay only some error codes,
216+
// only on a `continue`, and may change in any release, so a copy would silently hide real errors.
217+
test('keeps an error span for the attempt of a turn the thread recovered from', async () => {
218+
const chatSpans = await chatSpansUnder('thread_stream_recovered');
219+
220+
expect(chatSpans.map(span => [span.attributes[GEN_AI_RESPONSE_ID]?.value, span.status])).toEqual([
221+
['msg_thread_stream_recovered_1', 'ok'],
222+
[undefined, 'error'],
223+
['msg_thread_stream_recovered_2', 'ok'],
224+
]);
225+
});
226+
227+
test("ends the span of a request sent with a failed stream's signal", async () => {
228+
const chatSpans = await chatSpansUnder('thread_stream_failed_signal_reuse');
229+
const followUp = spanForCall(chatSpans, 'msg_thread_stream_failed_signal_reuse_final');
230+
231+
expect(chatSpans.map(span => span.status).sort()).toEqual(['error', 'ok']);
232+
expect(followUp.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(5);
233+
expect(followUp.attributes[GEN_AI_RESPONSE_TEXT]?.value).toBe('Sunny.');
234+
});
235+
});
236+
});
237+
119238
createEsmAndCjsTests(__dirname, 'scenario-session-tool-runner.mjs', 'instrument.mjs', (createRunner, test) => {
120239
for (const env of [{}, { OTEL_SETUP: 'true' }]) {
121240
test(`emits an execute_tool span for each tool a session runner runs with ${JSON.stringify(env)}`, async () => {

0 commit comments

Comments
 (0)