From c5f6f31712e5b049a6f04055d1de9276a5c28c54 Mon Sep 17 00:00:00 2001 From: pengjonas Date: Sun, 13 Sep 2026 16:29:51 -0400 Subject: [PATCH] Include thinking text in Anthropic streaming chunks Streamed thinking deltas carry only a thinking=true marker since 9fe9f9c, so a streaming caller cannot show thinking as it arrives. When the model calls a tool, the per-round aggregated message holding the thinking is consumed by the tool-calling loop, so the thinking text never reaches the caller at all. Expose each delta's text in a thinkingText metadata key, keeping the content null so it is still not aggregated into the assistant message text. Signed-off-by: pengjonas --- .../ai/anthropic/AnthropicChatModel.java | 4 ++- .../ai/anthropic/AnthropicChatModelTests.java | 31 +++++++++++++++++++ .../ROOT/pages/api/chat/anthropic-chat.adoc | 9 +++--- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/models/spring-ai-anthropic/src/main/java/org/springframework/ai/anthropic/AnthropicChatModel.java b/models/spring-ai-anthropic/src/main/java/org/springframework/ai/anthropic/AnthropicChatModel.java index 1255c89867..2cfa838c84 100644 --- a/models/spring-ai-anthropic/src/main/java/org/springframework/ai/anthropic/AnthropicChatModel.java +++ b/models/spring-ai-anthropic/src/main/java/org/springframework/ai/anthropic/AnthropicChatModel.java @@ -440,12 +440,14 @@ else if (contentBlock.isWebSearchToolResult()) { return null; } - // Thinking chunk — emit with thinking metadata + // Thinking chunk — emit with thinking metadata. The text is exposed in + // metadata, not content, so it is not aggregated into the message text. if (delta.isThinking()) { String thinkingText = delta.asThinking().thinking(); streamingState.appendThinking(thinkingText); Map thinkingProperties = new HashMap<>(); thinkingProperties.put("thinking", Boolean.TRUE); + thinkingProperties.put("thinkingText", thinkingText); AssistantMessage assistantMessage = AssistantMessage.builder().properties(thinkingProperties).build(); return new ChatResponse(List.of(new Generation(assistantMessage))); } diff --git a/models/spring-ai-anthropic/src/test/java/org/springframework/ai/anthropic/AnthropicChatModelTests.java b/models/spring-ai-anthropic/src/test/java/org/springframework/ai/anthropic/AnthropicChatModelTests.java index 30bc7e3e52..d116de4f6a 100644 --- a/models/spring-ai-anthropic/src/test/java/org/springframework/ai/anthropic/AnthropicChatModelTests.java +++ b/models/spring-ai-anthropic/src/test/java/org/springframework/ai/anthropic/AnthropicChatModelTests.java @@ -393,6 +393,37 @@ void streamingThinkingBlockIsReplayedBeforeToolUseBlock() { assertThat(replayedAssistantBlocks.get(1).isToolUse()).isTrue(); } + @Test + @SuppressWarnings("unchecked") + void streamingThinkingDeltasExposeThinkingTextInMetadata() { + List events = List.of(messageStartEvent(), thinkingStartEvent(), + thinkingDeltaEvent("thinking "), thinkingDeltaEvent("text"), signatureDeltaEvent("thinking-signature"), + contentBlockStopEvent(0), messageDeltaEvent(StopReason.END_TURN)); + StreamResponse streamResponse = mock(StreamResponse.class); + given(streamResponse.stream()).willReturn(events.stream()); + + HttpResponseFor> rawResponse = mock(HttpResponseFor.class); + given(rawResponse.parse()).willReturn(streamResponse); + given(rawResponse.headers()).willReturn(Headers.builder().build()); + + given(this.anthropicClientAsync.messages()).willReturn(this.messageServiceAsync); + given(this.messageServiceAsync.withRawResponse()).willReturn(this.messageServiceAsyncWithRawResponse); + given(this.messageServiceAsyncWithRawResponse.createStreaming(any(MessageCreateParams.class), + any(RequestOptions.class))) + .willReturn(CompletableFuture.completedFuture(rawResponse)); + + List responses = this.chatModel.stream(new Prompt("Think about it.")).collectList().block(); + + assertThat(responses).isNotNull(); + List thinkingDeltas = responses.stream() + .map(response -> response.getResult().getOutput()) + .filter(message -> message.getMetadata().containsKey("thinking")) + .toList(); + assertThat(thinkingDeltas).extracting(message -> message.getMetadata().get("thinkingText")) + .containsExactly("thinking ", "text"); + assertThat(thinkingDeltas).allSatisfy(message -> assertThat(message.getText()).isNull()); + } + @Test void cacheOptionsIsMergedFromRuntimePrompt() { AnthropicChatModel model = AnthropicChatModel.builder() diff --git a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/anthropic-chat.adoc b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/anthropic-chat.adoc index 89b7f60285..08998da29c 100644 --- a/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/anthropic-chat.adoc +++ b/spring-ai-docs/src/main/antora/modules/ROOT/pages/api/chat/anthropic-chat.adoc @@ -485,10 +485,9 @@ stream.subscribe(response -> { AssistantMessage message = generation.getOutput(); if (message.getMetadata().containsKey("thinking")) { - // Thinking delta — getText() returns null; the full accumulated thinking text - // is available via AnthropicAssistantMessage.getThinkingContents() on the - // aggregated response after streaming completes. - System.out.println("[thinking...]"); + // Thinking delta — getText() returns null so thinking does not mix into the + // answer; the incremental thinking text is in the "thinkingText" metadata key. + System.out.print(message.getMetadata().get("thinkingText")); } else if (message.getMetadata().containsKey("signature")) { // Thinking block signature (emitted at end of thinking) @@ -512,7 +511,7 @@ When thinking is enabled, the response contains different types of content: | **Thinking Block** | `signature` | Claude's reasoning text with a cryptographic signature. In sync mode, the thinking text is in `getText()` and the signature is in `getMetadata().get("signature")`. | **Redacted Thinking** | `data` | Safety-redacted reasoning. Contains only a `data` marker, no visible text. | **Signature (streaming)** | `signature` | In streaming mode, the signature arrives as a separate delta at the end of a thinking block. -| **Thinking Delta (streaming)** | `thinking` | Incremental thinking chunks during streaming. The `thinking` metadata key is set to `true`. `getText()` returns null on these chunks; the full thinking text is available via `AnthropicAssistantMessage.getThinkingContents()` on the aggregated response. +| **Thinking Delta (streaming)** | `thinking`, `thinkingText` | Incremental thinking chunks during streaming. The `thinking` metadata key is set to `true` and `thinkingText` holds the incremental thinking text. `getText()` returns null on these chunks, so thinking is not aggregated into the answer text; the full thinking text is also available via `AnthropicAssistantMessage.getThinkingContents()` on the aggregated response. | **Text Block** | _(none)_ | The final answer text in `getText()`. |====