Skip to content

Surface chat system_fingerprint and logprobs as response metadata #1104

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
PratikDhanave (PratikDhanave) wants to merge 4 commits into
base: main
Choose a base branch
from
Open

Surface chat system_fingerprint and logprobs as response metadata #1104

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8aa128d
Surface chat system_fingerprint and logprobs as response metadata
PratikDhanave Sep 18, 2026
51552c4
Address review: accumulate streaming chat logprobs across chunks
PratikDhanave Sep 21, 2026
ecbf3ca
Merge remote-tracking branch 'upstream/main' into fix/openai-chat-res…
PratikDhanave Oct 3, 2026
fe81313
Merge remote-tracking branch 'upstream/main' into fix/openai-chat-res…
PratikDhanave Nov 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 44 additions & 14 deletions provider/openaiprovider/chat.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ func (a *chatClient) run(ctx context.Context, messages []*message.Message, optio
// the streaming path below.
var contents []message.Content
var finishReason string
var additionalProperties map[string]any
if len(resp.Choices) > 0 {
choice := resp.Choices[0]
contents = make([]message.Content, 0, 1+len(choice.Message.ToolCalls))
Expand All @@ -168,19 +169,29 @@ func (a *chatClient) run(ctx context.Context, messages []*message.Message, optio
contents = append(contents, &message.ErrorContent{Message: choice.Message.Refusal, ErrorCode: "Refusal"})
}
finishReason = normalizeChatFinishReason(choice.FinishReason)
if len(choice.Logprobs.Content) > 0 || len(choice.Logprobs.Refusal) > 0 {
additionalProperties = map[string]any{"Logprobs": choice.Logprobs}
}
}
if resp.SystemFingerprint != "" {
if additionalProperties == nil {
additionalProperties = make(map[string]any)
}
additionalProperties["SystemFingerprint"] = resp.SystemFingerprint
}
if resp.JSON.Usage.Valid() {
contents = addUsage(contents, resp.Usage)
}
return func(yield func(*agent.ResponseUpdate, error) bool) {
update := &agent.ResponseUpdate{
Contents: contents,
Role: message.RoleAssistant,
ResponseID: resp.ID,
MessageID: resp.ID,
FinishReason: finishReason,
CreatedAt: time.Unix(resp.Created, 0),
RawRepresentation: resp,
Contents: contents,
Role: message.RoleAssistant,
ResponseID: resp.ID,
MessageID: resp.ID,
FinishReason: finishReason,
CreatedAt: time.Unix(resp.Created, 0),
AdditionalProperties: additionalProperties,

@github-actions github-actions Bot Oct 3, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Parity issue: SystemFingerprint/Logprobs are scoped to the message, not the response

Because this update sets MessageID: resp.ID (non-empty) alongside AdditionalProperties, Response.Update (see agent/response.go's own doc comment: "A non-empty MessageID scopes these properties to that message. Otherwise, they are response-level properties.") copies SystemFingerprint/Logprobs into msg.AdditionalProperties, not resp.AdditionalProperties. The PR's own test (lastMessageAdditionalProperties) confirms this — it has to scan resp.Messages rather than reading resp.AdditionalProperties.

Upstream Python treats both as response-level metadata: _get_metadata_from_chat_response/_get_metadata_from_chat_choice populate a single response_metadata dict that is passed as ChatResponse(..., additional_properties=response_metadata, ...) (_chat_completion_client.py lines ~879-904, 1007-1022), never on an individual Message.

This same provider package already has the correct pattern for this exact situation: responsesPopulateAdditionalProperties in provider/openaiprovider/responses.go (around line 1265) explicitly yields the message-bearing update first, then emits a second, metadata-only update with an empty MessageID so the properties land on Response.AdditionalProperties instead of the message.

Suggested fix: follow the same split used in responses.go — yield the content-bearing update with MessageID set, then emit a follow-up update (or restructure this single update) with MessageID unset so SystemFingerprint/Logprobs land in Response.AdditionalProperties, matching Python's ChatResponse.additional_properties semantics. The same issue applies to the streaming path at line 250.

RawRepresentation: resp,
}
if !yield(update, nil) {
return
Expand All @@ -198,6 +209,10 @@ func (a *chatClient) run(ctx context.Context, messages []*message.Message, optio
stream := a.client.Chat.Completions.NewStreaming(ctx, body, telemetryOption)
defer func() { _ = stream.Close() }()
var acc openai.ChatCompletionAccumulator
// Logprobs stream incrementally (one chunk carries only its own tokens),
// so accumulate them across chunks; each update carries the running total
// and the final message keeps the complete per-choice list.
var accLogprobs openai.ChatCompletionChunkChoiceLogprobs
var created int64
for stream.Next() {
chunk := stream.Current()
Expand All @@ -221,17 +236,32 @@ func (a *chatClient) run(ctx context.Context, messages []*message.Message, optio
contents = addUsage(contents, chunk.Usage)
}
var finishReason string
var additionalProperties map[string]any
if len(chunk.Choices) > 0 {
finishReason = normalizeChatFinishReason(chunk.Choices[0].FinishReason)
if logprobs := chunk.Choices[0].Logprobs; len(logprobs.Content) > 0 || len(logprobs.Refusal) > 0 {
accLogprobs.Content = append(accLogprobs.Content, logprobs.Content...)
accLogprobs.Refusal = append(accLogprobs.Refusal, logprobs.Refusal...)
}
}
if len(accLogprobs.Content) > 0 || len(accLogprobs.Refusal) > 0 {
additionalProperties = map[string]any{"Logprobs": accLogprobs}
}
if chunk.SystemFingerprint != "" {
if additionalProperties == nil {
additionalProperties = make(map[string]any)
}
additionalProperties["SystemFingerprint"] = chunk.SystemFingerprint
}
resp := &agent.ResponseUpdate{
Contents: contents,
Role: role,
ResponseID: chunk.ID,
MessageID: chunk.ID,
FinishReason: finishReason,
CreatedAt: time.Unix(chunk.Created, 0),
RawRepresentation: chunk,
Contents: contents,
Role: role,
ResponseID: chunk.ID,
MessageID: chunk.ID,
FinishReason: finishReason,
CreatedAt: time.Unix(chunk.Created, 0),
AdditionalProperties: additionalProperties,
RawRepresentation: chunk,
}
if !yield(resp, nil) {
return
Expand Down
79 changes: 79 additions & 0 deletions provider/openaiprovider/chat_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -837,6 +837,85 @@ func TestChatBasicRequestResponse_NonStreaming(t *testing.T) {
}
}

// system_fingerprint (response) and per-choice logprobs must be surfaced on the
// update's AdditionalProperties, matching the Python client which carries both
// as chat response metadata.
func TestChatResponseMetadataSurfaced_NonStreaming(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = io.WriteString(w, `{"id":"chatcmpl-md","object":"chat.completion","created":1727888631,"model":"gpt-4o-mini","system_fingerprint":"fp_test","choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"logprobs":{"content":[{"token":"ok","logprob":-0.1,"bytes":[111,107],"top_logprobs":[]}]},"finish_reason":"stop"}]}`)
}))
defer server.Close()

resp, err := newTestClient(server).RunText(t.Context(), "hi").Collect()
if err != nil {
t.Fatalf("error = %v", err)
}
props := lastMessageAdditionalProperties(t, resp)
if props["SystemFingerprint"] != "fp_test" {
t.Errorf("SystemFingerprint = %v, want %q", props["SystemFingerprint"], "fp_test")
}
if _, ok := props["Logprobs"]; !ok {
t.Errorf("Logprobs missing from AdditionalProperties: %#v", props)
}
}

func TestChatResponseMetadataSurfaced_Streaming(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "text/event-stream")
_, _ = io.WriteString(w, "data: {\"id\":\"chatcmpl-md\",\"object\":\"chat.completion.chunk\",\"created\":1727888631,\"model\":\"gpt-4o-mini\",\"system_fingerprint\":\"fp_test\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"ok\"},\"finish_reason\":\"stop\"}]}\n\n")
_, _ = io.WriteString(w, "data: [DONE]\n\n")
}))
defer server.Close()

resp, err := newTestClient(server).RunText(t.Context(), "hi", agent.Stream(true)).Collect()
if err != nil {
t.Fatalf("error = %v", err)
}
props := lastMessageAdditionalProperties(t, resp)
if props["SystemFingerprint"] != "fp_test" {
t.Errorf("SystemFingerprint = %v, want %q", props["SystemFingerprint"], "fp_test")
}
}

// Streaming logprobs arrive incrementally per chunk; the collected message must
// carry the accumulated per-choice list, not just the final chunk's tokens.
func TestChatStreamingLogprobsAccumulated(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "text/event-stream")
_, _ = io.WriteString(w, "data: {\"id\":\"c\",\"object\":\"chat.completion.chunk\",\"created\":1,\"model\":\"gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"content\":\"Hel\"},\"logprobs\":{\"content\":[{\"token\":\"Hel\",\"logprob\":-0.1,\"bytes\":[72],\"top_logprobs\":[]}]}}]}\n\n")
_, _ = io.WriteString(w, "data: {\"id\":\"c\",\"object\":\"chat.completion.chunk\",\"created\":1,\"model\":\"gpt-4o-mini\",\"choices\":[{\"index\":0,\"delta\":{\"content\":\"lo\"},\"logprobs\":{\"content\":[{\"token\":\"lo\",\"logprob\":-0.2,\"bytes\":[108],\"top_logprobs\":[]}]},\"finish_reason\":\"stop\"}]}\n\n")
_, _ = io.WriteString(w, "data: [DONE]\n\n")
}))
defer server.Close()

resp, err := newTestClient(server).RunText(t.Context(), "hi", agent.Stream(true)).Collect()
if err != nil {
t.Fatalf("error = %v", err)
}
props := lastMessageAdditionalProperties(t, resp)
lp, ok := props["Logprobs"].(openai.ChatCompletionChunkChoiceLogprobs)
if !ok {
t.Fatalf("Logprobs = %T, want ChatCompletionChunkChoiceLogprobs", props["Logprobs"])
}
if len(lp.Content) != 2 {
t.Errorf("accumulated logprob content = %d tokens, want 2 (one per chunk)", len(lp.Content))
}
}

// lastMessageAdditionalProperties returns the AdditionalProperties of the last
// message carrying any, so metadata assertions do not depend on message count.
func lastMessageAdditionalProperties(t *testing.T, resp *agent.Response) map[string]any {
t.Helper()
for i := len(resp.Messages) - 1; i >= 0; i-- {
if len(resp.Messages[i].AdditionalProperties) > 0 {
return resp.Messages[i].AdditionalProperties
}
}
t.Fatalf("no message carried AdditionalProperties: %#v", resp.Messages)
return nil
}

// cache_write_tokens (prompt-cache writes / cache creation) must be surfaced via AdditionalCounts,
// matching the Python client.
func TestChatCacheWriteTokensSurfaced(t *testing.T) {
Expand Down
Loading