Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions changelog/unreleased/kiro-usage-cache.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
### English

- Anthropic Messages streams now report `input_tokens` in the final `message_delta`; it was missing before, so clients saw 0 input. Across all providers, Anthropic `input_tokens` now counts only the uncached part of the prompt, with `cache_read_input_tokens` and `cache_creation_input_tokens` carrying the rest, matching Anthropic's own usage format.
- Kiro requests now report estimated prompt-cache reads and writes. Kiro does not return cache figures, so the gateway tracks prompt prefixes per account for five minutes, the way Kiro-Go does, and reports the repeated part as a cache read. The figures are estimates.
- Kiro output tokens are estimated more closely (separate weights for letters, digits, punctuation and CJK text), and token counts are now read from more upstream usage shapes when Kiro does send them.

### 中文

- Anthropic Messages 流式响应现在会在最后的 `message_delta` 中返回 `input_tokens`,之前缺失导致客户端看到的输入为 0。所有渠道的 Anthropic `input_tokens` 现在只计未命中缓存的部分,其余由 `cache_read_input_tokens` 和 `cache_creation_input_tokens` 表示,与 Anthropic 官方用量格式一致。
- Kiro 请求现在会返回估算的缓存读取和缓存写入。Kiro 本身不返回缓存数据,网关参照 Kiro-Go 按账号记录 5 分钟内的提示前缀,重复部分计为缓存读取。该数值为估算值。
- Kiro 输出 Tokens 的估算更准确(字母、数字、标点和中日韩文字分别计权),Kiro 返回 Token 数时也能识别更多上游用量格式。
11 changes: 1 addition & 10 deletions internal/gateway/anthropic.go
Original file line number Diff line number Diff line change
Expand Up @@ -186,16 +186,7 @@ func anthropicMessageResponse(requestID, model, content, reasoning string, toolC
if len(blocks) == 0 {
blocks = append(blocks, map[string]any{"type": "text", "text": ""})
}
usage := map[string]any{
"input_tokens": promptTokens,
"output_tokens": completionTokens,
}
if cacheReadTokens != nil {
usage["cache_read_input_tokens"] = *cacheReadTokens
}
if cacheWriteTokens != nil {
usage["cache_creation_input_tokens"] = *cacheWriteTokens
}
usage := anthropicUsage(promptTokens, completionTokens, cacheReadTokens, cacheWriteTokens)
return map[string]any{
"id": "msg_" + requestID, "type": "message", "role": "assistant", "model": model,
"content": blocks, "stop_reason": anthropicStopReason(finishReason, toolCalls), "stop_sequence": nil,
Expand Down
31 changes: 31 additions & 0 deletions internal/gateway/anthropic_usage.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package gateway

// Upstream usage arrives in OpenAI form, where prompt_tokens includes the
// cached share. Anthropic splits it: input_tokens counts only the uncached
// part, and cache_read_input_tokens / cache_creation_input_tokens carry the
// rest, so a client adding the three gets the whole prompt back.

// anthropicInputTokens is the uncached share of an OpenAI prompt count.
func anthropicInputTokens(promptTokens int, cacheReadTokens, cacheWriteTokens *int) int {
billed := promptTokens - derefInt(cacheReadTokens) - derefInt(cacheWriteTokens)
if billed < 0 {
return 0
}
return billed
}

// anthropicUsage is the usage object on a Messages response or the final
// message_delta of a Messages stream.
func anthropicUsage(promptTokens, completionTokens int, cacheReadTokens, cacheWriteTokens *int) map[string]any {
usage := map[string]any{
"input_tokens": anthropicInputTokens(promptTokens, cacheReadTokens, cacheWriteTokens),
"output_tokens": completionTokens,
}
if cacheReadTokens != nil {
usage["cache_read_input_tokens"] = *cacheReadTokens
}
if cacheWriteTokens != nil {
usage["cache_creation_input_tokens"] = *cacheWriteTokens
}
return usage
}
61 changes: 61 additions & 0 deletions internal/gateway/anthropic_usage_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package gateway

import (
"bytes"
"strings"
"testing"
)

func TestAnthropicInputTokensExcludeCacheShares(t *testing.T) {
cases := []struct {
name string
prompt int
read, write *int
want int
}{
{name: "no cache", prompt: 100, want: 100},
{name: "read and write", prompt: 100, read: ptrInt(64), write: ptrInt(12), want: 24},
{name: "never negative", prompt: 10, read: ptrInt(64), want: 0},
}
for _, tc := range cases {
if got := anthropicInputTokens(tc.prompt, tc.read, tc.write); got != tc.want {
t.Errorf("%s: input_tokens = %d, want %d", tc.name, got, tc.want)
}
}
}

func TestAnthropicMessageResponseSplitsInputFromCache(t *testing.T) {
resp := anthropicMessageResponse("req", "model", "ok", "", nil, "stop", 100, 20, ptrInt(64), ptrInt(12))
usage := resp["usage"].(map[string]any)
if usage["input_tokens"] != 24 || usage["output_tokens"] != 20 {
t.Fatalf("usage = %#v", usage)
}
}

func TestRelayAnthropicStreamReportsInputTokensInMessageDelta(t *testing.T) {
body := strings.NewReader("data: {\"model\":\"m\",\"choices\":[{\"delta\":{\"content\":\"hi\"}}]}\n\n" +
"data: {\"choices\":[{\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":100,\"completion_tokens\":20,\"cache_read_tokens\":64,\"cache_write_tokens\":12}}\n\ndata: [DONE]\n\n")
var out bytes.Buffer
if _, err := RelayAnthropicStream(&out, body, "req", "model"); err != nil {
t.Fatal(err)
}
stream := out.String()
delta := stream[strings.Index(stream, "event: message_delta"):]
for _, want := range []string{`"input_tokens":24`, `"output_tokens":20`, `"cache_read_input_tokens":64`, `"cache_creation_input_tokens":12`} {
if !strings.Contains(delta, want) {
t.Fatalf("message_delta missing %s:\n%s", want, delta)
}
}
}

func TestRelayAnthropicStreamWithoutCacheReportsFullPrompt(t *testing.T) {
body := strings.NewReader("data: {\"choices\":[{\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":1}}\n\ndata: [DONE]\n\n")
var out bytes.Buffer
if _, err := RelayAnthropicStream(&out, body, "req", "model"); err != nil {
t.Fatal(err)
}
delta := out.String()[strings.Index(out.String(), "event: message_delta"):]
if !strings.Contains(delta, `"input_tokens":4`) || strings.Contains(delta, "cache_read_input_tokens") {
t.Fatalf("message_delta:\n%s", delta)
}
}
10 changes: 3 additions & 7 deletions internal/gateway/compat_stream.go
Original file line number Diff line number Diff line change
Expand Up @@ -383,13 +383,9 @@ func RelayAnthropicStream(writer io.Writer, body io.Reader, requestID, model str
}
}
calls := output.calls()
deltaUsage := map[string]any{"output_tokens": derefInt(stats.CompletionTokens)}
if stats.CacheReadTokens != nil {
deltaUsage["cache_read_input_tokens"] = *stats.CacheReadTokens
}
if stats.CacheWriteTokens != nil {
deltaUsage["cache_creation_input_tokens"] = *stats.CacheWriteTokens
}
// message_start goes out before any usage is known, so the final counts,
// input included, ride on message_delta.
deltaUsage := anthropicUsage(derefInt(stats.PromptTokens), derefInt(stats.CompletionTokens), stats.CacheReadTokens, stats.CacheWriteTokens)
if err := writeSSEEvent(writer, "message_delta", map[string]any{"type": "message_delta", "delta": map[string]any{"stop_reason": anthropicStopReason(output.finishReason, calls), "stop_sequence": nil}, "usage": deltaUsage}); err != nil {
return stats, err
}
Expand Down
23 changes: 16 additions & 7 deletions internal/providers/kiro/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,14 @@ type Client struct {
mu sync.Mutex
models map[string][]providers.ModelInfo
logins map[string]*loginState

// cache estimates prompt-cache reads and writes Kiro does not report.
// A nil cache (tests build Client directly) estimates nothing.
cache *promptCache
}

func NewClient(store Store) *Client {
return &Client{store: store, http: &http.Client{}}
return &Client{store: store, http: &http.Client{}, cache: newPromptCache()}
}

func (c *Client) Adapter() providers.Adapter {
Expand Down Expand Up @@ -280,21 +284,22 @@ func (c *Client) ChatStream(ctx context.Context, accountID string, req translate
return nil, providers.ResolvedChat{}, err
}
resolved := providers.ResolvedChat{ReasoningLevel: payload.Level}
cache := c.cache.begin(accountID, payload)
if c.useResponses(ctx, accountID, cred) {
body, err := c.postResponses(ctx, client, cred, payload)
if err != nil {
return nil, providers.ResolvedChat{}, err
}
return writeOpenAIStream(body, model, payload.clientNames(), armMarkupFilter(payload.Tools)), resolved, nil
return writeOpenAIStream(body, model, payload.clientNames(), armMarkupFilter(payload.Tools), cache), resolved, nil
}
body, kind, err := c.postLegacy(ctx, client, accountID, cred, payload)
if err != nil {
return nil, providers.ResolvedChat{}, err
}
if kind == streamResponses {
return writeOpenAIStream(body, model, payload.clientNames(), armMarkupFilter(payload.Tools)), resolved, nil
return writeOpenAIStream(body, model, payload.clientNames(), armMarkupFilter(payload.Tools), cache), resolved, nil
}
return writeEventStream(body, model, payload.clientNames(), armMarkupFilter(payload.Tools)), resolved, nil
return writeEventStream(body, model, payload.clientNames(), armMarkupFilter(payload.Tools), cache), resolved, nil
}

func (c *Client) postResponses(ctx context.Context, client *http.Client, cred Credential, payload conversation) (io.ReadCloser, error) {
Expand Down Expand Up @@ -461,12 +466,12 @@ func applyLegacyHeaders(req *http.Request, cred Credential, endpoint legacyEndpo

// writeEventStream reads a generateAssistantResponse body and writes the same
// OpenAI chunks CreateResponse produces.
func writeEventStream(body io.ReadCloser, model string, names map[string]string, filter *markupFilter) *http.Response {
func writeEventStream(body io.ReadCloser, model string, names map[string]string, filter *markupFilter, cache *cacheTurn) *http.Response {
pr, pw := io.Pipe()
go func() {
defer body.Close()
defer pw.Close()
writeConsumed(pw, model, eventStreamSource(body), names, filter)
writeConsumed(pw, model, eventStreamSource(body), names, filter, cache)
}()
return &http.Response{
StatusCode: http.StatusOK,
Expand All @@ -485,6 +490,7 @@ func collectOutcome(body io.Reader, level string) (providers.ChatOutcome, error)
var calls []call
finish := "stop"
var prompt, completion int
var cacheRead, cacheWrite *int
var credits float64
sawCredits := false
source := ""
Expand Down Expand Up @@ -516,6 +522,8 @@ func collectOutcome(body io.Reader, level string) (providers.ChatOutcome, error)
Usage struct {
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
CacheReadTokens *int `json:"cache_read_tokens"`
CacheWriteTokens *int `json:"cache_write_tokens"`
Credits *float64 `json:"credits"`
Source string `json:"source"`
} `json:"usage"`
Expand All @@ -532,6 +540,7 @@ func collectOutcome(body io.Reader, level string) (providers.ChatOutcome, error)
if chunk.Usage.PromptTokens > 0 || chunk.Usage.CompletionTokens > 0 || chunk.Usage.Credits != nil {
prompt, completion = chunk.Usage.PromptTokens, chunk.Usage.CompletionTokens
source = chunk.Usage.Source
cacheRead, cacheWrite = chunk.Usage.CacheReadTokens, chunk.Usage.CacheWriteTokens
if chunk.Usage.Credits != nil {
credits, sawCredits = *chunk.Usage.Credits, true
}
Expand All @@ -555,7 +564,7 @@ func collectOutcome(body io.Reader, level string) (providers.ChatOutcome, error)
outcome := providers.ChatOutcome{
Content: content.String(), Reasoning: reasoning.String(), FinishReason: finish,
PromptTokens: prompt, CompletionTokens: completion, ReasoningLevel: level,
UsageSource: source,
UsageSource: source, CacheReadTokens: cacheRead, CacheWriteTokens: cacheWrite,
}
if sawCredits {
value := credits
Expand Down
12 changes: 0 additions & 12 deletions internal/providers/kiro/eventstream.go
Original file line number Diff line number Diff line change
Expand Up @@ -344,18 +344,6 @@ func consumeEvents(source eventSource, cb *streamCallbacks) error {
return nil
}

func updateTokens(event map[string]any, input, output int) (int, int) {
if usage, ok := event["usage"].(map[string]any); ok {
if n, ok := numberField(usage, "inputTokens", "input_tokens"); ok {
input = n
}
if n, ok := numberField(usage, "outputTokens", "output_tokens"); ok {
output = n
}
}
return input, output
}

func numberField(m map[string]any, keys ...string) (int, bool) {
for _, key := range keys {
switch value := m[key].(type) {
Expand Down
14 changes: 9 additions & 5 deletions internal/providers/kiro/openai_stream.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,12 @@ import (
// relay opens a tool_use block on the first chunk that carries a name and then
// treats every later arguments fragment as input_json_delta. Sending the name
// before the JSON is finished would leave the client with a partial object.
func writeOpenAIStream(body io.ReadCloser, model string, names map[string]string, filter *markupFilter) *http.Response {
func writeOpenAIStream(body io.ReadCloser, model string, names map[string]string, filter *markupFilter, cache *cacheTurn) *http.Response {
pr, pw := io.Pipe()
go func() {
defer body.Close()
defer pw.Close()
writeConsumed(pw, model, responsesSSESource(body), names, filter)
writeConsumed(pw, model, responsesSSESource(body), names, filter, cache)
}()
return &http.Response{
StatusCode: http.StatusOK,
Expand All @@ -30,8 +30,9 @@ func writeOpenAIStream(body io.ReadCloser, model string, names map[string]string

// writeConsumed turns one event source into OpenAI chat chunks. Tool calls are
// announced only after the upstream emits a complete call. names maps a Kiro
// tool name back to the name the client declared.
func writeConsumed(w io.Writer, model string, source eventSource, names map[string]string, filter *markupFilter) {
// tool name back to the name the client declared. cache, when set, adds the
// local prompt-cache estimate and is stored only after a clean stream.
func writeConsumed(w io.Writer, model string, source eventSource, names map[string]string, filter *markupFilter, cache *cacheTurn) {
index := 0
var inputTokens, outputTokens int
var credits, contextPct float64
Expand Down Expand Up @@ -90,7 +91,10 @@ func writeConsumed(w io.Writer, model string, source eventSource, names map[stri
payload, _ := json.Marshal(map[string]any{"error": map[string]any{"message": err.Error(), "type": "upstream_error"}})
fmt.Fprintf(w, "data: %s\n\n", payload)
}
usage := streamUsage(model, inputTokens, outputTokens, contextPct, produced.total(), credits, sawCredits)
usage := streamUsage(model, inputTokens, outputTokens, contextPct, produced.total(), credits, sawCredits, cache.estimate())
if err == nil {
cache.commit()
}
done := map[string]any{
"id": "chatcmpl-kiro", "object": "chat.completion.chunk", "created": time.Now().Unix(), "model": model,
"choices": []map[string]any{{"index": 0, "delta": map[string]any{}, "finish_reason": finish}},
Expand Down
Loading
Loading