From ba91f4f2a9f3bce06d87db3e958cf7c00d0f9c15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=94=A1=E5=8F=8A?= <522caiji@gmail.com> Date: Fri, 20 Nov 2026 22:35:09 +0800 Subject: [PATCH] fix(kiro): estimate prompt cache and report Anthropic input tokens - gateway: Anthropic input_tokens excludes cache read/creation; stream message_delta now carries input_tokens. - kiro: local per-account prompt-cache estimator (prompt_cache.go), Kiro-Go style token estimator (token_estimate.go), broader upstream usage parsing (upstream_tokens.go). - collectOutcome passes cache read/write through. --- changelog/unreleased/kiro-usage-cache.md | 11 + internal/gateway/anthropic.go | 11 +- internal/gateway/anthropic_usage.go | 31 +++ internal/gateway/anthropic_usage_test.go | 61 +++++ internal/gateway/compat_stream.go | 10 +- internal/providers/kiro/client.go | 23 +- internal/providers/kiro/eventstream.go | 12 - internal/providers/kiro/openai_stream.go | 14 +- internal/providers/kiro/prompt_cache.go | 256 ++++++++++++++++++ internal/providers/kiro/prompt_cache_test.go | 158 +++++++++++ internal/providers/kiro/token_estimate.go | 50 ++++ .../providers/kiro/token_estimate_test.go | 38 +++ internal/providers/kiro/tools_test.go | 6 +- internal/providers/kiro/upstream_tokens.go | 71 +++++ .../providers/kiro/upstream_tokens_test.go | 31 +++ internal/providers/kiro/usage.go | 34 +-- internal/providers/kiro/usage_test.go | 8 +- 17 files changed, 752 insertions(+), 73 deletions(-) create mode 100644 changelog/unreleased/kiro-usage-cache.md create mode 100644 internal/gateway/anthropic_usage.go create mode 100644 internal/gateway/anthropic_usage_test.go create mode 100644 internal/providers/kiro/prompt_cache.go create mode 100644 internal/providers/kiro/prompt_cache_test.go create mode 100644 internal/providers/kiro/token_estimate.go create mode 100644 internal/providers/kiro/token_estimate_test.go create mode 100644 internal/providers/kiro/upstream_tokens.go create mode 100644 internal/providers/kiro/upstream_tokens_test.go diff --git a/changelog/unreleased/kiro-usage-cache.md b/changelog/unreleased/kiro-usage-cache.md new file mode 100644 index 00000000..eda3c437 --- /dev/null +++ b/changelog/unreleased/kiro-usage-cache.md @@ -0,0 +1,11 @@ +### English + +- Anthropic Messages streams now report `input_tokens` in the final `message_delta`; it was missing before, so clients saw 0 input. Across all providers, Anthropic `input_tokens` now counts only the uncached part of the prompt, with `cache_read_input_tokens` and `cache_creation_input_tokens` carrying the rest, matching Anthropic's own usage format. +- Kiro requests now report estimated prompt-cache reads and writes. Kiro does not return cache figures, so the gateway tracks prompt prefixes per account for five minutes, the way Kiro-Go does, and reports the repeated part as a cache read. The figures are estimates. +- Kiro output tokens are estimated more closely (separate weights for letters, digits, punctuation and CJK text), and token counts are now read from more upstream usage shapes when Kiro does send them. + +### 中文 + +- Anthropic Messages 流式响应现在会在最后的 `message_delta` 中返回 `input_tokens`,之前缺失导致客户端看到的输入为 0。所有渠道的 Anthropic `input_tokens` 现在只计未命中缓存的部分,其余由 `cache_read_input_tokens` 和 `cache_creation_input_tokens` 表示,与 Anthropic 官方用量格式一致。 +- Kiro 请求现在会返回估算的缓存读取和缓存写入。Kiro 本身不返回缓存数据,网关参照 Kiro-Go 按账号记录 5 分钟内的提示前缀,重复部分计为缓存读取。该数值为估算值。 +- Kiro 输出 Tokens 的估算更准确(字母、数字、标点和中日韩文字分别计权),Kiro 返回 Token 数时也能识别更多上游用量格式。 diff --git a/internal/gateway/anthropic.go b/internal/gateway/anthropic.go index 6dc8a854..69236b37 100644 --- a/internal/gateway/anthropic.go +++ b/internal/gateway/anthropic.go @@ -186,16 +186,7 @@ func anthropicMessageResponse(requestID, model, content, reasoning string, toolC if len(blocks) == 0 { blocks = append(blocks, map[string]any{"type": "text", "text": ""}) } - usage := map[string]any{ - "input_tokens": promptTokens, - "output_tokens": completionTokens, - } - if cacheReadTokens != nil { - usage["cache_read_input_tokens"] = *cacheReadTokens - } - if cacheWriteTokens != nil { - usage["cache_creation_input_tokens"] = *cacheWriteTokens - } + usage := anthropicUsage(promptTokens, completionTokens, cacheReadTokens, cacheWriteTokens) return map[string]any{ "id": "msg_" + requestID, "type": "message", "role": "assistant", "model": model, "content": blocks, "stop_reason": anthropicStopReason(finishReason, toolCalls), "stop_sequence": nil, diff --git a/internal/gateway/anthropic_usage.go b/internal/gateway/anthropic_usage.go new file mode 100644 index 00000000..523f09b0 --- /dev/null +++ b/internal/gateway/anthropic_usage.go @@ -0,0 +1,31 @@ +package gateway + +// Upstream usage arrives in OpenAI form, where prompt_tokens includes the +// cached share. Anthropic splits it: input_tokens counts only the uncached +// part, and cache_read_input_tokens / cache_creation_input_tokens carry the +// rest, so a client adding the three gets the whole prompt back. + +// anthropicInputTokens is the uncached share of an OpenAI prompt count. +func anthropicInputTokens(promptTokens int, cacheReadTokens, cacheWriteTokens *int) int { + billed := promptTokens - derefInt(cacheReadTokens) - derefInt(cacheWriteTokens) + if billed < 0 { + return 0 + } + return billed +} + +// anthropicUsage is the usage object on a Messages response or the final +// message_delta of a Messages stream. +func anthropicUsage(promptTokens, completionTokens int, cacheReadTokens, cacheWriteTokens *int) map[string]any { + usage := map[string]any{ + "input_tokens": anthropicInputTokens(promptTokens, cacheReadTokens, cacheWriteTokens), + "output_tokens": completionTokens, + } + if cacheReadTokens != nil { + usage["cache_read_input_tokens"] = *cacheReadTokens + } + if cacheWriteTokens != nil { + usage["cache_creation_input_tokens"] = *cacheWriteTokens + } + return usage +} diff --git a/internal/gateway/anthropic_usage_test.go b/internal/gateway/anthropic_usage_test.go new file mode 100644 index 00000000..daaa56a3 --- /dev/null +++ b/internal/gateway/anthropic_usage_test.go @@ -0,0 +1,61 @@ +package gateway + +import ( + "bytes" + "strings" + "testing" +) + +func TestAnthropicInputTokensExcludeCacheShares(t *testing.T) { + cases := []struct { + name string + prompt int + read, write *int + want int + }{ + {name: "no cache", prompt: 100, want: 100}, + {name: "read and write", prompt: 100, read: ptrInt(64), write: ptrInt(12), want: 24}, + {name: "never negative", prompt: 10, read: ptrInt(64), want: 0}, + } + for _, tc := range cases { + if got := anthropicInputTokens(tc.prompt, tc.read, tc.write); got != tc.want { + t.Errorf("%s: input_tokens = %d, want %d", tc.name, got, tc.want) + } + } +} + +func TestAnthropicMessageResponseSplitsInputFromCache(t *testing.T) { + resp := anthropicMessageResponse("req", "model", "ok", "", nil, "stop", 100, 20, ptrInt(64), ptrInt(12)) + usage := resp["usage"].(map[string]any) + if usage["input_tokens"] != 24 || usage["output_tokens"] != 20 { + t.Fatalf("usage = %#v", usage) + } +} + +func TestRelayAnthropicStreamReportsInputTokensInMessageDelta(t *testing.T) { + body := strings.NewReader("data: {\"model\":\"m\",\"choices\":[{\"delta\":{\"content\":\"hi\"}}]}\n\n" + + "data: {\"choices\":[{\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":100,\"completion_tokens\":20,\"cache_read_tokens\":64,\"cache_write_tokens\":12}}\n\ndata: [DONE]\n\n") + var out bytes.Buffer + if _, err := RelayAnthropicStream(&out, body, "req", "model"); err != nil { + t.Fatal(err) + } + stream := out.String() + delta := stream[strings.Index(stream, "event: message_delta"):] + for _, want := range []string{`"input_tokens":24`, `"output_tokens":20`, `"cache_read_input_tokens":64`, `"cache_creation_input_tokens":12`} { + if !strings.Contains(delta, want) { + t.Fatalf("message_delta missing %s:\n%s", want, delta) + } + } +} + +func TestRelayAnthropicStreamWithoutCacheReportsFullPrompt(t *testing.T) { + body := strings.NewReader("data: {\"choices\":[{\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":1}}\n\ndata: [DONE]\n\n") + var out bytes.Buffer + if _, err := RelayAnthropicStream(&out, body, "req", "model"); err != nil { + t.Fatal(err) + } + delta := out.String()[strings.Index(out.String(), "event: message_delta"):] + if !strings.Contains(delta, `"input_tokens":4`) || strings.Contains(delta, "cache_read_input_tokens") { + t.Fatalf("message_delta:\n%s", delta) + } +} diff --git a/internal/gateway/compat_stream.go b/internal/gateway/compat_stream.go index 5615e6d9..039bc57b 100644 --- a/internal/gateway/compat_stream.go +++ b/internal/gateway/compat_stream.go @@ -383,13 +383,9 @@ func RelayAnthropicStream(writer io.Writer, body io.Reader, requestID, model str } } calls := output.calls() - deltaUsage := map[string]any{"output_tokens": derefInt(stats.CompletionTokens)} - if stats.CacheReadTokens != nil { - deltaUsage["cache_read_input_tokens"] = *stats.CacheReadTokens - } - if stats.CacheWriteTokens != nil { - deltaUsage["cache_creation_input_tokens"] = *stats.CacheWriteTokens - } + // message_start goes out before any usage is known, so the final counts, + // input included, ride on message_delta. + deltaUsage := anthropicUsage(derefInt(stats.PromptTokens), derefInt(stats.CompletionTokens), stats.CacheReadTokens, stats.CacheWriteTokens) if err := writeSSEEvent(writer, "message_delta", map[string]any{"type": "message_delta", "delta": map[string]any{"stop_reason": anthropicStopReason(output.finishReason, calls), "stop_sequence": nil}, "usage": deltaUsage}); err != nil { return stats, err } diff --git a/internal/providers/kiro/client.go b/internal/providers/kiro/client.go index 3241a226..76e10f7c 100644 --- a/internal/providers/kiro/client.go +++ b/internal/providers/kiro/client.go @@ -44,10 +44,14 @@ type Client struct { mu sync.Mutex models map[string][]providers.ModelInfo logins map[string]*loginState + + // cache estimates prompt-cache reads and writes Kiro does not report. + // A nil cache (tests build Client directly) estimates nothing. + cache *promptCache } func NewClient(store Store) *Client { - return &Client{store: store, http: &http.Client{}} + return &Client{store: store, http: &http.Client{}, cache: newPromptCache()} } func (c *Client) Adapter() providers.Adapter { @@ -280,21 +284,22 @@ func (c *Client) ChatStream(ctx context.Context, accountID string, req translate return nil, providers.ResolvedChat{}, err } resolved := providers.ResolvedChat{ReasoningLevel: payload.Level} + cache := c.cache.begin(accountID, payload) if c.useResponses(ctx, accountID, cred) { body, err := c.postResponses(ctx, client, cred, payload) if err != nil { return nil, providers.ResolvedChat{}, err } - return writeOpenAIStream(body, model, payload.clientNames(), armMarkupFilter(payload.Tools)), resolved, nil + return writeOpenAIStream(body, model, payload.clientNames(), armMarkupFilter(payload.Tools), cache), resolved, nil } body, kind, err := c.postLegacy(ctx, client, accountID, cred, payload) if err != nil { return nil, providers.ResolvedChat{}, err } if kind == streamResponses { - return writeOpenAIStream(body, model, payload.clientNames(), armMarkupFilter(payload.Tools)), resolved, nil + return writeOpenAIStream(body, model, payload.clientNames(), armMarkupFilter(payload.Tools), cache), resolved, nil } - return writeEventStream(body, model, payload.clientNames(), armMarkupFilter(payload.Tools)), resolved, nil + return writeEventStream(body, model, payload.clientNames(), armMarkupFilter(payload.Tools), cache), resolved, nil } func (c *Client) postResponses(ctx context.Context, client *http.Client, cred Credential, payload conversation) (io.ReadCloser, error) { @@ -461,12 +466,12 @@ func applyLegacyHeaders(req *http.Request, cred Credential, endpoint legacyEndpo // writeEventStream reads a generateAssistantResponse body and writes the same // OpenAI chunks CreateResponse produces. -func writeEventStream(body io.ReadCloser, model string, names map[string]string, filter *markupFilter) *http.Response { +func writeEventStream(body io.ReadCloser, model string, names map[string]string, filter *markupFilter, cache *cacheTurn) *http.Response { pr, pw := io.Pipe() go func() { defer body.Close() defer pw.Close() - writeConsumed(pw, model, eventStreamSource(body), names, filter) + writeConsumed(pw, model, eventStreamSource(body), names, filter, cache) }() return &http.Response{ StatusCode: http.StatusOK, @@ -485,6 +490,7 @@ func collectOutcome(body io.Reader, level string) (providers.ChatOutcome, error) var calls []call finish := "stop" var prompt, completion int + var cacheRead, cacheWrite *int var credits float64 sawCredits := false source := "" @@ -516,6 +522,8 @@ func collectOutcome(body io.Reader, level string) (providers.ChatOutcome, error) Usage struct { PromptTokens int `json:"prompt_tokens"` CompletionTokens int `json:"completion_tokens"` + CacheReadTokens *int `json:"cache_read_tokens"` + CacheWriteTokens *int `json:"cache_write_tokens"` Credits *float64 `json:"credits"` Source string `json:"source"` } `json:"usage"` @@ -532,6 +540,7 @@ func collectOutcome(body io.Reader, level string) (providers.ChatOutcome, error) if chunk.Usage.PromptTokens > 0 || chunk.Usage.CompletionTokens > 0 || chunk.Usage.Credits != nil { prompt, completion = chunk.Usage.PromptTokens, chunk.Usage.CompletionTokens source = chunk.Usage.Source + cacheRead, cacheWrite = chunk.Usage.CacheReadTokens, chunk.Usage.CacheWriteTokens if chunk.Usage.Credits != nil { credits, sawCredits = *chunk.Usage.Credits, true } @@ -555,7 +564,7 @@ func collectOutcome(body io.Reader, level string) (providers.ChatOutcome, error) outcome := providers.ChatOutcome{ Content: content.String(), Reasoning: reasoning.String(), FinishReason: finish, PromptTokens: prompt, CompletionTokens: completion, ReasoningLevel: level, - UsageSource: source, + UsageSource: source, CacheReadTokens: cacheRead, CacheWriteTokens: cacheWrite, } if sawCredits { value := credits diff --git a/internal/providers/kiro/eventstream.go b/internal/providers/kiro/eventstream.go index d2119b03..d158fbcd 100644 --- a/internal/providers/kiro/eventstream.go +++ b/internal/providers/kiro/eventstream.go @@ -344,18 +344,6 @@ func consumeEvents(source eventSource, cb *streamCallbacks) error { return nil } -func updateTokens(event map[string]any, input, output int) (int, int) { - if usage, ok := event["usage"].(map[string]any); ok { - if n, ok := numberField(usage, "inputTokens", "input_tokens"); ok { - input = n - } - if n, ok := numberField(usage, "outputTokens", "output_tokens"); ok { - output = n - } - } - return input, output -} - func numberField(m map[string]any, keys ...string) (int, bool) { for _, key := range keys { switch value := m[key].(type) { diff --git a/internal/providers/kiro/openai_stream.go b/internal/providers/kiro/openai_stream.go index c4358911..7e62e369 100644 --- a/internal/providers/kiro/openai_stream.go +++ b/internal/providers/kiro/openai_stream.go @@ -14,12 +14,12 @@ import ( // relay opens a tool_use block on the first chunk that carries a name and then // treats every later arguments fragment as input_json_delta. Sending the name // before the JSON is finished would leave the client with a partial object. -func writeOpenAIStream(body io.ReadCloser, model string, names map[string]string, filter *markupFilter) *http.Response { +func writeOpenAIStream(body io.ReadCloser, model string, names map[string]string, filter *markupFilter, cache *cacheTurn) *http.Response { pr, pw := io.Pipe() go func() { defer body.Close() defer pw.Close() - writeConsumed(pw, model, responsesSSESource(body), names, filter) + writeConsumed(pw, model, responsesSSESource(body), names, filter, cache) }() return &http.Response{ StatusCode: http.StatusOK, @@ -30,8 +30,9 @@ func writeOpenAIStream(body io.ReadCloser, model string, names map[string]string // writeConsumed turns one event source into OpenAI chat chunks. Tool calls are // announced only after the upstream emits a complete call. names maps a Kiro -// tool name back to the name the client declared. -func writeConsumed(w io.Writer, model string, source eventSource, names map[string]string, filter *markupFilter) { +// tool name back to the name the client declared. cache, when set, adds the +// local prompt-cache estimate and is stored only after a clean stream. +func writeConsumed(w io.Writer, model string, source eventSource, names map[string]string, filter *markupFilter, cache *cacheTurn) { index := 0 var inputTokens, outputTokens int var credits, contextPct float64 @@ -90,7 +91,10 @@ func writeConsumed(w io.Writer, model string, source eventSource, names map[stri payload, _ := json.Marshal(map[string]any{"error": map[string]any{"message": err.Error(), "type": "upstream_error"}}) fmt.Fprintf(w, "data: %s\n\n", payload) } - usage := streamUsage(model, inputTokens, outputTokens, contextPct, produced.total(), credits, sawCredits) + usage := streamUsage(model, inputTokens, outputTokens, contextPct, produced.total(), credits, sawCredits, cache.estimate()) + if err == nil { + cache.commit() + } done := map[string]any{ "id": "chatcmpl-kiro", "object": "chat.completion.chunk", "created": time.Now().Unix(), "model": model, "choices": []map[string]any{{"index": 0, "delta": map[string]any{}, "finish_reason": finish}}, diff --git a/internal/providers/kiro/prompt_cache.go b/internal/providers/kiro/prompt_cache.go new file mode 100644 index 00000000..773e7c2e --- /dev/null +++ b/internal/providers/kiro/prompt_cache.go @@ -0,0 +1,256 @@ +package kiro + +import ( + "crypto/sha256" + "encoding/json" + "math" + "strings" + "sync" + "time" +) + +// Kiro reports neither cache reads nor cache writes, so they are estimated +// locally the way Kiro-Go does (proxy/cache_tracker.go): each request is cut +// into a prelude (model, system, tools) and one block per turn, every block +// end is fingerprinted with a running SHA-256, and a later request on the same +// account that repeats a stored prefix reads it from "cache". Fingerprints +// live in memory for the Anthropic default TTL of five minutes. The figures +// are an estimate for the request log and the client, not a billing number. + +const ( + promptCacheTTL = 5 * time.Minute + // Anthropic does not cache prefixes shorter than this. + minCacheableTokens = 1024 + opusMinCacheableTokens = 4096 + // The newest content in a request is never all served from cache. + maxCacheReadShare = 0.85 + // Media is counted at a flat size instead of by its base64 length. + mediaTokenEstimate = 1600 +) + +// promptCacheUsage is one request's estimate. Total is the local estimate of +// the whole prompt that Read and Write are measured against. +type promptCacheUsage struct { + Read, Write, Total int +} + +// scaled re-expresses the estimate against input, the prompt size the usage +// chunk reports (Kiro's context share), so the shares stay the same while +// read plus write never exceeds input. +func (u *promptCacheUsage) scaled(input int) (int, int) { + if u == nil || input <= 0 || u.Total <= 0 { + return 0, 0 + } + ratio := float64(input) / float64(u.Total) + read := min(int(math.Round(float64(u.Read)*ratio)), input) + write := min(int(math.Round(float64(u.Write)*ratio)), input-read) + return read, write +} + +type cacheBreakpoint struct { + fingerprint [32]byte + tokens int // cumulative estimate up to and including this block +} + +type cacheProfile struct { + model string + breakpoints []cacheBreakpoint + total int +} + +// buildCacheProfile fingerprints a conversation block by block. A user turn +// hashes the same whether it is the current turn or a replayed history turn, +// so this request's full prompt is the prefix the next turn looks up. +func buildCacheProfile(c conversation) *cacheProfile { + hasher := sha256.New() + profile := &cacheProfile{model: c.Model} + add := func(value any, tokens int) { + encoded, err := json.Marshal(value) + if err != nil { + return + } + hasher.Write(encoded) + hasher.Write([]byte{0}) + profile.total += tokens + var fingerprint [32]byte + copy(fingerprint[:], hasher.Sum(nil)) + profile.breakpoints = append(profile.breakpoints, cacheBreakpoint{fingerprint: fingerprint, tokens: profile.total}) + } + prelude := struct { + Model, System string + Tools []toolSpec + }{c.Model, c.System, c.Tools} + add(prelude, preludeTokens(c)) + for _, turn := range c.History { + switch { + case turn.User != nil: + add(turn.User, userTokens(*turn.User)) + case turn.Assistant != nil: + add(turn.Assistant, assistantTokens(*turn.Assistant)) + } + } + add(c.Current, userTokens(c.Current)) + if len(profile.breakpoints) == 0 || profile.total <= 0 { + return nil + } + return profile +} + +func preludeTokens(c conversation) int { + total := estimateTokens(c.System) + for _, tool := range c.Tools { + total += estimateTokens(tool.Name) + estimateTokens(tool.Description) + if schema, err := json.Marshal(tool.Parameters); err == nil { + total += estimateTokens(string(schema)) + } + } + return total +} + +func userTokens(turn userTurn) int { + total := estimateTokens(turn.Text) + mediaTokenEstimate*(len(turn.Images)+len(turn.Documents)) + for _, result := range turn.Results { + total += estimateTokens(result.Output) + } + return total +} + +func assistantTokens(turn assistantTurn) int { + total := estimateTokens(turn.Text) + for _, block := range turn.Reasoning { + total += estimateTokens(block.Text) + } + for _, call := range turn.ToolUses { + total += estimateTokens(call.Name) + if input, err := json.Marshal(call.Input); err == nil { + total += estimateTokens(string(input)) + } + } + return total +} + +func minCacheableFor(model string) int { + if strings.Contains(strings.ToLower(model), "opus") { + return opusMinCacheableTokens + } + return minCacheableTokens +} + +// promptCache holds stored prefix fingerprints per account. The zero of a nil +// *promptCache is valid and estimates nothing. +type promptCache struct { + mu sync.Mutex + entries map[string]map[[32]byte]time.Time // account -> fingerprint -> expiry + now func() time.Time +} + +func newPromptCache() *promptCache { + return &promptCache{entries: map[string]map[[32]byte]time.Time{}, now: time.Now} +} + +// compute estimates cache use for profile without storing it. The first +// cacheable request on an account writes the whole prompt. Later requests read +// the longest stored prefix, capped so part of the prompt stays uncached, and +// write the rest up to that cap. +func (t *promptCache) compute(account string, profile *cacheProfile) promptCacheUsage { + usage := promptCacheUsage{Total: profile.total} + minTokens := minCacheableFor(profile.model) + if profile.total < minTokens { + return usage + } + now := t.now() + t.mu.Lock() + defer t.mu.Unlock() + t.pruneLocked(now) + entries := t.entries[account] + if len(entries) == 0 { + usage.Write = profile.total + return usage + } + cacheable := int(float64(profile.total) * maxCacheReadShare) + matched := 0 + for i := len(profile.breakpoints) - 1; i >= 0; i-- { + point := profile.breakpoints[i] + if point.tokens < minTokens { + break + } + if _, ok := entries[point.fingerprint]; ok { + matched = min(point.tokens, cacheable) + break + } + } + usage.Read = matched + usage.Write = max(cacheable-matched, 0) + return usage +} + +// update stores every cacheable prefix of profile for the TTL. +func (t *promptCache) update(account string, profile *cacheProfile) { + minTokens := minCacheableFor(profile.model) + now := t.now() + t.mu.Lock() + defer t.mu.Unlock() + t.pruneLocked(now) + entries := t.entries[account] + if entries == nil { + entries = map[[32]byte]time.Time{} + t.entries[account] = entries + } + for _, point := range profile.breakpoints { + if point.tokens >= minTokens { + entries[point.fingerprint] = now.Add(promptCacheTTL) + } + } + if len(entries) == 0 { + delete(t.entries, account) + } +} + +func (t *promptCache) pruneLocked(now time.Time) { + for account, entries := range t.entries { + for fingerprint, expires := range entries { + if !expires.After(now) { + delete(entries, fingerprint) + } + } + if len(entries) == 0 { + delete(t.entries, account) + } + } +} + +// cacheTurn is one request's estimate. It is computed before the request is +// sent and stored only after the stream completes, so a failed request never +// seeds a hit for the next one. +type cacheTurn struct { + tracker *promptCache + account string + profile *cacheProfile + usage promptCacheUsage +} + +func (t *promptCache) begin(account string, c conversation) *cacheTurn { + if t == nil || account == "" { + return nil + } + profile := buildCacheProfile(c) + if profile == nil { + return nil + } + return &cacheTurn{tracker: t, account: account, profile: profile, usage: t.compute(account, profile)} +} + +func (c *cacheTurn) estimate() *promptCacheUsage { + if c == nil { + return nil + } + usage := c.usage + return &usage +} + +func (c *cacheTurn) commit() { + if c == nil { + return + } + c.tracker.update(c.account, c.profile) +} diff --git a/internal/providers/kiro/prompt_cache_test.go b/internal/providers/kiro/prompt_cache_test.go new file mode 100644 index 00000000..db901149 --- /dev/null +++ b/internal/providers/kiro/prompt_cache_test.go @@ -0,0 +1,158 @@ +package kiro + +import ( + "strings" + "testing" + "time" +) + +func longText(word string, n int) string { + return strings.TrimSpace(strings.Repeat(word+" ", n)) +} + +func cacheConversation(model string, turns ...string) conversation { + c := conversation{Model: model, System: longText("system", 2000)} + for i, text := range turns[:len(turns)-1] { + if i%2 == 0 { + c.History = append(c.History, historyTurn{User: &userTurn{Text: text}}) + } else { + c.History = append(c.History, historyTurn{Assistant: &assistantTurn{Text: text}}) + } + } + c.Current = userTurn{Text: turns[len(turns)-1]} + return c +} + +func TestPromptCacheFirstRequestWritesOnly(t *testing.T) { + cache := newPromptCache() + turn := cache.begin("acct", cacheConversation("claude-sonnet-4.6", "hello")) + usage := turn.estimate() + if usage.Read != 0 || usage.Write != usage.Total || usage.Total < minCacheableTokens { + t.Fatalf("usage = %+v", usage) + } +} + +func TestPromptCacheNextTurnReadsStoredPrefix(t *testing.T) { + cache := newPromptCache() + first := cache.begin("acct", cacheConversation("claude-sonnet-4.6", "hello")) + first.commit() + + // The new turns are long enough that the stored prefix stays under the + // 85% read cap. + next := cache.begin("acct", cacheConversation("claude-sonnet-4.6", "hello", longText("reply", 600), longText("again", 600))) + usage := next.estimate() + if usage.Read != first.usage.Total { + t.Fatalf("read = %d, want previous prompt %d", usage.Read, first.usage.Total) + } + cacheable := int(float64(usage.Total) * maxCacheReadShare) + if usage.Read+usage.Write != cacheable { + t.Fatalf("usage = %+v, cacheable = %d", usage, cacheable) + } +} + +func TestPromptCacheIsPerAccount(t *testing.T) { + cache := newPromptCache() + cache.begin("a", cacheConversation("claude-sonnet-4.6", "hello")).commit() + usage := cache.begin("b", cacheConversation("claude-sonnet-4.6", "hello")).estimate() + if usage.Read != 0 { + t.Fatalf("account b read account a's cache: %+v", usage) + } +} + +func TestPromptCacheExpires(t *testing.T) { + now := time.Now() + cache := newPromptCache() + cache.now = func() time.Time { return now } + cache.begin("acct", cacheConversation("claude-sonnet-4.6", "hello")).commit() + now = now.Add(promptCacheTTL + time.Second) + usage := cache.begin("acct", cacheConversation("claude-sonnet-4.6", "hello", "hi", "again")).estimate() + if usage.Read != 0 || usage.Write != usage.Total { + t.Fatalf("expired cache still read: %+v", usage) + } +} + +func TestPromptCacheSkipsShortPrompts(t *testing.T) { + cache := newPromptCache() + c := conversation{Model: "claude-sonnet-4.6", Current: userTurn{Text: "hi"}} + usage := cache.begin("acct", c).estimate() + if usage.Read != 0 || usage.Write != 0 { + t.Fatalf("short prompt cached: %+v", usage) + } +} + +func TestPromptCacheOpusNeedsLongerPrefix(t *testing.T) { + c := conversation{Model: "claude-opus-4.6", System: longText("system", 1500), Current: userTurn{Text: "hi"}} + usage := newPromptCache().begin("acct", c).estimate() + if usage.Total < minCacheableTokens || usage.Total >= opusMinCacheableTokens || usage.Write != 0 { + t.Fatalf("usage = %+v", usage) + } +} + +func TestPromptCacheFailedStreamIsNotStored(t *testing.T) { + cache := newPromptCache() + turn := cache.begin("acct", cacheConversation("claude-sonnet-4.6", "hello")) + frames := []streamEvent{ + {Type: "assistantResponseEvent", Event: map[string]any{"content": "partial"}}, + } + failing := func() eventSource { + source := sliceSource(frames) + return func() (streamEvent, error) { + next, err := source() + if err != nil { + return streamEvent{}, errUpstreamTest + } + return next, nil + } + }() + var out strings.Builder + writeConsumed(&out, "claude-sonnet-4.6", failing, nil, nil, turn) + if usage := cache.begin("acct", cacheConversation("claude-sonnet-4.6", "hello")).estimate(); usage.Read != 0 { + t.Fatalf("failed stream seeded a hit: %+v", usage) + } +} + +func TestPromptCacheNilIsSafe(t *testing.T) { + var cache *promptCache + turn := cache.begin("acct", cacheConversation("claude-sonnet-4.6", "hello")) + if turn != nil || turn.estimate() != nil { + t.Fatal("nil cache produced an estimate") + } + turn.commit() +} + +func TestPromptCacheScaledStaysWithinInput(t *testing.T) { + usage := &promptCacheUsage{Read: 800, Write: 150, Total: 1000} + read, write := usage.scaled(4000) + if read != 3200 || write != 600 { + t.Fatalf("scaled = %d, %d", read, write) + } + read, write = (&promptCacheUsage{Read: 900, Write: 900, Total: 1000}).scaled(100) + if read+write > 100 { + t.Fatalf("read %d + write %d exceeds input", read, write) + } +} + +// The usage chunk carries the estimate in OpenAI form; prompt_tokens stays the +// whole prompt and collectOutcome passes the cache split on. +func TestStreamUsageCarriesCacheEstimate(t *testing.T) { + cache := newPromptCache() + cache.begin("acct", cacheConversation("claude-sonnet-4.6", "hello")).commit() + turn := cache.begin("acct", cacheConversation("claude-sonnet-4.6", "hello", "hi there", "and now?")) + + var out strings.Builder + writeConsumed(&out, "claude-sonnet-4.6", sliceSource(capturedLegacyStream()), nil, nil, turn) + usage := finalUsage(t, out.String()) + read, _ := usage["cache_read_tokens"].(float64) + write, _ := usage["cache_write_tokens"].(float64) + if usage["prompt_tokens"] != float64(4109) || read <= 0 || read+write > 4109 { + t.Fatalf("usage = %v", usage) + } + + outcome, err := collectOutcome(strings.NewReader(out.String()), "") + if err != nil { + t.Fatal(err) + } + if outcome.CacheReadTokens == nil || *outcome.CacheReadTokens != int(read) || outcome.CacheWriteTokens == nil || *outcome.CacheWriteTokens != int(write) { + t.Fatalf("outcome = %+v", outcome) + } +} diff --git a/internal/providers/kiro/token_estimate.go b/internal/providers/kiro/token_estimate.go new file mode 100644 index 00000000..183e0ff4 --- /dev/null +++ b/internal/providers/kiro/token_estimate.go @@ -0,0 +1,50 @@ +package kiro + +import "math" + +// Kiro never states output tokens on generateAssistantResponse, so they are +// estimated from the text the stream produced. The weights follow Kiro-Go +// (proxy/token_estimator.go estimateApproxTokens): ordinary ASCII about 4.5 +// characters per token, digits 2, punctuation 1.5, and non-ASCII (CJK, emoji) +// 1.5. It is a usage magnitude for the request log, not a billing figure. + +// tokenCounter accumulates character classes across stream fragments and +// rounds once at the end, so a reply split into many chunks is not rounded up +// once per chunk. +type tokenCounter struct { + runes, ascii, digits, symbols, other int +} + +func (c *tokenCounter) add(text string) { + for _, r := range text { + c.runes++ + switch { + case r >= 0x80: + c.other++ + case r >= '0' && r <= '9': + c.digits++ + case (r >= '!' && r <= '/') || (r >= ':' && r <= '@') || (r >= '[' && r <= '`') || (r >= '{' && r <= '~'): + c.symbols++ + default: + c.ascii++ + } + } +} + +func (c *tokenCounter) total() int { + if c.runes == 0 { + return 0 + } + if c.runes < 5 { + return max(1, int(math.Ceil(float64(c.runes)/3))) + } + estimated := int(math.Ceil(float64(c.ascii)/4.5 + float64(c.digits)/2 + float64(c.symbols)/1.5 + float64(c.other)/1.5)) + return max(1, estimated) +} + +// estimateTokens is the same estimate for one piece of text. +func estimateTokens(text string) int { + var counter tokenCounter + counter.add(text) + return counter.total() +} diff --git a/internal/providers/kiro/token_estimate_test.go b/internal/providers/kiro/token_estimate_test.go new file mode 100644 index 00000000..64f5ae7a --- /dev/null +++ b/internal/providers/kiro/token_estimate_test.go @@ -0,0 +1,38 @@ +package kiro + +import ( + "errors" + "testing" +) + +var errUpstreamTest = errors.New("upstream reset") + +func TestEstimateTokens(t *testing.T) { + cases := map[string]int{ + "": 0, + "ok": 1, + "abcd": 2, + "hello world": 3, // 11 ascii incl. space / 4.5 + "1234567890": 5, + "{}[]()": 4, + "你好世界你好": 4, // 6 non-ASCII / 1.5 + "hello, 世界123": 5, // 6 ascii incl. space/4.5 + 1 comma/1.5 + 2/1.5 + 3/2 = 4.83 + } + for text, want := range cases { + if got := estimateTokens(text); got != want { + t.Errorf("estimateTokens(%q) = %d, want %d", text, got, want) + } + } +} + +// Rounding happens once, so many small fragments add up to the same estimate +// as the joined text. +func TestTokenCounterRoundsOnce(t *testing.T) { + var counter tokenCounter + for range 9 { + counter.add("ab") + } + if got, want := counter.total(), estimateTokens("ababababababababab"); got != want { + t.Fatalf("total = %d, want %d", got, want) + } +} diff --git a/internal/providers/kiro/tools_test.go b/internal/providers/kiro/tools_test.go index 87ab0a0a..e63d0e3a 100644 --- a/internal/providers/kiro/tools_test.go +++ b/internal/providers/kiro/tools_test.go @@ -51,7 +51,7 @@ func TestCreateResponseToolStreamMatchesAnthropicMessages(t *testing.T) { "response": map[string]any{"usage": map[string]any{"input_tokens": 11, "output_tokens": 7}}, })) - resp := writeOpenAIStream(io.NopCloser(strings.NewReader(raw.String())), "claude-sonnet-4.5", nil, nil) + resp := writeOpenAIStream(io.NopCloser(strings.NewReader(raw.String())), "claude-sonnet-4.5", nil, nil, nil) defer resp.Body.Close() openai, err := io.ReadAll(resp.Body) if err != nil { @@ -123,7 +123,7 @@ func TestIncompleteToolCallIsNotEmitted(t *testing.T) { raw.WriteString(sseLine(t, map[string]any{"type": "response.output_item.done", "item": map[string]any{"type": "function_call", "id": "fc_bad", "status": "incomplete"}})) raw.WriteString(sseLine(t, map[string]any{"type": "response.completed"})) - resp := writeOpenAIStream(io.NopCloser(strings.NewReader(raw.String())), "claude-sonnet-4.5", nil, nil) + resp := writeOpenAIStream(io.NopCloser(strings.NewReader(raw.String())), "claude-sonnet-4.5", nil, nil, nil) defer resp.Body.Close() body, _ := io.ReadAll(resp.Body) if strings.Contains(string(body), "Write") || strings.Contains(string(body), "call_bad") { @@ -380,7 +380,7 @@ func TestMarkupRescueBecomesToolCall(t *testing.T) { raw.WriteString(sseLine(t, map[string]any{"type": "response.output_text.delta", "delta": "calling "})) raw.WriteString(sseLine(t, map[string]any{"type": "response.output_text.delta", "delta": `/tmp/a`})) raw.WriteString(sseLine(t, map[string]any{"type": "response.completed", "response": map[string]any{"usage": map[string]any{"input_tokens": 1, "output_tokens": 1}}})) - resp := writeOpenAIStream(io.NopCloser(strings.NewReader(raw.String())), "claude-sonnet-4.5", nil, filter) + resp := writeOpenAIStream(io.NopCloser(strings.NewReader(raw.String())), "claude-sonnet-4.5", nil, filter, nil) defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { diff --git a/internal/providers/kiro/upstream_tokens.go b/internal/providers/kiro/upstream_tokens.go new file mode 100644 index 00000000..3e3efa0c --- /dev/null +++ b/internal/providers/kiro/upstream_tokens.go @@ -0,0 +1,71 @@ +package kiro + +import "strings" + +// Kiro rarely states token counts, and when it does the shape differs by path +// and version. updateTokens reads every shape Kiro-Go accepts +// (proxy/kiro.go updateTokensFromEvent): the event itself and any nested +// usage / tokenUsage / token_usage object, with camelCase or snake_case keys. +// Input comes from a direct count first, then from uncached + cache read + +// cache write, then from total minus output. + +var ( + inputTokenKeys = []string{ + "inputTokens", "promptTokens", "totalInputTokens", + "input_tokens", "prompt_tokens", "total_input_tokens", + } + outputTokenKeys = []string{ + "outputTokens", "completionTokens", "totalOutputTokens", + "output_tokens", "completion_tokens", "total_output_tokens", + } +) + +// maxUsageDepth bounds the walk over one event; usage objects sit near the top. +const maxUsageDepth = 6 + +func updateTokens(event map[string]any, input, output int) (int, int) { + candidates := []map[string]any{event} + collectUsageMaps(event, &candidates, 0) + for _, usage := range candidates { + if n, ok := numberField(usage, outputTokenKeys...); ok { + output = n + } + if n, ok := numberField(usage, inputTokenKeys...); ok { + input = n + continue + } + uncached, _ := numberField(usage, "uncachedInputTokens", "uncached_input_tokens") + read, _ := numberField(usage, "cacheReadInputTokens", "cache_read_input_tokens") + write, _ := numberField(usage, "cacheWriteInputTokens", "cache_write_input_tokens", "cacheCreationInputTokens", "cache_creation_input_tokens") + if sum := uncached + read + write; sum > 0 { + input = sum + continue + } + if total, ok := numberField(usage, "totalTokens", "total_tokens"); ok && total > output { + input = total - output + } + } + return input, output +} + +func collectUsageMaps(value any, out *[]map[string]any, depth int) { + if depth >= maxUsageDepth { + return + } + switch v := value.(type) { + case map[string]any: + for key, child := range v { + switch strings.ToLower(key) { + case "usage", "tokenusage", "token_usage": + if m, ok := child.(map[string]any); ok { + *out = append(*out, m) + } + } + collectUsageMaps(child, out, depth+1) + } + case []any: + for _, child := range v { + collectUsageMaps(child, out, depth+1) + } + } +} diff --git a/internal/providers/kiro/upstream_tokens_test.go b/internal/providers/kiro/upstream_tokens_test.go new file mode 100644 index 00000000..5faef5b4 --- /dev/null +++ b/internal/providers/kiro/upstream_tokens_test.go @@ -0,0 +1,31 @@ +package kiro + +import "testing" + +func TestUpdateTokensShapes(t *testing.T) { + cases := []struct { + name string + event map[string]any + in, out int + }{ + {"snake case usage", map[string]any{"usage": map[string]any{"input_tokens": 11.0, "output_tokens": 7.0}}, 11, 7}, + {"camel case top level", map[string]any{"promptTokens": 20.0, "completionTokens": 4.0}, 20, 4}, + {"nested tokenUsage", map[string]any{"metadata": map[string]any{"tokenUsage": map[string]any{"totalInputTokens": 30.0, "totalOutputTokens": 5.0}}}, 30, 5}, + {"cache split", map[string]any{"usage": map[string]any{"uncachedInputTokens": 10.0, "cacheReadInputTokens": 80.0, "cacheWriteInputTokens": 10.0, "outputTokens": 3.0}}, 100, 3}, + {"total minus output", map[string]any{"usage": map[string]any{"totalTokens": 50.0, "outputTokens": 8.0}}, 42, 8}, + {"usage is credits", map[string]any{"usage": 0.25}, 0, 0}, + } + for _, tc := range cases { + in, out := updateTokens(tc.event, 0, 0) + if in != tc.in || out != tc.out { + t.Errorf("%s: got %d/%d, want %d/%d", tc.name, in, out, tc.in, tc.out) + } + } +} + +func TestUpdateTokensKeepsEarlierCounts(t *testing.T) { + in, out := updateTokens(map[string]any{"content": "hi"}, 12, 3) + if in != 12 || out != 3 { + t.Fatalf("got %d/%d", in, out) + } +} diff --git a/internal/providers/kiro/usage.go b/internal/providers/kiro/usage.go index 7bae245d..7dfbf9e6 100644 --- a/internal/providers/kiro/usage.go +++ b/internal/providers/kiro/usage.go @@ -46,29 +46,6 @@ func inputFromContext(model string, percentage float64) int { return int(math.Round(percentage / 100 * float64(contextWindow(model)))) } -// tokenCounter estimates tokens for streamed text: CJK runes count about one -// token each, other text about four characters per token. It is a usage -// magnitude for the request log, not a billing figure. -type tokenCounter struct { - cjk, other int -} - -func (c *tokenCounter) add(text string) { - for _, r := range text { - if (r >= 0x3040 && r <= 0x30ff) || (r >= 0x3400 && r <= 0x9fff) || - (r >= 0xac00 && r <= 0xd7af) || (r >= 0xf900 && r <= 0xfaff) || - (r >= 0xff00 && r <= 0xffef) { - c.cjk++ - } else { - c.other++ - } - } -} - -func (c *tokenCounter) total() int { - return c.cjk + (c.other+3)/4 -} - // resolveUsage fills in tokens Kiro did not send. Counts the upstream stated // are kept. A missing input count comes from the context percentage and a // missing output count from the produced text; either one marks the source @@ -87,13 +64,20 @@ func resolveUsage(model string, input, output int, contextPct float64, produced } // streamUsage is the usage object on the final OpenAI chunk. It is empty only -// when Kiro sent nothing at all to count. -func streamUsage(model string, input, output int, contextPct float64, produced int, credits float64, sawCredits bool) map[string]any { +// when Kiro sent nothing at all to count. prompt_tokens keeps OpenAI meaning: +// it includes the cache read and creation shares estimated in cache. +func streamUsage(model string, input, output int, contextPct float64, produced int, credits float64, sawCredits bool, cache *promptCacheUsage) map[string]any { input, output, estimated := resolveUsage(model, input, output, contextPct, produced) if input <= 0 && output <= 0 && !sawCredits { return map[string]any{} } usage := map[string]any{"prompt_tokens": input, "completion_tokens": output, "total_tokens": input + output} + if cache != nil && input > 0 { + read, write := cache.scaled(input) + usage["cache_read_tokens"] = read + usage["cache_write_tokens"] = write + usage["prompt_tokens_details"] = map[string]any{"cached_tokens": read} + } if sawCredits { usage["credits"] = credits } diff --git a/internal/providers/kiro/usage_test.go b/internal/providers/kiro/usage_test.go index b6c8f343..02d39bd1 100644 --- a/internal/providers/kiro/usage_test.go +++ b/internal/providers/kiro/usage_test.go @@ -33,7 +33,7 @@ func finalUsage(t *testing.T, raw string) map[string]any { func TestLegacyStreamReportsEstimatedTokensAndCredits(t *testing.T) { var out strings.Builder - writeConsumed(&out, "claude-sonnet-4.6", sliceSource(capturedLegacyStream()), nil, nil) + writeConsumed(&out, "claude-sonnet-4.6", sliceSource(capturedLegacyStream()), nil, nil, nil) usage := finalUsage(t, out.String()) // 0.4109% of the 1M window Claude 4.6 uses. @@ -53,7 +53,7 @@ func TestLegacyStreamReportsEstimatedTokensAndCredits(t *testing.T) { func TestLegacyNonStreamKeepsUsage(t *testing.T) { var out strings.Builder - writeConsumed(&out, "claude-sonnet-4.6", sliceSource(capturedLegacyStream()), nil, nil) + writeConsumed(&out, "claude-sonnet-4.6", sliceSource(capturedLegacyStream()), nil, nil, nil) outcome, err := collectOutcome(strings.NewReader(out.String()), "medium") if err != nil { t.Fatal(err) @@ -106,7 +106,7 @@ func TestStatedTokensWin(t *testing.T) { }}, } var out strings.Builder - writeConsumed(&out, "claude-sonnet-4.5", sliceSource(frames), nil, nil) + writeConsumed(&out, "claude-sonnet-4.5", sliceSource(frames), nil, nil, nil) usage := finalUsage(t, out.String()) if usage["prompt_tokens"] != float64(11) || usage["completion_tokens"] != float64(7) || usage["source"] != "provider" { t.Fatalf("usage = %v", usage) @@ -133,7 +133,7 @@ func TestContextWindow(t *testing.T) { func TestNoUsageWhenNothingToCount(t *testing.T) { var out strings.Builder - writeConsumed(&out, "claude-sonnet-4.6", sliceSource(nil), nil, nil) + writeConsumed(&out, "claude-sonnet-4.6", sliceSource(nil), nil, nil, nil) if strings.Contains(out.String(), `"usage"`) { t.Fatalf("unexpected usage:\n%s", out.String()) }