diff --git a/changelog/unreleased/kiro-usage-stats.md b/changelog/unreleased/kiro-usage-stats.md new file mode 100644 index 00000000..463d933b --- /dev/null +++ b/changelog/unreleased/kiro-usage-stats.md @@ -0,0 +1,7 @@ +### English + +- Kiro requests now record token usage and credits. Kiro does not send token counts, so input tokens are derived from its context-usage percentage and output tokens are estimated from the reply (logged as `estimate`). Credits come from Kiro's metering events. + +### 中文 + +- Kiro 请求现在会记录 token 用量和积分。Kiro 上游不返回 token 数,因此输入 token 由上下文占用百分比换算,输出 token 按回复内容估算(来源记为 `estimate`)。积分取自 Kiro 的计费事件。 diff --git a/internal/providers/kiro/client.go b/internal/providers/kiro/client.go index 47c9d8fc..3241a226 100644 --- a/internal/providers/kiro/client.go +++ b/internal/providers/kiro/client.go @@ -487,6 +487,7 @@ func collectOutcome(body io.Reader, level string) (providers.ChatOutcome, error) var prompt, completion int var credits float64 sawCredits := false + source := "" for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) data, ok := strings.CutPrefix(line, "data:") @@ -530,6 +531,7 @@ func collectOutcome(body io.Reader, level string) (providers.ChatOutcome, error) } if chunk.Usage.PromptTokens > 0 || chunk.Usage.CompletionTokens > 0 || chunk.Usage.Credits != nil { prompt, completion = chunk.Usage.PromptTokens, chunk.Usage.CompletionTokens + source = chunk.Usage.Source if chunk.Usage.Credits != nil { credits, sawCredits = *chunk.Usage.Credits, true } @@ -553,11 +555,11 @@ func collectOutcome(body io.Reader, level string) (providers.ChatOutcome, error) outcome := providers.ChatOutcome{ Content: content.String(), Reasoning: reasoning.String(), FinishReason: finish, PromptTokens: prompt, CompletionTokens: completion, ReasoningLevel: level, + UsageSource: source, } if sawCredits { value := credits outcome.Credits = &value - outcome.UsageSource = "provider" } if len(calls) > 0 { raw := make([]map[string]any, 0, len(calls)) diff --git a/internal/providers/kiro/eventstream.go b/internal/providers/kiro/eventstream.go index feef0141..d2119b03 100644 --- a/internal/providers/kiro/eventstream.go +++ b/internal/providers/kiro/eventstream.go @@ -32,8 +32,13 @@ type streamCallbacks struct { OnText func(text string, thinking bool) OnToolUse func(tool toolUse) OnComplete func(inputTokens, outputTokens int) - OnCredits func(credits float64) - OnError func(err error) + // OnCredits receives the running credit total. CreateResponse states it + // once; generateAssistantResponse sends one meteringEvent per charge. + OnCredits func(credits float64) + // OnContextUsage receives the share of the model's input window the + // request filled, in percent. Kiro sends it instead of input tokens. + OnContextUsage func(percentage float64) + OnError func(err error) } func eventStreamSource(body io.Reader) eventSource { @@ -278,6 +283,8 @@ func consumeEvents(source eventSource, cb *streamCallbacks) error { } pending := &pendingTools{} var inputTokens, outputTokens int + var metered float64 + stated := false for { next, err := source() if err == io.EOF { @@ -291,10 +298,25 @@ func consumeEvents(source eventSource, cb *streamCallbacks) error { event = map[string]any{} } inputTokens, outputTokens = updateTokens(event, inputTokens, outputTokens) - if credits, ok := numberAny(event["credits"]); ok && cb.OnCredits != nil { - cb.OnCredits(credits) + if credits, ok := numberAny(event["credits"]); ok { + stated = true + if cb.OnCredits != nil { + cb.OnCredits(credits) + } + } + if pct, ok := numberAny(event["contextUsagePercentage"]); ok && cb.OnContextUsage != nil { + cb.OnContextUsage(pct) } switch next.Type { + case "meteringEvent": + // usage is the credits this charge cost. A stated total wins so + // the two sources are never added together. + if used, ok := numberAny(event["usage"]); ok && !stated { + metered += used + if cb.OnCredits != nil { + cb.OnCredits(metered) + } + } case "assistantResponseEvent": if text := firstString(event, "content"); text != "" && cb.OnText != nil { cb.OnText(text, false) diff --git a/internal/providers/kiro/openai_stream.go b/internal/providers/kiro/openai_stream.go index a85d2180..c4358911 100644 --- a/internal/providers/kiro/openai_stream.go +++ b/internal/providers/kiro/openai_stream.go @@ -34,11 +34,13 @@ func writeOpenAIStream(body io.ReadCloser, model string, names map[string]string func writeConsumed(w io.Writer, model string, source eventSource, names map[string]string, filter *markupFilter) { index := 0 var inputTokens, outputTokens int - var credits float64 + var credits, contextPct float64 sawCredits := false + var produced tokenCounter finish := "stop" err := consumeEvents(source, &streamCallbacks{ OnText: func(text string, thinking bool) { + produced.add(text) field := "content" if thinking { field = "reasoning_content" @@ -58,6 +60,8 @@ func writeConsumed(w io.Writer, model string, source eventSource, names map[stri tool.Name = client } arguments, _ := json.Marshal(tool.Input) + produced.add(tool.Name) + produced.add(string(arguments)) writeChunk(w, model, map[string]any{}, []map[string]any{toolChunk(index, tool.ID, tool.Name, string(arguments))}, "") index++ }, @@ -65,6 +69,7 @@ func writeConsumed(w io.Writer, model string, source eventSource, names map[stri OnCredits: func(value float64) { credits, sawCredits = value, true }, + OnContextUsage: func(pct float64) { contextPct = pct }, }) if tail := filter.flush(); tail != "" && err == nil { writeChunk(w, model, map[string]any{"content": tail}, nil, "") @@ -85,14 +90,7 @@ func writeConsumed(w io.Writer, model string, source eventSource, names map[stri payload, _ := json.Marshal(map[string]any{"error": map[string]any{"message": err.Error(), "type": "upstream_error"}}) fmt.Fprintf(w, "data: %s\n\n", payload) } - usage := map[string]any{} - if inputTokens > 0 || outputTokens > 0 || sawCredits { - usage = map[string]any{"prompt_tokens": inputTokens, "completion_tokens": outputTokens, "total_tokens": inputTokens + outputTokens} - if sawCredits { - usage["credits"] = credits - usage["source"] = "provider" - } - } + usage := streamUsage(model, inputTokens, outputTokens, contextPct, produced.total(), credits, sawCredits) done := map[string]any{ "id": "chatcmpl-kiro", "object": "chat.completion.chunk", "created": time.Now().Unix(), "model": model, "choices": []map[string]any{{"index": 0, "delta": map[string]any{}, "finish_reason": finish}}, diff --git a/internal/providers/kiro/usage.go b/internal/providers/kiro/usage.go new file mode 100644 index 00000000..7bae245d --- /dev/null +++ b/internal/providers/kiro/usage.go @@ -0,0 +1,107 @@ +package kiro + +import ( + "math" + "regexp" + "strconv" + "strings" +) + +// Kiro does not send token counts on generateAssistantResponse. It sends +// contextUsageEvent (the share of the input window the request filled) and +// meteringEvent (credits). Input tokens are that share of the model's window; +// output tokens are estimated from what the stream produced. This follows +// Kiro-Go (proxy/kiro.go getContextWindowSize, estimateClaudeOutputTokens). + +var claudeVersion = regexp.MustCompile(`claude-(?:opus|sonnet|haiku)-(\d+)(?:[.-](\d+))?`) + +// contextWindow is the input window Kiro measures contextUsagePercentage +// against: 1M for Claude 4.6 and newer, 200K for 4.5 and earlier. +func contextWindow(model string) int { + match := claudeVersion.FindStringSubmatch(strings.ToLower(model)) + if match == nil { + return 200_000 + } + major, err := strconv.Atoi(match[1]) + if err != nil { + return 200_000 + } + minor := 0 + if match[2] != "" { + if minor, err = strconv.Atoi(match[2]); err != nil { + return 200_000 + } + } + if major > 4 || (major == 4 && minor >= 6) { + return 1_000_000 + } + return 200_000 +} + +// inputFromContext turns a context percentage into input tokens. +func inputFromContext(model string, percentage float64) int { + if percentage <= 0 { + return 0 + } + return int(math.Round(percentage / 100 * float64(contextWindow(model)))) +} + +// tokenCounter estimates tokens for streamed text: CJK runes count about one +// token each, other text about four characters per token. It is a usage +// magnitude for the request log, not a billing figure. +type tokenCounter struct { + cjk, other int +} + +func (c *tokenCounter) add(text string) { + for _, r := range text { + if (r >= 0x3040 && r <= 0x30ff) || (r >= 0x3400 && r <= 0x9fff) || + (r >= 0xac00 && r <= 0xd7af) || (r >= 0xf900 && r <= 0xfaff) || + (r >= 0xff00 && r <= 0xffef) { + c.cjk++ + } else { + c.other++ + } + } +} + +func (c *tokenCounter) total() int { + return c.cjk + (c.other+3)/4 +} + +// resolveUsage fills in tokens Kiro did not send. Counts the upstream stated +// are kept. A missing input count comes from the context percentage and a +// missing output count from the produced text; either one marks the source +// as an estimate. +func resolveUsage(model string, input, output int, contextPct float64, produced int) (int, int, bool) { + estimated := false + if input <= 0 { + if derived := inputFromContext(model, contextPct); derived > 0 { + input, estimated = derived, true + } + } + if output <= 0 && produced > 0 { + output, estimated = produced, true + } + return input, output, estimated +} + +// streamUsage is the usage object on the final OpenAI chunk. It is empty only +// when Kiro sent nothing at all to count. +func streamUsage(model string, input, output int, contextPct float64, produced int, credits float64, sawCredits bool) map[string]any { + input, output, estimated := resolveUsage(model, input, output, contextPct, produced) + if input <= 0 && output <= 0 && !sawCredits { + return map[string]any{} + } + usage := map[string]any{"prompt_tokens": input, "completion_tokens": output, "total_tokens": input + output} + if sawCredits { + usage["credits"] = credits + } + switch { + case estimated: + usage["source"] = "estimate" + case sawCredits: + usage["source"] = "provider" + } + return usage +} diff --git a/internal/providers/kiro/usage_test.go b/internal/providers/kiro/usage_test.go new file mode 100644 index 00000000..b6c8f343 --- /dev/null +++ b/internal/providers/kiro/usage_test.go @@ -0,0 +1,140 @@ +package kiro + +import ( + "math" + "strings" + "testing" +) + +// capturedLegacyStream is what generateAssistantResponse sent for one short +// reply on a live account: no token counts, only a context share and credits. +func capturedLegacyStream() []streamEvent { + return []streamEvent{ + {Type: "assistantResponseEvent", Event: map[string]any{"content": "ok", "modelId": "claude-sonnet-4.6"}}, + {Type: "metadataEvent", Event: map[string]any{"stopReason": "END_TURN"}}, + {Type: "contextUsageEvent", Event: map[string]any{"contextUsagePercentage": 0.4108999967575073}}, + {Type: "meteringEvent", Event: map[string]any{"unit": "credit", "unitPlural": "credits", "usage": 0.017389340696517413}}, + } +} + +func finalUsage(t *testing.T, raw string) map[string]any { + t.Helper() + var usage map[string]any + for _, event := range parseSSE(t, strings.ReplaceAll(raw, "data: [DONE]", "")) { + if value, ok := event["usage"].(map[string]any); ok { + usage = value + } + } + if usage == nil { + t.Fatalf("no usage chunk:\n%s", raw) + } + return usage +} + +func TestLegacyStreamReportsEstimatedTokensAndCredits(t *testing.T) { + var out strings.Builder + writeConsumed(&out, "claude-sonnet-4.6", sliceSource(capturedLegacyStream()), nil, nil) + usage := finalUsage(t, out.String()) + + // 0.4109% of the 1M window Claude 4.6 uses. + if got := usage["prompt_tokens"]; got != float64(4109) { + t.Fatalf("prompt_tokens = %v", got) + } + if got := usage["completion_tokens"]; got != float64(1) { + t.Fatalf("completion_tokens = %v", got) + } + if got, _ := usage["credits"].(float64); math.Abs(got-0.017389340696517413) > 1e-12 { + t.Fatalf("credits = %v", usage["credits"]) + } + if usage["source"] != "estimate" { + t.Fatalf("source = %v", usage["source"]) + } +} + +func TestLegacyNonStreamKeepsUsage(t *testing.T) { + var out strings.Builder + writeConsumed(&out, "claude-sonnet-4.6", sliceSource(capturedLegacyStream()), nil, nil) + outcome, err := collectOutcome(strings.NewReader(out.String()), "medium") + if err != nil { + t.Fatal(err) + } + if outcome.PromptTokens != 4109 || outcome.CompletionTokens != 1 || outcome.UsageSource != "estimate" { + t.Fatalf("outcome = %+v", outcome) + } + if outcome.Credits == nil || math.Abs(*outcome.Credits-0.017389340696517413) > 1e-12 { + t.Fatalf("credits = %v", outcome.Credits) + } +} + +func TestMeteringEventsAddUp(t *testing.T) { + frames := []streamEvent{ + {Type: "meteringEvent", Event: map[string]any{"usage": 0.25}}, + {Type: "meteringEvent", Event: map[string]any{"usage": 0.5}}, + } + var last float64 + if err := consumeEvents(sliceSource(frames), &streamCallbacks{OnCredits: func(v float64) { last = v }}); err != nil { + t.Fatal(err) + } + if last != 0.75 { + t.Fatalf("credits = %v", last) + } +} + +// CreateResponse states a credit total; a metering event after it must not +// be added on top. +func TestStatedCreditsAreNotDoubleCounted(t *testing.T) { + frames := []streamEvent{ + {Type: "metadataEvent", Event: map[string]any{"credits": 1.5}}, + {Type: "meteringEvent", Event: map[string]any{"usage": 0.25}}, + } + var last float64 + if err := consumeEvents(sliceSource(frames), &streamCallbacks{OnCredits: func(v float64) { last = v }}); err != nil { + t.Fatal(err) + } + if last != 1.5 { + t.Fatalf("credits = %v", last) + } +} + +// Token counts the upstream states are kept and not marked as an estimate. +func TestStatedTokensWin(t *testing.T) { + frames := []streamEvent{ + {Type: "assistantResponseEvent", Event: map[string]any{"content": "a much longer reply than one token"}}, + {Type: "metadataEvent", Event: map[string]any{ + // float64, as encoding/json decodes upstream numbers. + "usage": map[string]any{"input_tokens": float64(11), "output_tokens": float64(7)}, "credits": 0.5, "contextUsagePercentage": 3.0, + }}, + } + var out strings.Builder + writeConsumed(&out, "claude-sonnet-4.5", sliceSource(frames), nil, nil) + usage := finalUsage(t, out.String()) + if usage["prompt_tokens"] != float64(11) || usage["completion_tokens"] != float64(7) || usage["source"] != "provider" { + t.Fatalf("usage = %v", usage) + } +} + +func TestContextWindow(t *testing.T) { + cases := map[string]int{ + "claude-haiku-4.5": 200_000, + "claude-sonnet-4.5": 200_000, + "claude-sonnet-4": 200_000, + "claude-sonnet-4.6": 1_000_000, + "claude-opus-4-7": 1_000_000, + "claude-opus-5.5": 1_000_000, + "claude-opus-5": 1_000_000, + "auto": 200_000, + } + for model, want := range cases { + if got := contextWindow(model); got != want { + t.Errorf("contextWindow(%q) = %d, want %d", model, got, want) + } + } +} + +func TestNoUsageWhenNothingToCount(t *testing.T) { + var out strings.Builder + writeConsumed(&out, "claude-sonnet-4.6", sliceSource(nil), nil, nil) + if strings.Contains(out.String(), `"usage"`) { + t.Fatalf("unexpected usage:\n%s", out.String()) + } +}