From 9d6709bc6f771421cc949d47d903b37883b6ddee Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Mon, 5 Oct 2026 12:51:48 -0400 Subject: [PATCH 1/5] feat(anthropic): capture prompt cache diagnostics Log request diagnostics, cache miss reasons, and missed input tokens. Fixes #861 --- ...sages_create_prompt_cache_diagnostics.yaml | 214 ++++++++++++++++++ .../integrations/anthropic/test_anthropic.py | 30 ++- .../integrations/anthropic/tracing.py | 10 + 3 files changed, 253 insertions(+), 1 deletion(-) create mode 100644 py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_messages_create_prompt_cache_diagnostics.yaml diff --git a/py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_messages_create_prompt_cache_diagnostics.yaml b/py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_messages_create_prompt_cache_diagnostics.yaml new file mode 100644 index 000000000..c36dcd480 --- /dev/null +++ b/py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_messages_create_prompt_cache_diagnostics.yaml @@ -0,0 +1,214 @@ +interactions: +- request: + body: '{"max_tokens":16,"messages":[{"role":"user","content":"What is the capital + of France?"}],"model":"claude-haiku-4-5-20251001"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '125' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - Anthropic/Python 1.11.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.11.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CfjQCSSy4c26N4NkgDe8Q","type":"message","role":"assistant","content":[{"type":"text","text":"The + capital of France is Paris."}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":14,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":10,"service_tier":"standard","inference_geo":"not_available"},"diagnostics":null}' + headers: + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '10000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-10-05T16:24:35Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-10-05T16:24:35Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-10-05T16:24:34Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '12000000' + anthropic-ratelimit-tokens-reset: + - '2026-10-05T16:24:35Z' + anthropic-workspace-id: + - wrkspc_018bFG1FgNVWqx5pthtdGTzS + cf-cache-status: + - DYNAMIC + cf-ray: + - a45dc70009cbab63-YYZ + connection: + - keep-alive + content-length: + - '524' + content-security-policy: + - default-src 'none'; frame-ancestors 'none' + content-type: + - application/json + date: + - Mon, 05 Oct 2026 16:24:35 GMT + request-id: + - req_011CfjQCSDKpHQFdYa6XaXut + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + traceresponse: + - 00-79ff2c64f499313f7ebbfbf1c496bc7c-2686ad04e69d12bf-01 + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-robots-tag: + - none + status: + code: 200 + message: OK +- request: + body: '{"max_tokens":16,"messages":[{"role":"user","content":"What is the capital + of Spain?"}],"model":"claude-haiku-4-5-20251001","diagnostics":{"previous_message_id":"msg_011CfjQCSSy4c26N4NkgDe8Q"}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '193' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - Anthropic/Python 1.11.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.11.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CfjQCVSJn3HCxe9Xi4HdM","type":"message","role":"assistant","content":[{"type":"text","text":"The + capital of Spain is Madrid."}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":14,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":10,"service_tier":"standard","inference_geo":"not_available"},"diagnostics":{"cache_miss_reason":{"type":"previous_message_not_found"}}}' + headers: + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '10000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-10-05T16:24:35Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-10-05T16:24:35Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-10-05T16:24:35Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '12000000' + anthropic-ratelimit-tokens-reset: + - '2026-10-05T16:24:35Z' + anthropic-workspace-id: + - wrkspc_018bFG1FgNVWqx5pthtdGTzS + cf-cache-status: + - DYNAMIC + cf-ray: + - a45dc7048ebfab63-YYZ + connection: + - keep-alive + content-length: + - '579' + content-security-policy: + - default-src 'none'; frame-ancestors 'none' + content-type: + - application/json + date: + - Mon, 05 Oct 2026 16:24:35 GMT + request-id: + - req_011CfjQCVBRnnbvwydPFZzxr + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + traceresponse: + - 00-bfb77651723fcc7aace4c3503d96644d-c55222761107889a-01 + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-robots-tag: + - none + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/anthropic/test_anthropic.py b/py/src/braintrust/integrations/anthropic/test_anthropic.py index 2de40b295..5d17ee233 100644 --- a/py/src/braintrust/integrations/anthropic/test_anthropic.py +++ b/py/src/braintrust/integrations/anthropic/test_anthropic.py @@ -190,6 +190,9 @@ def test_log_message_to_span_includes_stop_reason_and_stop_sequence(): stop_reason="stop_sequence", stop_sequence="DONE", stop_details=None, + diagnostics=SimpleNamespace( + cache_miss_reason=SimpleNamespace(type="messages_changed", cache_missed_input_tokens=9) + ), usage={ "input_tokens": 11, "output_tokens": 7, @@ -221,8 +224,9 @@ def test_log_message_to_span_includes_stop_reason_and_stop_sequence(): "server_tool_use_web_fetch_requests": 1.0, "tokens": 18.0, "time_to_first_token": 0.123, + "prompt_cache_missed_tokens": 9.0, }, - metadata={}, + metadata={"cache_miss_reason": "messages_changed"}, ) @@ -624,6 +628,30 @@ def test_anthropic_messages_create_prompt_cache_metrics(memory_logger, ttl, vcr_ ) +@pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path", "body"]) +def test_anthropic_messages_create_prompt_cache_diagnostics(memory_logger): + if os.environ.get("BRAINTRUST_TEST_PACKAGE_VERSION") != "latest": + pytest.skip("Prompt cache diagnostics require the latest Anthropic SDK cassette") + + client = wrap_anthropic(_get_client()) + request = { + "model": LATEST_MODEL, + "max_tokens": 16, + "messages": [{"role": "user", "content": "What is the capital of France?"}], + } + first_response = client.messages.create(**request) + second_request = {**request, "messages": [{"role": "user", "content": "What is the capital of Spain?"}]} + second_response = client.messages.create(**second_request, diagnostics={"previous_message_id": first_response.id}) + + spans = memory_logger.pop() + message_spans = [span for span in spans if span["span_attributes"]["name"] == "anthropic.messages.create"] + assert len(message_spans) == 2 + second_span = message_spans[1] + assert second_span["metadata"]["diagnostics"] == {"previous_message_id": first_response.id} + cache_miss_reason = second_response.diagnostics.cache_miss_reason + assert second_span["metadata"]["cache_miss_reason"] == cache_miss_reason.type + + @pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path", "body"]) @pytest.mark.asyncio @pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) diff --git a/py/src/braintrust/integrations/anthropic/tracing.py b/py/src/braintrust/integrations/anthropic/tracing.py index c04f90d39..591d45285 100644 --- a/py/src/braintrust/integrations/anthropic/tracing.py +++ b/py/src/braintrust/integrations/anthropic/tracing.py @@ -78,6 +78,7 @@ def _accumulate_event_parameters(accumulator) -> set[str]: "output_format", "compaction", "context_management", + "diagnostics", ) @@ -1557,6 +1558,15 @@ def _log_message_to_span( content = getattr(message, "content", None) output = _message_output(message, include_parsed_output=include_parsed_output) + diagnostics = getattr(message, "diagnostics", None) + cache_miss_reason = getattr(diagnostics, "cache_miss_reason", None) + reason_type = getattr(cache_miss_reason, "type", None) + if reason_type is not None: + metadata["cache_miss_reason"] = reason_type + cache_missed_input_tokens = getattr(cache_miss_reason, "cache_missed_input_tokens", None) + if is_numeric(cache_missed_input_tokens): + metrics["prompt_cache_missed_tokens"] = float(cache_missed_input_tokens) + span.log(output=output, metrics=metrics, metadata=metadata) _log_server_tool_spans(content, span) From 2c41862287a701703ad63ebd0b2a06918331a34c Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Mon, 5 Oct 2026 12:55:10 -0400 Subject: [PATCH 2/5] test(anthropic): remove fake cache diagnostics assertions --- py/src/braintrust/integrations/anthropic/test_anthropic.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/py/src/braintrust/integrations/anthropic/test_anthropic.py b/py/src/braintrust/integrations/anthropic/test_anthropic.py index 5d17ee233..804149213 100644 --- a/py/src/braintrust/integrations/anthropic/test_anthropic.py +++ b/py/src/braintrust/integrations/anthropic/test_anthropic.py @@ -190,9 +190,6 @@ def test_log_message_to_span_includes_stop_reason_and_stop_sequence(): stop_reason="stop_sequence", stop_sequence="DONE", stop_details=None, - diagnostics=SimpleNamespace( - cache_miss_reason=SimpleNamespace(type="messages_changed", cache_missed_input_tokens=9) - ), usage={ "input_tokens": 11, "output_tokens": 7, @@ -224,9 +221,8 @@ def test_log_message_to_span_includes_stop_reason_and_stop_sequence(): "server_tool_use_web_fetch_requests": 1.0, "tokens": 18.0, "time_to_first_token": 0.123, - "prompt_cache_missed_tokens": 9.0, }, - metadata={"cache_miss_reason": "messages_changed"}, + metadata={}, ) From 04dbdc6f0075fdc312861d5b4b123f7d8c5587e7 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Mon, 5 Oct 2026 13:08:33 -0400 Subject: [PATCH 3/5] fix(anthropic): preserve cache estimates as metadata Seed cache diagnostics on the first request and exercise a real system-change response. Keep approximate missed-token counts out of usage metrics. --- ...sages_create_prompt_cache_diagnostics.yaml | 511 +++++++++++++++++- .../integrations/anthropic/test_anthropic.py | 25 +- .../integrations/anthropic/tracing.py | 2 +- 3 files changed, 504 insertions(+), 34 deletions(-) diff --git a/py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_messages_create_prompt_cache_diagnostics.yaml b/py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_messages_create_prompt_cache_diagnostics.yaml index c36dcd480..d9510a411 100644 --- a/py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_messages_create_prompt_cache_diagnostics.yaml +++ b/py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_messages_create_prompt_cache_diagnostics.yaml @@ -1,7 +1,232 @@ interactions: - request: - body: '{"max_tokens":16,"messages":[{"role":"user","content":"What is the capital - of France?"}],"model":"claude-haiku-4-5-20251001"}' + body: '{"max_tokens":16,"messages":[{"role":"user","content":"Summarize section + 1."}],"model":"claude-haiku-4-5-20251001","diagnostics":{"previous_message_id":null},"system":[{"type":"text","text":"Cached + geography fact 0: Paris is the capital of France.\nCached geography fact 1: + Paris is the capital of France.\nCached geography fact 2: Paris is the capital + of France.\nCached geography fact 3: Paris is the capital of France.\nCached + geography fact 4: Paris is the capital of France.\nCached geography fact 5: + Paris is the capital of France.\nCached geography fact 6: Paris is the capital + of France.\nCached geography fact 7: Paris is the capital of France.\nCached + geography fact 8: Paris is the capital of France.\nCached geography fact 9: + Paris is the capital of France.\nCached geography fact 10: Paris is the capital + of France.\nCached geography fact 11: Paris is the capital of France.\nCached + geography fact 12: Paris is the capital of France.\nCached geography fact 13: + Paris is the capital of France.\nCached geography fact 14: Paris is the capital + of France.\nCached geography fact 15: Paris is the capital of France.\nCached + geography fact 16: Paris is the capital of France.\nCached geography fact 17: + Paris is the capital of France.\nCached geography fact 18: Paris is the capital + of France.\nCached geography fact 19: Paris is the capital of France.\nCached + geography fact 20: Paris is the capital of France.\nCached geography fact 21: + Paris is the capital of France.\nCached geography fact 22: Paris is the capital + of France.\nCached geography fact 23: Paris is the capital of France.\nCached + geography fact 24: Paris is the capital of France.\nCached geography fact 25: + Paris is the capital of France.\nCached geography fact 26: Paris is the capital + of France.\nCached geography fact 27: Paris is the capital of France.\nCached + geography fact 28: Paris is the capital of France.\nCached geography fact 29: + Paris is the capital of France.\nCached geography fact 30: Paris is the capital + of France.\nCached geography fact 31: Paris is the capital of France.\nCached + geography fact 32: Paris is the capital of France.\nCached geography fact 33: + Paris is the capital of France.\nCached geography fact 34: Paris is the capital + of France.\nCached geography fact 35: Paris is the capital of France.\nCached + geography fact 36: Paris is the capital of France.\nCached geography fact 37: + Paris is the capital of France.\nCached geography fact 38: Paris is the capital + of France.\nCached geography fact 39: Paris is the capital of France.\nCached + geography fact 40: Paris is the capital of France.\nCached geography fact 41: + Paris is the capital of France.\nCached geography fact 42: Paris is the capital + of France.\nCached geography fact 43: Paris is the capital of France.\nCached + geography fact 44: Paris is the capital of France.\nCached geography fact 45: + Paris is the capital of France.\nCached geography fact 46: Paris is the capital + of France.\nCached geography fact 47: Paris is the capital of France.\nCached + geography fact 48: Paris is the capital of France.\nCached geography fact 49: + Paris is the capital of France.\nCached geography fact 50: Paris is the capital + of France.\nCached geography fact 51: Paris is the capital of France.\nCached + geography fact 52: Paris is the capital of France.\nCached geography fact 53: + Paris is the capital of France.\nCached geography fact 54: Paris is the capital + of France.\nCached geography fact 55: Paris is the capital of France.\nCached + geography fact 56: Paris is the capital of France.\nCached geography fact 57: + Paris is the capital of France.\nCached geography fact 58: Paris is the capital + of France.\nCached geography fact 59: Paris is the capital of France.\nCached + geography fact 60: Paris is the capital of France.\nCached geography fact 61: + Paris is the capital of France.\nCached geography fact 62: Paris is the capital + of France.\nCached geography fact 63: Paris is the capital of France.\nCached + geography fact 64: Paris is the capital of France.\nCached geography fact 65: + Paris is the capital of France.\nCached geography fact 66: Paris is the capital + of France.\nCached geography fact 67: Paris is the capital of France.\nCached + geography fact 68: Paris is the capital of France.\nCached geography fact 69: + Paris is the capital of France.\nCached geography fact 70: Paris is the capital + of France.\nCached geography fact 71: Paris is the capital of France.\nCached + geography fact 72: Paris is the capital of France.\nCached geography fact 73: + Paris is the capital of France.\nCached geography fact 74: Paris is the capital + of France.\nCached geography fact 75: Paris is the capital of France.\nCached + geography fact 76: Paris is the capital of France.\nCached geography fact 77: + Paris is the capital of France.\nCached geography fact 78: Paris is the capital + of France.\nCached geography fact 79: Paris is the capital of France.\nCached + geography fact 80: Paris is the capital of France.\nCached geography fact 81: + Paris is the capital of France.\nCached geography fact 82: Paris is the capital + of France.\nCached geography fact 83: Paris is the capital of France.\nCached + geography fact 84: Paris is the capital of France.\nCached geography fact 85: + Paris is the capital of France.\nCached geography fact 86: Paris is the capital + of France.\nCached geography fact 87: Paris is the capital of France.\nCached + geography fact 88: Paris is the capital of France.\nCached geography fact 89: + Paris is the capital of France.\nCached geography fact 90: Paris is the capital + of France.\nCached geography fact 91: Paris is the capital of France.\nCached + geography fact 92: Paris is the capital of France.\nCached geography fact 93: + Paris is the capital of France.\nCached geography fact 94: Paris is the capital + of France.\nCached geography fact 95: Paris is the capital of France.\nCached + geography fact 96: Paris is the capital of France.\nCached geography fact 97: + Paris is the capital of France.\nCached geography fact 98: Paris is the capital + of France.\nCached geography fact 99: Paris is the capital of France.\nCached + geography fact 100: Paris is the capital of France.\nCached geography fact 101: + Paris is the capital of France.\nCached geography fact 102: Paris is the capital + of France.\nCached geography fact 103: Paris is the capital of France.\nCached + geography fact 104: Paris is the capital of France.\nCached geography fact 105: + Paris is the capital of France.\nCached geography fact 106: Paris is the capital + of France.\nCached geography fact 107: Paris is the capital of France.\nCached + geography fact 108: Paris is the capital of France.\nCached geography fact 109: + Paris is the capital of France.\nCached geography fact 110: Paris is the capital + of France.\nCached geography fact 111: Paris is the capital of France.\nCached + geography fact 112: Paris is the capital of France.\nCached geography fact 113: + Paris is the capital of France.\nCached geography fact 114: Paris is the capital + of France.\nCached geography fact 115: Paris is the capital of France.\nCached + geography fact 116: Paris is the capital of France.\nCached geography fact 117: + Paris is the capital of France.\nCached geography fact 118: Paris is the capital + of France.\nCached geography fact 119: Paris is the capital of France.\nCached + geography fact 120: Paris is the capital of France.\nCached geography fact 121: + Paris is the capital of France.\nCached geography fact 122: Paris is the capital + of France.\nCached geography fact 123: Paris is the capital of France.\nCached + geography fact 124: Paris is the capital of France.\nCached geography fact 125: + Paris is the capital of France.\nCached geography fact 126: Paris is the capital + of France.\nCached geography fact 127: Paris is the capital of France.\nCached + geography fact 128: Paris is the capital of France.\nCached geography fact 129: + Paris is the capital of France.\nCached geography fact 130: Paris is the capital + of France.\nCached geography fact 131: Paris is the capital of France.\nCached + geography fact 132: Paris is the capital of France.\nCached geography fact 133: + Paris is the capital of France.\nCached geography fact 134: Paris is the capital + of France.\nCached geography fact 135: Paris is the capital of France.\nCached + geography fact 136: Paris is the capital of France.\nCached geography fact 137: + Paris is the capital of France.\nCached geography fact 138: Paris is the capital + of France.\nCached geography fact 139: Paris is the capital of France.\nCached + geography fact 140: Paris is the capital of France.\nCached geography fact 141: + Paris is the capital of France.\nCached geography fact 142: Paris is the capital + of France.\nCached geography fact 143: Paris is the capital of France.\nCached + geography fact 144: Paris is the capital of France.\nCached geography fact 145: + Paris is the capital of France.\nCached geography fact 146: Paris is the capital + of France.\nCached geography fact 147: Paris is the capital of France.\nCached + geography fact 148: Paris is the capital of France.\nCached geography fact 149: + Paris is the capital of France.\nCached geography fact 150: Paris is the capital + of France.\nCached geography fact 151: Paris is the capital of France.\nCached + geography fact 152: Paris is the capital of France.\nCached geography fact 153: + Paris is the capital of France.\nCached geography fact 154: Paris is the capital + of France.\nCached geography fact 155: Paris is the capital of France.\nCached + geography fact 156: Paris is the capital of France.\nCached geography fact 157: + Paris is the capital of France.\nCached geography fact 158: Paris is the capital + of France.\nCached geography fact 159: Paris is the capital of France.\nCached + geography fact 160: Paris is the capital of France.\nCached geography fact 161: + Paris is the capital of France.\nCached geography fact 162: Paris is the capital + of France.\nCached geography fact 163: Paris is the capital of France.\nCached + geography fact 164: Paris is the capital of France.\nCached geography fact 165: + Paris is the capital of France.\nCached geography fact 166: Paris is the capital + of France.\nCached geography fact 167: Paris is the capital of France.\nCached + geography fact 168: Paris is the capital of France.\nCached geography fact 169: + Paris is the capital of France.\nCached geography fact 170: Paris is the capital + of France.\nCached geography fact 171: Paris is the capital of France.\nCached + geography fact 172: Paris is the capital of France.\nCached geography fact 173: + Paris is the capital of France.\nCached geography fact 174: Paris is the capital + of France.\nCached geography fact 175: Paris is the capital of France.\nCached + geography fact 176: Paris is the capital of France.\nCached geography fact 177: + Paris is the capital of France.\nCached geography fact 178: Paris is the capital + of France.\nCached geography fact 179: Paris is the capital of France.\nCached + geography fact 180: Paris is the capital of France.\nCached geography fact 181: + Paris is the capital of France.\nCached geography fact 182: Paris is the capital + of France.\nCached geography fact 183: Paris is the capital of France.\nCached + geography fact 184: Paris is the capital of France.\nCached geography fact 185: + Paris is the capital of France.\nCached geography fact 186: Paris is the capital + of France.\nCached geography fact 187: Paris is the capital of France.\nCached + geography fact 188: Paris is the capital of France.\nCached geography fact 189: + Paris is the capital of France.\nCached geography fact 190: Paris is the capital + of France.\nCached geography fact 191: Paris is the capital of France.\nCached + geography fact 192: Paris is the capital of France.\nCached geography fact 193: + Paris is the capital of France.\nCached geography fact 194: Paris is the capital + of France.\nCached geography fact 195: Paris is the capital of France.\nCached + geography fact 196: Paris is the capital of France.\nCached geography fact 197: + Paris is the capital of France.\nCached geography fact 198: Paris is the capital + of France.\nCached geography fact 199: Paris is the capital of France.\nCached + geography fact 200: Paris is the capital of France.\nCached geography fact 201: + Paris is the capital of France.\nCached geography fact 202: Paris is the capital + of France.\nCached geography fact 203: Paris is the capital of France.\nCached + geography fact 204: Paris is the capital of France.\nCached geography fact 205: + Paris is the capital of France.\nCached geography fact 206: Paris is the capital + of France.\nCached geography fact 207: Paris is the capital of France.\nCached + geography fact 208: Paris is the capital of France.\nCached geography fact 209: + Paris is the capital of France.\nCached geography fact 210: Paris is the capital + of France.\nCached geography fact 211: Paris is the capital of France.\nCached + geography fact 212: Paris is the capital of France.\nCached geography fact 213: + Paris is the capital of France.\nCached geography fact 214: Paris is the capital + of France.\nCached geography fact 215: Paris is the capital of France.\nCached + geography fact 216: Paris is the capital of France.\nCached geography fact 217: + Paris is the capital of France.\nCached geography fact 218: Paris is the capital + of France.\nCached geography fact 219: Paris is the capital of France.\nCached + geography fact 220: Paris is the capital of France.\nCached geography fact 221: + Paris is the capital of France.\nCached geography fact 222: Paris is the capital + of France.\nCached geography fact 223: Paris is the capital of France.\nCached + geography fact 224: Paris is the capital of France.\nCached geography fact 225: + Paris is the capital of France.\nCached geography fact 226: Paris is the capital + of France.\nCached geography fact 227: Paris is the capital of France.\nCached + geography fact 228: Paris is the capital of France.\nCached geography fact 229: + Paris is the capital of France.\nCached geography fact 230: Paris is the capital + of France.\nCached geography fact 231: Paris is the capital of France.\nCached + geography fact 232: Paris is the capital of France.\nCached geography fact 233: + Paris is the capital of France.\nCached geography fact 234: Paris is the capital + of France.\nCached geography fact 235: Paris is the capital of France.\nCached + geography fact 236: Paris is the capital of France.\nCached geography fact 237: + Paris is the capital of France.\nCached geography fact 238: Paris is the capital + of France.\nCached geography fact 239: Paris is the capital of France.\nCached + geography fact 240: Paris is the capital of France.\nCached geography fact 241: + Paris is the capital of France.\nCached geography fact 242: Paris is the capital + of France.\nCached geography fact 243: Paris is the capital of France.\nCached + geography fact 244: Paris is the capital of France.\nCached geography fact 245: + Paris is the capital of France.\nCached geography fact 246: Paris is the capital + of France.\nCached geography fact 247: Paris is the capital of France.\nCached + geography fact 248: Paris is the capital of France.\nCached geography fact 249: + Paris is the capital of France.\nCached geography fact 250: Paris is the capital + of France.\nCached geography fact 251: Paris is the capital of France.\nCached + geography fact 252: Paris is the capital of France.\nCached geography fact 253: + Paris is the capital of France.\nCached geography fact 254: Paris is the capital + of France.\nCached geography fact 255: Paris is the capital of France.\nCached + geography fact 256: Paris is the capital of France.\nCached geography fact 257: + Paris is the capital of France.\nCached geography fact 258: Paris is the capital + of France.\nCached geography fact 259: Paris is the capital of France.\nCached + geography fact 260: Paris is the capital of France.\nCached geography fact 261: + Paris is the capital of France.\nCached geography fact 262: Paris is the capital + of France.\nCached geography fact 263: Paris is the capital of France.\nCached + geography fact 264: Paris is the capital of France.\nCached geography fact 265: + Paris is the capital of France.\nCached geography fact 266: Paris is the capital + of France.\nCached geography fact 267: Paris is the capital of France.\nCached + geography fact 268: Paris is the capital of France.\nCached geography fact 269: + Paris is the capital of France.\nCached geography fact 270: Paris is the capital + of France.\nCached geography fact 271: Paris is the capital of France.\nCached + geography fact 272: Paris is the capital of France.\nCached geography fact 273: + Paris is the capital of France.\nCached geography fact 274: Paris is the capital + of France.\nCached geography fact 275: Paris is the capital of France.\nCached + geography fact 276: Paris is the capital of France.\nCached geography fact 277: + Paris is the capital of France.\nCached geography fact 278: Paris is the capital + of France.\nCached geography fact 279: Paris is the capital of France.\nCached + geography fact 280: Paris is the capital of France.\nCached geography fact 281: + Paris is the capital of France.\nCached geography fact 282: Paris is the capital + of France.\nCached geography fact 283: Paris is the capital of France.\nCached + geography fact 284: Paris is the capital of France.\nCached geography fact 285: + Paris is the capital of France.\nCached geography fact 286: Paris is the capital + of France.\nCached geography fact 287: Paris is the capital of France.\nCached + geography fact 288: Paris is the capital of France.\nCached geography fact 289: + Paris is the capital of France.\nCached geography fact 290: Paris is the capital + of France.\nCached geography fact 291: Paris is the capital of France.\nCached + geography fact 292: Paris is the capital of France.\nCached geography fact 293: + Paris is the capital of France.\nCached geography fact 294: Paris is the capital + of France.\nCached geography fact 295: Paris is the capital of France.\nCached + geography fact 296: Paris is the capital of France.\nCached geography fact 297: + Paris is the capital of France.\nCached geography fact 298: Paris is the capital + of France.\nCached geography fact 299: Paris is the capital of France.","cache_control":{"type":"ephemeral"}}]}' headers: accept: - application/json @@ -12,7 +237,7 @@ interactions: connection: - keep-alive content-length: - - '125' + - '18120' content-type: - application/json host: @@ -43,8 +268,8 @@ interactions: uri: https://api.anthropic.com/v1/messages response: body: - string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CfjQCSSy4c26N4NkgDe8Q","type":"message","role":"assistant","content":[{"type":"text","text":"The - capital of France is Paris."}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":14,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":10,"service_tier":"standard","inference_geo":"not_available"},"diagnostics":null}' + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CfjTT8eAE43fvjE5rNJFr","type":"message","role":"assistant","content":[{"type":"text","text":"I + don''t see any document or section 1 in our conversation. The"}],"container":null,"stop_reason":"max_tokens","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":13,"cache_creation_input_tokens":0,"cache_read_input_tokens":4500,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":16,"service_tier":"standard","inference_geo":"not_available"},"diagnostics":null}' headers: anthropic-organization-id: - 27796668-7351-40ac-acc4-024aee8995a5 @@ -53,49 +278,49 @@ interactions: anthropic-ratelimit-input-tokens-remaining: - '10000000' anthropic-ratelimit-input-tokens-reset: - - '2026-10-05T16:24:35Z' + - '2026-10-05T17:07:14Z' anthropic-ratelimit-output-tokens-limit: - '2000000' anthropic-ratelimit-output-tokens-remaining: - '2000000' anthropic-ratelimit-output-tokens-reset: - - '2026-10-05T16:24:35Z' + - '2026-10-05T17:07:14Z' anthropic-ratelimit-requests-limit: - '20000' anthropic-ratelimit-requests-remaining: - '19999' anthropic-ratelimit-requests-reset: - - '2026-10-05T16:24:34Z' + - '2026-10-05T17:07:14Z' anthropic-ratelimit-tokens-limit: - '12000000' anthropic-ratelimit-tokens-remaining: - '12000000' anthropic-ratelimit-tokens-reset: - - '2026-10-05T16:24:35Z' + - '2026-10-05T17:07:14Z' anthropic-workspace-id: - wrkspc_018bFG1FgNVWqx5pthtdGTzS cf-cache-status: - DYNAMIC cf-ray: - - a45dc70009cbab63-YYZ + - a45e057e98408af6-YYZ connection: - keep-alive content-length: - - '524' + - '560' content-security-policy: - default-src 'none'; frame-ancestors 'none' content-type: - application/json date: - - Mon, 05 Oct 2026 16:24:35 GMT + - Mon, 05 Oct 2026 17:07:14 GMT request-id: - - req_011CfjQCSDKpHQFdYa6XaXut + - req_011CfjTT86gV7sFAiHLaqcqU server: - cloudflare strict-transport-security: - max-age=31536000; includeSubDomains; preload traceresponse: - - 00-79ff2c64f499313f7ebbfbf1c496bc7c-2686ad04e69d12bf-01 + - 00-33a6ff3f7dca25f9f1453b1181ff07d9-25abd3c5da8ef548-01 transfer-encoding: - chunked vary: @@ -106,8 +331,233 @@ interactions: code: 200 message: OK - request: - body: '{"max_tokens":16,"messages":[{"role":"user","content":"What is the capital - of Spain?"}],"model":"claude-haiku-4-5-20251001","diagnostics":{"previous_message_id":"msg_011CfjQCSSy4c26N4NkgDe8Q"}}' + body: '{"max_tokens":16,"messages":[{"role":"user","content":"Summarize section + 1."}],"model":"claude-haiku-4-5-20251001","diagnostics":{"previous_message_id":"msg_011CfjTT8eAE43fvjE5rNJFr"},"system":[{"type":"text","text":"Changed + geography fact 0: Paris is the capital of France.\nCached geography fact 1: + Paris is the capital of France.\nCached geography fact 2: Paris is the capital + of France.\nCached geography fact 3: Paris is the capital of France.\nCached + geography fact 4: Paris is the capital of France.\nCached geography fact 5: + Paris is the capital of France.\nCached geography fact 6: Paris is the capital + of France.\nCached geography fact 7: Paris is the capital of France.\nCached + geography fact 8: Paris is the capital of France.\nCached geography fact 9: + Paris is the capital of France.\nCached geography fact 10: Paris is the capital + of France.\nCached geography fact 11: Paris is the capital of France.\nCached + geography fact 12: Paris is the capital of France.\nCached geography fact 13: + Paris is the capital of France.\nCached geography fact 14: Paris is the capital + of France.\nCached geography fact 15: Paris is the capital of France.\nCached + geography fact 16: Paris is the capital of France.\nCached geography fact 17: + Paris is the capital of France.\nCached geography fact 18: Paris is the capital + of France.\nCached geography fact 19: Paris is the capital of France.\nCached + geography fact 20: Paris is the capital of France.\nCached geography fact 21: + Paris is the capital of France.\nCached geography fact 22: Paris is the capital + of France.\nCached geography fact 23: Paris is the capital of France.\nCached + geography fact 24: Paris is the capital of France.\nCached geography fact 25: + Paris is the capital of France.\nCached geography fact 26: Paris is the capital + of France.\nCached geography fact 27: Paris is the capital of France.\nCached + geography fact 28: Paris is the capital of France.\nCached geography fact 29: + Paris is the capital of France.\nCached geography fact 30: Paris is the capital + of France.\nCached geography fact 31: Paris is the capital of France.\nCached + geography fact 32: Paris is the capital of France.\nCached geography fact 33: + Paris is the capital of France.\nCached geography fact 34: Paris is the capital + of France.\nCached geography fact 35: Paris is the capital of France.\nCached + geography fact 36: Paris is the capital of France.\nCached geography fact 37: + Paris is the capital of France.\nCached geography fact 38: Paris is the capital + of France.\nCached geography fact 39: Paris is the capital of France.\nCached + geography fact 40: Paris is the capital of France.\nCached geography fact 41: + Paris is the capital of France.\nCached geography fact 42: Paris is the capital + of France.\nCached geography fact 43: Paris is the capital of France.\nCached + geography fact 44: Paris is the capital of France.\nCached geography fact 45: + Paris is the capital of France.\nCached geography fact 46: Paris is the capital + of France.\nCached geography fact 47: Paris is the capital of France.\nCached + geography fact 48: Paris is the capital of France.\nCached geography fact 49: + Paris is the capital of France.\nCached geography fact 50: Paris is the capital + of France.\nCached geography fact 51: Paris is the capital of France.\nCached + geography fact 52: Paris is the capital of France.\nCached geography fact 53: + Paris is the capital of France.\nCached geography fact 54: Paris is the capital + of France.\nCached geography fact 55: Paris is the capital of France.\nCached + geography fact 56: Paris is the capital of France.\nCached geography fact 57: + Paris is the capital of France.\nCached geography fact 58: Paris is the capital + of France.\nCached geography fact 59: Paris is the capital of France.\nCached + geography fact 60: Paris is the capital of France.\nCached geography fact 61: + Paris is the capital of France.\nCached geography fact 62: Paris is the capital + of France.\nCached geography fact 63: Paris is the capital of France.\nCached + geography fact 64: Paris is the capital of France.\nCached geography fact 65: + Paris is the capital of France.\nCached geography fact 66: Paris is the capital + of France.\nCached geography fact 67: Paris is the capital of France.\nCached + geography fact 68: Paris is the capital of France.\nCached geography fact 69: + Paris is the capital of France.\nCached geography fact 70: Paris is the capital + of France.\nCached geography fact 71: Paris is the capital of France.\nCached + geography fact 72: Paris is the capital of France.\nCached geography fact 73: + Paris is the capital of France.\nCached geography fact 74: Paris is the capital + of France.\nCached geography fact 75: Paris is the capital of France.\nCached + geography fact 76: Paris is the capital of France.\nCached geography fact 77: + Paris is the capital of France.\nCached geography fact 78: Paris is the capital + of France.\nCached geography fact 79: Paris is the capital of France.\nCached + geography fact 80: Paris is the capital of France.\nCached geography fact 81: + Paris is the capital of France.\nCached geography fact 82: Paris is the capital + of France.\nCached geography fact 83: Paris is the capital of France.\nCached + geography fact 84: Paris is the capital of France.\nCached geography fact 85: + Paris is the capital of France.\nCached geography fact 86: Paris is the capital + of France.\nCached geography fact 87: Paris is the capital of France.\nCached + geography fact 88: Paris is the capital of France.\nCached geography fact 89: + Paris is the capital of France.\nCached geography fact 90: Paris is the capital + of France.\nCached geography fact 91: Paris is the capital of France.\nCached + geography fact 92: Paris is the capital of France.\nCached geography fact 93: + Paris is the capital of France.\nCached geography fact 94: Paris is the capital + of France.\nCached geography fact 95: Paris is the capital of France.\nCached + geography fact 96: Paris is the capital of France.\nCached geography fact 97: + Paris is the capital of France.\nCached geography fact 98: Paris is the capital + of France.\nCached geography fact 99: Paris is the capital of France.\nCached + geography fact 100: Paris is the capital of France.\nCached geography fact 101: + Paris is the capital of France.\nCached geography fact 102: Paris is the capital + of France.\nCached geography fact 103: Paris is the capital of France.\nCached + geography fact 104: Paris is the capital of France.\nCached geography fact 105: + Paris is the capital of France.\nCached geography fact 106: Paris is the capital + of France.\nCached geography fact 107: Paris is the capital of France.\nCached + geography fact 108: Paris is the capital of France.\nCached geography fact 109: + Paris is the capital of France.\nCached geography fact 110: Paris is the capital + of France.\nCached geography fact 111: Paris is the capital of France.\nCached + geography fact 112: Paris is the capital of France.\nCached geography fact 113: + Paris is the capital of France.\nCached geography fact 114: Paris is the capital + of France.\nCached geography fact 115: Paris is the capital of France.\nCached + geography fact 116: Paris is the capital of France.\nCached geography fact 117: + Paris is the capital of France.\nCached geography fact 118: Paris is the capital + of France.\nCached geography fact 119: Paris is the capital of France.\nCached + geography fact 120: Paris is the capital of France.\nCached geography fact 121: + Paris is the capital of France.\nCached geography fact 122: Paris is the capital + of France.\nCached geography fact 123: Paris is the capital of France.\nCached + geography fact 124: Paris is the capital of France.\nCached geography fact 125: + Paris is the capital of France.\nCached geography fact 126: Paris is the capital + of France.\nCached geography fact 127: Paris is the capital of France.\nCached + geography fact 128: Paris is the capital of France.\nCached geography fact 129: + Paris is the capital of France.\nCached geography fact 130: Paris is the capital + of France.\nCached geography fact 131: Paris is the capital of France.\nCached + geography fact 132: Paris is the capital of France.\nCached geography fact 133: + Paris is the capital of France.\nCached geography fact 134: Paris is the capital + of France.\nCached geography fact 135: Paris is the capital of France.\nCached + geography fact 136: Paris is the capital of France.\nCached geography fact 137: + Paris is the capital of France.\nCached geography fact 138: Paris is the capital + of France.\nCached geography fact 139: Paris is the capital of France.\nCached + geography fact 140: Paris is the capital of France.\nCached geography fact 141: + Paris is the capital of France.\nCached geography fact 142: Paris is the capital + of France.\nCached geography fact 143: Paris is the capital of France.\nCached + geography fact 144: Paris is the capital of France.\nCached geography fact 145: + Paris is the capital of France.\nCached geography fact 146: Paris is the capital + of France.\nCached geography fact 147: Paris is the capital of France.\nCached + geography fact 148: Paris is the capital of France.\nCached geography fact 149: + Paris is the capital of France.\nCached geography fact 150: Paris is the capital + of France.\nCached geography fact 151: Paris is the capital of France.\nCached + geography fact 152: Paris is the capital of France.\nCached geography fact 153: + Paris is the capital of France.\nCached geography fact 154: Paris is the capital + of France.\nCached geography fact 155: Paris is the capital of France.\nCached + geography fact 156: Paris is the capital of France.\nCached geography fact 157: + Paris is the capital of France.\nCached geography fact 158: Paris is the capital + of France.\nCached geography fact 159: Paris is the capital of France.\nCached + geography fact 160: Paris is the capital of France.\nCached geography fact 161: + Paris is the capital of France.\nCached geography fact 162: Paris is the capital + of France.\nCached geography fact 163: Paris is the capital of France.\nCached + geography fact 164: Paris is the capital of France.\nCached geography fact 165: + Paris is the capital of France.\nCached geography fact 166: Paris is the capital + of France.\nCached geography fact 167: Paris is the capital of France.\nCached + geography fact 168: Paris is the capital of France.\nCached geography fact 169: + Paris is the capital of France.\nCached geography fact 170: Paris is the capital + of France.\nCached geography fact 171: Paris is the capital of France.\nCached + geography fact 172: Paris is the capital of France.\nCached geography fact 173: + Paris is the capital of France.\nCached geography fact 174: Paris is the capital + of France.\nCached geography fact 175: Paris is the capital of France.\nCached + geography fact 176: Paris is the capital of France.\nCached geography fact 177: + Paris is the capital of France.\nCached geography fact 178: Paris is the capital + of France.\nCached geography fact 179: Paris is the capital of France.\nCached + geography fact 180: Paris is the capital of France.\nCached geography fact 181: + Paris is the capital of France.\nCached geography fact 182: Paris is the capital + of France.\nCached geography fact 183: Paris is the capital of France.\nCached + geography fact 184: Paris is the capital of France.\nCached geography fact 185: + Paris is the capital of France.\nCached geography fact 186: Paris is the capital + of France.\nCached geography fact 187: Paris is the capital of France.\nCached + geography fact 188: Paris is the capital of France.\nCached geography fact 189: + Paris is the capital of France.\nCached geography fact 190: Paris is the capital + of France.\nCached geography fact 191: Paris is the capital of France.\nCached + geography fact 192: Paris is the capital of France.\nCached geography fact 193: + Paris is the capital of France.\nCached geography fact 194: Paris is the capital + of France.\nCached geography fact 195: Paris is the capital of France.\nCached + geography fact 196: Paris is the capital of France.\nCached geography fact 197: + Paris is the capital of France.\nCached geography fact 198: Paris is the capital + of France.\nCached geography fact 199: Paris is the capital of France.\nCached + geography fact 200: Paris is the capital of France.\nCached geography fact 201: + Paris is the capital of France.\nCached geography fact 202: Paris is the capital + of France.\nCached geography fact 203: Paris is the capital of France.\nCached + geography fact 204: Paris is the capital of France.\nCached geography fact 205: + Paris is the capital of France.\nCached geography fact 206: Paris is the capital + of France.\nCached geography fact 207: Paris is the capital of France.\nCached + geography fact 208: Paris is the capital of France.\nCached geography fact 209: + Paris is the capital of France.\nCached geography fact 210: Paris is the capital + of France.\nCached geography fact 211: Paris is the capital of France.\nCached + geography fact 212: Paris is the capital of France.\nCached geography fact 213: + Paris is the capital of France.\nCached geography fact 214: Paris is the capital + of France.\nCached geography fact 215: Paris is the capital of France.\nCached + geography fact 216: Paris is the capital of France.\nCached geography fact 217: + Paris is the capital of France.\nCached geography fact 218: Paris is the capital + of France.\nCached geography fact 219: Paris is the capital of France.\nCached + geography fact 220: Paris is the capital of France.\nCached geography fact 221: + Paris is the capital of France.\nCached geography fact 222: Paris is the capital + of France.\nCached geography fact 223: Paris is the capital of France.\nCached + geography fact 224: Paris is the capital of France.\nCached geography fact 225: + Paris is the capital of France.\nCached geography fact 226: Paris is the capital + of France.\nCached geography fact 227: Paris is the capital of France.\nCached + geography fact 228: Paris is the capital of France.\nCached geography fact 229: + Paris is the capital of France.\nCached geography fact 230: Paris is the capital + of France.\nCached geography fact 231: Paris is the capital of France.\nCached + geography fact 232: Paris is the capital of France.\nCached geography fact 233: + Paris is the capital of France.\nCached geography fact 234: Paris is the capital + of France.\nCached geography fact 235: Paris is the capital of France.\nCached + geography fact 236: Paris is the capital of France.\nCached geography fact 237: + Paris is the capital of France.\nCached geography fact 238: Paris is the capital + of France.\nCached geography fact 239: Paris is the capital of France.\nCached + geography fact 240: Paris is the capital of France.\nCached geography fact 241: + Paris is the capital of France.\nCached geography fact 242: Paris is the capital + of France.\nCached geography fact 243: Paris is the capital of France.\nCached + geography fact 244: Paris is the capital of France.\nCached geography fact 245: + Paris is the capital of France.\nCached geography fact 246: Paris is the capital + of France.\nCached geography fact 247: Paris is the capital of France.\nCached + geography fact 248: Paris is the capital of France.\nCached geography fact 249: + Paris is the capital of France.\nCached geography fact 250: Paris is the capital + of France.\nCached geography fact 251: Paris is the capital of France.\nCached + geography fact 252: Paris is the capital of France.\nCached geography fact 253: + Paris is the capital of France.\nCached geography fact 254: Paris is the capital + of France.\nCached geography fact 255: Paris is the capital of France.\nCached + geography fact 256: Paris is the capital of France.\nCached geography fact 257: + Paris is the capital of France.\nCached geography fact 258: Paris is the capital + of France.\nCached geography fact 259: Paris is the capital of France.\nCached + geography fact 260: Paris is the capital of France.\nCached geography fact 261: + Paris is the capital of France.\nCached geography fact 262: Paris is the capital + of France.\nCached geography fact 263: Paris is the capital of France.\nCached + geography fact 264: Paris is the capital of France.\nCached geography fact 265: + Paris is the capital of France.\nCached geography fact 266: Paris is the capital + of France.\nCached geography fact 267: Paris is the capital of France.\nCached + geography fact 268: Paris is the capital of France.\nCached geography fact 269: + Paris is the capital of France.\nCached geography fact 270: Paris is the capital + of France.\nCached geography fact 271: Paris is the capital of France.\nCached + geography fact 272: Paris is the capital of France.\nCached geography fact 273: + Paris is the capital of France.\nCached geography fact 274: Paris is the capital + of France.\nCached geography fact 275: Paris is the capital of France.\nCached + geography fact 276: Paris is the capital of France.\nCached geography fact 277: + Paris is the capital of France.\nCached geography fact 278: Paris is the capital + of France.\nCached geography fact 279: Paris is the capital of France.\nCached + geography fact 280: Paris is the capital of France.\nCached geography fact 281: + Paris is the capital of France.\nCached geography fact 282: Paris is the capital + of France.\nCached geography fact 283: Paris is the capital of France.\nCached + geography fact 284: Paris is the capital of France.\nCached geography fact 285: + Paris is the capital of France.\nCached geography fact 286: Paris is the capital + of France.\nCached geography fact 287: Paris is the capital of France.\nCached + geography fact 288: Paris is the capital of France.\nCached geography fact 289: + Paris is the capital of France.\nCached geography fact 290: Paris is the capital + of France.\nCached geography fact 291: Paris is the capital of France.\nCached + geography fact 292: Paris is the capital of France.\nCached geography fact 293: + Paris is the capital of France.\nCached geography fact 294: Paris is the capital + of France.\nCached geography fact 295: Paris is the capital of France.\nCached + geography fact 296: Paris is the capital of France.\nCached geography fact 297: + Paris is the capital of France.\nCached geography fact 298: Paris is the capital + of France.\nCached geography fact 299: Paris is the capital of France.","cache_control":{"type":"ephemeral"}}]}' headers: accept: - application/json @@ -118,7 +568,7 @@ interactions: connection: - keep-alive content-length: - - '193' + - '18147' content-type: - application/json host: @@ -149,59 +599,60 @@ interactions: uri: https://api.anthropic.com/v1/messages response: body: - string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CfjQCVSJn3HCxe9Xi4HdM","type":"message","role":"assistant","content":[{"type":"text","text":"The - capital of Spain is Madrid."}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":14,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":10,"service_tier":"standard","inference_geo":"not_available"},"diagnostics":{"cache_miss_reason":{"type":"previous_message_not_found"}}}' + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CfjTTBGg2PvZksL2utzVu","type":"message","role":"assistant","content":[{"type":"text","text":"I + don''t see any document or text with sections provided in our conversation. + Could"}],"container":null,"stop_reason":"max_tokens","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":13,"cache_creation_input_tokens":4499,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":4499,"ephemeral_1h_input_tokens":0},"output_tokens":16,"service_tier":"standard","inference_geo":"not_available"},"diagnostics":{"cache_miss_reason":{"type":"system_changed","cache_missed_input_tokens":4397}}}' headers: anthropic-organization-id: - 27796668-7351-40ac-acc4-024aee8995a5 anthropic-ratelimit-input-tokens-limit: - '10000000' anthropic-ratelimit-input-tokens-remaining: - - '10000000' + - '9998000' anthropic-ratelimit-input-tokens-reset: - - '2026-10-05T16:24:35Z' + - '2026-10-05T17:07:15Z' anthropic-ratelimit-output-tokens-limit: - '2000000' anthropic-ratelimit-output-tokens-remaining: - '2000000' anthropic-ratelimit-output-tokens-reset: - - '2026-10-05T16:24:35Z' + - '2026-10-05T17:07:15Z' anthropic-ratelimit-requests-limit: - '20000' anthropic-ratelimit-requests-remaining: - '19999' anthropic-ratelimit-requests-reset: - - '2026-10-05T16:24:35Z' + - '2026-10-05T17:07:15Z' anthropic-ratelimit-tokens-limit: - '12000000' anthropic-ratelimit-tokens-remaining: - - '12000000' + - '11998000' anthropic-ratelimit-tokens-reset: - - '2026-10-05T16:24:35Z' + - '2026-10-05T17:07:15Z' anthropic-workspace-id: - wrkspc_018bFG1FgNVWqx5pthtdGTzS cf-cache-status: - DYNAMIC cf-ray: - - a45dc7048ebfab63-YYZ + - a45e0582e9bf8af6-YYZ connection: - keep-alive content-length: - - '579' + - '659' content-security-policy: - default-src 'none'; frame-ancestors 'none' content-type: - application/json date: - - Mon, 05 Oct 2026 16:24:35 GMT + - Mon, 05 Oct 2026 17:07:15 GMT request-id: - - req_011CfjQCVBRnnbvwydPFZzxr + - req_011CfjTTB42NG92bQaUt6Q2a server: - cloudflare strict-transport-security: - max-age=31536000; includeSubDomains; preload traceresponse: - - 00-bfb77651723fcc7aace4c3503d96644d-c55222761107889a-01 + - 00-1c2eaf88b6d847457bc00ab34bfc7b61-7360e88d1395d2ab-01 transfer-encoding: - chunked vary: diff --git a/py/src/braintrust/integrations/anthropic/test_anthropic.py b/py/src/braintrust/integrations/anthropic/test_anthropic.py index 804149213..71e800130 100644 --- a/py/src/braintrust/integrations/anthropic/test_anthropic.py +++ b/py/src/braintrust/integrations/anthropic/test_anthropic.py @@ -633,10 +633,27 @@ def test_anthropic_messages_create_prompt_cache_diagnostics(memory_logger): request = { "model": LATEST_MODEL, "max_tokens": 16, - "messages": [{"role": "user", "content": "What is the capital of France?"}], + "system": [ + { + "type": "text", + "text": PROMPT_CACHE_TEST_TEXT, + "cache_control": {"type": "ephemeral"}, + } + ], + "messages": [{"role": "user", "content": "Summarize section 1."}], + } + first_response = client.messages.create(**request, diagnostics={"previous_message_id": None}) + second_request = { + **request, + "system": [ + { + **request["system"][0], + "text": request["system"][0]["text"].replace( + "Cached geography fact 0:", "Changed geography fact 0:", 1 + ), + } + ], } - first_response = client.messages.create(**request) - second_request = {**request, "messages": [{"role": "user", "content": "What is the capital of Spain?"}]} second_response = client.messages.create(**second_request, diagnostics={"previous_message_id": first_response.id}) spans = memory_logger.pop() @@ -645,7 +662,9 @@ def test_anthropic_messages_create_prompt_cache_diagnostics(memory_logger): second_span = message_spans[1] assert second_span["metadata"]["diagnostics"] == {"previous_message_id": first_response.id} cache_miss_reason = second_response.diagnostics.cache_miss_reason + assert cache_miss_reason.type == "system_changed" assert second_span["metadata"]["cache_miss_reason"] == cache_miss_reason.type + assert second_span["metadata"]["cache_missed_input_tokens"] == cache_miss_reason.cache_missed_input_tokens @pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path", "body"]) diff --git a/py/src/braintrust/integrations/anthropic/tracing.py b/py/src/braintrust/integrations/anthropic/tracing.py index 591d45285..8e304db41 100644 --- a/py/src/braintrust/integrations/anthropic/tracing.py +++ b/py/src/braintrust/integrations/anthropic/tracing.py @@ -1565,7 +1565,7 @@ def _log_message_to_span( metadata["cache_miss_reason"] = reason_type cache_missed_input_tokens = getattr(cache_miss_reason, "cache_missed_input_tokens", None) if is_numeric(cache_missed_input_tokens): - metrics["prompt_cache_missed_tokens"] = float(cache_missed_input_tokens) + metadata["cache_missed_input_tokens"] = cache_missed_input_tokens span.log(output=output, metrics=metrics, metadata=metadata) _log_server_tool_spans(content, span) From 038e7a370a263cee51a35881b3288f0ca46dd441 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Mon, 5 Oct 2026 13:24:15 -0400 Subject: [PATCH 4/5] test(anthropic): cover streaming cache diagnostics --- ...sages_create_prompt_cache_diagnostics.yaml | 454 +++++++++++++++++- .../integrations/anthropic/test_anthropic.py | 18 +- 2 files changed, 443 insertions(+), 29 deletions(-) diff --git a/py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_messages_create_prompt_cache_diagnostics.yaml b/py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_messages_create_prompt_cache_diagnostics.yaml index d9510a411..d582d10ae 100644 --- a/py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_messages_create_prompt_cache_diagnostics.yaml +++ b/py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_messages_create_prompt_cache_diagnostics.yaml @@ -268,59 +268,60 @@ interactions: uri: https://api.anthropic.com/v1/messages response: body: - string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CfjTT8eAE43fvjE5rNJFr","type":"message","role":"assistant","content":[{"type":"text","text":"I - don''t see any document or section 1 in our conversation. The"}],"container":null,"stop_reason":"max_tokens","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":13,"cache_creation_input_tokens":0,"cache_read_input_tokens":4500,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":16,"service_tier":"standard","inference_geo":"not_available"},"diagnostics":null}' + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CfjUaiyDZnMUa2LR3wym7","type":"message","role":"assistant","content":[{"type":"text","text":"I + don''t see any document or text with sections provided in our conversation. + Could"}],"container":null,"stop_reason":"max_tokens","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":13,"cache_creation_input_tokens":4500,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":4500,"ephemeral_1h_input_tokens":0},"output_tokens":16,"service_tier":"standard","inference_geo":"not_available"},"diagnostics":null}' headers: anthropic-organization-id: - 27796668-7351-40ac-acc4-024aee8995a5 anthropic-ratelimit-input-tokens-limit: - '10000000' anthropic-ratelimit-input-tokens-remaining: - - '10000000' + - '9998000' anthropic-ratelimit-input-tokens-reset: - - '2026-10-05T17:07:14Z' + - '2026-10-05T17:22:04Z' anthropic-ratelimit-output-tokens-limit: - '2000000' anthropic-ratelimit-output-tokens-remaining: - '2000000' anthropic-ratelimit-output-tokens-reset: - - '2026-10-05T17:07:14Z' + - '2026-10-05T17:22:04Z' anthropic-ratelimit-requests-limit: - '20000' anthropic-ratelimit-requests-remaining: - '19999' anthropic-ratelimit-requests-reset: - - '2026-10-05T17:07:14Z' + - '2026-10-05T17:22:04Z' anthropic-ratelimit-tokens-limit: - '12000000' anthropic-ratelimit-tokens-remaining: - - '12000000' + - '11998000' anthropic-ratelimit-tokens-reset: - - '2026-10-05T17:07:14Z' + - '2026-10-05T17:22:04Z' anthropic-workspace-id: - wrkspc_018bFG1FgNVWqx5pthtdGTzS cf-cache-status: - DYNAMIC cf-ray: - - a45e057e98408af6-YYZ + - a45e1b372affa392-YYZ connection: - keep-alive content-length: - - '560' + - '583' content-security-policy: - default-src 'none'; frame-ancestors 'none' content-type: - application/json date: - - Mon, 05 Oct 2026 17:07:14 GMT + - Mon, 05 Oct 2026 17:22:04 GMT request-id: - - req_011CfjTT86gV7sFAiHLaqcqU + - req_011CfjUai49ZvY1vaREGnCH2 server: - cloudflare strict-transport-security: - max-age=31536000; includeSubDomains; preload traceresponse: - - 00-33a6ff3f7dca25f9f1453b1181ff07d9-25abd3c5da8ef548-01 + - 00-d2758526142a0360f59d7fefc97d75a6-29fd523946125707-01 transfer-encoding: - chunked vary: @@ -332,7 +333,7 @@ interactions: message: OK - request: body: '{"max_tokens":16,"messages":[{"role":"user","content":"Summarize section - 1."}],"model":"claude-haiku-4-5-20251001","diagnostics":{"previous_message_id":"msg_011CfjTT8eAE43fvjE5rNJFr"},"system":[{"type":"text","text":"Changed + 1."}],"model":"claude-haiku-4-5-20251001","diagnostics":{"previous_message_id":"msg_011CfjUaiyDZnMUa2LR3wym7"},"system":[{"type":"text","text":"Changed geography fact 0: Paris is the capital of France.\nCached geography fact 1: Paris is the capital of France.\nCached geography fact 2: Paris is the capital of France.\nCached geography fact 3: Paris is the capital of France.\nCached @@ -599,9 +600,8 @@ interactions: uri: https://api.anthropic.com/v1/messages response: body: - string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CfjTTBGg2PvZksL2utzVu","type":"message","role":"assistant","content":[{"type":"text","text":"I - don''t see any document or text with sections provided in our conversation. - Could"}],"container":null,"stop_reason":"max_tokens","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":13,"cache_creation_input_tokens":4499,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":4499,"ephemeral_1h_input_tokens":0},"output_tokens":16,"service_tier":"standard","inference_geo":"not_available"},"diagnostics":{"cache_miss_reason":{"type":"system_changed","cache_missed_input_tokens":4397}}}' + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CfjUan31skCMJ949C3xZW","type":"message","role":"assistant","content":[{"type":"text","text":"I + don''t see a document or text with sections in our conversation. You''ve"}],"container":null,"stop_reason":"max_tokens","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":13,"cache_creation_input_tokens":4499,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":4499,"ephemeral_1h_input_tokens":0},"output_tokens":16,"service_tier":"standard","inference_geo":"not_available"},"diagnostics":{"cache_miss_reason":{"type":"system_changed","cache_missed_input_tokens":4397}}}' headers: anthropic-organization-id: - 27796668-7351-40ac-acc4-024aee8995a5 @@ -610,49 +610,451 @@ interactions: anthropic-ratelimit-input-tokens-remaining: - '9998000' anthropic-ratelimit-input-tokens-reset: - - '2026-10-05T17:07:15Z' + - '2026-10-05T17:22:05Z' anthropic-ratelimit-output-tokens-limit: - '2000000' anthropic-ratelimit-output-tokens-remaining: - '2000000' anthropic-ratelimit-output-tokens-reset: - - '2026-10-05T17:07:15Z' + - '2026-10-05T17:22:05Z' anthropic-ratelimit-requests-limit: - '20000' anthropic-ratelimit-requests-remaining: - '19999' anthropic-ratelimit-requests-reset: - - '2026-10-05T17:07:15Z' + - '2026-10-05T17:22:05Z' anthropic-ratelimit-tokens-limit: - '12000000' anthropic-ratelimit-tokens-remaining: - '11998000' anthropic-ratelimit-tokens-reset: - - '2026-10-05T17:07:15Z' + - '2026-10-05T17:22:05Z' anthropic-workspace-id: - wrkspc_018bFG1FgNVWqx5pthtdGTzS cf-cache-status: - DYNAMIC cf-ray: - - a45e0582e9bf8af6-YYZ + - a45e1b3cdbb2a392-YYZ connection: - keep-alive content-length: - - '659' + - '649' content-security-policy: - default-src 'none'; frame-ancestors 'none' content-type: - application/json date: - - Mon, 05 Oct 2026 17:07:15 GMT + - Mon, 05 Oct 2026 17:22:05 GMT + request-id: + - req_011CfjUamocy9sdJeC4yThPg + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + traceresponse: + - 00-73ca5988771a8cf9d6989b1bb79082ac-2ecde2401906f677-01 + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-robots-tag: + - none + status: + code: 200 + message: OK +- request: + body: '{"max_tokens":16,"messages":[{"role":"user","content":"Summarize section + 1."}],"model":"claude-haiku-4-5-20251001","diagnostics":{"previous_message_id":"msg_011CfjUan31skCMJ949C3xZW"},"system":[{"type":"text","text":"Cached + geography fact 0: Paris is the capital of France.\nCached geography fact 1: + Paris is the capital of France.\nCached geography fact 2: Paris is the capital + of France.\nCached geography fact 3: Paris is the capital of France.\nCached + geography fact 4: Paris is the capital of France.\nCached geography fact 5: + Paris is the capital of France.\nCached geography fact 6: Paris is the capital + of France.\nCached geography fact 7: Paris is the capital of France.\nCached + geography fact 8: Paris is the capital of France.\nCached geography fact 9: + Paris is the capital of France.\nCached geography fact 10: Paris is the capital + of France.\nCached geography fact 11: Paris is the capital of France.\nCached + geography fact 12: Paris is the capital of France.\nCached geography fact 13: + Paris is the capital of France.\nCached geography fact 14: Paris is the capital + of France.\nCached geography fact 15: Paris is the capital of France.\nCached + geography fact 16: Paris is the capital of France.\nCached geography fact 17: + Paris is the capital of France.\nCached geography fact 18: Paris is the capital + of France.\nCached geography fact 19: Paris is the capital of France.\nCached + geography fact 20: Paris is the capital of France.\nCached geography fact 21: + Paris is the capital of France.\nCached geography fact 22: Paris is the capital + of France.\nCached geography fact 23: Paris is the capital of France.\nCached + geography fact 24: Paris is the capital of France.\nCached geography fact 25: + Paris is the capital of France.\nCached geography fact 26: Paris is the capital + of France.\nCached geography fact 27: Paris is the capital of France.\nCached + geography fact 28: Paris is the capital of France.\nCached geography fact 29: + Paris is the capital of France.\nCached geography fact 30: Paris is the capital + of France.\nCached geography fact 31: Paris is the capital of France.\nCached + geography fact 32: Paris is the capital of France.\nCached geography fact 33: + Paris is the capital of France.\nCached geography fact 34: Paris is the capital + of France.\nCached geography fact 35: Paris is the capital of France.\nCached + geography fact 36: Paris is the capital of France.\nCached geography fact 37: + Paris is the capital of France.\nCached geography fact 38: Paris is the capital + of France.\nCached geography fact 39: Paris is the capital of France.\nCached + geography fact 40: Paris is the capital of France.\nCached geography fact 41: + Paris is the capital of France.\nCached geography fact 42: Paris is the capital + of France.\nCached geography fact 43: Paris is the capital of France.\nCached + geography fact 44: Paris is the capital of France.\nCached geography fact 45: + Paris is the capital of France.\nCached geography fact 46: Paris is the capital + of France.\nCached geography fact 47: Paris is the capital of France.\nCached + geography fact 48: Paris is the capital of France.\nCached geography fact 49: + Paris is the capital of France.\nCached geography fact 50: Paris is the capital + of France.\nCached geography fact 51: Paris is the capital of France.\nCached + geography fact 52: Paris is the capital of France.\nCached geography fact 53: + Paris is the capital of France.\nCached geography fact 54: Paris is the capital + of France.\nCached geography fact 55: Paris is the capital of France.\nCached + geography fact 56: Paris is the capital of France.\nCached geography fact 57: + Paris is the capital of France.\nCached geography fact 58: Paris is the capital + of France.\nCached geography fact 59: Paris is the capital of France.\nCached + geography fact 60: Paris is the capital of France.\nCached geography fact 61: + Paris is the capital of France.\nCached geography fact 62: Paris is the capital + of France.\nCached geography fact 63: Paris is the capital of France.\nCached + geography fact 64: Paris is the capital of France.\nCached geography fact 65: + Paris is the capital of France.\nCached geography fact 66: Paris is the capital + of France.\nCached geography fact 67: Paris is the capital of France.\nCached + geography fact 68: Paris is the capital of France.\nCached geography fact 69: + Paris is the capital of France.\nCached geography fact 70: Paris is the capital + of France.\nCached geography fact 71: Paris is the capital of France.\nCached + geography fact 72: Paris is the capital of France.\nCached geography fact 73: + Paris is the capital of France.\nCached geography fact 74: Paris is the capital + of France.\nCached geography fact 75: Paris is the capital of France.\nCached + geography fact 76: Paris is the capital of France.\nCached geography fact 77: + Paris is the capital of France.\nCached geography fact 78: Paris is the capital + of France.\nCached geography fact 79: Paris is the capital of France.\nCached + geography fact 80: Paris is the capital of France.\nCached geography fact 81: + Paris is the capital of France.\nCached geography fact 82: Paris is the capital + of France.\nCached geography fact 83: Paris is the capital of France.\nCached + geography fact 84: Paris is the capital of France.\nCached geography fact 85: + Paris is the capital of France.\nCached geography fact 86: Paris is the capital + of France.\nCached geography fact 87: Paris is the capital of France.\nCached + geography fact 88: Paris is the capital of France.\nCached geography fact 89: + Paris is the capital of France.\nCached geography fact 90: Paris is the capital + of France.\nCached geography fact 91: Paris is the capital of France.\nCached + geography fact 92: Paris is the capital of France.\nCached geography fact 93: + Paris is the capital of France.\nCached geography fact 94: Paris is the capital + of France.\nCached geography fact 95: Paris is the capital of France.\nCached + geography fact 96: Paris is the capital of France.\nCached geography fact 97: + Paris is the capital of France.\nCached geography fact 98: Paris is the capital + of France.\nCached geography fact 99: Paris is the capital of France.\nCached + geography fact 100: Paris is the capital of France.\nCached geography fact 101: + Paris is the capital of France.\nCached geography fact 102: Paris is the capital + of France.\nCached geography fact 103: Paris is the capital of France.\nCached + geography fact 104: Paris is the capital of France.\nCached geography fact 105: + Paris is the capital of France.\nCached geography fact 106: Paris is the capital + of France.\nCached geography fact 107: Paris is the capital of France.\nCached + geography fact 108: Paris is the capital of France.\nCached geography fact 109: + Paris is the capital of France.\nCached geography fact 110: Paris is the capital + of France.\nCached geography fact 111: Paris is the capital of France.\nCached + geography fact 112: Paris is the capital of France.\nCached geography fact 113: + Paris is the capital of France.\nCached geography fact 114: Paris is the capital + of France.\nCached geography fact 115: Paris is the capital of France.\nCached + geography fact 116: Paris is the capital of France.\nCached geography fact 117: + Paris is the capital of France.\nCached geography fact 118: Paris is the capital + of France.\nCached geography fact 119: Paris is the capital of France.\nCached + geography fact 120: Paris is the capital of France.\nCached geography fact 121: + Paris is the capital of France.\nCached geography fact 122: Paris is the capital + of France.\nCached geography fact 123: Paris is the capital of France.\nCached + geography fact 124: Paris is the capital of France.\nCached geography fact 125: + Paris is the capital of France.\nCached geography fact 126: Paris is the capital + of France.\nCached geography fact 127: Paris is the capital of France.\nCached + geography fact 128: Paris is the capital of France.\nCached geography fact 129: + Paris is the capital of France.\nCached geography fact 130: Paris is the capital + of France.\nCached geography fact 131: Paris is the capital of France.\nCached + geography fact 132: Paris is the capital of France.\nCached geography fact 133: + Paris is the capital of France.\nCached geography fact 134: Paris is the capital + of France.\nCached geography fact 135: Paris is the capital of France.\nCached + geography fact 136: Paris is the capital of France.\nCached geography fact 137: + Paris is the capital of France.\nCached geography fact 138: Paris is the capital + of France.\nCached geography fact 139: Paris is the capital of France.\nCached + geography fact 140: Paris is the capital of France.\nCached geography fact 141: + Paris is the capital of France.\nCached geography fact 142: Paris is the capital + of France.\nCached geography fact 143: Paris is the capital of France.\nCached + geography fact 144: Paris is the capital of France.\nCached geography fact 145: + Paris is the capital of France.\nCached geography fact 146: Paris is the capital + of France.\nCached geography fact 147: Paris is the capital of France.\nCached + geography fact 148: Paris is the capital of France.\nCached geography fact 149: + Paris is the capital of France.\nCached geography fact 150: Paris is the capital + of France.\nCached geography fact 151: Paris is the capital of France.\nCached + geography fact 152: Paris is the capital of France.\nCached geography fact 153: + Paris is the capital of France.\nCached geography fact 154: Paris is the capital + of France.\nCached geography fact 155: Paris is the capital of France.\nCached + geography fact 156: Paris is the capital of France.\nCached geography fact 157: + Paris is the capital of France.\nCached geography fact 158: Paris is the capital + of France.\nCached geography fact 159: Paris is the capital of France.\nCached + geography fact 160: Paris is the capital of France.\nCached geography fact 161: + Paris is the capital of France.\nCached geography fact 162: Paris is the capital + of France.\nCached geography fact 163: Paris is the capital of France.\nCached + geography fact 164: Paris is the capital of France.\nCached geography fact 165: + Paris is the capital of France.\nCached geography fact 166: Paris is the capital + of France.\nCached geography fact 167: Paris is the capital of France.\nCached + geography fact 168: Paris is the capital of France.\nCached geography fact 169: + Paris is the capital of France.\nCached geography fact 170: Paris is the capital + of France.\nCached geography fact 171: Paris is the capital of France.\nCached + geography fact 172: Paris is the capital of France.\nCached geography fact 173: + Paris is the capital of France.\nCached geography fact 174: Paris is the capital + of France.\nCached geography fact 175: Paris is the capital of France.\nCached + geography fact 176: Paris is the capital of France.\nCached geography fact 177: + Paris is the capital of France.\nCached geography fact 178: Paris is the capital + of France.\nCached geography fact 179: Paris is the capital of France.\nCached + geography fact 180: Paris is the capital of France.\nCached geography fact 181: + Paris is the capital of France.\nCached geography fact 182: Paris is the capital + of France.\nCached geography fact 183: Paris is the capital of France.\nCached + geography fact 184: Paris is the capital of France.\nCached geography fact 185: + Paris is the capital of France.\nCached geography fact 186: Paris is the capital + of France.\nCached geography fact 187: Paris is the capital of France.\nCached + geography fact 188: Paris is the capital of France.\nCached geography fact 189: + Paris is the capital of France.\nCached geography fact 190: Paris is the capital + of France.\nCached geography fact 191: Paris is the capital of France.\nCached + geography fact 192: Paris is the capital of France.\nCached geography fact 193: + Paris is the capital of France.\nCached geography fact 194: Paris is the capital + of France.\nCached geography fact 195: Paris is the capital of France.\nCached + geography fact 196: Paris is the capital of France.\nCached geography fact 197: + Paris is the capital of France.\nCached geography fact 198: Paris is the capital + of France.\nCached geography fact 199: Paris is the capital of France.\nCached + geography fact 200: Paris is the capital of France.\nCached geography fact 201: + Paris is the capital of France.\nCached geography fact 202: Paris is the capital + of France.\nCached geography fact 203: Paris is the capital of France.\nCached + geography fact 204: Paris is the capital of France.\nCached geography fact 205: + Paris is the capital of France.\nCached geography fact 206: Paris is the capital + of France.\nCached geography fact 207: Paris is the capital of France.\nCached + geography fact 208: Paris is the capital of France.\nCached geography fact 209: + Paris is the capital of France.\nCached geography fact 210: Paris is the capital + of France.\nCached geography fact 211: Paris is the capital of France.\nCached + geography fact 212: Paris is the capital of France.\nCached geography fact 213: + Paris is the capital of France.\nCached geography fact 214: Paris is the capital + of France.\nCached geography fact 215: Paris is the capital of France.\nCached + geography fact 216: Paris is the capital of France.\nCached geography fact 217: + Paris is the capital of France.\nCached geography fact 218: Paris is the capital + of France.\nCached geography fact 219: Paris is the capital of France.\nCached + geography fact 220: Paris is the capital of France.\nCached geography fact 221: + Paris is the capital of France.\nCached geography fact 222: Paris is the capital + of France.\nCached geography fact 223: Paris is the capital of France.\nCached + geography fact 224: Paris is the capital of France.\nCached geography fact 225: + Paris is the capital of France.\nCached geography fact 226: Paris is the capital + of France.\nCached geography fact 227: Paris is the capital of France.\nCached + geography fact 228: Paris is the capital of France.\nCached geography fact 229: + Paris is the capital of France.\nCached geography fact 230: Paris is the capital + of France.\nCached geography fact 231: Paris is the capital of France.\nCached + geography fact 232: Paris is the capital of France.\nCached geography fact 233: + Paris is the capital of France.\nCached geography fact 234: Paris is the capital + of France.\nCached geography fact 235: Paris is the capital of France.\nCached + geography fact 236: Paris is the capital of France.\nCached geography fact 237: + Paris is the capital of France.\nCached geography fact 238: Paris is the capital + of France.\nCached geography fact 239: Paris is the capital of France.\nCached + geography fact 240: Paris is the capital of France.\nCached geography fact 241: + Paris is the capital of France.\nCached geography fact 242: Paris is the capital + of France.\nCached geography fact 243: Paris is the capital of France.\nCached + geography fact 244: Paris is the capital of France.\nCached geography fact 245: + Paris is the capital of France.\nCached geography fact 246: Paris is the capital + of France.\nCached geography fact 247: Paris is the capital of France.\nCached + geography fact 248: Paris is the capital of France.\nCached geography fact 249: + Paris is the capital of France.\nCached geography fact 250: Paris is the capital + of France.\nCached geography fact 251: Paris is the capital of France.\nCached + geography fact 252: Paris is the capital of France.\nCached geography fact 253: + Paris is the capital of France.\nCached geography fact 254: Paris is the capital + of France.\nCached geography fact 255: Paris is the capital of France.\nCached + geography fact 256: Paris is the capital of France.\nCached geography fact 257: + Paris is the capital of France.\nCached geography fact 258: Paris is the capital + of France.\nCached geography fact 259: Paris is the capital of France.\nCached + geography fact 260: Paris is the capital of France.\nCached geography fact 261: + Paris is the capital of France.\nCached geography fact 262: Paris is the capital + of France.\nCached geography fact 263: Paris is the capital of France.\nCached + geography fact 264: Paris is the capital of France.\nCached geography fact 265: + Paris is the capital of France.\nCached geography fact 266: Paris is the capital + of France.\nCached geography fact 267: Paris is the capital of France.\nCached + geography fact 268: Paris is the capital of France.\nCached geography fact 269: + Paris is the capital of France.\nCached geography fact 270: Paris is the capital + of France.\nCached geography fact 271: Paris is the capital of France.\nCached + geography fact 272: Paris is the capital of France.\nCached geography fact 273: + Paris is the capital of France.\nCached geography fact 274: Paris is the capital + of France.\nCached geography fact 275: Paris is the capital of France.\nCached + geography fact 276: Paris is the capital of France.\nCached geography fact 277: + Paris is the capital of France.\nCached geography fact 278: Paris is the capital + of France.\nCached geography fact 279: Paris is the capital of France.\nCached + geography fact 280: Paris is the capital of France.\nCached geography fact 281: + Paris is the capital of France.\nCached geography fact 282: Paris is the capital + of France.\nCached geography fact 283: Paris is the capital of France.\nCached + geography fact 284: Paris is the capital of France.\nCached geography fact 285: + Paris is the capital of France.\nCached geography fact 286: Paris is the capital + of France.\nCached geography fact 287: Paris is the capital of France.\nCached + geography fact 288: Paris is the capital of France.\nCached geography fact 289: + Paris is the capital of France.\nCached geography fact 290: Paris is the capital + of France.\nCached geography fact 291: Paris is the capital of France.\nCached + geography fact 292: Paris is the capital of France.\nCached geography fact 293: + Paris is the capital of France.\nCached geography fact 294: Paris is the capital + of France.\nCached geography fact 295: Paris is the capital of France.\nCached + geography fact 296: Paris is the capital of France.\nCached geography fact 297: + Paris is the capital of France.\nCached geography fact 298: Paris is the capital + of France.\nCached geography fact 299: Paris is the capital of France.","cache_control":{"type":"ephemeral"}}],"stream":true}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '18160' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - Anthropic/Python 1.11.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-helper-method: + - stream + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.11.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + x-stainless-stream-helper: + - messages + x-stainless-timeout: + - NOT_GIVEN + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: 'event: message_start + + data: {"type":"message_start","message":{"model":"claude-haiku-4-5-20251001","id":"msg_011CfjUaqiXM9bZ6SXbowvTt","type":"message","role":"assistant","content":[],"container":null,"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":13,"cache_creation_input_tokens":0,"cache_read_input_tokens":4500,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":1,"service_tier":"standard","inference_geo":"not_available"},"diagnostics":{"cache_miss_reason":{"type":"system_changed","cache_missed_input_tokens":4397}}} + } + + + event: content_block_start + + data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} } + + + event: ping + + data: {"type": "ping"} + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"I"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" + don"}} + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"''t + see a \"Section "} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"1\" + in our"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" + conversation. The"} } + + + event: content_block_delta + + data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" + content"} } + + + event: content_block_stop + + data: {"type":"content_block_stop","index":0 } + + + event: message_delta + + data: {"type":"message_delta","delta":{"stop_reason":"max_tokens","stop_sequence":null,"stop_details":null,"container":null},"usage":{"input_tokens":13,"cache_creation_input_tokens":0,"cache_read_input_tokens":4500,"output_tokens":16} } + + + event: message_stop + + data: {"type":"message_stop" } + + + ' + headers: + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '9997000' + anthropic-ratelimit-input-tokens-reset: + - '2026-10-05T17:22:05Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-10-05T17:22:05Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-10-05T17:22:05Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '11997000' + anthropic-ratelimit-tokens-reset: + - '2026-10-05T17:22:05Z' + anthropic-workspace-id: + - wrkspc_018bFG1FgNVWqx5pthtdGTzS + cache-control: + - no-cache + cf-cache-status: + - DYNAMIC + cf-ray: + - a45e1b422a82a392-YYZ + connection: + - keep-alive + content-length: + - '1961' + content-security-policy: + - default-src 'none'; frame-ancestors 'none' + content-type: + - text/event-stream; charset=utf-8 + date: + - Mon, 05 Oct 2026 17:22:06 GMT request-id: - - req_011CfjTTB42NG92bQaUt6Q2a + - req_011CfjUaqVP4DRVcEPvDyutM server: - cloudflare strict-transport-security: - max-age=31536000; includeSubDomains; preload traceresponse: - - 00-1c2eaf88b6d847457bc00ab34bfc7b61-7360e88d1395d2ab-01 + - 00-630763f0e64154df942d963671f56da6-916afba76da8ce56-01 transfer-encoding: - chunked vary: diff --git a/py/src/braintrust/integrations/anthropic/test_anthropic.py b/py/src/braintrust/integrations/anthropic/test_anthropic.py index 71e800130..30d0ca77f 100644 --- a/py/src/braintrust/integrations/anthropic/test_anthropic.py +++ b/py/src/braintrust/integrations/anthropic/test_anthropic.py @@ -655,17 +655,29 @@ def test_anthropic_messages_create_prompt_cache_diagnostics(memory_logger): ], } second_response = client.messages.create(**second_request, diagnostics={"previous_message_id": first_response.id}) + with client.messages.stream( + **request, + diagnostics={"previous_message_id": second_response.id}, + ) as stream: + stream_events = list(stream) spans = memory_logger.pop() - message_spans = [span for span in spans if span["span_attributes"]["name"] == "anthropic.messages.create"] - assert len(message_spans) == 2 - second_span = message_spans[1] + create_spans = [span for span in spans if span["span_attributes"]["name"] == "anthropic.messages.create"] + stream_span = next(span for span in spans if span["span_attributes"]["name"] == "anthropic.messages.stream") + assert len(create_spans) == 2 + second_span = create_spans[1] assert second_span["metadata"]["diagnostics"] == {"previous_message_id": first_response.id} cache_miss_reason = second_response.diagnostics.cache_miss_reason assert cache_miss_reason.type == "system_changed" assert second_span["metadata"]["cache_miss_reason"] == cache_miss_reason.type assert second_span["metadata"]["cache_missed_input_tokens"] == cache_miss_reason.cache_missed_input_tokens + message_start = stream_events[0] + stream_cache_miss_reason = message_start.message.diagnostics.cache_miss_reason + assert stream_cache_miss_reason.type == "system_changed" + assert stream_span["metadata"]["cache_miss_reason"] == stream_cache_miss_reason.type + assert stream_span["metadata"]["cache_missed_input_tokens"] == stream_cache_miss_reason.cache_missed_input_tokens + @pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path", "body"]) @pytest.mark.asyncio From 6b078b5db485846bbe636a1693f42fc5641b1fb5 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Mon, 5 Oct 2026 14:40:24 -0400 Subject: [PATCH 5/5] test(anthropic): satisfy pylint for diagnostics requests --- .../integrations/anthropic/test_anthropic.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/py/src/braintrust/integrations/anthropic/test_anthropic.py b/py/src/braintrust/integrations/anthropic/test_anthropic.py index 30d0ca77f..5c5434d99 100644 --- a/py/src/braintrust/integrations/anthropic/test_anthropic.py +++ b/py/src/braintrust/integrations/anthropic/test_anthropic.py @@ -625,6 +625,8 @@ def test_anthropic_messages_create_prompt_cache_metrics(memory_logger, ttl, vcr_ @pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path", "body"]) +# The SDK's generated method signature omits the runtime-supported diagnostics parameter. +# pylint: disable=unexpected-keyword-arg def test_anthropic_messages_create_prompt_cache_diagnostics(memory_logger): if os.environ.get("BRAINTRUST_TEST_PACKAGE_VERSION") != "latest": pytest.skip("Prompt cache diagnostics require the latest Anthropic SDK cassette") @@ -642,7 +644,8 @@ def test_anthropic_messages_create_prompt_cache_diagnostics(memory_logger): ], "messages": [{"role": "user", "content": "Summarize section 1."}], } - first_response = client.messages.create(**request, diagnostics={"previous_message_id": None}) + first_request = {**request, "diagnostics": {"previous_message_id": None}} + first_response = client.messages.create(**first_request) second_request = { **request, "system": [ @@ -653,12 +656,11 @@ def test_anthropic_messages_create_prompt_cache_diagnostics(memory_logger): ), } ], + "diagnostics": {"previous_message_id": first_response.id}, } - second_response = client.messages.create(**second_request, diagnostics={"previous_message_id": first_response.id}) - with client.messages.stream( - **request, - diagnostics={"previous_message_id": second_response.id}, - ) as stream: + second_response = client.messages.create(**second_request) + stream_request = {**request, "diagnostics": {"previous_message_id": second_response.id}} + with client.messages.stream(**stream_request) as stream: stream_events = list(stream) spans = memory_logger.pop()