From 4654e618b166ab1c2bd0f4012f4e79e2108184ba Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Wed, 14 Oct 2026 11:41:54 -0400 Subject: [PATCH 1/3] fix(integrations): log dspy and litellm images as attachments Convert inline images before span serialization and preserve provider inputs. Consolidate VCR coverage across supported versions and remove redundant tests and unused cassettes. Fixes SDK-551. --- py/noxfile.py | 1 + py/pyproject.toml | 5 + .../auto_test_scripts/test_auto_dspy.py | 2 +- .../cassettes/2.6.0/test_dspy_callback.yaml | 123 -------- ...t_dspy_images_materialize_attachments.yaml | 121 ++++++++ ..._dspy_images_materialize_attachments.yaml} | 51 ++-- .../braintrust/integrations/dspy/test_dspy.py | 203 +++++++++---- .../braintrust/integrations/dspy/tracing.py | 70 ++++- ...st_litellm_async_streaming_with_break.yaml | 138 --------- ...test_litellm_completion_metrics[sync].yaml | 215 -------------- ...t_litellm_completion_streaming[async].yaml | 138 --------- ...st_litellm_completion_streaming[sync].yaml | 274 ------------------ ...erializes_attachment[blocking-async].yaml} | 65 +++-- ...aterializes_attachment[blocking-sync].yaml | 111 +++++++ ...erializes_attachment[streaming-async].yaml | 129 +++++++++ ...terializes_attachment[streaming-sync].yaml | 129 +++++++++ ...st_litellm_async_streaming_with_break.yaml | 132 --------- ...t_litellm_completion_streaming[async].yaml | 132 --------- ...st_litellm_completion_streaming[sync].yaml | 132 --------- ...erializes_attachment[blocking-async].yaml} | 39 ++- ...terializes_attachment[blocking-sync].yaml} | 37 ++- ...erializes_attachment[streaming-async].yaml | 129 +++++++++ ...terializes_attachment[streaming-sync].yaml | 129 +++++++++ .../integrations/litellm/test_litellm.py | 230 +++++---------- .../integrations/litellm/tracing.py | 4 +- py/uv.lock | 4 + 26 files changed, 1142 insertions(+), 1601 deletions(-) delete mode 100644 py/src/braintrust/integrations/dspy/cassettes/2.6.0/test_dspy_callback.yaml create mode 100644 py/src/braintrust/integrations/dspy/cassettes/2.6.0/test_dspy_images_materialize_attachments.yaml rename py/src/braintrust/integrations/dspy/cassettes/latest/{test_dspy_callback.yaml => test_dspy_images_materialize_attachments.yaml} (58%) delete mode 100644 py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_async_streaming_with_break.yaml delete mode 100644 py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_metrics[sync].yaml delete mode 100644 py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_streaming[async].yaml delete mode 100644 py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_streaming[sync].yaml rename py/src/braintrust/integrations/litellm/cassettes/1.74.0/{test_litellm_completion_metrics[async].yaml => test_litellm_image_input_materializes_attachment[blocking-async].yaml} (61%) create mode 100644 py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_input_materializes_attachment[blocking-sync].yaml create mode 100644 py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_input_materializes_attachment[streaming-async].yaml create mode 100644 py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_input_materializes_attachment[streaming-sync].yaml delete mode 100644 py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_async_streaming_with_break.yaml delete mode 100644 py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_streaming[async].yaml delete mode 100644 py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_streaming[sync].yaml rename py/src/braintrust/integrations/litellm/cassettes/latest/{test_litellm_completion_metrics[async].yaml => test_litellm_image_input_materializes_attachment[blocking-async].yaml} (68%) rename py/src/braintrust/integrations/litellm/cassettes/latest/{test_litellm_completion_metrics[sync].yaml => test_litellm_image_input_materializes_attachment[blocking-sync].yaml} (68%) create mode 100644 py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_input_materializes_attachment[streaming-async].yaml create mode 100644 py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_input_materializes_attachment[streaming-sync].yaml diff --git a/py/noxfile.py b/py/noxfile.py index e1e3a52f2..dd43b0ac7 100644 --- a/py/noxfile.py +++ b/py/noxfile.py @@ -632,6 +632,7 @@ def test_dspy(session, version): # on Python 3.10 due to unresolved Pydantic forward references. _skip_if_incompatible(session) _install_test_deps(session) + _install_group_locked(session, "test-dspy") if version == LATEST: # DSPy only lower-bounds LiteLLM, whose 1.92.0 release lacks Windows # and Python 3.14 wheels. Preinstall our portable matrix pin so DSPy's diff --git a/py/pyproject.toml b/py/pyproject.toml index fd4facfd6..f2b01895e 100644 --- a/py/pyproject.toml +++ b/py/pyproject.toml @@ -141,6 +141,11 @@ test-litellm = [ "orjson==3.12.0; python_version>='3.15'", ] +test-dspy = [ + # DSPy 2.6 requires Pillow to format image inputs but does not install it. + "pillow==12.3.0", +] + test-agno = [ {include-group = "test"}, "openai==2.31.0", diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_dspy.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_dspy.py index 6a4813aa0..feceeeeb3 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_dspy.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_dspy.py @@ -1,7 +1,7 @@ """Test auto_instrument for DSPy. Note: This test focuses on patching behavior only. Span verification for DSPy -is done in test_dspy.py::test_dspy_callback which uses pytest-vcr (supports httpx). +is done in test_dspy.py::test_dspy_images_materialize_attachments which uses pytest-vcr (supports httpx). The standalone VCR in test_utils doesn't capture httpx used by litellm/dspy. """ diff --git a/py/src/braintrust/integrations/dspy/cassettes/2.6.0/test_dspy_callback.yaml b/py/src/braintrust/integrations/dspy/cassettes/2.6.0/test_dspy_callback.yaml deleted file mode 100644 index 4759ff5ff..000000000 --- a/py/src/braintrust/integrations/dspy/cassettes/2.6.0/test_dspy_callback.yaml +++ /dev/null @@ -1,123 +0,0 @@ -interactions: -- request: - body: '{"messages":[{"role":"system","content":"Your input fields are:\n1. `question` - (str):\nYour output fields are:\n1. `reasoning` (str): \n2. `answer` (str):\nAll - interactions will be structured in the following way, with the appropriate values - filled in.\n\n[[ ## question ## ]]\n{question}\n\n[[ ## reasoning ## ]]\n{reasoning}\n\n[[ - ## answer ## ]]\n{answer}\n\n[[ ## completed ## ]]\nIn adhering to this structure, - your objective is: \n Given the fields `question`, produce the fields - `answer`."},{"role":"user","content":"[[ ## question ## ]]\nWhat is 2+2?\n\nRespond - with the corresponding output fields, starting with the field `[[ ## reasoning - ## ]]`, then `[[ ## answer ## ]]`, and then ending with the marker for `[[ ## - completed ## ]]`."}],"model":"gpt-4o-mini","max_tokens":4000,"temperature":0.0}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate - connection: - - keep-alive - content-length: - - '813' - content-type: - - application/json - host: - - api.openai.com - user-agent: - - OpenAI/Python 2.6.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - 'false' - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.6.0 - x-stainless-raw-response: - - 'true' - x-stainless-read-timeout: - - '600.0' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.9.21 - method: POST - uri: https://api.openai.com/v1/chat/completions - response: - body: - string: !!binary | - H4sIAAAAAAAAAwAAAP//jJPJbtswEIbveooBc6xk2KqX2NciB7cFgiZGEcA2BJocSWwokuFiNwjy - 7gUl27K7AL0I0Hzzj/5Z9JYAEMHJAgirqWeNkdmnR/twd/d1L748scl9ODzRh9vlo/l2/3n1fUnS - qNC7H8j8STVgujESvdCqw8wi9RirjmbT4Xw+nn6ctqDRHGWUVcZnY501QoksH+bjbDjLRrdHda0F - Q0cWsE4AAN7aZ/SpOP4kCximp0iDztEKyeKcBECsljFCqHPCeao8SXvItPKoWuvrNdzcgEXqtBKq - ii/b7UataoSXgC62A9Q9Oyi1BV8juNCALiEHqjjkA1gq2FEnGFArfN2gFyyFQ40KXnUAyjn4gwYV - mh1aB15X6Gu0aUuZbnZCYawrLOypDOgGsKrRYqktppDDB8gBXwKVDsaDjdqozjBV7oD25Hbcg+MW - kHfssmuLZXA0Tl4FKS8AVUp7Gltt5709kvfzhKWujNU795uUlEIJVxfd9OI0ndeGtPQ9Adi2mwxX - yyHG6sb4wutnbD83mh03SfoD6ulkdoReeyr7eJ6fwFW9gqOnQrqLWyCMshp5L+0PhwYu9AVILrr+ - 083fap/v5n/K94AxNB55YSxywa477tMsxv/rX2nnKbeGiUO7FwwLL9DGTXAsaZDd1RP36jw2RSlU - hdZY0Z1+aYrJdEjLKU4mc5K8J78AAAD//wMAHmfYmwgEAAA= - headers: - CF-RAY: - - 991b80ba5f531dd5-IAD - Connection: - - keep-alive - Content-Encoding: - - gzip - Content-Type: - - application/json - Date: - - Mon, 20 Oct 2025 21:10:37 GMT - Server: - - cloudflare - Set-Cookie: - - __cf_bm=NCarHmYOyIz1ZAIOne5DFJej0ZWtyIkscrsY1JN2xgU-1760994637-1.0.1.1-S4ijZSTcO6mty21tIa9pLH.RT_PbGhLHllaxLNjL.qAfWfKv8CzxzsKMrOwqSNdcr8XrFQZqX0N0k7NXrH129pMDQ8PsNZMyn1FG241fvS4; - path=/; expires=Mon, 20-Oct-25 21:40:37 GMT; domain=.api.openai.com; HttpOnly; - Secure; SameSite=None - - _cfuvid=Mp.a7GDzqU0CGQVrU3qeUynTdP6UBCXjJeHOoQ6jEDI-1760994637165-0.0.1.1-604800000; - path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None - Strict-Transport-Security: - - max-age=31536000; includeSubDomains; preload - Transfer-Encoding: - - chunked - X-Content-Type-Options: - - nosniff - access-control-expose-headers: - - X-Request-ID - alt-svc: - - h3=":443"; ma=86400 - cf-cache-status: - - DYNAMIC - openai-organization: - - braintrust-data - openai-processing-ms: - - '1008' - openai-project: - - proj_vsCSXafhhByzWOThMrJcZiw9 - openai-version: - - '2020-10-01' - x-envoy-upstream-service-time: - - '1171' - x-openai-proxy-wasm: - - v0.1 - x-ratelimit-limit-requests: - - '30000' - x-ratelimit-limit-tokens: - - '150000000' - x-ratelimit-remaining-requests: - - '29999' - x-ratelimit-remaining-tokens: - - '149999832' - x-ratelimit-reset-requests: - - 2ms - x-ratelimit-reset-tokens: - - 0s - x-request-id: - - req_3350c1e2f2d54aaf9923d2481dfaff3c - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/dspy/cassettes/2.6.0/test_dspy_images_materialize_attachments.yaml b/py/src/braintrust/integrations/dspy/cassettes/2.6.0/test_dspy_images_materialize_attachments.yaml new file mode 100644 index 000000000..ed169436f --- /dev/null +++ b/py/src/braintrust/integrations/dspy/cassettes/2.6.0/test_dspy_images_materialize_attachments.yaml @@ -0,0 +1,121 @@ +interactions: +- request: + body: '{"model":"gpt-4o-mini","messages":[{"role":"system","content":"Your input + fields are:\n1. `image` (Image)\n2. `page` (Image)\n\nYour output fields are:\n1. + `answer` (str)\n\nAll interactions will be structured in the following way, + with the appropriate values filled in.\n\n[[ ## image ## ]]\n{image}\n\n[[ ## + page ## ]]\n{page}\n\n[[ ## answer ## ]]\n{answer}\n\n[[ ## completed ## ]]\n\nIn + adhering to this structure, your objective is: \n Describe the colors + in these two page images."},{"role":"user","content":[{"type":"text","text":"[[ + ## image ## ]]\n\n"},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="}},{"type":"text","text":"\n\n[[ + ## page ## ]]\n\n"},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="}},{"type":"text","text":"\n\nRespond + with the corresponding output fields, starting with the field `[[ ## answer + ## ]]`, and then ending with the marker for `[[ ## completed ## ]]`."}]}],"max_tokens":1000,"temperature":0.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '1165' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 3.29.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 3.29.0 + x-stainless-raw-response: + - 'true' + x-stainless-read-timeout: + - '600.0' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-EYuEzFQDguHfWMglcxsl7uwpF32kg\",\n \"object\": + \"chat.completion\",\n \"created\": 1791988049,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"[[ ## answer ## ]]\\nThe colors in + both images are a solid, vibrant red. \\n\\n[[ ## completed ## ]]\",\n \"refusal\": + null,\n \"annotations\": []\n },\n \"logprobs\": null,\n + \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 17159,\n \"completion_tokens\": 23,\n \"total_tokens\": 17182,\n \"prompt_tokens_details\": + {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_bf0094d986\"\n}\n" + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a4a743dc5e60aaba-YYZ + connection: + - keep-alive + content-length: + - '910' + content-type: + - application/json + date: + - Wed, 14 Oct 2026 14:27:30 GMT + openai-processing-ms: + - '748' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-input-images: + - '50000' + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-input-images: + - '49998' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149998317' + x-ratelimit-reset-input-images: + - 2ms + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_97b1286a8ab44a50a486d7f5805b7982 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/dspy/cassettes/latest/test_dspy_callback.yaml b/py/src/braintrust/integrations/dspy/cassettes/latest/test_dspy_images_materialize_attachments.yaml similarity index 58% rename from py/src/braintrust/integrations/dspy/cassettes/latest/test_dspy_callback.yaml rename to py/src/braintrust/integrations/dspy/cassettes/latest/test_dspy_images_materialize_attachments.yaml index c338dcbf0..252817766 100644 --- a/py/src/braintrust/integrations/dspy/cassettes/latest/test_dspy_callback.yaml +++ b/py/src/braintrust/integrations/dspy/cassettes/latest/test_dspy_images_materialize_attachments.yaml @@ -1,15 +1,15 @@ interactions: - request: - body: '{"messages":[{"role":"system","content":"Your input fields are:\n1. `question` - (str):\nYour output fields are:\n1. `reasoning` (str): \n2. `answer` (str):\nAll + body: '{"messages":[{"role":"system","content":"Your input fields are:\n1. `image` + (Image): \n2. `page` (Page):\nYour output fields are:\n1. `answer` (str):\nAll interactions will be structured in the following way, with the appropriate values - filled in.\n\n[[ ## question ## ]]\n{question}\n\n[[ ## reasoning ## ]]\n{reasoning}\n\n[[ - ## answer ## ]]\n{answer}\n\n[[ ## completed ## ]]\nIn adhering to this structure, - your objective is: \n Given the fields `question`, produce the fields - `answer`."},{"role":"user","content":"[[ ## question ## ]]\nWhat is 2+2?\n\nRespond - with the corresponding output fields, starting with the field `[[ ## reasoning - ## ]]`, then `[[ ## answer ## ]]`, and then ending with the marker for `[[ ## - completed ## ]]`."}],"model":"gpt-4o-mini"}' + filled in.\n\n[[ ## image ## ]]\n{image}\n\n[[ ## page ## ]]\n{page}\n\n[[ ## + answer ## ]]\n{answer}\n\n[[ ## completed ## ]]\nIn adhering to this structure, + your objective is: \n Describe the colors in these two page images."},{"role":"user","content":[{"type":"text","text":"[[ + ## image ## ]]\n"},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="}},{"type":"text","text":"\n\n[[ + ## page ## ]]\n{\"number\": 2, \"image\": \""},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="}},{"type":"text","text":"\"}\n\nRespond + with the corresponding output fields, starting with the field `[[ ## answer + ## ]]`, and then ending with the marker for `[[ ## completed ## ]]`."}]}],"model":"gpt-4o-mini"}' headers: accept: - application/json @@ -18,7 +18,7 @@ interactions: connection: - keep-alive content-length: - - '777' + - '1154' content-type: - application/json host: @@ -49,19 +49,18 @@ interactions: uri: https://api.openai.com/v1/chat/completions response: body: - string: "{\n \"id\": \"chatcmpl-EYXW4udLtlqintb4oZb1YgnuW3wPU\",\n \"object\": - \"chat.completion\",\n \"created\": 1791900696,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + string: "{\n \"id\": \"chatcmpl-EYtp520w14OX3nAIFuclWlAvVUt0Q\",\n \"object\": + \"chat.completion\",\n \"created\": 1791986443,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": - \"assistant\",\n \"content\": \"[[ ## reasoning ## ]]\\nTo solve the - equation 2 + 2, we simply add the two numbers together. The sum of 2 and 2 - is 4.\\n\\n[[ ## answer ## ]]\\n4\\n\\n[[ ## completed ## ]]\",\n \"refusal\": + \"assistant\",\n \"content\": \"[[ ## answer ## ]]\\nThe color in the + images is a solid, vibrant red.\\n\\n[[ ## completed ## ]]\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": - 170,\n \"completion_tokens\": 47,\n \"total_tokens\": 217,\n \"prompt_tokens_details\": + 17167,\n \"completion_tokens\": 22,\n \"total_tokens\": 17189,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": - \"default\",\n \"system_fingerprint\": \"fp_df9611cffa\"\n}\n" + \"default\",\n \"system_fingerprint\": \"fp_bf0094d986\"\n}\n" headers: access-control-expose-headers: - X-Request-ID @@ -72,17 +71,17 @@ interactions: cf-cache-status: - DYNAMIC cf-ray: - - a49eef390eec5327-YYZ + - a4a71ca40932ebc0-YYZ connection: - keep-alive content-length: - - '975' + - '906' content-type: - application/json date: - - Tue, 13 Oct 2026 14:11:37 GMT + - Wed, 14 Oct 2026 14:00:44 GMT openai-processing-ms: - - '808' + - '636' openai-version: - '2020-10-01' server: @@ -95,20 +94,26 @@ interactions: - nosniff x-openai-proxy-wasm: - v0.1 + x-ratelimit-limit-input-images: + - '50000' x-ratelimit-limit-requests: - '30000' x-ratelimit-limit-tokens: - '150000000' + x-ratelimit-remaining-input-images: + - '49997' x-ratelimit-remaining-requests: - '29998' x-ratelimit-remaining-tokens: - - '149999812' + - '149997537' + x-ratelimit-reset-input-images: + - 3ms x-ratelimit-reset-requests: - 4ms x-ratelimit-reset-tokens: - 0s x-request-id: - - req_f9b517b5c9b04fb993dd2b9b70450d54 + - req_7e7d9e781211446b8482c8a294249204 status: code: 200 message: OK diff --git a/py/src/braintrust/integrations/dspy/test_dspy.py b/py/src/braintrust/integrations/dspy/test_dspy.py index c3f10d112..4bdc1d1c1 100644 --- a/py/src/braintrust/integrations/dspy/test_dspy.py +++ b/py/src/braintrust/integrations/dspy/test_dspy.py @@ -2,16 +2,21 @@ Tests for DSPy integration with Braintrust. """ +import base64 import inspect import os from types import SimpleNamespace import dspy import pytest -from braintrust import logger +from braintrust import Attachment, logger +from braintrust.bt_json import bt_dumps from braintrust.integrations.dspy import BraintrustDSpyCallback +from braintrust.integrations.dspy.tracing import _process_images +from braintrust.integrations.litellm import patch_litellm from braintrust.integrations.test_utils import run_in_subprocess, verify_autoinstrument_script from braintrust.test_helpers import init_test_logger +from pydantic import BaseModel, Field, field_serializer PROJECT_NAME = "test-dspy-app" @@ -22,77 +27,149 @@ LM_KWARGS = {"engine": "litellm"} if "engine" in inspect.signature(dspy.LM.__init__).parameters else {} -@pytest.fixture -def memory_logger(): - init_test_logger(PROJECT_NAME) - with logger._internal_with_memory_background_logger() as bgl: - yield bgl +def test_dspy_image_conversion_preserves_other_model_serializers(): + class Page(BaseModel): + image: dspy.Image + number: int + secret: str = Field(exclude=True) + + @field_serializer("number") + def serialize_number(self, number): + return f"page-{number}" + + image = dspy.Image(url="data:image/png;base64,aW1hZ2U=") + page = Page(image=image, number=2, secret="private") + converted = _process_images(page) + assert isinstance(converted["image"]["image_url"]["url"], Attachment) + assert converted["number"] == "page-2" + assert "secret" not in converted + assert page.image is image + + +@pytest.mark.parametrize("url", ["https://example.com/page.png", "data:image/png;base64,not base64!"]) +def test_dspy_image_conversion_preserves_unmaterialized_images(url): + image = dspy.Image(url=url) + assert _process_images(image) is image + assert _process_images({"image": image, "url": url}) == {"image": image, "url": url} + + +@pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path", "query", "body"]) +def test_dspy_images_materialize_attachments(memory_logger): + """Images stay out of inline logs throughout a real DSPy/LiteLLM call.""" + image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==" + image_url = f"data:image/png;base64,{image_base64}" + + class Page(BaseModel): + number: int + image: dspy.Image + + # DSPy 2.6 supports direct image fields, but does not expand images nested + # in Pydantic fields into message content parts. + nested_images = os.environ.get("BRAINTRUST_TEST_PACKAGE_VERSION") != "2.6.0" + describe = dspy.Signature( + { + "image": (dspy.Image, dspy.InputField()), + "page": (Page if nested_images else dspy.Image, dspy.InputField()), + "answer": (str, dspy.OutputField()), + }, + instructions="Describe the colors in these two page images.", + ) + class DescribePages(dspy.Module): + def __init__(self): + super().__init__() + self.predict = dspy.Predict(describe) -@pytest.mark.vcr -def test_dspy_callback(memory_logger): - """Test DSPy callback logs spans correctly.""" - assert not memory_logger.pop() + def forward(self, image, page): + result = self.predict(image=image, page=page if nested_images else page.image) + return dspy.Prediction(answer=result.answer, image=image, page=page) - # Configure DSPy with Braintrust callback + patch_litellm() lm = dspy.LM(MODEL, cache=False, **LM_KWARGS) - dspy.configure(lm=lm, callbacks=[BraintrustDSpyCallback()]) - - # Use ChainOfThought for a more interesting test - cot = dspy.ChainOfThought("question -> answer") - result = cot(question="What is 2+2?") - - assert result.answer # Verify we got a response - - # Check logged spans - spans = memory_logger.pop() - assert len(spans) >= 4 # Should have module, adapter format, LM, and adapter parse spans - - spans_by_name = {span["span_attributes"]["name"]: span for span in spans} - - module_span = spans_by_name["dspy.module.ChainOfThought"] - assert module_span["span_attributes"]["type"] == "task" - assert module_span["metadata"]["module_class"].endswith("ChainOfThought") - - lm_span = spans_by_name["dspy.lm"] + image = dspy.Image(url=image_url) + page = Page(number=2, image=dspy.Image(url=image_url)) + with dspy.context(lm=lm, callbacks=[BraintrustDSpyCallback()]): + result = DescribePages()(image=image, page=page) + + assert result.answer + assert result.image is image + assert result.page is page + assert image.url == page.image.url == image_url + spans = {span["span_attributes"]["name"]: span for span in memory_logger.pop()} + + for name in ("dspy.module.DescribePages", "dspy.module.Predict"): + span = spans[name] + assert span["span_attributes"]["type"] == "task" + assert span["metadata"]["module_class"].endswith(name.removeprefix("dspy.module.")) + + lm_span = spans["dspy.lm"] assert lm_span["context"]["span_origin"]["instrumentation"]["name"] == "dspy-auto" assert lm_span["span_attributes"].get("type") != "llm" - assert "metadata" in lm_span - assert "model" in lm_span["metadata"] - assert MODEL in lm_span["metadata"]["model"] + assert lm_span["metadata"]["model"] == MODEL assert lm_span["metadata"]["provider"] == "openai" - assert "input" in lm_span - assert "output" in lm_span - # The latest cassette contains a real OpenAI usage payload. DSPy 2.6's - # recorded response predates usage being included. - if os.environ.get("BRAINTRUST_TEST_PACKAGE_VERSION") == "latest": - assert lm_span["metrics"]["prompt_tokens"] == 170 - assert lm_span["metrics"]["completion_tokens"] == 47 - assert lm_span["metrics"]["tokens"] == 217 + assert lm_span["output"] + metrics = lm_span["metrics"] + assert metrics["prompt_tokens"] > 0 + assert metrics["completion_tokens"] > 0 + assert metrics["tokens"] == metrics["prompt_tokens"] + metrics["completion_tokens"] + + for name in ("dspy.adapter.format", "dspy.adapter.parse"): + span = spans[name] + assert span["span_attributes"]["type"] == "task" + assert span["metadata"]["adapter_class"].endswith("ChatAdapter") + assert "signature" in span["input"] + assert "demos" in spans["dspy.adapter.format"]["input"] + assert "completion" in spans["dspy.adapter.parse"]["input"] + assert isinstance(spans["dspy.adapter.parse"]["output"], dict) + span_ids = {span["span_id"] for span in spans.values()} + for name in ("dspy.lm", "dspy.adapter.format", "dspy.adapter.parse"): + assert spans[name]["span_parents"][0] in span_ids + + def assert_image(value): + attachment = value["image_url"]["url"] + assert isinstance(attachment, Attachment) + assert attachment.data == base64.b64decode(image_base64) + assert attachment.reference["content_type"] == "image/png" + + for name in ("dspy.module.DescribePages", "dspy.module.Predict"): + inputs = spans[name]["input"] + inputs = inputs.get("kwargs", inputs) + assert_image(inputs["image"]) + if nested_images or name == "dspy.module.DescribePages": + assert_image(inputs["page"]["image"]) + assert inputs["page"]["number"] == 2 + else: + assert_image(inputs["page"]) + assert_image(spans["dspy.module.DescribePages"]["output"]["image"]) + assert_image(spans["dspy.module.DescribePages"]["output"]["page"]["image"]) + assert_image(spans["dspy.adapter.format"]["input"]["inputs"]["image"]) + logged_page = spans["dspy.adapter.format"]["input"]["inputs"]["page"] + assert_image(logged_page["image"] if nested_images else logged_page) + for messages in ( + spans["dspy.adapter.format"]["output"], + spans["dspy.lm"]["input"]["messages"], + spans["Completion"]["input"], + ): + image_parts = [ + part + for message in messages + if isinstance(message["content"], list) + for part in message["content"] + if part["type"] == "image_url" + ] + assert len(image_parts) == 2 + for part in image_parts: + assert_image(part) + serialized = bt_dumps(spans) + assert image_base64 not in serialized + assert "CUSTOM-TYPE" not in serialized - format_span = spans_by_name["dspy.adapter.format"] - assert format_span["span_attributes"]["type"] == "task" - assert format_span["metadata"]["adapter_class"].endswith("ChatAdapter") - assert "signature" in format_span["input"] - assert "demos" in format_span["input"] - assert "inputs" in format_span["input"] - assert isinstance(format_span["output"], list) - parse_span = spans_by_name["dspy.adapter.parse"] - assert parse_span["span_attributes"]["type"] == "task" - assert parse_span["metadata"]["adapter_class"].endswith("ChatAdapter") - assert "signature" in parse_span["input"] - assert "completion" in parse_span["input"] - assert isinstance(parse_span["output"], dict) - - # Verify spans are nested under the broader DSPy execution - span_ids = {span["span_id"] for span in spans} - assert lm_span.get("span_parents") - assert format_span.get("span_parents") - assert parse_span.get("span_parents") - assert format_span["span_parents"][0] in span_ids - assert lm_span["span_parents"][0] in span_ids - assert parse_span["span_parents"][0] in span_ids +@pytest.fixture +def memory_logger(): + init_test_logger(PROJECT_NAME) + with logger._internal_with_memory_background_logger() as bgl: + yield bgl def test_dspy_callback_ignores_usage_from_previous_lm_call(memory_logger): diff --git a/py/src/braintrust/integrations/dspy/tracing.py b/py/src/braintrust/integrations/dspy/tracing.py index 3a1c23818..6a4a17f2b 100644 --- a/py/src/braintrust/integrations/dspy/tracing.py +++ b/py/src/braintrust/integrations/dspy/tracing.py @@ -1,12 +1,15 @@ """DSPy-specific callback, span creation, and metadata extraction.""" +import logging from typing import Any +from braintrust.integrations.utils import _materialize_attachment from braintrust.logger import start_span as _bt_start_span from braintrust.span_types import SpanTypeAttribute _INSTRUMENTATION = "dspy-auto" +_log = logging.getLogger(__name__) _LM_METADATA_PARAM_ALLOWLIST = ("temperature", "max_tokens", "top_p", "top_k", "stop") _EVALUATE_METADATA_ALLOWLIST = ("accuracy", "score", "total", "correct") @@ -29,7 +32,9 @@ def _extract_provider(instance: Any, model: Any) -> str | None: try: + from dspy import Example, Image from dspy.utils.callback import BaseCallback + from pydantic import BaseModel _HAS_DSPY = True except ImportError: @@ -37,6 +42,57 @@ def _extract_provider(instance: Any, model: Any) -> str | None: BaseCallback = object # type: ignore[assignment,misc] +def _process_images(value: Any) -> Any: + """Materialize images before DSPy/Pydantic serializers turn them into prompt strings. + + Walk raw model and Example fields so nested images remain recognizable. + Only the logged copy changes; provider inputs and outputs stay intact. + """ + try: + return _process_images_impl(value) + except Exception: + _log.debug("Failed to materialize DSPy images", exc_info=True) + return value + + +def _process_images_impl(value: Any) -> Any: + if isinstance(value, Image): + url = value.url + if isinstance(url, str) and url.startswith("data:image/"): + resolved = _materialize_attachment(url) + if resolved is not None: + return {"image_url": {"url": resolved.attachment}} + return value + if isinstance(value, str) and value.startswith("data:image/"): + resolved = _materialize_attachment(value) + return resolved.attachment if resolved is not None else value + if isinstance(value, (list, tuple)): + converted = [_process_images(item) for item in value] + if all(item is original for item, original in zip(converted, value)): + return value + return tuple(converted) if isinstance(value, tuple) else converted + if isinstance(value, Example): + # toDict() serializes nested Pydantic fields in DSPy >= 3.4. + return {key: _process_images(item) for key, item in value.items()} + if isinstance(value, BaseModel): + # dict(model) exposes raw values instead of invoking field serializers. + fields = dict(value) + converted_fields = _process_images(fields) + if converted_fields is fields: + return value + changed = {key for key in fields if converted_fields[key] is not fields[key]} + result = value.model_dump(exclude=changed, exclude_none=True) + model_fields = type(value).model_fields + result.update( + (key, converted_fields[key]) for key in changed if key not in model_fields or not model_fields[key].exclude + ) + return result + if isinstance(value, dict): + converted_dict = {key: _process_images(item) for key, item in value.items()} + return value if all(converted_dict[key] is item for key, item in value.items()) else converted_dict + return value + + class BraintrustDSpyCallback(BaseCallback): """Callback handler that logs DSPy execution traces to Braintrust. @@ -137,7 +193,7 @@ def on_lm_start( span = start_span( name="dspy.lm", - input=inputs, + input=_process_images(inputs), metadata=metadata, ) span.set_current() @@ -166,7 +222,7 @@ def _end_span( if exception: log_data["error"] = exception if outputs is not None: - log_data["output"] = outputs + log_data["output"] = _process_images(outputs) if metrics: log_data["metrics"] = metrics @@ -252,7 +308,7 @@ def on_module_start( span = start_span( name=f"dspy.module.{cls_name}", type=SpanTypeAttribute.TASK, - input=inputs, + input=_process_images(inputs), metadata={"module_class": module_name}, ) span.set_current() @@ -287,7 +343,7 @@ def _start_adapter_span( span = start_span( name=span_name, type=SpanTypeAttribute.TASK, - input=inputs, + input=_process_images(inputs), metadata=metadata, ) span.set_current() @@ -377,7 +433,7 @@ def on_tool_start( span = start_span( name=tool_name, type=SpanTypeAttribute.TOOL, - input=inputs, + input=_process_images(inputs), ) span.set_current() self._spans[call_id] = span @@ -420,7 +476,7 @@ def on_evaluate_start( span = start_span( name="dspy.evaluate", type=SpanTypeAttribute.TASK, - input=inputs, + input=_process_images(inputs), metadata=metadata, ) span.set_current() @@ -448,7 +504,7 @@ def on_evaluate_end( if exception: log_data["error"] = exception if outputs is not None: - log_data["output"] = outputs + log_data["output"] = _process_images(outputs) if isinstance(outputs, dict): metadata: dict[str, Any] = {} for key in _EVALUATE_METADATA_ALLOWLIST: diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_async_streaming_with_break.yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_async_streaming_with_break.yaml deleted file mode 100644 index d50872b33..000000000 --- a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_async_streaming_with_break.yaml +++ /dev/null @@ -1,138 +0,0 @@ -interactions: -- request: - body: '{"messages":[{"role":"user","content":"What''s 12 + 12?"}],"model":"gpt-4o-mini","stream":true,"stream_options":{"include_usage":true}}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate - connection: - - keep-alive - content-length: - - '134' - content-type: - - application/json - host: - - api.openai.com - user-agent: - - AsyncOpenAI/Python 2.24.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - async:asyncio - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.24.0 - x-stainless-raw-response: - - 'true' - x-stainless-read-timeout: - - '600.0' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.13.3 - method: POST - uri: https://api.openai.com/v1/chat/completions - response: - body: - string: 'data: {"id":"chatcmpl-DFld8139VUAiMKuqKwYSqrvuCxt4n","object":"chat.completion.chunk","created":1772650382,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"pnUPoD0pg"} - - - data: {"id":"chatcmpl-DFld8139VUAiMKuqKwYSqrvuCxt4n","object":"chat.completion.chunk","created":1772650382,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"37zFEOP4I"} - - - data: {"id":"chatcmpl-DFld8139VUAiMKuqKwYSqrvuCxt4n","object":"chat.completion.chunk","created":1772650382,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":" - +"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"4WPyeDQnv"} - - - data: {"id":"chatcmpl-DFld8139VUAiMKuqKwYSqrvuCxt4n","object":"chat.completion.chunk","created":1772650382,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":" - "},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"sOxR63A12m"} - - - data: {"id":"chatcmpl-DFld8139VUAiMKuqKwYSqrvuCxt4n","object":"chat.completion.chunk","created":1772650382,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"bHyDGfLF5"} - - - data: {"id":"chatcmpl-DFld8139VUAiMKuqKwYSqrvuCxt4n","object":"chat.completion.chunk","created":1772650382,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":" - equals"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"1SVl"} - - - data: {"id":"chatcmpl-DFld8139VUAiMKuqKwYSqrvuCxt4n","object":"chat.completion.chunk","created":1772650382,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":" - "},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"mHNJKCDuMZ"} - - - data: {"id":"chatcmpl-DFld8139VUAiMKuqKwYSqrvuCxt4n","object":"chat.completion.chunk","created":1772650382,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":"24"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"ABixefOK3"} - - - data: {"id":"chatcmpl-DFld8139VUAiMKuqKwYSqrvuCxt4n","object":"chat.completion.chunk","created":1772650382,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"8hKx5NPN8n"} - - - data: {"id":"chatcmpl-DFld8139VUAiMKuqKwYSqrvuCxt4n","object":"chat.completion.chunk","created":1772650382,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"tdXAE"} - - - data: {"id":"chatcmpl-DFld8139VUAiMKuqKwYSqrvuCxt4n","object":"chat.completion.chunk","created":1772650382,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[],"usage":{"prompt_tokens":14,"completion_tokens":8,"total_tokens":22,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"bstubRw2p2F"} - - - data: [DONE] - - - ' - headers: - Access-Control-Expose-Headers: - - X-Request-ID - CF-RAY: - - 9d7314d87c81aa96-YYZ - Connection: - - keep-alive - Content-Type: - - text/event-stream; charset=utf-8 - Date: - - Wed, 04 Mar 2026 18:53:02 GMT - Server: - - cloudflare - Set-Cookie: - - __cf_bm=rXpr5trNRw1Qr3qbQJW.KHLQVMw77ivBL8WgtL_WzJo-1772650382.1581635-1.0.1.1-FmrGGx6rgCOI7Eo8DmohZe.rH15CYM_m1ffbwb.i_njpQwLnRzsFohZzV9dn0E9BEhVdvGXyfn1IRurlydiZLLFw2rNGb9ivmJkBlNxGLCMQalVVWZ_U2k2kk3YOqpHd; - HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 04 Mar 2026 - 19:23:02 GMT - Strict-Transport-Security: - - max-age=31536000; includeSubDomains; preload - Transfer-Encoding: - - chunked - X-Content-Type-Options: - - nosniff - alt-svc: - - h3=":443"; ma=86400 - cf-cache-status: - - DYNAMIC - openai-organization: - - braintrust-data - openai-processing-ms: - - '295' - openai-project: - - proj_vsCSXafhhByzWOThMrJcZiw9 - openai-version: - - '2020-10-01' - x-openai-proxy-wasm: - - v0.1 - x-ratelimit-limit-requests: - - '30000' - x-ratelimit-limit-tokens: - - '150000000' - x-ratelimit-remaining-requests: - - '29999' - x-ratelimit-remaining-tokens: - - '149999992' - x-ratelimit-reset-requests: - - 2ms - x-ratelimit-reset-tokens: - - 0s - x-request-id: - - req_057c10d296a342fba68072d8c99ce937 - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_metrics[sync].yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_metrics[sync].yaml deleted file mode 100644 index f2443b2cd..000000000 --- a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_metrics[sync].yaml +++ /dev/null @@ -1,215 +0,0 @@ -interactions: - - request: - body: - '{"messages": [{"role": "user", "content": "What''s 12 + 12?"}], "model": - "gpt-4o-mini"}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate, zstd - connection: - - keep-alive - content-length: - - "86" - content-type: - - application/json - host: - - api.openai.com - user-agent: - - OpenAI/Python 1.92.3 - x-stainless-arch: - - arm64 - x-stainless-async: - - "false" - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 1.92.3 - x-stainless-raw-response: - - "true" - x-stainless-read-timeout: - - "600.0" - x-stainless-retry-count: - - "0" - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.11.10 - method: POST - uri: https://api.openai.com/v1/chat/completions - response: - body: - string: !!binary | - H4sIAAAAAAAAAwAAAP//jJJBb9QwEIXv+RXWXNlUiderbfdGj/SA4IJUVEWuPUlcHNvYExZU7X9H - TrabFIrEJYf55k3eG89zwRgYDQcGqpekhmDL2/gl3d5X20/HnaW7u/cfno497+7lx89JHGGTFf7x - CRW9qK6UH4JFMt7NWEWUhHlqvd/x6mYvxH4Cg9dos6wLVApfDsaZkldclNW+rK/P6t4bhQkO7GvB - GGPP0zf7dBp/woFVm5fKgCnJDuFwaWIMore5AjIlk0g6gs0ClXeEbrJec/aO1Zzh91HaxLi4WjdG - bMcks1k3WrsC0jlPMoedLD6cyeliyvouRP+Y/pBCa5xJfRNRJu+ygUQ+wERPBWMPU/jxVR4I0Q+B - GvLfcPpdLeZxsKx8gddnRp6kXcqcb94Y1mgkaWxa7Q6UVD3qRbksWo7a+BUoVpH/9vLW7Dm2cd3/ - jF+AUhgIdRMiaqNe513aIuZ7/FfbZcWTYUgYfxiFDRmM+Rk0tnK085VA+pUIh6Y1rsMYoplPpQ3N - VsidkHizVVCcit8AAAD//wMA0gHr9zgDAAA= - headers: - CF-RAY: - - 95cb00386a5a14f6-LAX - Connection: - - keep-alive - Content-Encoding: - - gzip - Content-Type: - - application/json - Date: - - Wed, 09 Jul 2025 21:44:08 GMT - Server: - - cloudflare - Set-Cookie: - - __cf_bm=LhJr7iTKhj3HkuIqsYYQzHnLMYlAsrPO_8DONvOoa1s-1752097448-1.0.1.1-3QZBiExLtV3A83cyPK4CeYqjpSygQ6uYDZ0v7kYNlmEHZIOldYiM088zk99s107ShpZgiHzSJwcw0Fviy3roAk.Ifv1OH0UzhIx2VTAKaS4; - path=/; expires=Wed, 09-Jul-25 22:14:08 GMT; domain=.api.openai.com; HttpOnly; - Secure; SameSite=None - - _cfuvid=1FpkTfHv2KC0I5jKg65yXzjPU21dBkiuyT323kob89c-1752097448243-0.0.1.1-604800000; - path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None - Transfer-Encoding: - - chunked - X-Content-Type-Options: - - nosniff - access-control-expose-headers: - - X-Request-ID - alt-svc: - - h3=":443"; ma=86400 - cf-cache-status: - - DYNAMIC - openai-organization: - - braintrust-data - openai-processing-ms: - - "319" - openai-version: - - "2020-10-01" - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - x-envoy-upstream-service-time: - - "322" - x-ratelimit-limit-requests: - - "30000" - x-ratelimit-limit-tokens: - - "150000000" - x-ratelimit-remaining-requests: - - "29999" - x-ratelimit-remaining-tokens: - - "149999993" - x-ratelimit-reset-requests: - - 2ms - x-ratelimit-reset-tokens: - - 0s - x-request-id: - - req_0c9ffafb565ae4556d4374eaba259452 - status: - code: 200 - message: OK - - request: - body: - '{"messages": [{"role": "user", "content": "What''s 12 + 12?"}], "model": - "gpt-4o-mini"}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate, zstd - connection: - - keep-alive - content-length: - - "86" - content-type: - - application/json - cookie: - - __cf_bm=LhJr7iTKhj3HkuIqsYYQzHnLMYlAsrPO_8DONvOoa1s-1752097448-1.0.1.1-3QZBiExLtV3A83cyPK4CeYqjpSygQ6uYDZ0v7kYNlmEHZIOldYiM088zk99s107ShpZgiHzSJwcw0Fviy3roAk.Ifv1OH0UzhIx2VTAKaS4; - _cfuvid=1FpkTfHv2KC0I5jKg65yXzjPU21dBkiuyT323kob89c-1752097448243-0.0.1.1-604800000 - host: - - api.openai.com - user-agent: - - OpenAI/Python 1.92.3 - x-stainless-arch: - - arm64 - x-stainless-async: - - "false" - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 1.92.3 - x-stainless-raw-response: - - "true" - x-stainless-read-timeout: - - "600.0" - x-stainless-retry-count: - - "0" - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.11.10 - method: POST - uri: https://api.openai.com/v1/chat/completions - response: - body: - string: !!binary | - H4sIAAAAAAAAAwAAAP//jJJPb9QwEMXv+RSWr2yqxOtq/xxZcQIhTnBAVeS1J4nB8Rh7gkDVfnfk - ZLtJS5G45DC/eZP3xvNYMMat4UfGda9ID8GVb+OXdHq3/STfG1F9FP2Hz1tvxEnWJzyc+SYr8PwN - ND2p7jQOwQFZ9DPWERRBnlrv7kV12Em5n8CABlyWdYFKieVgvS1FJWRZ7cp6f1X3aDUkfmRfC8YY - e5y+2ac38IsfWbV5qgyQkuqAH29NjPGILle4SskmUp74ZoEaPYGfrNeCvWG1YPBjVC4xIe/WjRHa - Mals1o/OrYDyHknlsJPFhyu53Ew57ELEc3oh5a31NvVNBJXQZwOJMPCJXgrGHqbw47M8PEQcAjWE - 32H6XS3ncXxZ+QL3V0ZIyi1lITavDGsMkLIurXbHtdI9mEW5LFqNxuIKFKvIf3t5bfYc2/ruf8Yv - QGsIBKYJEYzVz/MubRHyPf6r7bbiyTBPEH9aDQ1ZiPkZDLRqdPOV8PQ7EQxNa30HMUQ7n0obmq1U - 91LBYat5cSn+AAAA//8DAN/D0ms4AwAA - headers: - CF-RAY: - - 95cb003c2e2414f6-LAX - Connection: - - keep-alive - Content-Encoding: - - gzip - Content-Type: - - application/json - Date: - - Wed, 09 Jul 2025 21:44:09 GMT - Server: - - cloudflare - Transfer-Encoding: - - chunked - X-Content-Type-Options: - - nosniff - access-control-expose-headers: - - X-Request-ID - alt-svc: - - h3=":443"; ma=86400 - cf-cache-status: - - DYNAMIC - openai-organization: - - braintrust-data - openai-processing-ms: - - "675" - openai-version: - - "2020-10-01" - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - x-envoy-upstream-service-time: - - "693" - x-ratelimit-limit-requests: - - "30000" - x-ratelimit-limit-tokens: - - "150000000" - x-ratelimit-remaining-requests: - - "29999" - x-ratelimit-remaining-tokens: - - "149999993" - x-ratelimit-reset-requests: - - 2ms - x-ratelimit-reset-tokens: - - 0s - x-request-id: - - req_8eb436911ec46aaf3095f96c82730abe - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_streaming[async].yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_streaming[async].yaml deleted file mode 100644 index c2010a57e..000000000 --- a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_streaming[async].yaml +++ /dev/null @@ -1,138 +0,0 @@ -interactions: -- request: - body: '{"messages":[{"role":"user","content":"What''s 12 + 12?"}],"model":"gpt-4o-mini","stream":true,"stream_options":{"include_usage":true}}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate - connection: - - keep-alive - content-length: - - '134' - content-type: - - application/json - host: - - api.openai.com - user-agent: - - AsyncOpenAI/Python 2.24.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - async:asyncio - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.24.0 - x-stainless-raw-response: - - 'true' - x-stainless-read-timeout: - - '600.0' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.13.3 - method: POST - uri: https://api.openai.com/v1/chat/completions - response: - body: - string: 'data: {"id":"chatcmpl-DFld4pdjHTYDiYVVqutj5XkGk3LPM","object":"chat.completion.chunk","created":1772650378,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"qCUq24aUP"} - - - data: {"id":"chatcmpl-DFld4pdjHTYDiYVVqutj5XkGk3LPM","object":"chat.completion.chunk","created":1772650378,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"tyBUuLlXP"} - - - data: {"id":"chatcmpl-DFld4pdjHTYDiYVVqutj5XkGk3LPM","object":"chat.completion.chunk","created":1772650378,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":" - +"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"AccQ6DEkI"} - - - data: {"id":"chatcmpl-DFld4pdjHTYDiYVVqutj5XkGk3LPM","object":"chat.completion.chunk","created":1772650378,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":" - "},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"4HK9XGcEQh"} - - - data: {"id":"chatcmpl-DFld4pdjHTYDiYVVqutj5XkGk3LPM","object":"chat.completion.chunk","created":1772650378,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"ZrKglFglG"} - - - data: {"id":"chatcmpl-DFld4pdjHTYDiYVVqutj5XkGk3LPM","object":"chat.completion.chunk","created":1772650378,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":" - equals"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"Megf"} - - - data: {"id":"chatcmpl-DFld4pdjHTYDiYVVqutj5XkGk3LPM","object":"chat.completion.chunk","created":1772650378,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":" - "},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"TAGN0Y1TQL"} - - - data: {"id":"chatcmpl-DFld4pdjHTYDiYVVqutj5XkGk3LPM","object":"chat.completion.chunk","created":1772650378,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":"24"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"0T8blqEVA"} - - - data: {"id":"chatcmpl-DFld4pdjHTYDiYVVqutj5XkGk3LPM","object":"chat.completion.chunk","created":1772650378,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"ber8oHdwqa"} - - - data: {"id":"chatcmpl-DFld4pdjHTYDiYVVqutj5XkGk3LPM","object":"chat.completion.chunk","created":1772650378,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"BPBAy"} - - - data: {"id":"chatcmpl-DFld4pdjHTYDiYVVqutj5XkGk3LPM","object":"chat.completion.chunk","created":1772650378,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[],"usage":{"prompt_tokens":14,"completion_tokens":8,"total_tokens":22,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"xmssdH5hQck"} - - - data: [DONE] - - - ' - headers: - Access-Control-Expose-Headers: - - X-Request-ID - CF-RAY: - - 9d7314c2cc23b407-YYZ - Connection: - - keep-alive - Content-Type: - - text/event-stream; charset=utf-8 - Date: - - Wed, 04 Mar 2026 18:52:58 GMT - Server: - - cloudflare - Set-Cookie: - - __cf_bm=16nX.lrlnS0MVxy6hFRb78u914aYPaUfIvgFXxMZEI8-1772650378.6869698-1.0.1.1-NB_.XAMuCAXL7hlX46NSshfJwOcB9qTdNGhWnSp_G2pcRTyMt.gwjEX_kjL5lKU0prJTyJn5fTgP_HjE_OiDcg8lBni08Fs8nNdCSw0u2dQWdCqHw.usO2s587EcmVS2; - HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 04 Mar 2026 - 19:22:58 GMT - Strict-Transport-Security: - - max-age=31536000; includeSubDomains; preload - Transfer-Encoding: - - chunked - X-Content-Type-Options: - - nosniff - alt-svc: - - h3=":443"; ma=86400 - cf-cache-status: - - DYNAMIC - openai-organization: - - braintrust-data - openai-processing-ms: - - '178' - openai-project: - - proj_vsCSXafhhByzWOThMrJcZiw9 - openai-version: - - '2020-10-01' - x-openai-proxy-wasm: - - v0.1 - x-ratelimit-limit-requests: - - '30000' - x-ratelimit-limit-tokens: - - '150000000' - x-ratelimit-remaining-requests: - - '29999' - x-ratelimit-remaining-tokens: - - '149999995' - x-ratelimit-reset-requests: - - 2ms - x-ratelimit-reset-tokens: - - 0s - x-request-id: - - req_98a341ea8970475ca3bcabd4bacf04bf - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_streaming[sync].yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_streaming[sync].yaml deleted file mode 100644 index 1a6b3a9f7..000000000 --- a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_streaming[sync].yaml +++ /dev/null @@ -1,274 +0,0 @@ -interactions: - - request: - body: - '{"messages": [{"role": "user", "content": "What''s 12 + 12?"}], "model": - "gpt-4o-mini", "stream": true, "stream_options": {"include_usage": true}}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate, zstd - connection: - - keep-alive - content-length: - - "145" - content-type: - - application/json - cookie: - - __cf_bm=LhJr7iTKhj3HkuIqsYYQzHnLMYlAsrPO_8DONvOoa1s-1752097448-1.0.1.1-3QZBiExLtV3A83cyPK4CeYqjpSygQ6uYDZ0v7kYNlmEHZIOldYiM088zk99s107ShpZgiHzSJwcw0Fviy3roAk.Ifv1OH0UzhIx2VTAKaS4; - _cfuvid=1FpkTfHv2KC0I5jKg65yXzjPU21dBkiuyT323kob89c-1752097448243-0.0.1.1-604800000 - host: - - api.openai.com - user-agent: - - OpenAI/Python 1.92.3 - x-stainless-arch: - - arm64 - x-stainless-async: - - "false" - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 1.92.3 - x-stainless-raw-response: - - "true" - x-stainless-read-timeout: - - "600.0" - x-stainless-retry-count: - - "0" - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.11.10 - method: POST - uri: https://api.openai.com/v1/chat/completions - response: - body: - string: - 'data: {"id":"chatcmpl-BrWsECewLnZpwYk2y6TeJIT6H7F48","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsECewLnZpwYk2y6TeJIT6H7F48","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsECewLnZpwYk2y6TeJIT6H7F48","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" - +"},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsECewLnZpwYk2y6TeJIT6H7F48","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" - "},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsECewLnZpwYk2y6TeJIT6H7F48","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsECewLnZpwYk2y6TeJIT6H7F48","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" - equals"},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsECewLnZpwYk2y6TeJIT6H7F48","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" - "},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsECewLnZpwYk2y6TeJIT6H7F48","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"24"},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsECewLnZpwYk2y6TeJIT6H7F48","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsECewLnZpwYk2y6TeJIT6H7F48","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null} - - - data: {"id":"chatcmpl-BrWsECewLnZpwYk2y6TeJIT6H7F48","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[],"usage":{"prompt_tokens":14,"completion_tokens":8,"total_tokens":22,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}}} - - - data: [DONE] - - - ' - headers: - CF-RAY: - - 95cb00484b3214f6-LAX - Connection: - - keep-alive - Content-Type: - - text/event-stream; charset=utf-8 - Date: - - Wed, 09 Jul 2025 21:44:10 GMT - Server: - - cloudflare - Transfer-Encoding: - - chunked - X-Content-Type-Options: - - nosniff - access-control-expose-headers: - - X-Request-ID - alt-svc: - - h3=":443"; ma=86400 - cf-cache-status: - - DYNAMIC - openai-organization: - - braintrust-data - openai-processing-ms: - - "327" - openai-version: - - "2020-10-01" - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - x-envoy-upstream-service-time: - - "332" - x-ratelimit-limit-requests: - - "30000" - x-ratelimit-limit-tokens: - - "150000000" - x-ratelimit-remaining-requests: - - "29999" - x-ratelimit-remaining-tokens: - - "149999993" - x-ratelimit-reset-requests: - - 2ms - x-ratelimit-reset-tokens: - - 0s - x-request-id: - - req_82e49575f67bda7dced7355056a8aef0 - status: - code: 200 - message: OK - - request: - body: - '{"messages": [{"role": "user", "content": "What''s 12 + 12?"}], "model": - "gpt-4o-mini", "stream": true, "stream_options": {"include_usage": true}}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate, zstd - connection: - - keep-alive - content-length: - - "145" - content-type: - - application/json - cookie: - - __cf_bm=LhJr7iTKhj3HkuIqsYYQzHnLMYlAsrPO_8DONvOoa1s-1752097448-1.0.1.1-3QZBiExLtV3A83cyPK4CeYqjpSygQ6uYDZ0v7kYNlmEHZIOldYiM088zk99s107ShpZgiHzSJwcw0Fviy3roAk.Ifv1OH0UzhIx2VTAKaS4; - _cfuvid=1FpkTfHv2KC0I5jKg65yXzjPU21dBkiuyT323kob89c-1752097448243-0.0.1.1-604800000 - host: - - api.openai.com - user-agent: - - OpenAI/Python 1.92.3 - x-stainless-arch: - - arm64 - x-stainless-async: - - "false" - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 1.92.3 - x-stainless-raw-response: - - "true" - x-stainless-read-timeout: - - "600.0" - x-stainless-retry-count: - - "0" - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.11.10 - method: POST - uri: https://api.openai.com/v1/chat/completions - response: - body: - string: - 'data: {"id":"chatcmpl-BrWsEMMFl7DBEVA7ID65Tklpkbnbs","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsEMMFl7DBEVA7ID65Tklpkbnbs","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsEMMFl7DBEVA7ID65Tklpkbnbs","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" - +"},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsEMMFl7DBEVA7ID65Tklpkbnbs","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" - "},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsEMMFl7DBEVA7ID65Tklpkbnbs","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsEMMFl7DBEVA7ID65Tklpkbnbs","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" - equals"},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsEMMFl7DBEVA7ID65Tklpkbnbs","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" - "},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsEMMFl7DBEVA7ID65Tklpkbnbs","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"24"},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsEMMFl7DBEVA7ID65Tklpkbnbs","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsEMMFl7DBEVA7ID65Tklpkbnbs","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null} - - - data: {"id":"chatcmpl-BrWsEMMFl7DBEVA7ID65Tklpkbnbs","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[],"usage":{"prompt_tokens":14,"completion_tokens":8,"total_tokens":22,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}}} - - - data: [DONE] - - - ' - headers: - CF-RAY: - - 95cb004baed214f6-LAX - Connection: - - keep-alive - Content-Type: - - text/event-stream; charset=utf-8 - Date: - - Wed, 09 Jul 2025 21:44:11 GMT - Server: - - cloudflare - Transfer-Encoding: - - chunked - X-Content-Type-Options: - - nosniff - access-control-expose-headers: - - X-Request-ID - alt-svc: - - h3=":443"; ma=86400 - cf-cache-status: - - DYNAMIC - openai-organization: - - braintrust-data - openai-processing-ms: - - "203" - openai-version: - - "2020-10-01" - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - x-envoy-upstream-service-time: - - "208" - x-ratelimit-limit-requests: - - "30000" - x-ratelimit-limit-tokens: - - "150000000" - x-ratelimit-remaining-requests: - - "29999" - x-ratelimit-remaining-tokens: - - "149999993" - x-ratelimit-reset-requests: - - 2ms - x-ratelimit-reset-tokens: - - 0s - x-request-id: - - req_6a6ea838af6542b8572e83b621c03f90 - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_metrics[async].yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_input_materializes_attachment[blocking-async].yaml similarity index 61% rename from py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_metrics[async].yaml rename to py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_input_materializes_attachment[blocking-async].yaml index 81a0aface..c9c00f8fa 100644 --- a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_metrics[async].yaml +++ b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_input_materializes_attachment[blocking-async].yaml @@ -1,6 +1,7 @@ interactions: - request: - body: '{"messages":[{"role":"user","content":"What''s 12 + 12?"}],"model":"gpt-4o-mini"}' + body: '{"messages":[{"role":"user","content":[{"type":"text","text":"What color + is this image?"},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==","detail":"low"}}]}],"model":"gpt-4o-mini"}' headers: accept: - application/json @@ -9,13 +10,13 @@ interactions: connection: - keep-alive content-length: - - '80' + - '292' content-type: - application/json host: - api.openai.com user-agent: - - AsyncOpenAI/Python 2.24.0 + - AsyncOpenAI/Python 1.99.9 x-stainless-arch: - arm64 x-stainless-async: @@ -25,7 +26,7 @@ interactions: x-stainless-os: - MacOS x-stainless-package-version: - - 2.24.0 + - 1.99.9 x-stainless-raw-response: - 'true' x-stainless-read-timeout: @@ -35,75 +36,75 @@ interactions: x-stainless-runtime: - CPython x-stainless-runtime-version: - - 3.13.3 + - 3.14.7 method: POST uri: https://api.openai.com/v1/chat/completions response: body: - string: "{\n \"id\": \"chatcmpl-DFlcrhg78gJ6cwXCo9bHgDtJvZfyJ\",\n \"object\": - \"chat.completion\",\n \"created\": 1772650365,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + string: "{\n \"id\": \"chatcmpl-EYu9kVBUD5NXdNPINDruLZ6Dp4b7v\",\n \"object\": + \"chat.completion\",\n \"created\": 1791987724,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": - \"assistant\",\n \"content\": \"12 + 12 equals 24.\",\n \"refusal\": + \"assistant\",\n \"content\": \"The image is red.\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": - 14,\n \"completion_tokens\": 8,\n \"total_tokens\": 22,\n \"prompt_tokens_details\": - {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + 2846,\n \"completion_tokens\": 5,\n \"total_tokens\": 2851,\n \"prompt_tokens_details\": + {\n \"cached_tokens\": 2816,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": - \"default\",\n \"system_fingerprint\": \"fp_373a14eb6f\"\n}\n" + \"default\",\n \"system_fingerprint\": \"fp_bf0094d986\"\n}\n" headers: Access-Control-Expose-Headers: - X-Request-ID - CF-RAY: - - 9d73146f9953ec71-YYZ + - CF-Ray + - CF-Ray + Alt-Svc: + - h3=":443"; ma=86400 + CF-Cache-Status: + - DYNAMIC + CF-Ray: + - a4a73be998b9aad0-YYZ Connection: - keep-alive Content-Type: - application/json Date: - - Wed, 04 Mar 2026 18:52:45 GMT + - Wed, 14 Oct 2026 14:22:04 GMT Server: - cloudflare - Set-Cookie: - - __cf_bm=V_yDANNrUk_AKhRc3JMD1PiBDa9.W0AopRO95256b8E-1772650365.3751957-1.0.1.1-rr5DmxlLKEWnz6279TOurNHTLABqVASjVIhHAKczPI96o54yejvrdU4ScLdL_x_Gac9DOP0W5SCGWRL9TiZ9ouKagJlXEBbC1TxshGOdmnDPW66QcddzsGwFYyyNEPMQ; - HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 04 Mar 2026 - 19:22:45 GMT Strict-Transport-Security: - max-age=31536000; includeSubDomains; preload Transfer-Encoding: - chunked X-Content-Type-Options: - nosniff - alt-svc: - - h3=":443"; ma=86400 - cf-cache-status: - - DYNAMIC + X-Request-ID: + - req_bf70a332dc234d0382ce79e61705092e content-length: - - '824' - openai-organization: - - braintrust-data + - '830' openai-processing-ms: - - '356' - openai-project: - - proj_vsCSXafhhByzWOThMrJcZiw9 + - '507' openai-version: - '2020-10-01' x-openai-proxy-wasm: - v0.1 + x-ratelimit-limit-input-images: + - '50000' x-ratelimit-limit-requests: - '30000' x-ratelimit-limit-tokens: - '150000000' + x-ratelimit-remaining-input-images: + - '49999' x-ratelimit-remaining-requests: - '29999' x-ratelimit-remaining-tokens: - - '149999992' + - '149997145' + x-ratelimit-reset-input-images: + - 1ms x-ratelimit-reset-requests: - 2ms x-ratelimit-reset-tokens: - - 0s - x-request-id: - - req_906cbaef60c6445dbf09994daa66d913 + - 1ms status: code: 200 message: OK diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_input_materializes_attachment[blocking-sync].yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_input_materializes_attachment[blocking-sync].yaml new file mode 100644 index 000000000..1a21f300e --- /dev/null +++ b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_input_materializes_attachment[blocking-sync].yaml @@ -0,0 +1,111 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":[{"type":"text","text":"What color + is this image?"},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==","detail":"low"}}]}],"model":"gpt-4o-mini"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '292' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.99.9 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.99.9 + x-stainless-raw-response: + - 'true' + x-stainless-read-timeout: + - '600.0' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-EYu9jl6WYyB5qpK2VhI2RxCU5vNPr\",\n \"object\": + \"chat.completion\",\n \"created\": 1791987723,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"The image is a solid shade of red.\",\n + \ \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": + null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 2846,\n \"completion_tokens\": 9,\n \"total_tokens\": 2855,\n \"prompt_tokens_details\": + {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_bf0094d986\"\n}\n" + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a4a73be048e524ee-YYZ + connection: + - keep-alive + content-length: + - '844' + content-type: + - application/json + date: + - Wed, 14 Oct 2026 14:22:03 GMT + openai-processing-ms: + - '563' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-input-images: + - '50000' + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-input-images: + - '49999' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999227' + x-ratelimit-reset-input-images: + - 1ms + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_745443a771ed4ce08eb85228e08d659f + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_input_materializes_attachment[streaming-async].yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_input_materializes_attachment[streaming-async].yaml new file mode 100644 index 000000000..72f9376aa --- /dev/null +++ b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_input_materializes_attachment[streaming-async].yaml @@ -0,0 +1,129 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":[{"type":"text","text":"What color + is this image?"},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==","detail":"low"}}]}],"model":"gpt-4o-mini","stream":true,"stream_options":{"include_usage":true}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '346' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.99.9 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.99.9 + x-stainless-raw-response: + - 'true' + x-stainless-read-timeout: + - '600.0' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-EYu9mwUsvcT2JYlurPM4DuiEKPlv9","object":"chat.completion.chunk","created":1791987726,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"eqGrxHf0D"} + + + data: {"id":"chatcmpl-EYu9mwUsvcT2JYlurPM4DuiEKPlv9","object":"chat.completion.chunk","created":1791987726,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"13w8nzb7"} + + + data: {"id":"chatcmpl-EYu9mwUsvcT2JYlurPM4DuiEKPlv9","object":"chat.completion.chunk","created":1791987726,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":" + image"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"4UyTu"} + + + data: {"id":"chatcmpl-EYu9mwUsvcT2JYlurPM4DuiEKPlv9","object":"chat.completion.chunk","created":1791987726,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":" + is"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"IkZeH63t"} + + + data: {"id":"chatcmpl-EYu9mwUsvcT2JYlurPM4DuiEKPlv9","object":"chat.completion.chunk","created":1791987726,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":" + red"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"wfO3uly"} + + + data: {"id":"chatcmpl-EYu9mwUsvcT2JYlurPM4DuiEKPlv9","object":"chat.completion.chunk","created":1791987726,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"tlKVCmGw3M"} + + + data: {"id":"chatcmpl-EYu9mwUsvcT2JYlurPM4DuiEKPlv9","object":"chat.completion.chunk","created":1791987726,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"PlPQt"} + + + data: {"id":"chatcmpl-EYu9mwUsvcT2JYlurPM4DuiEKPlv9","object":"chat.completion.chunk","created":1791987726,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[],"usage":{"prompt_tokens":2846,"completion_tokens":5,"total_tokens":2851,"prompt_tokens_details":{"cached_tokens":2816,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"7whN"} + + + data: [DONE] + + + ' + headers: + Access-Control-Expose-Headers: + - X-Request-ID + - CF-Ray + - CF-Ray + Alt-Svc: + - h3=":443"; ma=86400 + CF-Cache-Status: + - DYNAMIC + CF-Ray: + - a4a73bf6ae90c85f-YYZ + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Wed, 14 Oct 2026 14:22:06 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + X-Request-ID: + - req_1722642ac3344354b0b9bfcc99824be5 + openai-processing-ms: + - '346' + openai-version: + - '2020-10-01' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-input-images: + - '50000' + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-input-images: + - '49999' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999227' + x-ratelimit-reset-input-images: + - 1ms + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_input_materializes_attachment[streaming-sync].yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_input_materializes_attachment[streaming-sync].yaml new file mode 100644 index 000000000..d8f6ba67c --- /dev/null +++ b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_input_materializes_attachment[streaming-sync].yaml @@ -0,0 +1,129 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":[{"type":"text","text":"What color + is this image?"},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==","detail":"low"}}]}],"model":"gpt-4o-mini","stream":true,"stream_options":{"include_usage":true}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '346' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.99.9 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.99.9 + x-stainless-raw-response: + - 'true' + x-stainless-read-timeout: + - '600.0' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-EYu9lYfA1vyWZLYnHL9aAIJany9mz","object":"chat.completion.chunk","created":1791987725,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"OtjYHSaOf"} + + + data: {"id":"chatcmpl-EYu9lYfA1vyWZLYnHL9aAIJany9mz","object":"chat.completion.chunk","created":1791987725,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"Y9YxcuFq"} + + + data: {"id":"chatcmpl-EYu9lYfA1vyWZLYnHL9aAIJany9mz","object":"chat.completion.chunk","created":1791987725,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":" + image"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"y9w9i"} + + + data: {"id":"chatcmpl-EYu9lYfA1vyWZLYnHL9aAIJany9mz","object":"chat.completion.chunk","created":1791987725,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":" + is"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"WNi3XRpU"} + + + data: {"id":"chatcmpl-EYu9lYfA1vyWZLYnHL9aAIJany9mz","object":"chat.completion.chunk","created":1791987725,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":" + red"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"j4myJAb"} + + + data: {"id":"chatcmpl-EYu9lYfA1vyWZLYnHL9aAIJany9mz","object":"chat.completion.chunk","created":1791987725,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"NaWNm0sLwt"} + + + data: {"id":"chatcmpl-EYu9lYfA1vyWZLYnHL9aAIJany9mz","object":"chat.completion.chunk","created":1791987725,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"IF0BH"} + + + data: {"id":"chatcmpl-EYu9lYfA1vyWZLYnHL9aAIJany9mz","object":"chat.completion.chunk","created":1791987725,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[],"usage":{"prompt_tokens":2846,"completion_tokens":5,"total_tokens":2851,"prompt_tokens_details":{"cached_tokens":2816,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"UgiA"} + + + data: [DONE] + + + ' + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a4a73beeca9924ee-YYZ + connection: + - keep-alive + content-type: + - text/event-stream; charset=utf-8 + date: + - Wed, 14 Oct 2026 14:22:05 GMT + openai-processing-ms: + - '397' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-input-images: + - '50000' + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-input-images: + - '49998' + x-ratelimit-remaining-requests: + - '29998' + x-ratelimit-remaining-tokens: + - '149994295' + x-ratelimit-reset-input-images: + - 2ms + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 2ms + x-request-id: + - req_c54b29776a544e7395e0fbda66847fe7 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_async_streaming_with_break.yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_async_streaming_with_break.yaml deleted file mode 100644 index 6a05cdae1..000000000 --- a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_async_streaming_with_break.yaml +++ /dev/null @@ -1,132 +0,0 @@ -interactions: -- request: - body: '{"messages":[{"role":"user","content":"What''s 12 + 12?"}],"model":"gpt-4o-mini","stream":true,"stream_options":{"include_usage":true}}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate - connection: - - keep-alive - content-length: - - '134' - content-type: - - application/json - host: - - api.openai.com - user-agent: - - AsyncOpenAI/Python 2.54.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - async:asyncio - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.54.0 - x-stainless-raw-response: - - 'true' - x-stainless-read-timeout: - - '600.0' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.14.7 - method: POST - uri: https://api.openai.com/v1/chat/completions - response: - body: - string: 'data: {"id":"chatcmpl-EYXVmvIiGUSdc5i1uvZNMfj2Jt06w","object":"chat.completion.chunk","created":1791900678,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"86oPdoSHv"} - - - data: {"id":"chatcmpl-EYXVmvIiGUSdc5i1uvZNMfj2Jt06w","object":"chat.completion.chunk","created":1791900678,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"NcLYdwGwi"} - - - data: {"id":"chatcmpl-EYXVmvIiGUSdc5i1uvZNMfj2Jt06w","object":"chat.completion.chunk","created":1791900678,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":" - +"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"AJsEn2EKa"} - - - data: {"id":"chatcmpl-EYXVmvIiGUSdc5i1uvZNMfj2Jt06w","object":"chat.completion.chunk","created":1791900678,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":" - "},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"x90vYXDKLS"} - - - data: {"id":"chatcmpl-EYXVmvIiGUSdc5i1uvZNMfj2Jt06w","object":"chat.completion.chunk","created":1791900678,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"edzu4o841"} - - - data: {"id":"chatcmpl-EYXVmvIiGUSdc5i1uvZNMfj2Jt06w","object":"chat.completion.chunk","created":1791900678,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":" - equals"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"7pAK"} - - - data: {"id":"chatcmpl-EYXVmvIiGUSdc5i1uvZNMfj2Jt06w","object":"chat.completion.chunk","created":1791900678,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":" - "},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"VvBycIqVjT"} - - - data: {"id":"chatcmpl-EYXVmvIiGUSdc5i1uvZNMfj2Jt06w","object":"chat.completion.chunk","created":1791900678,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":"24"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"apXgr1Z4r"} - - - data: {"id":"chatcmpl-EYXVmvIiGUSdc5i1uvZNMfj2Jt06w","object":"chat.completion.chunk","created":1791900678,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"sHtvMah9Cg"} - - - data: {"id":"chatcmpl-EYXVmvIiGUSdc5i1uvZNMfj2Jt06w","object":"chat.completion.chunk","created":1791900678,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"fIT79"} - - - data: {"id":"chatcmpl-EYXVmvIiGUSdc5i1uvZNMfj2Jt06w","object":"chat.completion.chunk","created":1791900678,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[],"usage":{"prompt_tokens":14,"completion_tokens":8,"total_tokens":22,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"WhUZOPFh67J"} - - - data: [DONE] - - - ' - headers: - Access-Control-Expose-Headers: - - X-Request-ID - - CF-Ray - - CF-Ray - Alt-Svc: - - h3=":443"; ma=86400 - CF-Cache-Status: - - DYNAMIC - CF-Ray: - - a49eeec68ce8175c-YYZ - Connection: - - keep-alive - Content-Type: - - text/event-stream; charset=utf-8 - Date: - - Tue, 13 Oct 2026 14:11:18 GMT - Server: - - cloudflare - Strict-Transport-Security: - - max-age=31536000; includeSubDomains; preload - Transfer-Encoding: - - chunked - X-Content-Type-Options: - - nosniff - X-Request-ID: - - req_fbfb7605216349f78876a000f3eceb35 - openai-processing-ms: - - '197' - openai-version: - - '2020-10-01' - x-openai-proxy-wasm: - - v0.1 - x-ratelimit-limit-requests: - - '30000' - x-ratelimit-limit-tokens: - - '150000000' - x-ratelimit-remaining-requests: - - '29999' - x-ratelimit-remaining-tokens: - - '149999995' - x-ratelimit-reset-requests: - - 2ms - x-ratelimit-reset-tokens: - - 0s - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_streaming[async].yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_streaming[async].yaml deleted file mode 100644 index 563faf1cf..000000000 --- a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_streaming[async].yaml +++ /dev/null @@ -1,132 +0,0 @@ -interactions: -- request: - body: '{"messages":[{"role":"user","content":"What''s 12 + 12?"}],"model":"gpt-4o-mini","stream":true,"stream_options":{"include_usage":true}}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate - connection: - - keep-alive - content-length: - - '134' - content-type: - - application/json - host: - - api.openai.com - user-agent: - - AsyncOpenAI/Python 2.54.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - async:asyncio - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.54.0 - x-stainless-raw-response: - - 'true' - x-stainless-read-timeout: - - '600.0' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.14.7 - method: POST - uri: https://api.openai.com/v1/chat/completions - response: - body: - string: 'data: {"id":"chatcmpl-EYXUd9BxNE1C6BNgLrUUVgroBi53r","object":"chat.completion.chunk","created":1791900607,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"az4SZGSlo"} - - - data: {"id":"chatcmpl-EYXUd9BxNE1C6BNgLrUUVgroBi53r","object":"chat.completion.chunk","created":1791900607,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"nGETNsphG"} - - - data: {"id":"chatcmpl-EYXUd9BxNE1C6BNgLrUUVgroBi53r","object":"chat.completion.chunk","created":1791900607,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":" - +"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"UiTHZCs76"} - - - data: {"id":"chatcmpl-EYXUd9BxNE1C6BNgLrUUVgroBi53r","object":"chat.completion.chunk","created":1791900607,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":" - "},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"rRUsQQKfu5"} - - - data: {"id":"chatcmpl-EYXUd9BxNE1C6BNgLrUUVgroBi53r","object":"chat.completion.chunk","created":1791900607,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"xIQq040r9"} - - - data: {"id":"chatcmpl-EYXUd9BxNE1C6BNgLrUUVgroBi53r","object":"chat.completion.chunk","created":1791900607,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":" - equals"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"SYsw"} - - - data: {"id":"chatcmpl-EYXUd9BxNE1C6BNgLrUUVgroBi53r","object":"chat.completion.chunk","created":1791900607,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":" - "},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"cCm3Yn8Y4g"} - - - data: {"id":"chatcmpl-EYXUd9BxNE1C6BNgLrUUVgroBi53r","object":"chat.completion.chunk","created":1791900607,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":"24"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"Rj5qprBk3"} - - - data: {"id":"chatcmpl-EYXUd9BxNE1C6BNgLrUUVgroBi53r","object":"chat.completion.chunk","created":1791900607,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"wAz8qEZFkJ"} - - - data: {"id":"chatcmpl-EYXUd9BxNE1C6BNgLrUUVgroBi53r","object":"chat.completion.chunk","created":1791900607,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"Cb4Xv"} - - - data: {"id":"chatcmpl-EYXUd9BxNE1C6BNgLrUUVgroBi53r","object":"chat.completion.chunk","created":1791900607,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[],"usage":{"prompt_tokens":14,"completion_tokens":8,"total_tokens":22,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"KT5mK3wfFp8"} - - - data: [DONE] - - - ' - headers: - Access-Control-Expose-Headers: - - X-Request-ID - - CF-Ray - - CF-Ray - Alt-Svc: - - h3=":443"; ma=86400 - CF-Cache-Status: - - DYNAMIC - CF-Ray: - - a49eed0b8867e21a-YYZ - Connection: - - keep-alive - Content-Type: - - text/event-stream; charset=utf-8 - Date: - - Tue, 13 Oct 2026 14:10:07 GMT - Server: - - cloudflare - Strict-Transport-Security: - - max-age=31536000; includeSubDomains; preload - Transfer-Encoding: - - chunked - X-Content-Type-Options: - - nosniff - X-Request-ID: - - req_b2f0fd6aca6d4e88a7bd7c535d16ff23 - openai-processing-ms: - - '271' - openai-version: - - '2020-10-01' - x-openai-proxy-wasm: - - v0.1 - x-ratelimit-limit-requests: - - '30000' - x-ratelimit-limit-tokens: - - '150000000' - x-ratelimit-remaining-requests: - - '29997' - x-ratelimit-remaining-tokens: - - '149999917' - x-ratelimit-reset-requests: - - 6ms - x-ratelimit-reset-tokens: - - 0s - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_streaming[sync].yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_streaming[sync].yaml deleted file mode 100644 index 2f4beaf09..000000000 --- a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_streaming[sync].yaml +++ /dev/null @@ -1,132 +0,0 @@ -interactions: -- request: - body: '{"messages":[{"role":"user","content":"What''s 12 + 12?"}],"model":"gpt-4o-mini","stream":true,"stream_options":{"include_usage":true}}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate - connection: - - keep-alive - content-length: - - '134' - content-type: - - application/json - host: - - api.openai.com - user-agent: - - OpenAI/Python 2.54.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - 'false' - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.54.0 - x-stainless-raw-response: - - 'true' - x-stainless-read-timeout: - - '600.0' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.14.7 - method: POST - uri: https://api.openai.com/v1/chat/completions - response: - body: - string: 'data: {"id":"chatcmpl-EYXUcaJJFE5TWou2BQMf7zku6gcAW","object":"chat.completion.chunk","created":1791900606,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"6WWKtsopF"} - - - data: {"id":"chatcmpl-EYXUcaJJFE5TWou2BQMf7zku6gcAW","object":"chat.completion.chunk","created":1791900606,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"hXX4hklk2"} - - - data: {"id":"chatcmpl-EYXUcaJJFE5TWou2BQMf7zku6gcAW","object":"chat.completion.chunk","created":1791900606,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":" - +"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"7rNYiBG5x"} - - - data: {"id":"chatcmpl-EYXUcaJJFE5TWou2BQMf7zku6gcAW","object":"chat.completion.chunk","created":1791900606,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":" - "},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"bwuiERvMOR"} - - - data: {"id":"chatcmpl-EYXUcaJJFE5TWou2BQMf7zku6gcAW","object":"chat.completion.chunk","created":1791900606,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"3UiklAQHE"} - - - data: {"id":"chatcmpl-EYXUcaJJFE5TWou2BQMf7zku6gcAW","object":"chat.completion.chunk","created":1791900606,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":" - equals"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"XimM"} - - - data: {"id":"chatcmpl-EYXUcaJJFE5TWou2BQMf7zku6gcAW","object":"chat.completion.chunk","created":1791900606,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":" - "},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"bGFtdg4LW1"} - - - data: {"id":"chatcmpl-EYXUcaJJFE5TWou2BQMf7zku6gcAW","object":"chat.completion.chunk","created":1791900606,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":"24"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"9i7fnUBTF"} - - - data: {"id":"chatcmpl-EYXUcaJJFE5TWou2BQMf7zku6gcAW","object":"chat.completion.chunk","created":1791900606,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"nW5Mi0qnYH"} - - - data: {"id":"chatcmpl-EYXUcaJJFE5TWou2BQMf7zku6gcAW","object":"chat.completion.chunk","created":1791900606,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"YtuLt"} - - - data: {"id":"chatcmpl-EYXUcaJJFE5TWou2BQMf7zku6gcAW","object":"chat.completion.chunk","created":1791900606,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[],"usage":{"prompt_tokens":14,"completion_tokens":8,"total_tokens":22,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"BuOktpcHB3k"} - - - data: [DONE] - - - ' - headers: - access-control-expose-headers: - - X-Request-ID - - CF-Ray - - CF-Ray - alt-svc: - - h3=":443"; ma=86400 - cf-cache-status: - - DYNAMIC - cf-ray: - - a49eed050e3de7e3-YYZ - connection: - - keep-alive - content-type: - - text/event-stream; charset=utf-8 - date: - - Tue, 13 Oct 2026 14:10:06 GMT - openai-processing-ms: - - '557' - openai-version: - - '2020-10-01' - server: - - cloudflare - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - transfer-encoding: - - chunked - x-content-type-options: - - nosniff - x-openai-proxy-wasm: - - v0.1 - x-ratelimit-limit-requests: - - '30000' - x-ratelimit-limit-tokens: - - '150000000' - x-ratelimit-remaining-requests: - - '29999' - x-ratelimit-remaining-tokens: - - '149999980' - x-ratelimit-reset-requests: - - 2ms - x-ratelimit-reset-tokens: - - 0s - x-request-id: - - req_2498b2107dde491c944d456d2954d01f - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_metrics[async].yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_input_materializes_attachment[blocking-async].yaml similarity index 68% rename from py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_metrics[async].yaml rename to py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_input_materializes_attachment[blocking-async].yaml index ba0b8a6ae..3362d6a0c 100644 --- a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_metrics[async].yaml +++ b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_input_materializes_attachment[blocking-async].yaml @@ -1,6 +1,7 @@ interactions: - request: - body: '{"messages":[{"role":"user","content":"What''s 12 + 12?"}],"model":"gpt-4o-mini"}' + body: '{"messages":[{"role":"user","content":[{"type":"text","text":"What color + is this image?"},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==","detail":"low"}}]}],"model":"gpt-4o-mini"}' headers: accept: - application/json @@ -9,7 +10,7 @@ interactions: connection: - keep-alive content-length: - - '80' + - '292' content-type: - application/json host: @@ -40,17 +41,17 @@ interactions: uri: https://api.openai.com/v1/chat/completions response: body: - string: "{\n \"id\": \"chatcmpl-EYXUaCyzKCjce6yPt9BwPzTa6sKS1\",\n \"object\": - \"chat.completion\",\n \"created\": 1791900604,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + string: "{\n \"id\": \"chatcmpl-EYtp6pxfy9Is2XiCiF7ImwR9E72RB\",\n \"object\": + \"chat.completion\",\n \"created\": 1791986444,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": - \"assistant\",\n \"content\": \"12 + 12 equals 24.\",\n \"refusal\": + \"assistant\",\n \"content\": \"The image is red.\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": - 14,\n \"completion_tokens\": 8,\n \"total_tokens\": 22,\n \"prompt_tokens_details\": + 2846,\n \"completion_tokens\": 5,\n \"total_tokens\": 2851,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": - \"default\",\n \"system_fingerprint\": \"fp_98b84877d4\"\n}\n" + \"default\",\n \"system_fingerprint\": \"fp_bf0094d986\"\n}\n" headers: Access-Control-Expose-Headers: - X-Request-ID @@ -61,13 +62,13 @@ interactions: CF-Cache-Status: - DYNAMIC CF-Ray: - - a49eecf9be56b405-YYZ + - a4a71cabdc43ac34-YYZ Connection: - keep-alive Content-Type: - application/json Date: - - Tue, 13 Oct 2026 14:10:04 GMT + - Wed, 14 Oct 2026 14:00:44 GMT Server: - cloudflare Strict-Transport-Security: @@ -77,27 +78,33 @@ interactions: X-Content-Type-Options: - nosniff X-Request-ID: - - req_0fb014b849db409ebb518b29fc611349 + - req_1e999d2fdaa049acb128a87b20f1af11 content-length: - - '824' + - '827' openai-processing-ms: - - '301' + - '577' openai-version: - '2020-10-01' x-openai-proxy-wasm: - v0.1 + x-ratelimit-limit-input-images: + - '50000' x-ratelimit-limit-requests: - '30000' x-ratelimit-limit-tokens: - '150000000' + x-ratelimit-remaining-input-images: + - '49996' x-ratelimit-remaining-requests: - - '29996' + - '29997' x-ratelimit-remaining-tokens: - - '149999897' + - '149994685' + x-ratelimit-reset-input-images: + - 4ms x-ratelimit-reset-requests: - - 8ms + - 6ms x-ratelimit-reset-tokens: - - 0s + - 2ms status: code: 200 message: OK diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_metrics[sync].yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_input_materializes_attachment[blocking-sync].yaml similarity index 68% rename from py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_metrics[sync].yaml rename to py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_input_materializes_attachment[blocking-sync].yaml index 0ea032694..27db33068 100644 --- a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_metrics[sync].yaml +++ b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_input_materializes_attachment[blocking-sync].yaml @@ -1,6 +1,7 @@ interactions: - request: - body: '{"messages":[{"role":"user","content":"What''s 12 + 12?"}],"model":"gpt-4o-mini"}' + body: '{"messages":[{"role":"user","content":[{"type":"text","text":"What color + is this image?"},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==","detail":"low"}}]}],"model":"gpt-4o-mini"}' headers: accept: - application/json @@ -9,7 +10,7 @@ interactions: connection: - keep-alive content-length: - - '80' + - '292' content-type: - application/json host: @@ -40,17 +41,17 @@ interactions: uri: https://api.openai.com/v1/chat/completions response: body: - string: "{\n \"id\": \"chatcmpl-EYXUZvCgLxv6iQXr6Wou0fKqz6SQH\",\n \"object\": - \"chat.completion\",\n \"created\": 1791900603,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + string: "{\n \"id\": \"chatcmpl-EYtp5K46aNHS6fYXH8ZjtNSwZ8Das\",\n \"object\": + \"chat.completion\",\n \"created\": 1791986443,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": - \"assistant\",\n \"content\": \"12 + 12 equals 24.\",\n \"refusal\": + \"assistant\",\n \"content\": \"The image is red.\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": - 14,\n \"completion_tokens\": 8,\n \"total_tokens\": 22,\n \"prompt_tokens_details\": + 2846,\n \"completion_tokens\": 5,\n \"total_tokens\": 2851,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": - \"default\",\n \"system_fingerprint\": \"fp_98b84877d4\"\n}\n" + \"default\",\n \"system_fingerprint\": \"fp_bf0094d986\"\n}\n" headers: access-control-expose-headers: - X-Request-ID @@ -61,17 +62,17 @@ interactions: cf-cache-status: - DYNAMIC cf-ray: - - a49eecf3ebe2e7e3-YYZ + - a4a71ca1be1f1d7b-YYZ connection: - keep-alive content-length: - - '824' + - '827' content-type: - application/json date: - - Tue, 13 Oct 2026 14:10:04 GMT + - Wed, 14 Oct 2026 14:00:43 GMT openai-processing-ms: - - '403' + - '550' openai-version: - '2020-10-01' server: @@ -84,20 +85,26 @@ interactions: - nosniff x-openai-proxy-wasm: - v0.1 + x-ratelimit-limit-input-images: + - '50000' x-ratelimit-limit-requests: - '30000' x-ratelimit-limit-tokens: - '150000000' + x-ratelimit-remaining-input-images: + - '49999' x-ratelimit-remaining-requests: - - '29998' + - '29999' x-ratelimit-remaining-tokens: - - '149999965' + - '149999227' + x-ratelimit-reset-input-images: + - 1ms x-ratelimit-reset-requests: - - 4ms + - 2ms x-ratelimit-reset-tokens: - 0s x-request-id: - - req_4a22d3d84f3a4d43b9383871d54fffc5 + - req_35b324b193a445beb964830795ce33d2 status: code: 200 message: OK diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_input_materializes_attachment[streaming-async].yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_input_materializes_attachment[streaming-async].yaml new file mode 100644 index 000000000..5f485c47f --- /dev/null +++ b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_input_materializes_attachment[streaming-async].yaml @@ -0,0 +1,129 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":[{"type":"text","text":"What color + is this image?"},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==","detail":"low"}}]}],"model":"gpt-4o-mini","stream":true,"stream_options":{"include_usage":true}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '346' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 2.54.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.54.0 + x-stainless-raw-response: + - 'true' + x-stainless-read-timeout: + - '600.0' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-EYtp7vbmKNUlLGVuGA5bDynWdllQy","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"Wd8bPGPWn"} + + + data: {"id":"chatcmpl-EYtp7vbmKNUlLGVuGA5bDynWdllQy","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"IwJMmW4N"} + + + data: {"id":"chatcmpl-EYtp7vbmKNUlLGVuGA5bDynWdllQy","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":" + image"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"mVtZP"} + + + data: {"id":"chatcmpl-EYtp7vbmKNUlLGVuGA5bDynWdllQy","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":" + is"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"WimwpwqF"} + + + data: {"id":"chatcmpl-EYtp7vbmKNUlLGVuGA5bDynWdllQy","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":" + red"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"FiPsrzf"} + + + data: {"id":"chatcmpl-EYtp7vbmKNUlLGVuGA5bDynWdllQy","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"QPdKT8nTQG"} + + + data: {"id":"chatcmpl-EYtp7vbmKNUlLGVuGA5bDynWdllQy","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"VNbaB"} + + + data: {"id":"chatcmpl-EYtp7vbmKNUlLGVuGA5bDynWdllQy","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[],"usage":{"prompt_tokens":2846,"completion_tokens":5,"total_tokens":2851,"prompt_tokens_details":{"cached_tokens":2816,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"zl4O"} + + + data: [DONE] + + + ' + headers: + Access-Control-Expose-Headers: + - X-Request-ID + - CF-Ray + - CF-Ray + Alt-Svc: + - h3=":443"; ma=86400 + CF-Cache-Status: + - DYNAMIC + CF-Ray: + - a4a71cb5cff3c7b6-YYZ + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Wed, 14 Oct 2026 14:00:46 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + X-Request-ID: + - req_64ad238576f047de83ea74158ca67b52 + openai-processing-ms: + - '410' + openai-version: + - '2020-10-01' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-input-images: + - '50000' + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-input-images: + - '49998' + x-ratelimit-remaining-requests: + - '29998' + x-ratelimit-remaining-tokens: + - '149996375' + x-ratelimit-reset-input-images: + - 2ms + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 1ms + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_input_materializes_attachment[streaming-sync].yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_input_materializes_attachment[streaming-sync].yaml new file mode 100644 index 000000000..53f9bcf42 --- /dev/null +++ b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_input_materializes_attachment[streaming-sync].yaml @@ -0,0 +1,129 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":[{"type":"text","text":"What color + is this image?"},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==","detail":"low"}}]}],"model":"gpt-4o-mini","stream":true,"stream_options":{"include_usage":true}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '346' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 2.54.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.54.0 + x-stainless-raw-response: + - 'true' + x-stainless-read-timeout: + - '600.0' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-EYtp7uw24Hy9MLO6l2oPPeFnzNl9V","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"TW41ygRYE"} + + + data: {"id":"chatcmpl-EYtp7uw24Hy9MLO6l2oPPeFnzNl9V","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"mrbL1cbQ"} + + + data: {"id":"chatcmpl-EYtp7uw24Hy9MLO6l2oPPeFnzNl9V","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":" + image"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"mjsbU"} + + + data: {"id":"chatcmpl-EYtp7uw24Hy9MLO6l2oPPeFnzNl9V","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":" + is"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"hsqE9Cfd"} + + + data: {"id":"chatcmpl-EYtp7uw24Hy9MLO6l2oPPeFnzNl9V","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":" + red"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"5UMOUsM"} + + + data: {"id":"chatcmpl-EYtp7uw24Hy9MLO6l2oPPeFnzNl9V","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"j94dYx8OTD"} + + + data: {"id":"chatcmpl-EYtp7uw24Hy9MLO6l2oPPeFnzNl9V","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"Gg3XJ"} + + + data: {"id":"chatcmpl-EYtp7uw24Hy9MLO6l2oPPeFnzNl9V","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[],"usage":{"prompt_tokens":2846,"completion_tokens":5,"total_tokens":2851,"prompt_tokens_details":{"cached_tokens":2816,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"s3fl"} + + + data: [DONE] + + + ' + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a4a71cb03dcb1d7b-YYZ + connection: + - keep-alive + content-type: + - text/event-stream; charset=utf-8 + date: + - Wed, 14 Oct 2026 14:00:45 GMT + openai-processing-ms: + - '354' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-input-images: + - '50000' + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-input-images: + - '49999' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999227' + x-ratelimit-reset-input-images: + - 1ms + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_442f2b5c55e24a9f921afdaceadd84a1 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/litellm/test_litellm.py b/py/src/braintrust/integrations/litellm/test_litellm.py index ead3681c8..9f97e3775 100644 --- a/py/src/braintrust/integrations/litellm/test_litellm.py +++ b/py/src/braintrust/integrations/litellm/test_litellm.py @@ -1,4 +1,5 @@ import asyncio +import base64 import os import time import uuid @@ -106,35 +107,54 @@ def memory_logger(): yield bgl -@pytest.mark.vcr +@pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path", "query", "body"]) @sync_async -def test_litellm_completion_metrics(memory_logger, is_async) -> None: - assert not memory_logger.pop() - +@pytest.mark.parametrize("stream", [False, True], ids=["blocking", "streaming"]) +def test_litellm_image_input_materializes_attachment(memory_logger, is_async, stream): + """Completion spans capture output, usage, and attachments in every call mode.""" + image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==" + image_url = f"data:image/png;base64,{image_base64}" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What color is this image?"}, + {"type": "image_url", "image_url": {"url": image_url, "detail": "low"}}, + ], + } + ] + run = _run_stream if stream else _run start = time.time() - response = _run( - is_async, - litellm.completion, - litellm.acompletion, - model=TEST_MODEL, - messages=[{"role": "user", "content": TEST_PROMPT}], + response = run( + is_async, litellm.completion, litellm.acompletion, model=TEST_MODEL, messages=messages, stream=stream ) end = time.time() - - assert response - assert response.choices[0].message.content - assert "24" in response.choices[0].message.content or "twenty-four" in response.choices[0].message.content.lower() - + if stream: + assert len(response) > 1 + content = "".join(chunk.choices[0].delta.content or "" for chunk in response if chunk.choices) + else: + content = response.choices[0].message.content + assert content + assert messages[0]["content"][1]["image_url"] == {"url": image_url, "detail": "low"} spans = memory_logger.pop() assert len(spans) == 1 span = spans[0] - assert span assert span["context"]["span_origin"]["instrumentation"]["name"] == "litellm-auto" - metrics = span["metrics"] - assert_metrics_are_valid(metrics, start, end) + assert span["span_attributes"]["name"] == "Completion" + assert span["span_attributes"]["type"] == "llm" assert span["metadata"]["model"] == TEST_MODEL assert span["metadata"]["provider"] == "openai" - assert span["input"] == [{"role": "user", "content": TEST_PROMPT}] + assert_metrics_are_valid(span["metrics"], start, end) + assert span["metrics"]["time_to_first_token"] >= 0 + assert span["output"][0]["message"]["content"] == content + assert span["input"][0]["role"] == "user" + assert span["input"][0]["content"][0] == messages[0]["content"][0] + logged_image = span["input"][0]["content"][1]["image_url"] + assert logged_image["detail"] == "low" + attachment = logged_image["url"] + assert isinstance(attachment, Attachment) + assert attachment.data == base64.b64decode(image_base64) + assert attachment.reference["content_type"] == "image/png" @pytest.mark.vcr @@ -241,48 +261,6 @@ def test_litellm_text_completion_metrics(memory_logger, is_async) -> None: assert "text" in span["output"][0] -@pytest.mark.vcr -@sync_async -def test_litellm_completion_streaming(memory_logger, is_async): - assert not memory_logger.pop() - - start = time.time() - chunks = _run_stream( - is_async, - litellm.completion, - litellm.acompletion, - model=TEST_MODEL, - messages=[{"role": "user", "content": TEST_PROMPT}], - stream=True, - ) - end = time.time() - - # Verify streaming works - assert chunks - assert len(chunks) > 1 - - # Concatenate content from chunks to verify - content = "" - for chunk in chunks: - if chunk.choices and chunk.choices[0].delta.content: - content += chunk.choices[0].delta.content - - # Make sure we got a valid answer in the content - assert "24" in content or "twenty-four" in content.lower() - - # Verify spans were created - spans = memory_logger.pop() - assert len(spans) == 1 - span = spans[0] - assert span - metrics = span["metrics"] - assert_metrics_are_valid(metrics, start, end) - assert span["metadata"]["model"] == TEST_MODEL - assert span["metadata"]["provider"] == "openai" - assert TEST_PROMPT in str(span["input"]) - assert "24" in str(span["output"]) or "twenty-four" in str(span["output"]).lower() - - @pytest.mark.vcr @sync_async def test_litellm_decisions_metrics(memory_logger, is_async): @@ -331,38 +309,56 @@ def test_litellm_decisions_image(memory_logger, is_async): assert_decisions_image_span(memory_logger.pop(), response, messages) -@pytest.mark.vcr -@sync_async -def test_litellm_responses_metrics(memory_logger, is_async): - assert not memory_logger.pop() - +@pytest.mark.parametrize( + "stream,is_async", + [ + pytest.param( + stream, + is_async, + id=f"{'streaming' if stream else 'blocking'}-{'async' if is_async else 'sync'}", + marks=pytest.mark.vcr( + cassette_name=f"test_litellm_responses_{'streaming' if stream else 'metrics'}[{'async' if is_async else 'sync'}]" + ), + ) + for stream in (False, True) + for is_async in (False, True) + ], +) +def test_litellm_responses(memory_logger, is_async, stream): + """Reuse the existing blocking/streaming recordings for the Responses API.""" + run = _run_stream if stream else _run + params = {"stream": True} if stream else {"instructions": "Just the number please"} start = time.time() - response = _run( + response = run( is_async, litellm.responses, litellm.aresponses, model=TEST_MODEL, input=TEST_PROMPT, - instructions="Just the number please", + **params, ) end = time.time() - assert response - assert response.output - assert len(response.output) > 0 - content = response.output[0].content[0].text + if stream: + assert len(response) > 1 + content = "".join( + chunk.delta for chunk in response if getattr(chunk, "type", None) == "response.output_text.delta" + ) + else: + content = response.output[0].content[0].text assert "24" in content or "twenty-four" in content.lower() - # Verify spans were created spans = memory_logger.pop() assert len(spans) == 1 span = spans[0] - assert span metrics = span["metrics"] assert_metrics_are_valid(metrics, start, end) assert span["metadata"]["model"] == TEST_MODEL assert span["metadata"]["provider"] == "openai" assert TEST_PROMPT in str(span["input"]) + assert "24" in str(span["output"]) or "twenty-four" in str(span["output"]).lower() + if stream: + assert span["metadata"]["stream"] is True @pytest.mark.vcr @@ -500,7 +496,6 @@ def test_litellm_speech(memory_logger, is_async): _assert_speech_output_attachment(span) -@pytest.mark.vcr @sync_async def test_litellm_completion_error(memory_logger, is_async): assert not memory_logger.pop() @@ -508,7 +503,7 @@ def test_litellm_completion_error(memory_logger, is_async): # Use a non-existent model to force an error fake_model = "non-existent-model" - try: + with pytest.raises(litellm.BadRequestError): _run( is_async, litellm.completion, @@ -516,17 +511,12 @@ def test_litellm_completion_error(memory_logger, is_async): model=fake_model, messages=[{"role": "user", "content": TEST_PROMPT}], ) - pytest.fail("Expected an exception but none was raised") - except Exception: - # We expect an error here - pass logs = memory_logger.pop() assert len(logs) == 1 log = logs[0] assert log["project_id"] == PROJECT_NAME - # Check that we got a log entry with the fake model - assert fake_model in str(log) + assert log["metadata"]["model"] == fake_model @pytest.mark.vcr @@ -620,82 +610,6 @@ def test_litellm_tool_calls(memory_logger): ) -@pytest.mark.vcr -@sync_async -def test_litellm_responses_streaming(memory_logger, is_async): - """Test the responses API with streaming.""" - assert not memory_logger.pop() - - start = time.time() - chunks = _run_stream( - is_async, - litellm.responses, - litellm.aresponses, - model=TEST_MODEL, - input="What's 12 + 12?", - stream=True, - ) - end = time.time() - - assert chunks - assert len(chunks) > 1 - output = "".join(chunk.delta for chunk in chunks if getattr(chunk, "type", None) == "response.output_text.delta") - assert "24" in output - - # Verify the span is created - spans = memory_logger.pop() - assert len(spans) == 1 - span = spans[0] - metrics = span["metrics"] - assert_metrics_are_valid(metrics, start, end) - assert span["metadata"]["stream"] is True - assert "What's 12 + 12?" in str(span["input"]) - assert "24" in str(span["output"]) - - -@pytest.mark.vcr -@pytest.mark.asyncio -async def test_litellm_async_streaming_with_break(memory_logger): - """Test breaking out of the async streaming loop early.""" - assert not memory_logger.pop() - - start = time.time() - stream = await litellm.acompletion( - model=TEST_MODEL, messages=[{"role": "user", "content": TEST_PROMPT}], stream=True - ) - - # Only process the first few chunks - counter = 0 - async for chunk in stream: - counter += 1 - if counter >= 2: - break - end = time.time() - - # We should still get valid metrics even with early break - spans = memory_logger.pop() - assert len(spans) == 1 - span = spans[0] - metrics = span["metrics"] - assert metrics["time_to_first_token"] >= 0 - - -def test_litellm_is_numeric_excludes_booleans(): - """Reproduce issue #1357: _is_numeric should exclude booleans. - - OpenRouter returns `is_byok: true` in usage data. Since Python's bool is - a subclass of int, isinstance(True, int) is True. The _is_numeric function - must explicitly exclude booleans so they don't end up in metrics, which - causes a 400 from the API (expected number, received boolean). - """ - from braintrust.util import is_numeric - - assert is_numeric(1) - assert is_numeric(1.0) - assert not is_numeric(True) - assert not is_numeric(False) - - @pytest.mark.vcr def test_litellm_openrouter_no_booleans_in_metrics(memory_logger): """Reproduce issue #1357: OpenRouter returns is_byok boolean in usage. diff --git a/py/src/braintrust/integrations/litellm/tracing.py b/py/src/braintrust/integrations/litellm/tracing.py index d01a9c2b3..abde530fd 100644 --- a/py/src/braintrust/integrations/litellm/tracing.py +++ b/py/src/braintrust/integrations/litellm/tracing.py @@ -7,7 +7,7 @@ from types import TracebackType from typing import Any -from braintrust.integrations.openai.tracing import _decisions_input, _decisions_output +from braintrust.integrations.openai.tracing import _decisions_input, _decisions_output, _process_attachments_in_input from braintrust.integrations.utils import ( _extract_audio_output, _materialize_attachment, @@ -862,7 +862,7 @@ def _update_span_payload_from_params(params: dict[str, Any], input_key: str = "i return merge_dicts( span_info_d, - {"input": input_data, "metadata": metadata}, + {"input": _process_attachments_in_input(input_data), "metadata": metadata}, ) diff --git a/py/uv.lock b/py/uv.lock index f9652322d..2010a36b8 100644 --- a/py/uv.lock +++ b/py/uv.lock @@ -1095,6 +1095,9 @@ test-deepagents = [ { name = "pytest-asyncio" }, { name = "pytest-vcr" }, ] +test-dspy = [ + { name = "pillow" }, +] test-huggingface-hub = [ { name = "aiohttp" }, { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-instructor') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-llamaindex') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-llamaindex-0-13') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-instructor') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-llamaindex') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-llamaindex-0-13') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-instructor') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-llamaindex') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-llamaindex-0-13') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-instructor') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-llamaindex') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-llamaindex-0-13') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-instructor') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-llamaindex') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-llamaindex-0-13') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-instructor' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-instructor' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-instructor' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-instructor' and extra == 'group-10-braintrust-test-llamaindex') or (extra == 'group-10-braintrust-test-instructor' and extra == 'group-10-braintrust-test-llamaindex-0-13') or (extra == 'group-10-braintrust-test-instructor' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-instructor' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-llamaindex') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-llamaindex-0-13') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-llamaindex') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-llamaindex-0-13') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-llamaindex') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-llamaindex-0-13') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire-constraints') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-llamaindex' and extra == 'group-10-braintrust-test-llamaindex-0-13') or (extra == 'group-10-braintrust-test-llamaindex' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-llamaindex' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-llamaindex-0-13' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-llamaindex-0-13' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands')" }, @@ -1368,6 +1371,7 @@ test-deepagents = [ { name = "pytest-asyncio", specifier = "==1.3.0" }, { name = "pytest-vcr", specifier = "==1.0.2" }, ] +test-dspy = [{ name = "pillow", specifier = "==12.3.0" }] test-huggingface-hub = [ { name = "aiohttp" }, { name = "numpy" }, From dc711ab4b780617c57189c702145dedba38885ca Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Wed, 14 Oct 2026 11:55:06 -0400 Subject: [PATCH 2/3] fix(dspy): handle non-mapping image model serialization Materialize RootModel roots directly and use converted raw fields for list/scalar model serializers, respecting field exclusions. Extend the image VCR regression while reusing existing provider recordings. --- .../braintrust/integrations/dspy/test_dspy.py | 54 ++++++++++++++++--- .../braintrust/integrations/dspy/tracing.py | 14 ++++- 2 files changed, 60 insertions(+), 8 deletions(-) diff --git a/py/src/braintrust/integrations/dspy/test_dspy.py b/py/src/braintrust/integrations/dspy/test_dspy.py index 4bdc1d1c1..13ef0ca85 100644 --- a/py/src/braintrust/integrations/dspy/test_dspy.py +++ b/py/src/braintrust/integrations/dspy/test_dspy.py @@ -16,7 +16,7 @@ from braintrust.integrations.litellm import patch_litellm from braintrust.integrations.test_utils import run_in_subprocess, verify_autoinstrument_script from braintrust.test_helpers import init_test_logger -from pydantic import BaseModel, Field, field_serializer +from pydantic import BaseModel, Field, RootModel, field_serializer, model_serializer PROJECT_NAME = "test-dspy-app" @@ -51,10 +51,16 @@ def test_dspy_image_conversion_preserves_unmaterialized_images(url): image = dspy.Image(url=url) assert _process_images(image) is image assert _process_images({"image": image, "url": url}) == {"image": image, "url": url} + model = RootModel[list[dspy.Image]]([image]) + assert _process_images(model) is model -@pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path", "query", "body"]) -def test_dspy_images_materialize_attachments(memory_logger): +@pytest.mark.vcr( + cassette_name="test_dspy_images_materialize_attachments", + match_on=["method", "scheme", "host", "port", "path", "query", "body"], +) +@pytest.mark.parametrize("model_kind", ["model", "root", "list", "scalar"]) +def test_dspy_images_materialize_attachments(memory_logger, model_kind): """Images stay out of inline logs throughout a real DSPy/LiteLLM call.""" image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==" image_url = f"data:image/png;base64,{image_base64}" @@ -63,6 +69,14 @@ class Page(BaseModel): number: int image: dspy.Image + class SerializedPage(BaseModel): + page: Page + secret: str = Field(default="private", exclude=True) + + @model_serializer(mode="plain") + def serialize(self): + return [self.page] if model_kind == "list" else self.page.image.url + # DSPy 2.6 supports direct image fields, but does not expand images nested # in Pydantic fields into message content parts. nested_images = os.environ.get("BRAINTRUST_TEST_PACKAGE_VERSION") != "2.6.0" @@ -81,20 +95,32 @@ def __init__(self): self.predict = dspy.Predict(describe) def forward(self, image, page): - result = self.predict(image=image, page=page if nested_images else page.image) + # The outer module accepts several model shapes, while every case + # sends the same request to the provider and reuses its recording. + if isinstance(page, RootModel): + predictor_page = page.root[0] + elif isinstance(page, SerializedPage): + predictor_page = page.page + else: + predictor_page = page + result = self.predict(image=image, page=predictor_page if nested_images else predictor_page.image) return dspy.Prediction(answer=result.answer, image=image, page=page) patch_litellm() lm = dspy.LM(MODEL, cache=False, **LM_KWARGS) image = dspy.Image(url=image_url) page = Page(number=2, image=dspy.Image(url=image_url)) + if model_kind == "root": + page = RootModel[list[Page]]([page]) + elif model_kind in ("list", "scalar"): + page = SerializedPage(page=page) with dspy.context(lm=lm, callbacks=[BraintrustDSpyCallback()]): result = DescribePages()(image=image, page=page) assert result.answer assert result.image is image assert result.page is page - assert image.url == page.image.url == image_url + assert image.url == image_url spans = {span["span_attributes"]["name"]: span for span in memory_logger.pop()} for name in ("dspy.module.DescribePages", "dspy.module.Predict"): @@ -131,17 +157,31 @@ def assert_image(value): assert attachment.data == base64.b64decode(image_base64) assert attachment.reference["content_type"] == "image/png" + def assert_page(value): + if model_kind == "root": + assert len(value) == 1 + value = value[0] + elif model_kind in ("list", "scalar"): + # Non-mapping serializers are logged using their raw fields so + # attachments survive, while explicitly excluded fields stay out. + assert "secret" not in value + value = value["page"] + assert value["number"] == 2 + assert_image(value["image"]) + for name in ("dspy.module.DescribePages", "dspy.module.Predict"): inputs = spans[name]["input"] inputs = inputs.get("kwargs", inputs) assert_image(inputs["image"]) - if nested_images or name == "dspy.module.DescribePages": + if name == "dspy.module.DescribePages": + assert_page(inputs["page"]) + elif nested_images: assert_image(inputs["page"]["image"]) assert inputs["page"]["number"] == 2 else: assert_image(inputs["page"]) assert_image(spans["dspy.module.DescribePages"]["output"]["image"]) - assert_image(spans["dspy.module.DescribePages"]["output"]["page"]["image"]) + assert_page(spans["dspy.module.DescribePages"]["output"]["page"]) assert_image(spans["dspy.adapter.format"]["input"]["inputs"]["image"]) logged_page = spans["dspy.adapter.format"]["input"]["inputs"]["page"] assert_image(logged_page["image"] if nested_images else logged_page) diff --git a/py/src/braintrust/integrations/dspy/tracing.py b/py/src/braintrust/integrations/dspy/tracing.py index 6a4a17f2b..f97faf6dc 100644 --- a/py/src/braintrust/integrations/dspy/tracing.py +++ b/py/src/braintrust/integrations/dspy/tracing.py @@ -34,7 +34,7 @@ def _extract_provider(instance: Any, model: Any) -> str | None: try: from dspy import Example, Image from dspy.utils.callback import BaseCallback - from pydantic import BaseModel + from pydantic import BaseModel, RootModel _HAS_DSPY = True except ImportError: @@ -74,6 +74,9 @@ def _process_images_impl(value: Any) -> Any: if isinstance(value, Example): # toDict() serializes nested Pydantic fields in DSPy >= 3.4. return {key: _process_images(item) for key, item in value.items()} + if isinstance(value, RootModel): + converted_root = _process_images(value.root) + return value if converted_root is value.root else converted_root if isinstance(value, BaseModel): # dict(model) exposes raw values instead of invoking field serializers. fields = dict(value) @@ -83,6 +86,15 @@ def _process_images_impl(value: Any) -> Any: changed = {key for key in fields if converted_fields[key] is not fields[key]} result = value.model_dump(exclude=changed, exclude_none=True) model_fields = type(value).model_fields + if not isinstance(result, dict): + # Plain model serializers can ignore exclude=changed and return + # lists/scalars containing inline images. Log the converted raw + # fields in that case, respecting explicit field exclusions. + return { + key: item + for key, item in converted_fields.items() + if item is not None and (key not in model_fields or not model_fields[key].exclude) + } result.update( (key, converted_fields[key]) for key in changed if key not in model_fields or not model_fields[key].exclude ) From 090b157db31c82ec2cab4f12278b39babd43f991 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Wed, 14 Oct 2026 12:21:35 -0400 Subject: [PATCH 3/3] fix(dspy): preserve serializers when materializing images Materialize only the serialized representation of Pydantic models. Decode DSPy image prompt strings after serialization so redacted fields remain omitted and custom list, scalar, and mapping schemas are preserved. Extend the existing VCR regression to cover redactions and reshaping. --- .../braintrust/integrations/dspy/test_dspy.py | 44 +++++++++---- .../braintrust/integrations/dspy/tracing.py | 63 +++++++++---------- 2 files changed, 62 insertions(+), 45 deletions(-) diff --git a/py/src/braintrust/integrations/dspy/test_dspy.py b/py/src/braintrust/integrations/dspy/test_dspy.py index 13ef0ca85..79dffea76 100644 --- a/py/src/braintrust/integrations/dspy/test_dspy.py +++ b/py/src/braintrust/integrations/dspy/test_dspy.py @@ -30,6 +30,7 @@ def test_dspy_image_conversion_preserves_other_model_serializers(): class Page(BaseModel): image: dspy.Image + redacted_image: dspy.Image number: int secret: str = Field(exclude=True) @@ -37,11 +38,17 @@ class Page(BaseModel): def serialize_number(self, number): return f"page-{number}" + @field_serializer("redacted_image") + def redact_image(self, image): + return "[redacted]" + image = dspy.Image(url="data:image/png;base64,aW1hZ2U=") - page = Page(image=image, number=2, secret="private") + page = Page(image=image, redacted_image=image, number=2, secret="private") converted = _process_images(page) - assert isinstance(converted["image"]["image_url"]["url"], Attachment) + logged_image = converted["image"].get("image_url", converted["image"]) + assert isinstance(logged_image["url"], Attachment) assert converted["number"] == "page-2" + assert converted["redacted_image"] == "[redacted]" assert "secret" not in converted assert page.image is image @@ -59,7 +66,7 @@ def test_dspy_image_conversion_preserves_unmaterialized_images(url): cassette_name="test_dspy_images_materialize_attachments", match_on=["method", "scheme", "host", "port", "path", "query", "body"], ) -@pytest.mark.parametrize("model_kind", ["model", "root", "list", "scalar"]) +@pytest.mark.parametrize("model_kind", ["model", "root", "list", "scalar", "dict", "redacted"]) def test_dspy_images_materialize_attachments(memory_logger, model_kind): """Images stay out of inline logs throughout a real DSPy/LiteLLM call.""" image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==" @@ -72,10 +79,17 @@ class Page(BaseModel): class SerializedPage(BaseModel): page: Page secret: str = Field(default="private", exclude=True) + private_notes: str = "serializer-hidden-value" @model_serializer(mode="plain") def serialize(self): - return [self.page] if model_kind == "list" else self.page.image.url + if model_kind == "list": + return [self.page] + if model_kind == "dict": + return {"visible": self.page} + if model_kind == "redacted": + return "[redacted]" + return self.page.image.url # DSPy 2.6 supports direct image fields, but does not expand images nested # in Pydantic fields into message content parts. @@ -112,7 +126,7 @@ def forward(self, image, page): page = Page(number=2, image=dspy.Image(url=image_url)) if model_kind == "root": page = RootModel[list[Page]]([page]) - elif model_kind in ("list", "scalar"): + elif model_kind in ("list", "scalar", "dict", "redacted"): page = SerializedPage(page=page) with dspy.context(lm=lm, callbacks=[BraintrustDSpyCallback()]): result = DescribePages()(image=image, page=page) @@ -152,20 +166,25 @@ def forward(self, image, page): assert spans[name]["span_parents"][0] in span_ids def assert_image(value): - attachment = value["image_url"]["url"] + attachment = value.get("image_url", value)["url"] assert isinstance(attachment, Attachment) assert attachment.data == base64.b64decode(image_base64) assert attachment.reference["content_type"] == "image/png" def assert_page(value): - if model_kind == "root": + if model_kind in ("root", "list"): assert len(value) == 1 value = value[0] - elif model_kind in ("list", "scalar"): - # Non-mapping serializers are logged using their raw fields so - # attachments survive, while explicitly excluded fields stay out. - assert "secret" not in value - value = value["page"] + elif model_kind == "dict": + assert set(value) == {"visible"} + value = value["visible"] + elif model_kind == "scalar": + assert isinstance(value, Attachment) + assert value.data == base64.b64decode(image_base64) + return + elif model_kind == "redacted": + assert value == "[redacted]" + return assert value["number"] == 2 assert_image(value["image"]) @@ -203,6 +222,7 @@ def assert_page(value): serialized = bt_dumps(spans) assert image_base64 not in serialized assert "CUSTOM-TYPE" not in serialized + assert "serializer-hidden-value" not in serialized @pytest.fixture diff --git a/py/src/braintrust/integrations/dspy/tracing.py b/py/src/braintrust/integrations/dspy/tracing.py index f97faf6dc..dae8fb841 100644 --- a/py/src/braintrust/integrations/dspy/tracing.py +++ b/py/src/braintrust/integrations/dspy/tracing.py @@ -1,5 +1,6 @@ """DSPy-specific callback, span creation, and metadata extraction.""" +import json import logging from typing import Any @@ -10,6 +11,8 @@ _INSTRUMENTATION = "dspy-auto" _log = logging.getLogger(__name__) +_CUSTOM_TYPE_START = "<>" +_CUSTOM_TYPE_END = "<>" _LM_METADATA_PARAM_ALLOWLIST = ("temperature", "max_tokens", "top_p", "top_k", "stop") _EVALUATE_METADATA_ALLOWLIST = ("accuracy", "score", "total", "correct") @@ -34,7 +37,7 @@ def _extract_provider(instance: Any, model: Any) -> str | None: try: from dspy import Example, Image from dspy.utils.callback import BaseCallback - from pydantic import BaseModel, RootModel + from pydantic import BaseModel _HAS_DSPY = True except ImportError: @@ -43,9 +46,9 @@ def _extract_provider(instance: Any, model: Any) -> str | None: def _process_images(value: Any) -> Any: - """Materialize images before DSPy/Pydantic serializers turn them into prompt strings. + """Materialize images in DSPy values while preserving Pydantic serialization. - Walk raw model and Example fields so nested images remain recognizable. + Honor Pydantic serializers, then materialize their image representations. Only the logged copy changes; provider inputs and outputs stay intact. """ try: @@ -56,16 +59,25 @@ def _process_images(value: Any) -> Any: def _process_images_impl(value: Any) -> Any: - if isinstance(value, Image): - url = value.url - if isinstance(url, str) and url.startswith("data:image/"): - resolved = _materialize_attachment(url) - if resolved is not None: - return {"image_url": {"url": resolved.attachment}} - return value if isinstance(value, str) and value.startswith("data:image/"): resolved = _materialize_attachment(value) return resolved.attachment if resolved is not None else value + if isinstance(value, str) and value.startswith(_CUSTOM_TYPE_START) and value.endswith(_CUSTOM_TYPE_END): + # DSPy serializes nested Image fields as reserved prompt strings. + # Decode the complete wrapper after the user's serializers have run, + # so omitted fields stay omitted and custom output shapes survive. + parts = json.loads(value[len(_CUSTOM_TYPE_START) : -len(_CUSTOM_TYPE_END)]) + if isinstance(parts, list): + converted_parts = _process_images(parts) + if converted_parts is not parts: + if ( + len(converted_parts) == 1 + and isinstance(converted_parts[0], dict) + and converted_parts[0].get("type") == "image_url" + ): + return converted_parts[0] + return converted_parts + return value if isinstance(value, (list, tuple)): converted = [_process_images(item) for item in value] if all(item is original for item, original in zip(converted, value)): @@ -74,31 +86,16 @@ def _process_images_impl(value: Any) -> Any: if isinstance(value, Example): # toDict() serializes nested Pydantic fields in DSPy >= 3.4. return {key: _process_images(item) for key, item in value.items()} - if isinstance(value, RootModel): - converted_root = _process_images(value.root) - return value if converted_root is value.root else converted_root if isinstance(value, BaseModel): - # dict(model) exposes raw values instead of invoking field serializers. - fields = dict(value) - converted_fields = _process_images(fields) - if converted_fields is fields: + serialized = value.model_dump(exclude_none=True) + converted_model = _process_images(serialized) + if converted_model is serialized: return value - changed = {key for key in fields if converted_fields[key] is not fields[key]} - result = value.model_dump(exclude=changed, exclude_none=True) - model_fields = type(value).model_fields - if not isinstance(result, dict): - # Plain model serializers can ignore exclude=changed and return - # lists/scalars containing inline images. Log the converted raw - # fields in that case, respecting explicit field exclusions. - return { - key: item - for key, item in converted_fields.items() - if item is not None and (key not in model_fields or not model_fields[key].exclude) - } - result.update( - (key, converted_fields[key]) for key in changed if key not in model_fields or not model_fields[key].exclude - ) - return result + # DSPy 2.6 serializes Image as {"url": ...}; later versions use the + # reserved prompt wrapper handled above. Normalize only this SDK type. + if type(value) is Image and isinstance(converted_model, dict) and set(converted_model) == {"url"}: + return {"image_url": converted_model} + return converted_model if isinstance(value, dict): converted_dict = {key: _process_images(item) for key, item in value.items()} return value if all(converted_dict[key] is item for key, item in value.items()) else converted_dict