diff --git a/py/noxfile.py b/py/noxfile.py index e1e3a52f2..dd43b0ac7 100644 --- a/py/noxfile.py +++ b/py/noxfile.py @@ -632,6 +632,7 @@ def test_dspy(session, version): # on Python 3.10 due to unresolved Pydantic forward references. _skip_if_incompatible(session) _install_test_deps(session) + _install_group_locked(session, "test-dspy") if version == LATEST: # DSPy only lower-bounds LiteLLM, whose 1.92.0 release lacks Windows # and Python 3.14 wheels. Preinstall our portable matrix pin so DSPy's diff --git a/py/pyproject.toml b/py/pyproject.toml index fd4facfd6..f2b01895e 100644 --- a/py/pyproject.toml +++ b/py/pyproject.toml @@ -141,6 +141,11 @@ test-litellm = [ "orjson==3.12.0; python_version>='3.15'", ] +test-dspy = [ + # DSPy 2.6 requires Pillow to format image inputs but does not install it. + "pillow==12.3.0", +] + test-agno = [ {include-group = "test"}, "openai==2.31.0", diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_dspy.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_dspy.py index 6a4813aa0..feceeeeb3 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_dspy.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_dspy.py @@ -1,7 +1,7 @@ """Test auto_instrument for DSPy. Note: This test focuses on patching behavior only. Span verification for DSPy -is done in test_dspy.py::test_dspy_callback which uses pytest-vcr (supports httpx). +is done in test_dspy.py::test_dspy_images_materialize_attachments which uses pytest-vcr (supports httpx). The standalone VCR in test_utils doesn't capture httpx used by litellm/dspy. """ diff --git a/py/src/braintrust/integrations/dspy/cassettes/2.6.0/test_dspy_callback.yaml b/py/src/braintrust/integrations/dspy/cassettes/2.6.0/test_dspy_callback.yaml deleted file mode 100644 index 4759ff5ff..000000000 --- a/py/src/braintrust/integrations/dspy/cassettes/2.6.0/test_dspy_callback.yaml +++ /dev/null @@ -1,123 +0,0 @@ -interactions: -- request: - body: '{"messages":[{"role":"system","content":"Your input fields are:\n1. `question` - (str):\nYour output fields are:\n1. `reasoning` (str): \n2. `answer` (str):\nAll - interactions will be structured in the following way, with the appropriate values - filled in.\n\n[[ ## question ## ]]\n{question}\n\n[[ ## reasoning ## ]]\n{reasoning}\n\n[[ - ## answer ## ]]\n{answer}\n\n[[ ## completed ## ]]\nIn adhering to this structure, - your objective is: \n Given the fields `question`, produce the fields - `answer`."},{"role":"user","content":"[[ ## question ## ]]\nWhat is 2+2?\n\nRespond - with the corresponding output fields, starting with the field `[[ ## reasoning - ## ]]`, then `[[ ## answer ## ]]`, and then ending with the marker for `[[ ## - completed ## ]]`."}],"model":"gpt-4o-mini","max_tokens":4000,"temperature":0.0}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate - connection: - - keep-alive - content-length: - - '813' - content-type: - - application/json - host: - - api.openai.com - user-agent: - - OpenAI/Python 2.6.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - 'false' - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.6.0 - x-stainless-raw-response: - - 'true' - x-stainless-read-timeout: - - '600.0' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.9.21 - method: POST - uri: https://api.openai.com/v1/chat/completions - response: - body: - string: !!binary | - H4sIAAAAAAAAAwAAAP//jJPJbtswEIbveooBc6xk2KqX2NciB7cFgiZGEcA2BJocSWwokuFiNwjy - 7gUl27K7AL0I0Hzzj/5Z9JYAEMHJAgirqWeNkdmnR/twd/d1L748scl9ODzRh9vlo/l2/3n1fUnS - qNC7H8j8STVgujESvdCqw8wi9RirjmbT4Xw+nn6ctqDRHGWUVcZnY501QoksH+bjbDjLRrdHda0F - Q0cWsE4AAN7aZ/SpOP4kCximp0iDztEKyeKcBECsljFCqHPCeao8SXvItPKoWuvrNdzcgEXqtBKq - ii/b7UataoSXgC62A9Q9Oyi1BV8juNCALiEHqjjkA1gq2FEnGFArfN2gFyyFQ40KXnUAyjn4gwYV - mh1aB15X6Gu0aUuZbnZCYawrLOypDOgGsKrRYqktppDDB8gBXwKVDsaDjdqozjBV7oD25Hbcg+MW - kHfssmuLZXA0Tl4FKS8AVUp7Gltt5709kvfzhKWujNU795uUlEIJVxfd9OI0ndeGtPQ9Adi2mwxX - yyHG6sb4wutnbD83mh03SfoD6ulkdoReeyr7eJ6fwFW9gqOnQrqLWyCMshp5L+0PhwYu9AVILrr+ - 083fap/v5n/K94AxNB55YSxywa477tMsxv/rX2nnKbeGiUO7FwwLL9DGTXAsaZDd1RP36jw2RSlU - hdZY0Z1+aYrJdEjLKU4mc5K8J78AAAD//wMAHmfYmwgEAAA= - headers: - CF-RAY: - - 991b80ba5f531dd5-IAD - Connection: - - keep-alive - Content-Encoding: - - gzip - Content-Type: - - application/json - Date: - - Mon, 20 Oct 2025 21:10:37 GMT - Server: - - cloudflare - Set-Cookie: - - __cf_bm=NCarHmYOyIz1ZAIOne5DFJej0ZWtyIkscrsY1JN2xgU-1760994637-1.0.1.1-S4ijZSTcO6mty21tIa9pLH.RT_PbGhLHllaxLNjL.qAfWfKv8CzxzsKMrOwqSNdcr8XrFQZqX0N0k7NXrH129pMDQ8PsNZMyn1FG241fvS4; - path=/; expires=Mon, 20-Oct-25 21:40:37 GMT; domain=.api.openai.com; HttpOnly; - Secure; SameSite=None - - _cfuvid=Mp.a7GDzqU0CGQVrU3qeUynTdP6UBCXjJeHOoQ6jEDI-1760994637165-0.0.1.1-604800000; - path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None - Strict-Transport-Security: - - max-age=31536000; includeSubDomains; preload - Transfer-Encoding: - - chunked - X-Content-Type-Options: - - nosniff - access-control-expose-headers: - - X-Request-ID - alt-svc: - - h3=":443"; ma=86400 - cf-cache-status: - - DYNAMIC - openai-organization: - - braintrust-data - openai-processing-ms: - - '1008' - openai-project: - - proj_vsCSXafhhByzWOThMrJcZiw9 - openai-version: - - '2020-10-01' - x-envoy-upstream-service-time: - - '1171' - x-openai-proxy-wasm: - - v0.1 - x-ratelimit-limit-requests: - - '30000' - x-ratelimit-limit-tokens: - - '150000000' - x-ratelimit-remaining-requests: - - '29999' - x-ratelimit-remaining-tokens: - - '149999832' - x-ratelimit-reset-requests: - - 2ms - x-ratelimit-reset-tokens: - - 0s - x-request-id: - - req_3350c1e2f2d54aaf9923d2481dfaff3c - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/dspy/cassettes/2.6.0/test_dspy_images_materialize_attachments.yaml b/py/src/braintrust/integrations/dspy/cassettes/2.6.0/test_dspy_images_materialize_attachments.yaml new file mode 100644 index 000000000..ed169436f --- /dev/null +++ b/py/src/braintrust/integrations/dspy/cassettes/2.6.0/test_dspy_images_materialize_attachments.yaml @@ -0,0 +1,121 @@ +interactions: +- request: + body: '{"model":"gpt-4o-mini","messages":[{"role":"system","content":"Your input + fields are:\n1. `image` (Image)\n2. `page` (Image)\n\nYour output fields are:\n1. + `answer` (str)\n\nAll interactions will be structured in the following way, + with the appropriate values filled in.\n\n[[ ## image ## ]]\n{image}\n\n[[ ## + page ## ]]\n{page}\n\n[[ ## answer ## ]]\n{answer}\n\n[[ ## completed ## ]]\n\nIn + adhering to this structure, your objective is: \n Describe the colors + in these two page images."},{"role":"user","content":[{"type":"text","text":"[[ + ## image ## ]]\n\n"},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="}},{"type":"text","text":"\n\n[[ + ## page ## ]]\n\n"},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="}},{"type":"text","text":"\n\nRespond + with the corresponding output fields, starting with the field `[[ ## answer + ## ]]`, and then ending with the marker for `[[ ## completed ## ]]`."}]}],"max_tokens":1000,"temperature":0.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '1165' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 3.29.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 3.29.0 + x-stainless-raw-response: + - 'true' + x-stainless-read-timeout: + - '600.0' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-EYuEzFQDguHfWMglcxsl7uwpF32kg\",\n \"object\": + \"chat.completion\",\n \"created\": 1791988049,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"[[ ## answer ## ]]\\nThe colors in + both images are a solid, vibrant red. \\n\\n[[ ## completed ## ]]\",\n \"refusal\": + null,\n \"annotations\": []\n },\n \"logprobs\": null,\n + \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 17159,\n \"completion_tokens\": 23,\n \"total_tokens\": 17182,\n \"prompt_tokens_details\": + {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_bf0094d986\"\n}\n" + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a4a743dc5e60aaba-YYZ + connection: + - keep-alive + content-length: + - '910' + content-type: + - application/json + date: + - Wed, 14 Oct 2026 14:27:30 GMT + openai-processing-ms: + - '748' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-input-images: + - '50000' + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-input-images: + - '49998' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149998317' + x-ratelimit-reset-input-images: + - 2ms + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_97b1286a8ab44a50a486d7f5805b7982 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/dspy/cassettes/latest/test_dspy_callback.yaml b/py/src/braintrust/integrations/dspy/cassettes/latest/test_dspy_images_materialize_attachments.yaml similarity index 58% rename from py/src/braintrust/integrations/dspy/cassettes/latest/test_dspy_callback.yaml rename to py/src/braintrust/integrations/dspy/cassettes/latest/test_dspy_images_materialize_attachments.yaml index c338dcbf0..252817766 100644 --- a/py/src/braintrust/integrations/dspy/cassettes/latest/test_dspy_callback.yaml +++ b/py/src/braintrust/integrations/dspy/cassettes/latest/test_dspy_images_materialize_attachments.yaml @@ -1,15 +1,15 @@ interactions: - request: - body: '{"messages":[{"role":"system","content":"Your input fields are:\n1. `question` - (str):\nYour output fields are:\n1. `reasoning` (str): \n2. `answer` (str):\nAll + body: '{"messages":[{"role":"system","content":"Your input fields are:\n1. `image` + (Image): \n2. `page` (Page):\nYour output fields are:\n1. `answer` (str):\nAll interactions will be structured in the following way, with the appropriate values - filled in.\n\n[[ ## question ## ]]\n{question}\n\n[[ ## reasoning ## ]]\n{reasoning}\n\n[[ - ## answer ## ]]\n{answer}\n\n[[ ## completed ## ]]\nIn adhering to this structure, - your objective is: \n Given the fields `question`, produce the fields - `answer`."},{"role":"user","content":"[[ ## question ## ]]\nWhat is 2+2?\n\nRespond - with the corresponding output fields, starting with the field `[[ ## reasoning - ## ]]`, then `[[ ## answer ## ]]`, and then ending with the marker for `[[ ## - completed ## ]]`."}],"model":"gpt-4o-mini"}' + filled in.\n\n[[ ## image ## ]]\n{image}\n\n[[ ## page ## ]]\n{page}\n\n[[ ## + answer ## ]]\n{answer}\n\n[[ ## completed ## ]]\nIn adhering to this structure, + your objective is: \n Describe the colors in these two page images."},{"role":"user","content":[{"type":"text","text":"[[ + ## image ## ]]\n"},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="}},{"type":"text","text":"\n\n[[ + ## page ## ]]\n{\"number\": 2, \"image\": \""},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="}},{"type":"text","text":"\"}\n\nRespond + with the corresponding output fields, starting with the field `[[ ## answer + ## ]]`, and then ending with the marker for `[[ ## completed ## ]]`."}]}],"model":"gpt-4o-mini"}' headers: accept: - application/json @@ -18,7 +18,7 @@ interactions: connection: - keep-alive content-length: - - '777' + - '1154' content-type: - application/json host: @@ -49,19 +49,18 @@ interactions: uri: https://api.openai.com/v1/chat/completions response: body: - string: "{\n \"id\": \"chatcmpl-EYXW4udLtlqintb4oZb1YgnuW3wPU\",\n \"object\": - \"chat.completion\",\n \"created\": 1791900696,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + string: "{\n \"id\": \"chatcmpl-EYtp520w14OX3nAIFuclWlAvVUt0Q\",\n \"object\": + \"chat.completion\",\n \"created\": 1791986443,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": - \"assistant\",\n \"content\": \"[[ ## reasoning ## ]]\\nTo solve the - equation 2 + 2, we simply add the two numbers together. The sum of 2 and 2 - is 4.\\n\\n[[ ## answer ## ]]\\n4\\n\\n[[ ## completed ## ]]\",\n \"refusal\": + \"assistant\",\n \"content\": \"[[ ## answer ## ]]\\nThe color in the + images is a solid, vibrant red.\\n\\n[[ ## completed ## ]]\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": - 170,\n \"completion_tokens\": 47,\n \"total_tokens\": 217,\n \"prompt_tokens_details\": + 17167,\n \"completion_tokens\": 22,\n \"total_tokens\": 17189,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": - \"default\",\n \"system_fingerprint\": \"fp_df9611cffa\"\n}\n" + \"default\",\n \"system_fingerprint\": \"fp_bf0094d986\"\n}\n" headers: access-control-expose-headers: - X-Request-ID @@ -72,17 +71,17 @@ interactions: cf-cache-status: - DYNAMIC cf-ray: - - a49eef390eec5327-YYZ + - a4a71ca40932ebc0-YYZ connection: - keep-alive content-length: - - '975' + - '906' content-type: - application/json date: - - Tue, 13 Oct 2026 14:11:37 GMT + - Wed, 14 Oct 2026 14:00:44 GMT openai-processing-ms: - - '808' + - '636' openai-version: - '2020-10-01' server: @@ -95,20 +94,26 @@ interactions: - nosniff x-openai-proxy-wasm: - v0.1 + x-ratelimit-limit-input-images: + - '50000' x-ratelimit-limit-requests: - '30000' x-ratelimit-limit-tokens: - '150000000' + x-ratelimit-remaining-input-images: + - '49997' x-ratelimit-remaining-requests: - '29998' x-ratelimit-remaining-tokens: - - '149999812' + - '149997537' + x-ratelimit-reset-input-images: + - 3ms x-ratelimit-reset-requests: - 4ms x-ratelimit-reset-tokens: - 0s x-request-id: - - req_f9b517b5c9b04fb993dd2b9b70450d54 + - req_7e7d9e781211446b8482c8a294249204 status: code: 200 message: OK diff --git a/py/src/braintrust/integrations/dspy/test_dspy.py b/py/src/braintrust/integrations/dspy/test_dspy.py index c3f10d112..79dffea76 100644 --- a/py/src/braintrust/integrations/dspy/test_dspy.py +++ b/py/src/braintrust/integrations/dspy/test_dspy.py @@ -2,16 +2,21 @@ Tests for DSPy integration with Braintrust. """ +import base64 import inspect import os from types import SimpleNamespace import dspy import pytest -from braintrust import logger +from braintrust import Attachment, logger +from braintrust.bt_json import bt_dumps from braintrust.integrations.dspy import BraintrustDSpyCallback +from braintrust.integrations.dspy.tracing import _process_images +from braintrust.integrations.litellm import patch_litellm from braintrust.integrations.test_utils import run_in_subprocess, verify_autoinstrument_script from braintrust.test_helpers import init_test_logger +from pydantic import BaseModel, Field, RootModel, field_serializer, model_serializer PROJECT_NAME = "test-dspy-app" @@ -22,77 +27,209 @@ LM_KWARGS = {"engine": "litellm"} if "engine" in inspect.signature(dspy.LM.__init__).parameters else {} -@pytest.fixture -def memory_logger(): - init_test_logger(PROJECT_NAME) - with logger._internal_with_memory_background_logger() as bgl: - yield bgl - - -@pytest.mark.vcr -def test_dspy_callback(memory_logger): - """Test DSPy callback logs spans correctly.""" - assert not memory_logger.pop() +def test_dspy_image_conversion_preserves_other_model_serializers(): + class Page(BaseModel): + image: dspy.Image + redacted_image: dspy.Image + number: int + secret: str = Field(exclude=True) + + @field_serializer("number") + def serialize_number(self, number): + return f"page-{number}" + + @field_serializer("redacted_image") + def redact_image(self, image): + return "[redacted]" + + image = dspy.Image(url="data:image/png;base64,aW1hZ2U=") + page = Page(image=image, redacted_image=image, number=2, secret="private") + converted = _process_images(page) + logged_image = converted["image"].get("image_url", converted["image"]) + assert isinstance(logged_image["url"], Attachment) + assert converted["number"] == "page-2" + assert converted["redacted_image"] == "[redacted]" + assert "secret" not in converted + assert page.image is image + + +@pytest.mark.parametrize("url", ["https://example.com/page.png", "data:image/png;base64,not base64!"]) +def test_dspy_image_conversion_preserves_unmaterialized_images(url): + image = dspy.Image(url=url) + assert _process_images(image) is image + assert _process_images({"image": image, "url": url}) == {"image": image, "url": url} + model = RootModel[list[dspy.Image]]([image]) + assert _process_images(model) is model + + +@pytest.mark.vcr( + cassette_name="test_dspy_images_materialize_attachments", + match_on=["method", "scheme", "host", "port", "path", "query", "body"], +) +@pytest.mark.parametrize("model_kind", ["model", "root", "list", "scalar", "dict", "redacted"]) +def test_dspy_images_materialize_attachments(memory_logger, model_kind): + """Images stay out of inline logs throughout a real DSPy/LiteLLM call.""" + image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==" + image_url = f"data:image/png;base64,{image_base64}" + + class Page(BaseModel): + number: int + image: dspy.Image + + class SerializedPage(BaseModel): + page: Page + secret: str = Field(default="private", exclude=True) + private_notes: str = "serializer-hidden-value" + + @model_serializer(mode="plain") + def serialize(self): + if model_kind == "list": + return [self.page] + if model_kind == "dict": + return {"visible": self.page} + if model_kind == "redacted": + return "[redacted]" + return self.page.image.url + + # DSPy 2.6 supports direct image fields, but does not expand images nested + # in Pydantic fields into message content parts. + nested_images = os.environ.get("BRAINTRUST_TEST_PACKAGE_VERSION") != "2.6.0" + describe = dspy.Signature( + { + "image": (dspy.Image, dspy.InputField()), + "page": (Page if nested_images else dspy.Image, dspy.InputField()), + "answer": (str, dspy.OutputField()), + }, + instructions="Describe the colors in these two page images.", + ) - # Configure DSPy with Braintrust callback + class DescribePages(dspy.Module): + def __init__(self): + super().__init__() + self.predict = dspy.Predict(describe) + + def forward(self, image, page): + # The outer module accepts several model shapes, while every case + # sends the same request to the provider and reuses its recording. + if isinstance(page, RootModel): + predictor_page = page.root[0] + elif isinstance(page, SerializedPage): + predictor_page = page.page + else: + predictor_page = page + result = self.predict(image=image, page=predictor_page if nested_images else predictor_page.image) + return dspy.Prediction(answer=result.answer, image=image, page=page) + + patch_litellm() lm = dspy.LM(MODEL, cache=False, **LM_KWARGS) - dspy.configure(lm=lm, callbacks=[BraintrustDSpyCallback()]) - - # Use ChainOfThought for a more interesting test - cot = dspy.ChainOfThought("question -> answer") - result = cot(question="What is 2+2?") - - assert result.answer # Verify we got a response - - # Check logged spans - spans = memory_logger.pop() - assert len(spans) >= 4 # Should have module, adapter format, LM, and adapter parse spans - - spans_by_name = {span["span_attributes"]["name"]: span for span in spans} - - module_span = spans_by_name["dspy.module.ChainOfThought"] - assert module_span["span_attributes"]["type"] == "task" - assert module_span["metadata"]["module_class"].endswith("ChainOfThought") - - lm_span = spans_by_name["dspy.lm"] + image = dspy.Image(url=image_url) + page = Page(number=2, image=dspy.Image(url=image_url)) + if model_kind == "root": + page = RootModel[list[Page]]([page]) + elif model_kind in ("list", "scalar", "dict", "redacted"): + page = SerializedPage(page=page) + with dspy.context(lm=lm, callbacks=[BraintrustDSpyCallback()]): + result = DescribePages()(image=image, page=page) + + assert result.answer + assert result.image is image + assert result.page is page + assert image.url == image_url + spans = {span["span_attributes"]["name"]: span for span in memory_logger.pop()} + + for name in ("dspy.module.DescribePages", "dspy.module.Predict"): + span = spans[name] + assert span["span_attributes"]["type"] == "task" + assert span["metadata"]["module_class"].endswith(name.removeprefix("dspy.module.")) + + lm_span = spans["dspy.lm"] assert lm_span["context"]["span_origin"]["instrumentation"]["name"] == "dspy-auto" assert lm_span["span_attributes"].get("type") != "llm" - assert "metadata" in lm_span - assert "model" in lm_span["metadata"] - assert MODEL in lm_span["metadata"]["model"] + assert lm_span["metadata"]["model"] == MODEL assert lm_span["metadata"]["provider"] == "openai" - assert "input" in lm_span - assert "output" in lm_span - # The latest cassette contains a real OpenAI usage payload. DSPy 2.6's - # recorded response predates usage being included. - if os.environ.get("BRAINTRUST_TEST_PACKAGE_VERSION") == "latest": - assert lm_span["metrics"]["prompt_tokens"] == 170 - assert lm_span["metrics"]["completion_tokens"] == 47 - assert lm_span["metrics"]["tokens"] == 217 + assert lm_span["output"] + metrics = lm_span["metrics"] + assert metrics["prompt_tokens"] > 0 + assert metrics["completion_tokens"] > 0 + assert metrics["tokens"] == metrics["prompt_tokens"] + metrics["completion_tokens"] + + for name in ("dspy.adapter.format", "dspy.adapter.parse"): + span = spans[name] + assert span["span_attributes"]["type"] == "task" + assert span["metadata"]["adapter_class"].endswith("ChatAdapter") + assert "signature" in span["input"] + assert "demos" in spans["dspy.adapter.format"]["input"] + assert "completion" in spans["dspy.adapter.parse"]["input"] + assert isinstance(spans["dspy.adapter.parse"]["output"], dict) + span_ids = {span["span_id"] for span in spans.values()} + for name in ("dspy.lm", "dspy.adapter.format", "dspy.adapter.parse"): + assert spans[name]["span_parents"][0] in span_ids + + def assert_image(value): + attachment = value.get("image_url", value)["url"] + assert isinstance(attachment, Attachment) + assert attachment.data == base64.b64decode(image_base64) + assert attachment.reference["content_type"] == "image/png" + + def assert_page(value): + if model_kind in ("root", "list"): + assert len(value) == 1 + value = value[0] + elif model_kind == "dict": + assert set(value) == {"visible"} + value = value["visible"] + elif model_kind == "scalar": + assert isinstance(value, Attachment) + assert value.data == base64.b64decode(image_base64) + return + elif model_kind == "redacted": + assert value == "[redacted]" + return + assert value["number"] == 2 + assert_image(value["image"]) + + for name in ("dspy.module.DescribePages", "dspy.module.Predict"): + inputs = spans[name]["input"] + inputs = inputs.get("kwargs", inputs) + assert_image(inputs["image"]) + if name == "dspy.module.DescribePages": + assert_page(inputs["page"]) + elif nested_images: + assert_image(inputs["page"]["image"]) + assert inputs["page"]["number"] == 2 + else: + assert_image(inputs["page"]) + assert_image(spans["dspy.module.DescribePages"]["output"]["image"]) + assert_page(spans["dspy.module.DescribePages"]["output"]["page"]) + assert_image(spans["dspy.adapter.format"]["input"]["inputs"]["image"]) + logged_page = spans["dspy.adapter.format"]["input"]["inputs"]["page"] + assert_image(logged_page["image"] if nested_images else logged_page) + for messages in ( + spans["dspy.adapter.format"]["output"], + spans["dspy.lm"]["input"]["messages"], + spans["Completion"]["input"], + ): + image_parts = [ + part + for message in messages + if isinstance(message["content"], list) + for part in message["content"] + if part["type"] == "image_url" + ] + assert len(image_parts) == 2 + for part in image_parts: + assert_image(part) + serialized = bt_dumps(spans) + assert image_base64 not in serialized + assert "CUSTOM-TYPE" not in serialized + assert "serializer-hidden-value" not in serialized - format_span = spans_by_name["dspy.adapter.format"] - assert format_span["span_attributes"]["type"] == "task" - assert format_span["metadata"]["adapter_class"].endswith("ChatAdapter") - assert "signature" in format_span["input"] - assert "demos" in format_span["input"] - assert "inputs" in format_span["input"] - assert isinstance(format_span["output"], list) - parse_span = spans_by_name["dspy.adapter.parse"] - assert parse_span["span_attributes"]["type"] == "task" - assert parse_span["metadata"]["adapter_class"].endswith("ChatAdapter") - assert "signature" in parse_span["input"] - assert "completion" in parse_span["input"] - assert isinstance(parse_span["output"], dict) - - # Verify spans are nested under the broader DSPy execution - span_ids = {span["span_id"] for span in spans} - assert lm_span.get("span_parents") - assert format_span.get("span_parents") - assert parse_span.get("span_parents") - assert format_span["span_parents"][0] in span_ids - assert lm_span["span_parents"][0] in span_ids - assert parse_span["span_parents"][0] in span_ids +@pytest.fixture +def memory_logger(): + init_test_logger(PROJECT_NAME) + with logger._internal_with_memory_background_logger() as bgl: + yield bgl def test_dspy_callback_ignores_usage_from_previous_lm_call(memory_logger): diff --git a/py/src/braintrust/integrations/dspy/tracing.py b/py/src/braintrust/integrations/dspy/tracing.py index 3a1c23818..dae8fb841 100644 --- a/py/src/braintrust/integrations/dspy/tracing.py +++ b/py/src/braintrust/integrations/dspy/tracing.py @@ -1,12 +1,18 @@ """DSPy-specific callback, span creation, and metadata extraction.""" +import json +import logging from typing import Any +from braintrust.integrations.utils import _materialize_attachment from braintrust.logger import start_span as _bt_start_span from braintrust.span_types import SpanTypeAttribute _INSTRUMENTATION = "dspy-auto" +_log = logging.getLogger(__name__) +_CUSTOM_TYPE_START = "<>" +_CUSTOM_TYPE_END = "<>" _LM_METADATA_PARAM_ALLOWLIST = ("temperature", "max_tokens", "top_p", "top_k", "stop") _EVALUATE_METADATA_ALLOWLIST = ("accuracy", "score", "total", "correct") @@ -29,7 +35,9 @@ def _extract_provider(instance: Any, model: Any) -> str | None: try: + from dspy import Example, Image from dspy.utils.callback import BaseCallback + from pydantic import BaseModel _HAS_DSPY = True except ImportError: @@ -37,6 +45,63 @@ def _extract_provider(instance: Any, model: Any) -> str | None: BaseCallback = object # type: ignore[assignment,misc] +def _process_images(value: Any) -> Any: + """Materialize images in DSPy values while preserving Pydantic serialization. + + Honor Pydantic serializers, then materialize their image representations. + Only the logged copy changes; provider inputs and outputs stay intact. + """ + try: + return _process_images_impl(value) + except Exception: + _log.debug("Failed to materialize DSPy images", exc_info=True) + return value + + +def _process_images_impl(value: Any) -> Any: + if isinstance(value, str) and value.startswith("data:image/"): + resolved = _materialize_attachment(value) + return resolved.attachment if resolved is not None else value + if isinstance(value, str) and value.startswith(_CUSTOM_TYPE_START) and value.endswith(_CUSTOM_TYPE_END): + # DSPy serializes nested Image fields as reserved prompt strings. + # Decode the complete wrapper after the user's serializers have run, + # so omitted fields stay omitted and custom output shapes survive. + parts = json.loads(value[len(_CUSTOM_TYPE_START) : -len(_CUSTOM_TYPE_END)]) + if isinstance(parts, list): + converted_parts = _process_images(parts) + if converted_parts is not parts: + if ( + len(converted_parts) == 1 + and isinstance(converted_parts[0], dict) + and converted_parts[0].get("type") == "image_url" + ): + return converted_parts[0] + return converted_parts + return value + if isinstance(value, (list, tuple)): + converted = [_process_images(item) for item in value] + if all(item is original for item, original in zip(converted, value)): + return value + return tuple(converted) if isinstance(value, tuple) else converted + if isinstance(value, Example): + # toDict() serializes nested Pydantic fields in DSPy >= 3.4. + return {key: _process_images(item) for key, item in value.items()} + if isinstance(value, BaseModel): + serialized = value.model_dump(exclude_none=True) + converted_model = _process_images(serialized) + if converted_model is serialized: + return value + # DSPy 2.6 serializes Image as {"url": ...}; later versions use the + # reserved prompt wrapper handled above. Normalize only this SDK type. + if type(value) is Image and isinstance(converted_model, dict) and set(converted_model) == {"url"}: + return {"image_url": converted_model} + return converted_model + if isinstance(value, dict): + converted_dict = {key: _process_images(item) for key, item in value.items()} + return value if all(converted_dict[key] is item for key, item in value.items()) else converted_dict + return value + + class BraintrustDSpyCallback(BaseCallback): """Callback handler that logs DSPy execution traces to Braintrust. @@ -137,7 +202,7 @@ def on_lm_start( span = start_span( name="dspy.lm", - input=inputs, + input=_process_images(inputs), metadata=metadata, ) span.set_current() @@ -166,7 +231,7 @@ def _end_span( if exception: log_data["error"] = exception if outputs is not None: - log_data["output"] = outputs + log_data["output"] = _process_images(outputs) if metrics: log_data["metrics"] = metrics @@ -252,7 +317,7 @@ def on_module_start( span = start_span( name=f"dspy.module.{cls_name}", type=SpanTypeAttribute.TASK, - input=inputs, + input=_process_images(inputs), metadata={"module_class": module_name}, ) span.set_current() @@ -287,7 +352,7 @@ def _start_adapter_span( span = start_span( name=span_name, type=SpanTypeAttribute.TASK, - input=inputs, + input=_process_images(inputs), metadata=metadata, ) span.set_current() @@ -377,7 +442,7 @@ def on_tool_start( span = start_span( name=tool_name, type=SpanTypeAttribute.TOOL, - input=inputs, + input=_process_images(inputs), ) span.set_current() self._spans[call_id] = span @@ -420,7 +485,7 @@ def on_evaluate_start( span = start_span( name="dspy.evaluate", type=SpanTypeAttribute.TASK, - input=inputs, + input=_process_images(inputs), metadata=metadata, ) span.set_current() @@ -448,7 +513,7 @@ def on_evaluate_end( if exception: log_data["error"] = exception if outputs is not None: - log_data["output"] = outputs + log_data["output"] = _process_images(outputs) if isinstance(outputs, dict): metadata: dict[str, Any] = {} for key in _EVALUATE_METADATA_ALLOWLIST: diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_async_streaming_with_break.yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_async_streaming_with_break.yaml deleted file mode 100644 index d50872b33..000000000 --- a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_async_streaming_with_break.yaml +++ /dev/null @@ -1,138 +0,0 @@ -interactions: -- request: - body: '{"messages":[{"role":"user","content":"What''s 12 + 12?"}],"model":"gpt-4o-mini","stream":true,"stream_options":{"include_usage":true}}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate - connection: - - keep-alive - content-length: - - '134' - content-type: - - application/json - host: - - api.openai.com - user-agent: - - AsyncOpenAI/Python 2.24.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - async:asyncio - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.24.0 - x-stainless-raw-response: - - 'true' - x-stainless-read-timeout: - - '600.0' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.13.3 - method: POST - uri: https://api.openai.com/v1/chat/completions - response: - body: - string: 'data: {"id":"chatcmpl-DFld8139VUAiMKuqKwYSqrvuCxt4n","object":"chat.completion.chunk","created":1772650382,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"pnUPoD0pg"} - - - data: {"id":"chatcmpl-DFld8139VUAiMKuqKwYSqrvuCxt4n","object":"chat.completion.chunk","created":1772650382,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"37zFEOP4I"} - - - data: {"id":"chatcmpl-DFld8139VUAiMKuqKwYSqrvuCxt4n","object":"chat.completion.chunk","created":1772650382,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":" - +"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"4WPyeDQnv"} - - - data: {"id":"chatcmpl-DFld8139VUAiMKuqKwYSqrvuCxt4n","object":"chat.completion.chunk","created":1772650382,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":" - "},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"sOxR63A12m"} - - - data: {"id":"chatcmpl-DFld8139VUAiMKuqKwYSqrvuCxt4n","object":"chat.completion.chunk","created":1772650382,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"bHyDGfLF5"} - - - data: {"id":"chatcmpl-DFld8139VUAiMKuqKwYSqrvuCxt4n","object":"chat.completion.chunk","created":1772650382,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":" - equals"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"1SVl"} - - - data: {"id":"chatcmpl-DFld8139VUAiMKuqKwYSqrvuCxt4n","object":"chat.completion.chunk","created":1772650382,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":" - "},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"mHNJKCDuMZ"} - - - data: {"id":"chatcmpl-DFld8139VUAiMKuqKwYSqrvuCxt4n","object":"chat.completion.chunk","created":1772650382,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":"24"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"ABixefOK3"} - - - data: {"id":"chatcmpl-DFld8139VUAiMKuqKwYSqrvuCxt4n","object":"chat.completion.chunk","created":1772650382,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"8hKx5NPN8n"} - - - data: {"id":"chatcmpl-DFld8139VUAiMKuqKwYSqrvuCxt4n","object":"chat.completion.chunk","created":1772650382,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"tdXAE"} - - - data: {"id":"chatcmpl-DFld8139VUAiMKuqKwYSqrvuCxt4n","object":"chat.completion.chunk","created":1772650382,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[],"usage":{"prompt_tokens":14,"completion_tokens":8,"total_tokens":22,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"bstubRw2p2F"} - - - data: [DONE] - - - ' - headers: - Access-Control-Expose-Headers: - - X-Request-ID - CF-RAY: - - 9d7314d87c81aa96-YYZ - Connection: - - keep-alive - Content-Type: - - text/event-stream; charset=utf-8 - Date: - - Wed, 04 Mar 2026 18:53:02 GMT - Server: - - cloudflare - Set-Cookie: - - __cf_bm=rXpr5trNRw1Qr3qbQJW.KHLQVMw77ivBL8WgtL_WzJo-1772650382.1581635-1.0.1.1-FmrGGx6rgCOI7Eo8DmohZe.rH15CYM_m1ffbwb.i_njpQwLnRzsFohZzV9dn0E9BEhVdvGXyfn1IRurlydiZLLFw2rNGb9ivmJkBlNxGLCMQalVVWZ_U2k2kk3YOqpHd; - HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 04 Mar 2026 - 19:23:02 GMT - Strict-Transport-Security: - - max-age=31536000; includeSubDomains; preload - Transfer-Encoding: - - chunked - X-Content-Type-Options: - - nosniff - alt-svc: - - h3=":443"; ma=86400 - cf-cache-status: - - DYNAMIC - openai-organization: - - braintrust-data - openai-processing-ms: - - '295' - openai-project: - - proj_vsCSXafhhByzWOThMrJcZiw9 - openai-version: - - '2020-10-01' - x-openai-proxy-wasm: - - v0.1 - x-ratelimit-limit-requests: - - '30000' - x-ratelimit-limit-tokens: - - '150000000' - x-ratelimit-remaining-requests: - - '29999' - x-ratelimit-remaining-tokens: - - '149999992' - x-ratelimit-reset-requests: - - 2ms - x-ratelimit-reset-tokens: - - 0s - x-request-id: - - req_057c10d296a342fba68072d8c99ce937 - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_metrics[sync].yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_metrics[sync].yaml deleted file mode 100644 index f2443b2cd..000000000 --- a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_metrics[sync].yaml +++ /dev/null @@ -1,215 +0,0 @@ -interactions: - - request: - body: - '{"messages": [{"role": "user", "content": "What''s 12 + 12?"}], "model": - "gpt-4o-mini"}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate, zstd - connection: - - keep-alive - content-length: - - "86" - content-type: - - application/json - host: - - api.openai.com - user-agent: - - OpenAI/Python 1.92.3 - x-stainless-arch: - - arm64 - x-stainless-async: - - "false" - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 1.92.3 - x-stainless-raw-response: - - "true" - x-stainless-read-timeout: - - "600.0" - x-stainless-retry-count: - - "0" - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.11.10 - method: POST - uri: https://api.openai.com/v1/chat/completions - response: - body: - string: !!binary | - H4sIAAAAAAAAAwAAAP//jJJBb9QwEIXv+RXWXNlUiderbfdGj/SA4IJUVEWuPUlcHNvYExZU7X9H - TrabFIrEJYf55k3eG89zwRgYDQcGqpekhmDL2/gl3d5X20/HnaW7u/cfno497+7lx89JHGGTFf7x - CRW9qK6UH4JFMt7NWEWUhHlqvd/x6mYvxH4Cg9dos6wLVApfDsaZkldclNW+rK/P6t4bhQkO7GvB - GGPP0zf7dBp/woFVm5fKgCnJDuFwaWIMore5AjIlk0g6gs0ClXeEbrJec/aO1Zzh91HaxLi4WjdG - bMcks1k3WrsC0jlPMoedLD6cyeliyvouRP+Y/pBCa5xJfRNRJu+ygUQ+wERPBWMPU/jxVR4I0Q+B - GvLfcPpdLeZxsKx8gddnRp6kXcqcb94Y1mgkaWxa7Q6UVD3qRbksWo7a+BUoVpH/9vLW7Dm2cd3/ - jF+AUhgIdRMiaqNe513aIuZ7/FfbZcWTYUgYfxiFDRmM+Rk0tnK085VA+pUIh6Y1rsMYoplPpQ3N - VsidkHizVVCcit8AAAD//wMA0gHr9zgDAAA= - headers: - CF-RAY: - - 95cb00386a5a14f6-LAX - Connection: - - keep-alive - Content-Encoding: - - gzip - Content-Type: - - application/json - Date: - - Wed, 09 Jul 2025 21:44:08 GMT - Server: - - cloudflare - Set-Cookie: - - __cf_bm=LhJr7iTKhj3HkuIqsYYQzHnLMYlAsrPO_8DONvOoa1s-1752097448-1.0.1.1-3QZBiExLtV3A83cyPK4CeYqjpSygQ6uYDZ0v7kYNlmEHZIOldYiM088zk99s107ShpZgiHzSJwcw0Fviy3roAk.Ifv1OH0UzhIx2VTAKaS4; - path=/; expires=Wed, 09-Jul-25 22:14:08 GMT; domain=.api.openai.com; HttpOnly; - Secure; SameSite=None - - _cfuvid=1FpkTfHv2KC0I5jKg65yXzjPU21dBkiuyT323kob89c-1752097448243-0.0.1.1-604800000; - path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None - Transfer-Encoding: - - chunked - X-Content-Type-Options: - - nosniff - access-control-expose-headers: - - X-Request-ID - alt-svc: - - h3=":443"; ma=86400 - cf-cache-status: - - DYNAMIC - openai-organization: - - braintrust-data - openai-processing-ms: - - "319" - openai-version: - - "2020-10-01" - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - x-envoy-upstream-service-time: - - "322" - x-ratelimit-limit-requests: - - "30000" - x-ratelimit-limit-tokens: - - "150000000" - x-ratelimit-remaining-requests: - - "29999" - x-ratelimit-remaining-tokens: - - "149999993" - x-ratelimit-reset-requests: - - 2ms - x-ratelimit-reset-tokens: - - 0s - x-request-id: - - req_0c9ffafb565ae4556d4374eaba259452 - status: - code: 200 - message: OK - - request: - body: - '{"messages": [{"role": "user", "content": "What''s 12 + 12?"}], "model": - "gpt-4o-mini"}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate, zstd - connection: - - keep-alive - content-length: - - "86" - content-type: - - application/json - cookie: - - __cf_bm=LhJr7iTKhj3HkuIqsYYQzHnLMYlAsrPO_8DONvOoa1s-1752097448-1.0.1.1-3QZBiExLtV3A83cyPK4CeYqjpSygQ6uYDZ0v7kYNlmEHZIOldYiM088zk99s107ShpZgiHzSJwcw0Fviy3roAk.Ifv1OH0UzhIx2VTAKaS4; - _cfuvid=1FpkTfHv2KC0I5jKg65yXzjPU21dBkiuyT323kob89c-1752097448243-0.0.1.1-604800000 - host: - - api.openai.com - user-agent: - - OpenAI/Python 1.92.3 - x-stainless-arch: - - arm64 - x-stainless-async: - - "false" - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 1.92.3 - x-stainless-raw-response: - - "true" - x-stainless-read-timeout: - - "600.0" - x-stainless-retry-count: - - "0" - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.11.10 - method: POST - uri: https://api.openai.com/v1/chat/completions - response: - body: - string: !!binary | - H4sIAAAAAAAAAwAAAP//jJJPb9QwEMXv+RSWr2yqxOtq/xxZcQIhTnBAVeS1J4nB8Rh7gkDVfnfk - ZLtJS5G45DC/eZP3xvNYMMat4UfGda9ID8GVb+OXdHq3/STfG1F9FP2Hz1tvxEnWJzyc+SYr8PwN - ND2p7jQOwQFZ9DPWERRBnlrv7kV12Em5n8CABlyWdYFKieVgvS1FJWRZ7cp6f1X3aDUkfmRfC8YY - e5y+2ac38IsfWbV5qgyQkuqAH29NjPGILle4SskmUp74ZoEaPYGfrNeCvWG1YPBjVC4xIe/WjRHa - Mals1o/OrYDyHknlsJPFhyu53Ew57ELEc3oh5a31NvVNBJXQZwOJMPCJXgrGHqbw47M8PEQcAjWE - 32H6XS3ncXxZ+QL3V0ZIyi1lITavDGsMkLIurXbHtdI9mEW5LFqNxuIKFKvIf3t5bfYc2/ruf8Yv - QGsIBKYJEYzVz/MubRHyPf6r7bbiyTBPEH9aDQ1ZiPkZDLRqdPOV8PQ7EQxNa30HMUQ7n0obmq1U - 91LBYat5cSn+AAAA//8DAN/D0ms4AwAA - headers: - CF-RAY: - - 95cb003c2e2414f6-LAX - Connection: - - keep-alive - Content-Encoding: - - gzip - Content-Type: - - application/json - Date: - - Wed, 09 Jul 2025 21:44:09 GMT - Server: - - cloudflare - Transfer-Encoding: - - chunked - X-Content-Type-Options: - - nosniff - access-control-expose-headers: - - X-Request-ID - alt-svc: - - h3=":443"; ma=86400 - cf-cache-status: - - DYNAMIC - openai-organization: - - braintrust-data - openai-processing-ms: - - "675" - openai-version: - - "2020-10-01" - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - x-envoy-upstream-service-time: - - "693" - x-ratelimit-limit-requests: - - "30000" - x-ratelimit-limit-tokens: - - "150000000" - x-ratelimit-remaining-requests: - - "29999" - x-ratelimit-remaining-tokens: - - "149999993" - x-ratelimit-reset-requests: - - 2ms - x-ratelimit-reset-tokens: - - 0s - x-request-id: - - req_8eb436911ec46aaf3095f96c82730abe - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_streaming[async].yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_streaming[async].yaml deleted file mode 100644 index c2010a57e..000000000 --- a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_streaming[async].yaml +++ /dev/null @@ -1,138 +0,0 @@ -interactions: -- request: - body: '{"messages":[{"role":"user","content":"What''s 12 + 12?"}],"model":"gpt-4o-mini","stream":true,"stream_options":{"include_usage":true}}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate - connection: - - keep-alive - content-length: - - '134' - content-type: - - application/json - host: - - api.openai.com - user-agent: - - AsyncOpenAI/Python 2.24.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - async:asyncio - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.24.0 - x-stainless-raw-response: - - 'true' - x-stainless-read-timeout: - - '600.0' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.13.3 - method: POST - uri: https://api.openai.com/v1/chat/completions - response: - body: - string: 'data: {"id":"chatcmpl-DFld4pdjHTYDiYVVqutj5XkGk3LPM","object":"chat.completion.chunk","created":1772650378,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"qCUq24aUP"} - - - data: {"id":"chatcmpl-DFld4pdjHTYDiYVVqutj5XkGk3LPM","object":"chat.completion.chunk","created":1772650378,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"tyBUuLlXP"} - - - data: {"id":"chatcmpl-DFld4pdjHTYDiYVVqutj5XkGk3LPM","object":"chat.completion.chunk","created":1772650378,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":" - +"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"AccQ6DEkI"} - - - data: {"id":"chatcmpl-DFld4pdjHTYDiYVVqutj5XkGk3LPM","object":"chat.completion.chunk","created":1772650378,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":" - "},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"4HK9XGcEQh"} - - - data: {"id":"chatcmpl-DFld4pdjHTYDiYVVqutj5XkGk3LPM","object":"chat.completion.chunk","created":1772650378,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"ZrKglFglG"} - - - data: {"id":"chatcmpl-DFld4pdjHTYDiYVVqutj5XkGk3LPM","object":"chat.completion.chunk","created":1772650378,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":" - equals"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"Megf"} - - - data: {"id":"chatcmpl-DFld4pdjHTYDiYVVqutj5XkGk3LPM","object":"chat.completion.chunk","created":1772650378,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":" - "},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"TAGN0Y1TQL"} - - - data: {"id":"chatcmpl-DFld4pdjHTYDiYVVqutj5XkGk3LPM","object":"chat.completion.chunk","created":1772650378,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":"24"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"0T8blqEVA"} - - - data: {"id":"chatcmpl-DFld4pdjHTYDiYVVqutj5XkGk3LPM","object":"chat.completion.chunk","created":1772650378,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"ber8oHdwqa"} - - - data: {"id":"chatcmpl-DFld4pdjHTYDiYVVqutj5XkGk3LPM","object":"chat.completion.chunk","created":1772650378,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"BPBAy"} - - - data: {"id":"chatcmpl-DFld4pdjHTYDiYVVqutj5XkGk3LPM","object":"chat.completion.chunk","created":1772650378,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_373a14eb6f","choices":[],"usage":{"prompt_tokens":14,"completion_tokens":8,"total_tokens":22,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"xmssdH5hQck"} - - - data: [DONE] - - - ' - headers: - Access-Control-Expose-Headers: - - X-Request-ID - CF-RAY: - - 9d7314c2cc23b407-YYZ - Connection: - - keep-alive - Content-Type: - - text/event-stream; charset=utf-8 - Date: - - Wed, 04 Mar 2026 18:52:58 GMT - Server: - - cloudflare - Set-Cookie: - - __cf_bm=16nX.lrlnS0MVxy6hFRb78u914aYPaUfIvgFXxMZEI8-1772650378.6869698-1.0.1.1-NB_.XAMuCAXL7hlX46NSshfJwOcB9qTdNGhWnSp_G2pcRTyMt.gwjEX_kjL5lKU0prJTyJn5fTgP_HjE_OiDcg8lBni08Fs8nNdCSw0u2dQWdCqHw.usO2s587EcmVS2; - HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 04 Mar 2026 - 19:22:58 GMT - Strict-Transport-Security: - - max-age=31536000; includeSubDomains; preload - Transfer-Encoding: - - chunked - X-Content-Type-Options: - - nosniff - alt-svc: - - h3=":443"; ma=86400 - cf-cache-status: - - DYNAMIC - openai-organization: - - braintrust-data - openai-processing-ms: - - '178' - openai-project: - - proj_vsCSXafhhByzWOThMrJcZiw9 - openai-version: - - '2020-10-01' - x-openai-proxy-wasm: - - v0.1 - x-ratelimit-limit-requests: - - '30000' - x-ratelimit-limit-tokens: - - '150000000' - x-ratelimit-remaining-requests: - - '29999' - x-ratelimit-remaining-tokens: - - '149999995' - x-ratelimit-reset-requests: - - 2ms - x-ratelimit-reset-tokens: - - 0s - x-request-id: - - req_98a341ea8970475ca3bcabd4bacf04bf - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_streaming[sync].yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_streaming[sync].yaml deleted file mode 100644 index 1a6b3a9f7..000000000 --- a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_streaming[sync].yaml +++ /dev/null @@ -1,274 +0,0 @@ -interactions: - - request: - body: - '{"messages": [{"role": "user", "content": "What''s 12 + 12?"}], "model": - "gpt-4o-mini", "stream": true, "stream_options": {"include_usage": true}}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate, zstd - connection: - - keep-alive - content-length: - - "145" - content-type: - - application/json - cookie: - - __cf_bm=LhJr7iTKhj3HkuIqsYYQzHnLMYlAsrPO_8DONvOoa1s-1752097448-1.0.1.1-3QZBiExLtV3A83cyPK4CeYqjpSygQ6uYDZ0v7kYNlmEHZIOldYiM088zk99s107ShpZgiHzSJwcw0Fviy3roAk.Ifv1OH0UzhIx2VTAKaS4; - _cfuvid=1FpkTfHv2KC0I5jKg65yXzjPU21dBkiuyT323kob89c-1752097448243-0.0.1.1-604800000 - host: - - api.openai.com - user-agent: - - OpenAI/Python 1.92.3 - x-stainless-arch: - - arm64 - x-stainless-async: - - "false" - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 1.92.3 - x-stainless-raw-response: - - "true" - x-stainless-read-timeout: - - "600.0" - x-stainless-retry-count: - - "0" - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.11.10 - method: POST - uri: https://api.openai.com/v1/chat/completions - response: - body: - string: - 'data: {"id":"chatcmpl-BrWsECewLnZpwYk2y6TeJIT6H7F48","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsECewLnZpwYk2y6TeJIT6H7F48","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsECewLnZpwYk2y6TeJIT6H7F48","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" - +"},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsECewLnZpwYk2y6TeJIT6H7F48","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" - "},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsECewLnZpwYk2y6TeJIT6H7F48","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsECewLnZpwYk2y6TeJIT6H7F48","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" - equals"},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsECewLnZpwYk2y6TeJIT6H7F48","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" - "},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsECewLnZpwYk2y6TeJIT6H7F48","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"24"},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsECewLnZpwYk2y6TeJIT6H7F48","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsECewLnZpwYk2y6TeJIT6H7F48","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null} - - - data: {"id":"chatcmpl-BrWsECewLnZpwYk2y6TeJIT6H7F48","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[],"usage":{"prompt_tokens":14,"completion_tokens":8,"total_tokens":22,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}}} - - - data: [DONE] - - - ' - headers: - CF-RAY: - - 95cb00484b3214f6-LAX - Connection: - - keep-alive - Content-Type: - - text/event-stream; charset=utf-8 - Date: - - Wed, 09 Jul 2025 21:44:10 GMT - Server: - - cloudflare - Transfer-Encoding: - - chunked - X-Content-Type-Options: - - nosniff - access-control-expose-headers: - - X-Request-ID - alt-svc: - - h3=":443"; ma=86400 - cf-cache-status: - - DYNAMIC - openai-organization: - - braintrust-data - openai-processing-ms: - - "327" - openai-version: - - "2020-10-01" - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - x-envoy-upstream-service-time: - - "332" - x-ratelimit-limit-requests: - - "30000" - x-ratelimit-limit-tokens: - - "150000000" - x-ratelimit-remaining-requests: - - "29999" - x-ratelimit-remaining-tokens: - - "149999993" - x-ratelimit-reset-requests: - - 2ms - x-ratelimit-reset-tokens: - - 0s - x-request-id: - - req_82e49575f67bda7dced7355056a8aef0 - status: - code: 200 - message: OK - - request: - body: - '{"messages": [{"role": "user", "content": "What''s 12 + 12?"}], "model": - "gpt-4o-mini", "stream": true, "stream_options": {"include_usage": true}}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate, zstd - connection: - - keep-alive - content-length: - - "145" - content-type: - - application/json - cookie: - - __cf_bm=LhJr7iTKhj3HkuIqsYYQzHnLMYlAsrPO_8DONvOoa1s-1752097448-1.0.1.1-3QZBiExLtV3A83cyPK4CeYqjpSygQ6uYDZ0v7kYNlmEHZIOldYiM088zk99s107ShpZgiHzSJwcw0Fviy3roAk.Ifv1OH0UzhIx2VTAKaS4; - _cfuvid=1FpkTfHv2KC0I5jKg65yXzjPU21dBkiuyT323kob89c-1752097448243-0.0.1.1-604800000 - host: - - api.openai.com - user-agent: - - OpenAI/Python 1.92.3 - x-stainless-arch: - - arm64 - x-stainless-async: - - "false" - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 1.92.3 - x-stainless-raw-response: - - "true" - x-stainless-read-timeout: - - "600.0" - x-stainless-retry-count: - - "0" - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.11.10 - method: POST - uri: https://api.openai.com/v1/chat/completions - response: - body: - string: - 'data: {"id":"chatcmpl-BrWsEMMFl7DBEVA7ID65Tklpkbnbs","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsEMMFl7DBEVA7ID65Tklpkbnbs","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsEMMFl7DBEVA7ID65Tklpkbnbs","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" - +"},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsEMMFl7DBEVA7ID65Tklpkbnbs","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" - "},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsEMMFl7DBEVA7ID65Tklpkbnbs","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsEMMFl7DBEVA7ID65Tklpkbnbs","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" - equals"},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsEMMFl7DBEVA7ID65Tklpkbnbs","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":" - "},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsEMMFl7DBEVA7ID65Tklpkbnbs","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"24"},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsEMMFl7DBEVA7ID65Tklpkbnbs","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-BrWsEMMFl7DBEVA7ID65Tklpkbnbs","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null} - - - data: {"id":"chatcmpl-BrWsEMMFl7DBEVA7ID65Tklpkbnbs","object":"chat.completion.chunk","created":1752097450,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_62a23a81ef","choices":[],"usage":{"prompt_tokens":14,"completion_tokens":8,"total_tokens":22,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}}} - - - data: [DONE] - - - ' - headers: - CF-RAY: - - 95cb004baed214f6-LAX - Connection: - - keep-alive - Content-Type: - - text/event-stream; charset=utf-8 - Date: - - Wed, 09 Jul 2025 21:44:11 GMT - Server: - - cloudflare - Transfer-Encoding: - - chunked - X-Content-Type-Options: - - nosniff - access-control-expose-headers: - - X-Request-ID - alt-svc: - - h3=":443"; ma=86400 - cf-cache-status: - - DYNAMIC - openai-organization: - - braintrust-data - openai-processing-ms: - - "203" - openai-version: - - "2020-10-01" - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - x-envoy-upstream-service-time: - - "208" - x-ratelimit-limit-requests: - - "30000" - x-ratelimit-limit-tokens: - - "150000000" - x-ratelimit-remaining-requests: - - "29999" - x-ratelimit-remaining-tokens: - - "149999993" - x-ratelimit-reset-requests: - - 2ms - x-ratelimit-reset-tokens: - - 0s - x-request-id: - - req_6a6ea838af6542b8572e83b621c03f90 - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_metrics[async].yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_input_materializes_attachment[blocking-async].yaml similarity index 61% rename from py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_metrics[async].yaml rename to py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_input_materializes_attachment[blocking-async].yaml index 81a0aface..c9c00f8fa 100644 --- a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_metrics[async].yaml +++ b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_input_materializes_attachment[blocking-async].yaml @@ -1,6 +1,7 @@ interactions: - request: - body: '{"messages":[{"role":"user","content":"What''s 12 + 12?"}],"model":"gpt-4o-mini"}' + body: '{"messages":[{"role":"user","content":[{"type":"text","text":"What color + is this image?"},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==","detail":"low"}}]}],"model":"gpt-4o-mini"}' headers: accept: - application/json @@ -9,13 +10,13 @@ interactions: connection: - keep-alive content-length: - - '80' + - '292' content-type: - application/json host: - api.openai.com user-agent: - - AsyncOpenAI/Python 2.24.0 + - AsyncOpenAI/Python 1.99.9 x-stainless-arch: - arm64 x-stainless-async: @@ -25,7 +26,7 @@ interactions: x-stainless-os: - MacOS x-stainless-package-version: - - 2.24.0 + - 1.99.9 x-stainless-raw-response: - 'true' x-stainless-read-timeout: @@ -35,75 +36,75 @@ interactions: x-stainless-runtime: - CPython x-stainless-runtime-version: - - 3.13.3 + - 3.14.7 method: POST uri: https://api.openai.com/v1/chat/completions response: body: - string: "{\n \"id\": \"chatcmpl-DFlcrhg78gJ6cwXCo9bHgDtJvZfyJ\",\n \"object\": - \"chat.completion\",\n \"created\": 1772650365,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + string: "{\n \"id\": \"chatcmpl-EYu9kVBUD5NXdNPINDruLZ6Dp4b7v\",\n \"object\": + \"chat.completion\",\n \"created\": 1791987724,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": - \"assistant\",\n \"content\": \"12 + 12 equals 24.\",\n \"refusal\": + \"assistant\",\n \"content\": \"The image is red.\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": - 14,\n \"completion_tokens\": 8,\n \"total_tokens\": 22,\n \"prompt_tokens_details\": - {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + 2846,\n \"completion_tokens\": 5,\n \"total_tokens\": 2851,\n \"prompt_tokens_details\": + {\n \"cached_tokens\": 2816,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": - \"default\",\n \"system_fingerprint\": \"fp_373a14eb6f\"\n}\n" + \"default\",\n \"system_fingerprint\": \"fp_bf0094d986\"\n}\n" headers: Access-Control-Expose-Headers: - X-Request-ID - CF-RAY: - - 9d73146f9953ec71-YYZ + - CF-Ray + - CF-Ray + Alt-Svc: + - h3=":443"; ma=86400 + CF-Cache-Status: + - DYNAMIC + CF-Ray: + - a4a73be998b9aad0-YYZ Connection: - keep-alive Content-Type: - application/json Date: - - Wed, 04 Mar 2026 18:52:45 GMT + - Wed, 14 Oct 2026 14:22:04 GMT Server: - cloudflare - Set-Cookie: - - __cf_bm=V_yDANNrUk_AKhRc3JMD1PiBDa9.W0AopRO95256b8E-1772650365.3751957-1.0.1.1-rr5DmxlLKEWnz6279TOurNHTLABqVASjVIhHAKczPI96o54yejvrdU4ScLdL_x_Gac9DOP0W5SCGWRL9TiZ9ouKagJlXEBbC1TxshGOdmnDPW66QcddzsGwFYyyNEPMQ; - HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 04 Mar 2026 - 19:22:45 GMT Strict-Transport-Security: - max-age=31536000; includeSubDomains; preload Transfer-Encoding: - chunked X-Content-Type-Options: - nosniff - alt-svc: - - h3=":443"; ma=86400 - cf-cache-status: - - DYNAMIC + X-Request-ID: + - req_bf70a332dc234d0382ce79e61705092e content-length: - - '824' - openai-organization: - - braintrust-data + - '830' openai-processing-ms: - - '356' - openai-project: - - proj_vsCSXafhhByzWOThMrJcZiw9 + - '507' openai-version: - '2020-10-01' x-openai-proxy-wasm: - v0.1 + x-ratelimit-limit-input-images: + - '50000' x-ratelimit-limit-requests: - '30000' x-ratelimit-limit-tokens: - '150000000' + x-ratelimit-remaining-input-images: + - '49999' x-ratelimit-remaining-requests: - '29999' x-ratelimit-remaining-tokens: - - '149999992' + - '149997145' + x-ratelimit-reset-input-images: + - 1ms x-ratelimit-reset-requests: - 2ms x-ratelimit-reset-tokens: - - 0s - x-request-id: - - req_906cbaef60c6445dbf09994daa66d913 + - 1ms status: code: 200 message: OK diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_input_materializes_attachment[blocking-sync].yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_input_materializes_attachment[blocking-sync].yaml new file mode 100644 index 000000000..1a21f300e --- /dev/null +++ b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_input_materializes_attachment[blocking-sync].yaml @@ -0,0 +1,111 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":[{"type":"text","text":"What color + is this image?"},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==","detail":"low"}}]}],"model":"gpt-4o-mini"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '292' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.99.9 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.99.9 + x-stainless-raw-response: + - 'true' + x-stainless-read-timeout: + - '600.0' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-EYu9jl6WYyB5qpK2VhI2RxCU5vNPr\",\n \"object\": + \"chat.completion\",\n \"created\": 1791987723,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"The image is a solid shade of red.\",\n + \ \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": + null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 2846,\n \"completion_tokens\": 9,\n \"total_tokens\": 2855,\n \"prompt_tokens_details\": + {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_bf0094d986\"\n}\n" + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a4a73be048e524ee-YYZ + connection: + - keep-alive + content-length: + - '844' + content-type: + - application/json + date: + - Wed, 14 Oct 2026 14:22:03 GMT + openai-processing-ms: + - '563' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-input-images: + - '50000' + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-input-images: + - '49999' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999227' + x-ratelimit-reset-input-images: + - 1ms + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_745443a771ed4ce08eb85228e08d659f + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_input_materializes_attachment[streaming-async].yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_input_materializes_attachment[streaming-async].yaml new file mode 100644 index 000000000..72f9376aa --- /dev/null +++ b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_input_materializes_attachment[streaming-async].yaml @@ -0,0 +1,129 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":[{"type":"text","text":"What color + is this image?"},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==","detail":"low"}}]}],"model":"gpt-4o-mini","stream":true,"stream_options":{"include_usage":true}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '346' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.99.9 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.99.9 + x-stainless-raw-response: + - 'true' + x-stainless-read-timeout: + - '600.0' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-EYu9mwUsvcT2JYlurPM4DuiEKPlv9","object":"chat.completion.chunk","created":1791987726,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"eqGrxHf0D"} + + + data: {"id":"chatcmpl-EYu9mwUsvcT2JYlurPM4DuiEKPlv9","object":"chat.completion.chunk","created":1791987726,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"13w8nzb7"} + + + data: {"id":"chatcmpl-EYu9mwUsvcT2JYlurPM4DuiEKPlv9","object":"chat.completion.chunk","created":1791987726,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":" + image"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"4UyTu"} + + + data: {"id":"chatcmpl-EYu9mwUsvcT2JYlurPM4DuiEKPlv9","object":"chat.completion.chunk","created":1791987726,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":" + is"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"IkZeH63t"} + + + data: {"id":"chatcmpl-EYu9mwUsvcT2JYlurPM4DuiEKPlv9","object":"chat.completion.chunk","created":1791987726,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":" + red"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"wfO3uly"} + + + data: {"id":"chatcmpl-EYu9mwUsvcT2JYlurPM4DuiEKPlv9","object":"chat.completion.chunk","created":1791987726,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"tlKVCmGw3M"} + + + data: {"id":"chatcmpl-EYu9mwUsvcT2JYlurPM4DuiEKPlv9","object":"chat.completion.chunk","created":1791987726,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"PlPQt"} + + + data: {"id":"chatcmpl-EYu9mwUsvcT2JYlurPM4DuiEKPlv9","object":"chat.completion.chunk","created":1791987726,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[],"usage":{"prompt_tokens":2846,"completion_tokens":5,"total_tokens":2851,"prompt_tokens_details":{"cached_tokens":2816,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"7whN"} + + + data: [DONE] + + + ' + headers: + Access-Control-Expose-Headers: + - X-Request-ID + - CF-Ray + - CF-Ray + Alt-Svc: + - h3=":443"; ma=86400 + CF-Cache-Status: + - DYNAMIC + CF-Ray: + - a4a73bf6ae90c85f-YYZ + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Wed, 14 Oct 2026 14:22:06 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + X-Request-ID: + - req_1722642ac3344354b0b9bfcc99824be5 + openai-processing-ms: + - '346' + openai-version: + - '2020-10-01' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-input-images: + - '50000' + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-input-images: + - '49999' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999227' + x-ratelimit-reset-input-images: + - 1ms + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_input_materializes_attachment[streaming-sync].yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_input_materializes_attachment[streaming-sync].yaml new file mode 100644 index 000000000..d8f6ba67c --- /dev/null +++ b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_input_materializes_attachment[streaming-sync].yaml @@ -0,0 +1,129 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":[{"type":"text","text":"What color + is this image?"},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==","detail":"low"}}]}],"model":"gpt-4o-mini","stream":true,"stream_options":{"include_usage":true}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '346' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.99.9 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.99.9 + x-stainless-raw-response: + - 'true' + x-stainless-read-timeout: + - '600.0' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-EYu9lYfA1vyWZLYnHL9aAIJany9mz","object":"chat.completion.chunk","created":1791987725,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"OtjYHSaOf"} + + + data: {"id":"chatcmpl-EYu9lYfA1vyWZLYnHL9aAIJany9mz","object":"chat.completion.chunk","created":1791987725,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"Y9YxcuFq"} + + + data: {"id":"chatcmpl-EYu9lYfA1vyWZLYnHL9aAIJany9mz","object":"chat.completion.chunk","created":1791987725,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":" + image"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"y9w9i"} + + + data: {"id":"chatcmpl-EYu9lYfA1vyWZLYnHL9aAIJany9mz","object":"chat.completion.chunk","created":1791987725,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":" + is"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"WNi3XRpU"} + + + data: {"id":"chatcmpl-EYu9lYfA1vyWZLYnHL9aAIJany9mz","object":"chat.completion.chunk","created":1791987725,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":" + red"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"j4myJAb"} + + + data: {"id":"chatcmpl-EYu9lYfA1vyWZLYnHL9aAIJany9mz","object":"chat.completion.chunk","created":1791987725,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"NaWNm0sLwt"} + + + data: {"id":"chatcmpl-EYu9lYfA1vyWZLYnHL9aAIJany9mz","object":"chat.completion.chunk","created":1791987725,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"IF0BH"} + + + data: {"id":"chatcmpl-EYu9lYfA1vyWZLYnHL9aAIJany9mz","object":"chat.completion.chunk","created":1791987725,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[],"usage":{"prompt_tokens":2846,"completion_tokens":5,"total_tokens":2851,"prompt_tokens_details":{"cached_tokens":2816,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"UgiA"} + + + data: [DONE] + + + ' + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a4a73beeca9924ee-YYZ + connection: + - keep-alive + content-type: + - text/event-stream; charset=utf-8 + date: + - Wed, 14 Oct 2026 14:22:05 GMT + openai-processing-ms: + - '397' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-input-images: + - '50000' + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-input-images: + - '49998' + x-ratelimit-remaining-requests: + - '29998' + x-ratelimit-remaining-tokens: + - '149994295' + x-ratelimit-reset-input-images: + - 2ms + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 2ms + x-request-id: + - req_c54b29776a544e7395e0fbda66847fe7 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_async_streaming_with_break.yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_async_streaming_with_break.yaml deleted file mode 100644 index 6a05cdae1..000000000 --- a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_async_streaming_with_break.yaml +++ /dev/null @@ -1,132 +0,0 @@ -interactions: -- request: - body: '{"messages":[{"role":"user","content":"What''s 12 + 12?"}],"model":"gpt-4o-mini","stream":true,"stream_options":{"include_usage":true}}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate - connection: - - keep-alive - content-length: - - '134' - content-type: - - application/json - host: - - api.openai.com - user-agent: - - AsyncOpenAI/Python 2.54.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - async:asyncio - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.54.0 - x-stainless-raw-response: - - 'true' - x-stainless-read-timeout: - - '600.0' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.14.7 - method: POST - uri: https://api.openai.com/v1/chat/completions - response: - body: - string: 'data: {"id":"chatcmpl-EYXVmvIiGUSdc5i1uvZNMfj2Jt06w","object":"chat.completion.chunk","created":1791900678,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"86oPdoSHv"} - - - data: {"id":"chatcmpl-EYXVmvIiGUSdc5i1uvZNMfj2Jt06w","object":"chat.completion.chunk","created":1791900678,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"NcLYdwGwi"} - - - data: {"id":"chatcmpl-EYXVmvIiGUSdc5i1uvZNMfj2Jt06w","object":"chat.completion.chunk","created":1791900678,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":" - +"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"AJsEn2EKa"} - - - data: {"id":"chatcmpl-EYXVmvIiGUSdc5i1uvZNMfj2Jt06w","object":"chat.completion.chunk","created":1791900678,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":" - "},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"x90vYXDKLS"} - - - data: {"id":"chatcmpl-EYXVmvIiGUSdc5i1uvZNMfj2Jt06w","object":"chat.completion.chunk","created":1791900678,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"edzu4o841"} - - - data: {"id":"chatcmpl-EYXVmvIiGUSdc5i1uvZNMfj2Jt06w","object":"chat.completion.chunk","created":1791900678,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":" - equals"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"7pAK"} - - - data: {"id":"chatcmpl-EYXVmvIiGUSdc5i1uvZNMfj2Jt06w","object":"chat.completion.chunk","created":1791900678,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":" - "},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"VvBycIqVjT"} - - - data: {"id":"chatcmpl-EYXVmvIiGUSdc5i1uvZNMfj2Jt06w","object":"chat.completion.chunk","created":1791900678,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":"24"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"apXgr1Z4r"} - - - data: {"id":"chatcmpl-EYXVmvIiGUSdc5i1uvZNMfj2Jt06w","object":"chat.completion.chunk","created":1791900678,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"sHtvMah9Cg"} - - - data: {"id":"chatcmpl-EYXVmvIiGUSdc5i1uvZNMfj2Jt06w","object":"chat.completion.chunk","created":1791900678,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"fIT79"} - - - data: {"id":"chatcmpl-EYXVmvIiGUSdc5i1uvZNMfj2Jt06w","object":"chat.completion.chunk","created":1791900678,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[],"usage":{"prompt_tokens":14,"completion_tokens":8,"total_tokens":22,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"WhUZOPFh67J"} - - - data: [DONE] - - - ' - headers: - Access-Control-Expose-Headers: - - X-Request-ID - - CF-Ray - - CF-Ray - Alt-Svc: - - h3=":443"; ma=86400 - CF-Cache-Status: - - DYNAMIC - CF-Ray: - - a49eeec68ce8175c-YYZ - Connection: - - keep-alive - Content-Type: - - text/event-stream; charset=utf-8 - Date: - - Tue, 13 Oct 2026 14:11:18 GMT - Server: - - cloudflare - Strict-Transport-Security: - - max-age=31536000; includeSubDomains; preload - Transfer-Encoding: - - chunked - X-Content-Type-Options: - - nosniff - X-Request-ID: - - req_fbfb7605216349f78876a000f3eceb35 - openai-processing-ms: - - '197' - openai-version: - - '2020-10-01' - x-openai-proxy-wasm: - - v0.1 - x-ratelimit-limit-requests: - - '30000' - x-ratelimit-limit-tokens: - - '150000000' - x-ratelimit-remaining-requests: - - '29999' - x-ratelimit-remaining-tokens: - - '149999995' - x-ratelimit-reset-requests: - - 2ms - x-ratelimit-reset-tokens: - - 0s - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_streaming[async].yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_streaming[async].yaml deleted file mode 100644 index 563faf1cf..000000000 --- a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_streaming[async].yaml +++ /dev/null @@ -1,132 +0,0 @@ -interactions: -- request: - body: '{"messages":[{"role":"user","content":"What''s 12 + 12?"}],"model":"gpt-4o-mini","stream":true,"stream_options":{"include_usage":true}}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate - connection: - - keep-alive - content-length: - - '134' - content-type: - - application/json - host: - - api.openai.com - user-agent: - - AsyncOpenAI/Python 2.54.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - async:asyncio - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.54.0 - x-stainless-raw-response: - - 'true' - x-stainless-read-timeout: - - '600.0' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.14.7 - method: POST - uri: https://api.openai.com/v1/chat/completions - response: - body: - string: 'data: {"id":"chatcmpl-EYXUd9BxNE1C6BNgLrUUVgroBi53r","object":"chat.completion.chunk","created":1791900607,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"az4SZGSlo"} - - - data: {"id":"chatcmpl-EYXUd9BxNE1C6BNgLrUUVgroBi53r","object":"chat.completion.chunk","created":1791900607,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"nGETNsphG"} - - - data: {"id":"chatcmpl-EYXUd9BxNE1C6BNgLrUUVgroBi53r","object":"chat.completion.chunk","created":1791900607,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":" - +"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"UiTHZCs76"} - - - data: {"id":"chatcmpl-EYXUd9BxNE1C6BNgLrUUVgroBi53r","object":"chat.completion.chunk","created":1791900607,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":" - "},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"rRUsQQKfu5"} - - - data: {"id":"chatcmpl-EYXUd9BxNE1C6BNgLrUUVgroBi53r","object":"chat.completion.chunk","created":1791900607,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"xIQq040r9"} - - - data: {"id":"chatcmpl-EYXUd9BxNE1C6BNgLrUUVgroBi53r","object":"chat.completion.chunk","created":1791900607,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":" - equals"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"SYsw"} - - - data: {"id":"chatcmpl-EYXUd9BxNE1C6BNgLrUUVgroBi53r","object":"chat.completion.chunk","created":1791900607,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":" - "},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"cCm3Yn8Y4g"} - - - data: {"id":"chatcmpl-EYXUd9BxNE1C6BNgLrUUVgroBi53r","object":"chat.completion.chunk","created":1791900607,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":"24"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"Rj5qprBk3"} - - - data: {"id":"chatcmpl-EYXUd9BxNE1C6BNgLrUUVgroBi53r","object":"chat.completion.chunk","created":1791900607,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"wAz8qEZFkJ"} - - - data: {"id":"chatcmpl-EYXUd9BxNE1C6BNgLrUUVgroBi53r","object":"chat.completion.chunk","created":1791900607,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"Cb4Xv"} - - - data: {"id":"chatcmpl-EYXUd9BxNE1C6BNgLrUUVgroBi53r","object":"chat.completion.chunk","created":1791900607,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[],"usage":{"prompt_tokens":14,"completion_tokens":8,"total_tokens":22,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"KT5mK3wfFp8"} - - - data: [DONE] - - - ' - headers: - Access-Control-Expose-Headers: - - X-Request-ID - - CF-Ray - - CF-Ray - Alt-Svc: - - h3=":443"; ma=86400 - CF-Cache-Status: - - DYNAMIC - CF-Ray: - - a49eed0b8867e21a-YYZ - Connection: - - keep-alive - Content-Type: - - text/event-stream; charset=utf-8 - Date: - - Tue, 13 Oct 2026 14:10:07 GMT - Server: - - cloudflare - Strict-Transport-Security: - - max-age=31536000; includeSubDomains; preload - Transfer-Encoding: - - chunked - X-Content-Type-Options: - - nosniff - X-Request-ID: - - req_b2f0fd6aca6d4e88a7bd7c535d16ff23 - openai-processing-ms: - - '271' - openai-version: - - '2020-10-01' - x-openai-proxy-wasm: - - v0.1 - x-ratelimit-limit-requests: - - '30000' - x-ratelimit-limit-tokens: - - '150000000' - x-ratelimit-remaining-requests: - - '29997' - x-ratelimit-remaining-tokens: - - '149999917' - x-ratelimit-reset-requests: - - 6ms - x-ratelimit-reset-tokens: - - 0s - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_streaming[sync].yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_streaming[sync].yaml deleted file mode 100644 index 2f4beaf09..000000000 --- a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_streaming[sync].yaml +++ /dev/null @@ -1,132 +0,0 @@ -interactions: -- request: - body: '{"messages":[{"role":"user","content":"What''s 12 + 12?"}],"model":"gpt-4o-mini","stream":true,"stream_options":{"include_usage":true}}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate - connection: - - keep-alive - content-length: - - '134' - content-type: - - application/json - host: - - api.openai.com - user-agent: - - OpenAI/Python 2.54.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - 'false' - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.54.0 - x-stainless-raw-response: - - 'true' - x-stainless-read-timeout: - - '600.0' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.14.7 - method: POST - uri: https://api.openai.com/v1/chat/completions - response: - body: - string: 'data: {"id":"chatcmpl-EYXUcaJJFE5TWou2BQMf7zku6gcAW","object":"chat.completion.chunk","created":1791900606,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"6WWKtsopF"} - - - data: {"id":"chatcmpl-EYXUcaJJFE5TWou2BQMf7zku6gcAW","object":"chat.completion.chunk","created":1791900606,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"hXX4hklk2"} - - - data: {"id":"chatcmpl-EYXUcaJJFE5TWou2BQMf7zku6gcAW","object":"chat.completion.chunk","created":1791900606,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":" - +"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"7rNYiBG5x"} - - - data: {"id":"chatcmpl-EYXUcaJJFE5TWou2BQMf7zku6gcAW","object":"chat.completion.chunk","created":1791900606,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":" - "},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"bwuiERvMOR"} - - - data: {"id":"chatcmpl-EYXUcaJJFE5TWou2BQMf7zku6gcAW","object":"chat.completion.chunk","created":1791900606,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":"12"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"3UiklAQHE"} - - - data: {"id":"chatcmpl-EYXUcaJJFE5TWou2BQMf7zku6gcAW","object":"chat.completion.chunk","created":1791900606,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":" - equals"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"XimM"} - - - data: {"id":"chatcmpl-EYXUcaJJFE5TWou2BQMf7zku6gcAW","object":"chat.completion.chunk","created":1791900606,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":" - "},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"bGFtdg4LW1"} - - - data: {"id":"chatcmpl-EYXUcaJJFE5TWou2BQMf7zku6gcAW","object":"chat.completion.chunk","created":1791900606,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":"24"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"9i7fnUBTF"} - - - data: {"id":"chatcmpl-EYXUcaJJFE5TWou2BQMf7zku6gcAW","object":"chat.completion.chunk","created":1791900606,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"nW5Mi0qnYH"} - - - data: {"id":"chatcmpl-EYXUcaJJFE5TWou2BQMf7zku6gcAW","object":"chat.completion.chunk","created":1791900606,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"YtuLt"} - - - data: {"id":"chatcmpl-EYXUcaJJFE5TWou2BQMf7zku6gcAW","object":"chat.completion.chunk","created":1791900606,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_98b84877d4","choices":[],"usage":{"prompt_tokens":14,"completion_tokens":8,"total_tokens":22,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"BuOktpcHB3k"} - - - data: [DONE] - - - ' - headers: - access-control-expose-headers: - - X-Request-ID - - CF-Ray - - CF-Ray - alt-svc: - - h3=":443"; ma=86400 - cf-cache-status: - - DYNAMIC - cf-ray: - - a49eed050e3de7e3-YYZ - connection: - - keep-alive - content-type: - - text/event-stream; charset=utf-8 - date: - - Tue, 13 Oct 2026 14:10:06 GMT - openai-processing-ms: - - '557' - openai-version: - - '2020-10-01' - server: - - cloudflare - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - transfer-encoding: - - chunked - x-content-type-options: - - nosniff - x-openai-proxy-wasm: - - v0.1 - x-ratelimit-limit-requests: - - '30000' - x-ratelimit-limit-tokens: - - '150000000' - x-ratelimit-remaining-requests: - - '29999' - x-ratelimit-remaining-tokens: - - '149999980' - x-ratelimit-reset-requests: - - 2ms - x-ratelimit-reset-tokens: - - 0s - x-request-id: - - req_2498b2107dde491c944d456d2954d01f - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_metrics[async].yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_input_materializes_attachment[blocking-async].yaml similarity index 68% rename from py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_metrics[async].yaml rename to py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_input_materializes_attachment[blocking-async].yaml index ba0b8a6ae..3362d6a0c 100644 --- a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_metrics[async].yaml +++ b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_input_materializes_attachment[blocking-async].yaml @@ -1,6 +1,7 @@ interactions: - request: - body: '{"messages":[{"role":"user","content":"What''s 12 + 12?"}],"model":"gpt-4o-mini"}' + body: '{"messages":[{"role":"user","content":[{"type":"text","text":"What color + is this image?"},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==","detail":"low"}}]}],"model":"gpt-4o-mini"}' headers: accept: - application/json @@ -9,7 +10,7 @@ interactions: connection: - keep-alive content-length: - - '80' + - '292' content-type: - application/json host: @@ -40,17 +41,17 @@ interactions: uri: https://api.openai.com/v1/chat/completions response: body: - string: "{\n \"id\": \"chatcmpl-EYXUaCyzKCjce6yPt9BwPzTa6sKS1\",\n \"object\": - \"chat.completion\",\n \"created\": 1791900604,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + string: "{\n \"id\": \"chatcmpl-EYtp6pxfy9Is2XiCiF7ImwR9E72RB\",\n \"object\": + \"chat.completion\",\n \"created\": 1791986444,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": - \"assistant\",\n \"content\": \"12 + 12 equals 24.\",\n \"refusal\": + \"assistant\",\n \"content\": \"The image is red.\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": - 14,\n \"completion_tokens\": 8,\n \"total_tokens\": 22,\n \"prompt_tokens_details\": + 2846,\n \"completion_tokens\": 5,\n \"total_tokens\": 2851,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": - \"default\",\n \"system_fingerprint\": \"fp_98b84877d4\"\n}\n" + \"default\",\n \"system_fingerprint\": \"fp_bf0094d986\"\n}\n" headers: Access-Control-Expose-Headers: - X-Request-ID @@ -61,13 +62,13 @@ interactions: CF-Cache-Status: - DYNAMIC CF-Ray: - - a49eecf9be56b405-YYZ + - a4a71cabdc43ac34-YYZ Connection: - keep-alive Content-Type: - application/json Date: - - Tue, 13 Oct 2026 14:10:04 GMT + - Wed, 14 Oct 2026 14:00:44 GMT Server: - cloudflare Strict-Transport-Security: @@ -77,27 +78,33 @@ interactions: X-Content-Type-Options: - nosniff X-Request-ID: - - req_0fb014b849db409ebb518b29fc611349 + - req_1e999d2fdaa049acb128a87b20f1af11 content-length: - - '824' + - '827' openai-processing-ms: - - '301' + - '577' openai-version: - '2020-10-01' x-openai-proxy-wasm: - v0.1 + x-ratelimit-limit-input-images: + - '50000' x-ratelimit-limit-requests: - '30000' x-ratelimit-limit-tokens: - '150000000' + x-ratelimit-remaining-input-images: + - '49996' x-ratelimit-remaining-requests: - - '29996' + - '29997' x-ratelimit-remaining-tokens: - - '149999897' + - '149994685' + x-ratelimit-reset-input-images: + - 4ms x-ratelimit-reset-requests: - - 8ms + - 6ms x-ratelimit-reset-tokens: - - 0s + - 2ms status: code: 200 message: OK diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_metrics[sync].yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_input_materializes_attachment[blocking-sync].yaml similarity index 68% rename from py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_metrics[sync].yaml rename to py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_input_materializes_attachment[blocking-sync].yaml index 0ea032694..27db33068 100644 --- a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_metrics[sync].yaml +++ b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_input_materializes_attachment[blocking-sync].yaml @@ -1,6 +1,7 @@ interactions: - request: - body: '{"messages":[{"role":"user","content":"What''s 12 + 12?"}],"model":"gpt-4o-mini"}' + body: '{"messages":[{"role":"user","content":[{"type":"text","text":"What color + is this image?"},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==","detail":"low"}}]}],"model":"gpt-4o-mini"}' headers: accept: - application/json @@ -9,7 +10,7 @@ interactions: connection: - keep-alive content-length: - - '80' + - '292' content-type: - application/json host: @@ -40,17 +41,17 @@ interactions: uri: https://api.openai.com/v1/chat/completions response: body: - string: "{\n \"id\": \"chatcmpl-EYXUZvCgLxv6iQXr6Wou0fKqz6SQH\",\n \"object\": - \"chat.completion\",\n \"created\": 1791900603,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + string: "{\n \"id\": \"chatcmpl-EYtp5K46aNHS6fYXH8ZjtNSwZ8Das\",\n \"object\": + \"chat.completion\",\n \"created\": 1791986443,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": - \"assistant\",\n \"content\": \"12 + 12 equals 24.\",\n \"refusal\": + \"assistant\",\n \"content\": \"The image is red.\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": - 14,\n \"completion_tokens\": 8,\n \"total_tokens\": 22,\n \"prompt_tokens_details\": + 2846,\n \"completion_tokens\": 5,\n \"total_tokens\": 2851,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": - \"default\",\n \"system_fingerprint\": \"fp_98b84877d4\"\n}\n" + \"default\",\n \"system_fingerprint\": \"fp_bf0094d986\"\n}\n" headers: access-control-expose-headers: - X-Request-ID @@ -61,17 +62,17 @@ interactions: cf-cache-status: - DYNAMIC cf-ray: - - a49eecf3ebe2e7e3-YYZ + - a4a71ca1be1f1d7b-YYZ connection: - keep-alive content-length: - - '824' + - '827' content-type: - application/json date: - - Tue, 13 Oct 2026 14:10:04 GMT + - Wed, 14 Oct 2026 14:00:43 GMT openai-processing-ms: - - '403' + - '550' openai-version: - '2020-10-01' server: @@ -84,20 +85,26 @@ interactions: - nosniff x-openai-proxy-wasm: - v0.1 + x-ratelimit-limit-input-images: + - '50000' x-ratelimit-limit-requests: - '30000' x-ratelimit-limit-tokens: - '150000000' + x-ratelimit-remaining-input-images: + - '49999' x-ratelimit-remaining-requests: - - '29998' + - '29999' x-ratelimit-remaining-tokens: - - '149999965' + - '149999227' + x-ratelimit-reset-input-images: + - 1ms x-ratelimit-reset-requests: - - 4ms + - 2ms x-ratelimit-reset-tokens: - 0s x-request-id: - - req_4a22d3d84f3a4d43b9383871d54fffc5 + - req_35b324b193a445beb964830795ce33d2 status: code: 200 message: OK diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_input_materializes_attachment[streaming-async].yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_input_materializes_attachment[streaming-async].yaml new file mode 100644 index 000000000..5f485c47f --- /dev/null +++ b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_input_materializes_attachment[streaming-async].yaml @@ -0,0 +1,129 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":[{"type":"text","text":"What color + is this image?"},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==","detail":"low"}}]}],"model":"gpt-4o-mini","stream":true,"stream_options":{"include_usage":true}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '346' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 2.54.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.54.0 + x-stainless-raw-response: + - 'true' + x-stainless-read-timeout: + - '600.0' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-EYtp7vbmKNUlLGVuGA5bDynWdllQy","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"Wd8bPGPWn"} + + + data: {"id":"chatcmpl-EYtp7vbmKNUlLGVuGA5bDynWdllQy","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"IwJMmW4N"} + + + data: {"id":"chatcmpl-EYtp7vbmKNUlLGVuGA5bDynWdllQy","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":" + image"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"mVtZP"} + + + data: {"id":"chatcmpl-EYtp7vbmKNUlLGVuGA5bDynWdllQy","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":" + is"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"WimwpwqF"} + + + data: {"id":"chatcmpl-EYtp7vbmKNUlLGVuGA5bDynWdllQy","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":" + red"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"FiPsrzf"} + + + data: {"id":"chatcmpl-EYtp7vbmKNUlLGVuGA5bDynWdllQy","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"QPdKT8nTQG"} + + + data: {"id":"chatcmpl-EYtp7vbmKNUlLGVuGA5bDynWdllQy","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"VNbaB"} + + + data: {"id":"chatcmpl-EYtp7vbmKNUlLGVuGA5bDynWdllQy","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[],"usage":{"prompt_tokens":2846,"completion_tokens":5,"total_tokens":2851,"prompt_tokens_details":{"cached_tokens":2816,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"zl4O"} + + + data: [DONE] + + + ' + headers: + Access-Control-Expose-Headers: + - X-Request-ID + - CF-Ray + - CF-Ray + Alt-Svc: + - h3=":443"; ma=86400 + CF-Cache-Status: + - DYNAMIC + CF-Ray: + - a4a71cb5cff3c7b6-YYZ + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Wed, 14 Oct 2026 14:00:46 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + X-Request-ID: + - req_64ad238576f047de83ea74158ca67b52 + openai-processing-ms: + - '410' + openai-version: + - '2020-10-01' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-input-images: + - '50000' + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-input-images: + - '49998' + x-ratelimit-remaining-requests: + - '29998' + x-ratelimit-remaining-tokens: + - '149996375' + x-ratelimit-reset-input-images: + - 2ms + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 1ms + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_input_materializes_attachment[streaming-sync].yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_input_materializes_attachment[streaming-sync].yaml new file mode 100644 index 000000000..53f9bcf42 --- /dev/null +++ b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_input_materializes_attachment[streaming-sync].yaml @@ -0,0 +1,129 @@ +interactions: +- request: + body: '{"messages":[{"role":"user","content":[{"type":"text","text":"What color + is this image?"},{"type":"image_url","image_url":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==","detail":"low"}}]}],"model":"gpt-4o-mini","stream":true,"stream_options":{"include_usage":true}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '346' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 2.54.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.54.0 + x-stainless-raw-response: + - 'true' + x-stainless-read-timeout: + - '600.0' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-EYtp7uw24Hy9MLO6l2oPPeFnzNl9V","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"TW41ygRYE"} + + + data: {"id":"chatcmpl-EYtp7uw24Hy9MLO6l2oPPeFnzNl9V","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"mrbL1cbQ"} + + + data: {"id":"chatcmpl-EYtp7uw24Hy9MLO6l2oPPeFnzNl9V","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":" + image"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"mjsbU"} + + + data: {"id":"chatcmpl-EYtp7uw24Hy9MLO6l2oPPeFnzNl9V","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":" + is"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"hsqE9Cfd"} + + + data: {"id":"chatcmpl-EYtp7uw24Hy9MLO6l2oPPeFnzNl9V","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":" + red"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"5UMOUsM"} + + + data: {"id":"chatcmpl-EYtp7uw24Hy9MLO6l2oPPeFnzNl9V","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"j94dYx8OTD"} + + + data: {"id":"chatcmpl-EYtp7uw24Hy9MLO6l2oPPeFnzNl9V","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"Gg3XJ"} + + + data: {"id":"chatcmpl-EYtp7uw24Hy9MLO6l2oPPeFnzNl9V","object":"chat.completion.chunk","created":1791986445,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_bf0094d986","choices":[],"usage":{"prompt_tokens":2846,"completion_tokens":5,"total_tokens":2851,"prompt_tokens_details":{"cached_tokens":2816,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"s3fl"} + + + data: [DONE] + + + ' + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a4a71cb03dcb1d7b-YYZ + connection: + - keep-alive + content-type: + - text/event-stream; charset=utf-8 + date: + - Wed, 14 Oct 2026 14:00:45 GMT + openai-processing-ms: + - '354' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-input-images: + - '50000' + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-input-images: + - '49999' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999227' + x-ratelimit-reset-input-images: + - 1ms + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_442f2b5c55e24a9f921afdaceadd84a1 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/litellm/test_litellm.py b/py/src/braintrust/integrations/litellm/test_litellm.py index ead3681c8..9f97e3775 100644 --- a/py/src/braintrust/integrations/litellm/test_litellm.py +++ b/py/src/braintrust/integrations/litellm/test_litellm.py @@ -1,4 +1,5 @@ import asyncio +import base64 import os import time import uuid @@ -106,35 +107,54 @@ def memory_logger(): yield bgl -@pytest.mark.vcr +@pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path", "query", "body"]) @sync_async -def test_litellm_completion_metrics(memory_logger, is_async) -> None: - assert not memory_logger.pop() - +@pytest.mark.parametrize("stream", [False, True], ids=["blocking", "streaming"]) +def test_litellm_image_input_materializes_attachment(memory_logger, is_async, stream): + """Completion spans capture output, usage, and attachments in every call mode.""" + image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==" + image_url = f"data:image/png;base64,{image_base64}" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What color is this image?"}, + {"type": "image_url", "image_url": {"url": image_url, "detail": "low"}}, + ], + } + ] + run = _run_stream if stream else _run start = time.time() - response = _run( - is_async, - litellm.completion, - litellm.acompletion, - model=TEST_MODEL, - messages=[{"role": "user", "content": TEST_PROMPT}], + response = run( + is_async, litellm.completion, litellm.acompletion, model=TEST_MODEL, messages=messages, stream=stream ) end = time.time() - - assert response - assert response.choices[0].message.content - assert "24" in response.choices[0].message.content or "twenty-four" in response.choices[0].message.content.lower() - + if stream: + assert len(response) > 1 + content = "".join(chunk.choices[0].delta.content or "" for chunk in response if chunk.choices) + else: + content = response.choices[0].message.content + assert content + assert messages[0]["content"][1]["image_url"] == {"url": image_url, "detail": "low"} spans = memory_logger.pop() assert len(spans) == 1 span = spans[0] - assert span assert span["context"]["span_origin"]["instrumentation"]["name"] == "litellm-auto" - metrics = span["metrics"] - assert_metrics_are_valid(metrics, start, end) + assert span["span_attributes"]["name"] == "Completion" + assert span["span_attributes"]["type"] == "llm" assert span["metadata"]["model"] == TEST_MODEL assert span["metadata"]["provider"] == "openai" - assert span["input"] == [{"role": "user", "content": TEST_PROMPT}] + assert_metrics_are_valid(span["metrics"], start, end) + assert span["metrics"]["time_to_first_token"] >= 0 + assert span["output"][0]["message"]["content"] == content + assert span["input"][0]["role"] == "user" + assert span["input"][0]["content"][0] == messages[0]["content"][0] + logged_image = span["input"][0]["content"][1]["image_url"] + assert logged_image["detail"] == "low" + attachment = logged_image["url"] + assert isinstance(attachment, Attachment) + assert attachment.data == base64.b64decode(image_base64) + assert attachment.reference["content_type"] == "image/png" @pytest.mark.vcr @@ -241,48 +261,6 @@ def test_litellm_text_completion_metrics(memory_logger, is_async) -> None: assert "text" in span["output"][0] -@pytest.mark.vcr -@sync_async -def test_litellm_completion_streaming(memory_logger, is_async): - assert not memory_logger.pop() - - start = time.time() - chunks = _run_stream( - is_async, - litellm.completion, - litellm.acompletion, - model=TEST_MODEL, - messages=[{"role": "user", "content": TEST_PROMPT}], - stream=True, - ) - end = time.time() - - # Verify streaming works - assert chunks - assert len(chunks) > 1 - - # Concatenate content from chunks to verify - content = "" - for chunk in chunks: - if chunk.choices and chunk.choices[0].delta.content: - content += chunk.choices[0].delta.content - - # Make sure we got a valid answer in the content - assert "24" in content or "twenty-four" in content.lower() - - # Verify spans were created - spans = memory_logger.pop() - assert len(spans) == 1 - span = spans[0] - assert span - metrics = span["metrics"] - assert_metrics_are_valid(metrics, start, end) - assert span["metadata"]["model"] == TEST_MODEL - assert span["metadata"]["provider"] == "openai" - assert TEST_PROMPT in str(span["input"]) - assert "24" in str(span["output"]) or "twenty-four" in str(span["output"]).lower() - - @pytest.mark.vcr @sync_async def test_litellm_decisions_metrics(memory_logger, is_async): @@ -331,38 +309,56 @@ def test_litellm_decisions_image(memory_logger, is_async): assert_decisions_image_span(memory_logger.pop(), response, messages) -@pytest.mark.vcr -@sync_async -def test_litellm_responses_metrics(memory_logger, is_async): - assert not memory_logger.pop() - +@pytest.mark.parametrize( + "stream,is_async", + [ + pytest.param( + stream, + is_async, + id=f"{'streaming' if stream else 'blocking'}-{'async' if is_async else 'sync'}", + marks=pytest.mark.vcr( + cassette_name=f"test_litellm_responses_{'streaming' if stream else 'metrics'}[{'async' if is_async else 'sync'}]" + ), + ) + for stream in (False, True) + for is_async in (False, True) + ], +) +def test_litellm_responses(memory_logger, is_async, stream): + """Reuse the existing blocking/streaming recordings for the Responses API.""" + run = _run_stream if stream else _run + params = {"stream": True} if stream else {"instructions": "Just the number please"} start = time.time() - response = _run( + response = run( is_async, litellm.responses, litellm.aresponses, model=TEST_MODEL, input=TEST_PROMPT, - instructions="Just the number please", + **params, ) end = time.time() - assert response - assert response.output - assert len(response.output) > 0 - content = response.output[0].content[0].text + if stream: + assert len(response) > 1 + content = "".join( + chunk.delta for chunk in response if getattr(chunk, "type", None) == "response.output_text.delta" + ) + else: + content = response.output[0].content[0].text assert "24" in content or "twenty-four" in content.lower() - # Verify spans were created spans = memory_logger.pop() assert len(spans) == 1 span = spans[0] - assert span metrics = span["metrics"] assert_metrics_are_valid(metrics, start, end) assert span["metadata"]["model"] == TEST_MODEL assert span["metadata"]["provider"] == "openai" assert TEST_PROMPT in str(span["input"]) + assert "24" in str(span["output"]) or "twenty-four" in str(span["output"]).lower() + if stream: + assert span["metadata"]["stream"] is True @pytest.mark.vcr @@ -500,7 +496,6 @@ def test_litellm_speech(memory_logger, is_async): _assert_speech_output_attachment(span) -@pytest.mark.vcr @sync_async def test_litellm_completion_error(memory_logger, is_async): assert not memory_logger.pop() @@ -508,7 +503,7 @@ def test_litellm_completion_error(memory_logger, is_async): # Use a non-existent model to force an error fake_model = "non-existent-model" - try: + with pytest.raises(litellm.BadRequestError): _run( is_async, litellm.completion, @@ -516,17 +511,12 @@ def test_litellm_completion_error(memory_logger, is_async): model=fake_model, messages=[{"role": "user", "content": TEST_PROMPT}], ) - pytest.fail("Expected an exception but none was raised") - except Exception: - # We expect an error here - pass logs = memory_logger.pop() assert len(logs) == 1 log = logs[0] assert log["project_id"] == PROJECT_NAME - # Check that we got a log entry with the fake model - assert fake_model in str(log) + assert log["metadata"]["model"] == fake_model @pytest.mark.vcr @@ -620,82 +610,6 @@ def test_litellm_tool_calls(memory_logger): ) -@pytest.mark.vcr -@sync_async -def test_litellm_responses_streaming(memory_logger, is_async): - """Test the responses API with streaming.""" - assert not memory_logger.pop() - - start = time.time() - chunks = _run_stream( - is_async, - litellm.responses, - litellm.aresponses, - model=TEST_MODEL, - input="What's 12 + 12?", - stream=True, - ) - end = time.time() - - assert chunks - assert len(chunks) > 1 - output = "".join(chunk.delta for chunk in chunks if getattr(chunk, "type", None) == "response.output_text.delta") - assert "24" in output - - # Verify the span is created - spans = memory_logger.pop() - assert len(spans) == 1 - span = spans[0] - metrics = span["metrics"] - assert_metrics_are_valid(metrics, start, end) - assert span["metadata"]["stream"] is True - assert "What's 12 + 12?" in str(span["input"]) - assert "24" in str(span["output"]) - - -@pytest.mark.vcr -@pytest.mark.asyncio -async def test_litellm_async_streaming_with_break(memory_logger): - """Test breaking out of the async streaming loop early.""" - assert not memory_logger.pop() - - start = time.time() - stream = await litellm.acompletion( - model=TEST_MODEL, messages=[{"role": "user", "content": TEST_PROMPT}], stream=True - ) - - # Only process the first few chunks - counter = 0 - async for chunk in stream: - counter += 1 - if counter >= 2: - break - end = time.time() - - # We should still get valid metrics even with early break - spans = memory_logger.pop() - assert len(spans) == 1 - span = spans[0] - metrics = span["metrics"] - assert metrics["time_to_first_token"] >= 0 - - -def test_litellm_is_numeric_excludes_booleans(): - """Reproduce issue #1357: _is_numeric should exclude booleans. - - OpenRouter returns `is_byok: true` in usage data. Since Python's bool is - a subclass of int, isinstance(True, int) is True. The _is_numeric function - must explicitly exclude booleans so they don't end up in metrics, which - causes a 400 from the API (expected number, received boolean). - """ - from braintrust.util import is_numeric - - assert is_numeric(1) - assert is_numeric(1.0) - assert not is_numeric(True) - assert not is_numeric(False) - - @pytest.mark.vcr def test_litellm_openrouter_no_booleans_in_metrics(memory_logger): """Reproduce issue #1357: OpenRouter returns is_byok boolean in usage. diff --git a/py/src/braintrust/integrations/litellm/tracing.py b/py/src/braintrust/integrations/litellm/tracing.py index d01a9c2b3..abde530fd 100644 --- a/py/src/braintrust/integrations/litellm/tracing.py +++ b/py/src/braintrust/integrations/litellm/tracing.py @@ -7,7 +7,7 @@ from types import TracebackType from typing import Any -from braintrust.integrations.openai.tracing import _decisions_input, _decisions_output +from braintrust.integrations.openai.tracing import _decisions_input, _decisions_output, _process_attachments_in_input from braintrust.integrations.utils import ( _extract_audio_output, _materialize_attachment, @@ -862,7 +862,7 @@ def _update_span_payload_from_params(params: dict[str, Any], input_key: str = "i return merge_dicts( span_info_d, - {"input": input_data, "metadata": metadata}, + {"input": _process_attachments_in_input(input_data), "metadata": metadata}, ) diff --git a/py/uv.lock b/py/uv.lock index f9652322d..2010a36b8 100644 --- a/py/uv.lock +++ b/py/uv.lock @@ -1095,6 +1095,9 @@ test-deepagents = [ { name = "pytest-asyncio" }, { name = "pytest-vcr" }, ] +test-dspy = [ + { name = "pillow" }, +] test-huggingface-hub = [ { name = "aiohttp" }, { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-instructor') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-llamaindex') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-llamaindex-0-13') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-instructor') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-llamaindex') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-llamaindex-0-13') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-instructor') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-llamaindex') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-llamaindex-0-13') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-instructor') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-llamaindex') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-llamaindex-0-13') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-instructor') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-llamaindex') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-llamaindex-0-13') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-instructor' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-instructor' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-instructor' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-instructor' and extra == 'group-10-braintrust-test-llamaindex') or (extra == 'group-10-braintrust-test-instructor' and extra == 'group-10-braintrust-test-llamaindex-0-13') or (extra == 'group-10-braintrust-test-instructor' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-instructor' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-llamaindex') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-llamaindex-0-13') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-llamaindex') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-llamaindex-0-13') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-llamaindex') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-llamaindex-0-13') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire-constraints') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-llamaindex' and extra == 'group-10-braintrust-test-llamaindex-0-13') or (extra == 'group-10-braintrust-test-llamaindex' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-llamaindex' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-llamaindex-0-13' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-llamaindex-0-13' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands')" }, @@ -1368,6 +1371,7 @@ test-deepagents = [ { name = "pytest-asyncio", specifier = "==1.3.0" }, { name = "pytest-vcr", specifier = "==1.0.2" }, ] +test-dspy = [{ name = "pillow", specifier = "==12.3.0" }] test-huggingface-hub = [ { name = "aiohttp" }, { name = "numpy" },