From 472a8e728de5a4ff8a86b73e81eecbaebed9947c Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Tue, 13 Oct 2026 11:37:53 -0400 Subject: [PATCH 1/3] feat(integrations): trace openai and litellm decisions Emit question spans for sync and async Decisions calls using the shared question and answer structure used by TypeSafe. Capture reported token usage, cache writes, confidence, and probability distributions. Materialize iterable inputs and question criteria without mutating caller objects or changing SDK argument validation. Add VCR-backed regression and auto-instrumentation coverage. Closes #921 --- .../auto_test_scripts/test_auto_litellm.py | 27 +++ .../auto_test_scripts/test_auto_openai.py | 31 +++ ...test_litellm_decisions_metrics[async].yaml | 82 ++++++++ .../test_litellm_decisions_metrics[sync].yaml | 82 ++++++++ .../integrations/litellm/patchers.py | 16 ++ .../integrations/litellm/test_litellm.py | 37 ++++ .../integrations/litellm/tracing.py | 48 +++++ ...penai_decisions_iterable_input[async].yaml | 97 ++++++++++ ...openai_decisions_iterable_input[sync].yaml | 97 ++++++++++ .../test_openai_decisions_metrics[async].yaml | 88 +++++++++ .../test_openai_decisions_metrics[sync].yaml | 100 ++++++++++ .../integrations/openai/integration.py | 2 + .../integrations/openai/patchers.py | 27 +++ .../integrations/openai/test_openai.py | 74 ++++++++ .../braintrust/integrations/openai/tracing.py | 177 ++++++++++++++++- py/src/braintrust/integrations/question.py | 33 ++++ .../braintrust/integrations/test_decisions.py | 179 ++++++++++++++++++ .../integrations/typesafe/tracing.py | 25 ++- 18 files changed, 1208 insertions(+), 14 deletions(-) create mode 100644 py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_decisions_metrics[async].yaml create mode 100644 py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_decisions_metrics[sync].yaml create mode 100644 py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_iterable_input[async].yaml create mode 100644 py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_iterable_input[sync].yaml create mode 100644 py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_metrics[async].yaml create mode 100644 py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_metrics[sync].yaml create mode 100644 py/src/braintrust/integrations/question.py create mode 100644 py/src/braintrust/integrations/test_decisions.py diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_litellm.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_litellm.py index d5cdd667e..8dd7f529a 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_litellm.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_litellm.py @@ -1,9 +1,18 @@ """Test auto_instrument for LiteLLM.""" +import asyncio + import litellm from braintrust.auto import auto_instrument from braintrust.integrations.litellm import LiteLLMIntegration +from braintrust.integrations.test_decisions import ( + DECISIONS_INPUT, + DECISIONS_MODEL, + DECISIONS_QUESTIONS, + assert_decisions_span, +) from braintrust.integrations.test_utils import autoinstrument_test_context +from braintrust.integrations.versioning import detect_module_version, version_satisfies # 1. Verify not patched initially @@ -34,4 +43,22 @@ span = spans[0] assert span["metadata"]["provider"] == "openai" +if version_satisfies(detect_module_version(litellm, ("litellm",)), ">=1.105.0"): + with autoinstrument_test_context("test_litellm_decisions_metrics[async]", integration="litellm") as memory_logger: + response = asyncio.run( + litellm.adecisions( + model=f"openai/{DECISIONS_MODEL}", + input=DECISIONS_INPUT, + questions=DECISIONS_QUESTIONS, + safety_identifier="do-not-log-this-identifier", + ) + ) + assert_decisions_span( + memory_logger.pop(), + response, + name="Decision", + model=f"openai/{DECISIONS_MODEL}", + instrumentation="litellm-auto", + ) + print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_openai.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_openai.py index b988640a6..ac9e63710 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_openai.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_openai.py @@ -1,11 +1,19 @@ """Test auto_instrument for OpenAI.""" +import asyncio import inspect import openai from braintrust import wrap_openai from braintrust.auto import auto_instrument +from braintrust.integrations.test_decisions import ( + DECISIONS_INPUT, + DECISIONS_MODEL, + DECISIONS_QUESTIONS, + assert_decisions_span, +) from braintrust.integrations.test_utils import autoinstrument_test_context +from braintrust.integrations.versioning import version_satisfies from wrapt import FunctionWrapper @@ -15,6 +23,10 @@ def _is_braintrust_wrapped(resource) -> bool: resources = [openai.resources.chat.completions.Completions] +if version_satisfies(openai.__version__, ">=3.26.0"): + from openai.resources.decisions import AsyncDecisions, Decisions + + resources.extend((Decisions, AsyncDecisions)) try: from openai.resources.beta.agents.sessions.sessions import Sessions @@ -66,4 +78,23 @@ def _is_braintrust_wrapped(resource) -> bool: assert spans[0]["metrics"]["prompt_tokens"] == response.usage.input_tokens assert spans[0]["metrics"]["completion_tokens"] == response.usage.output_tokens +if version_satisfies(openai.__version__, ">=3.26.0"): + with autoinstrument_test_context("test_openai_decisions_metrics[async]", integration="openai") as memory_logger: + client = wrap_openai(wrap_openai(openai.AsyncOpenAI())) + response = asyncio.run( + client.decisions.create( + model=DECISIONS_MODEL, + input=DECISIONS_INPUT, + questions=DECISIONS_QUESTIONS, + safety_identifier="do-not-log-this-identifier", + ) + ) + assert_decisions_span( + memory_logger.pop(), + response, + name="openai.decisions.create", + model=DECISIONS_MODEL, + instrumentation="openai-auto", + ) + print("SUCCESS") diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_decisions_metrics[async].yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_decisions_metrics[async].yaml new file mode 100644 index 000000000..49366214f --- /dev/null +++ b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_decisions_metrics[async].yaml @@ -0,0 +1,82 @@ +interactions: +- request: + body: '{"model":"gpt-6-luna","input":"The customer says: I love this product!","questions":[{"type":"predicate","name":"positive","instructions":"Is + the sentiment positive?"},{"type":"choice","name":"sentiment","instructions":"What + is the sentiment?","choices":[{"value":"positive","description":"Happy feedback"},{"value":"negative"}]},{"type":"score","name":"enthusiasm","instructions":"How + enthusiastic is the customer?","levels":[{"label":"low"},{"label":"high","description":"Very + enthusiastic"}]},{"type":"predicate","instructions":"Does the customer like + the product?"}],"safety_identifier":"do-not-log-this-identifier"}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '620' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - litellm/1.105.0 + method: POST + uri: https://api.openai.com/v1/decisions + response: + body: + string: '{"model":"gpt-6-luna","answers":[{"type":"predicate","name":"positive","probability":1.0},{"type":"choice","name":"sentiment","choice":"positive","probabilities":[{"value":"positive","probability":1.0},{"value":"negative","probability":0.0}],"confidence":1.0},{"type":"score","name":"enthusiasm","score":1.0,"probabilities":[{"value":0,"label":"low","probability":0.0},{"value":1,"label":"high","probability":1.0}],"confidence":1.0},{"type":"predicate","name":null,"probability":1.0}],"usage":{"input_tokens":524,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":0,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":524}}' + headers: + Access-Control-Allow-Origin: + - '*' + Access-Control-Expose-Headers: + - x-request-id,retry-after,server-timing-total + - CF-Ray + Alt-Svc: + - h3=":443"; ma=86400 + CF-RAY: + - a49f0ec66fc3180e-YYZ + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Tue, 13 Oct 2026 14:33:09 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + Vary: + - origin, access-control-request-method, access-control-request-headers + X-Content-Type-Options: + - nosniff + X-Request-ID: + - req_ad2cbcdd424f4f08adde160d303c9c40 + cf-cache-status: + - DYNAMIC + content-length: + - '664' + openai-processing-ms: + - '134' + server-timing-total: + - '134.317' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '180000000' + x-ratelimit-remaining-requests: + - '29996' + x-ratelimit-remaining-tokens: + - '179997900' + x-ratelimit-reset-requests: + - 8ms + x-ratelimit-reset-tokens: + - 0s + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_decisions_metrics[sync].yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_decisions_metrics[sync].yaml new file mode 100644 index 000000000..b1c1d63d2 --- /dev/null +++ b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_decisions_metrics[sync].yaml @@ -0,0 +1,82 @@ +interactions: +- request: + body: '{"model":"gpt-6-luna","input":"The customer says: I love this product!","questions":[{"type":"predicate","name":"positive","instructions":"Is + the sentiment positive?"},{"type":"choice","name":"sentiment","instructions":"What + is the sentiment?","choices":[{"value":"positive","description":"Happy feedback"},{"value":"negative"}]},{"type":"score","name":"enthusiasm","instructions":"How + enthusiastic is the customer?","levels":[{"label":"low"},{"label":"high","description":"Very + enthusiastic"}]},{"type":"predicate","instructions":"Does the customer like + the product?"}],"safety_identifier":"do-not-log-this-identifier"}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '620' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - litellm/1.105.0 + method: POST + uri: https://api.openai.com/v1/decisions + response: + body: + string: '{"model":"gpt-6-luna","answers":[{"type":"predicate","name":"positive","probability":1.0},{"type":"choice","name":"sentiment","choice":"positive","probabilities":[{"value":"positive","probability":1.0},{"value":"negative","probability":0.0}],"confidence":1.0},{"type":"score","name":"enthusiasm","score":1.0,"probabilities":[{"value":0,"label":"low","probability":0.0},{"value":1,"label":"high","probability":1.0}],"confidence":1.0},{"type":"predicate","name":null,"probability":1.0}],"usage":{"input_tokens":524,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":0,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":524}}' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - x-request-id,retry-after,server-timing-total + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a49f0ec4b8af592d-YYZ + connection: + - keep-alive + content-length: + - '664' + content-type: + - application/json + date: + - Tue, 13 Oct 2026 14:33:08 GMT + openai-processing-ms: + - '62' + server: + - cloudflare + server-timing-total: + - '62.746' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - origin, access-control-request-method, access-control-request-headers + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '180000000' + x-ratelimit-remaining-requests: + - '29997' + x-ratelimit-remaining-tokens: + - '179998425' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_a88e743b0474432c9b7a5898edd89a6c + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/litellm/patchers.py b/py/src/braintrust/integrations/litellm/patchers.py index 2c1d487ed..a05b18881 100644 --- a/py/src/braintrust/integrations/litellm/patchers.py +++ b/py/src/braintrust/integrations/litellm/patchers.py @@ -6,6 +6,7 @@ from .tracing import ( _acompletion_wrapper_async, + _adecisions_wrapper_async, _aembedding_wrapper_async, _aimage_generation_wrapper_async, _amoderation_wrapper_async, @@ -15,6 +16,7 @@ _atext_completion_wrapper_async, _atranscription_wrapper_async, _completion_wrapper, + _decisions_wrapper, _embedding_wrapper, _image_generation_wrapper, _moderation_wrapper, @@ -67,6 +69,18 @@ class LiteLLMAresponsesPatcher(FunctionWrapperPatcher): wrapper = _aresponses_wrapper_async +class LiteLLMDecisionsPatcher(FunctionWrapperPatcher): + name = "litellm.decisions" + target_path = "decisions" + wrapper = _decisions_wrapper + + +class LiteLLMAdecisionsPatcher(FunctionWrapperPatcher): + name = "litellm.adecisions" + target_path = "adecisions" + wrapper = _adecisions_wrapper_async + + class LiteLLMImageGenerationPatcher(FunctionWrapperPatcher): name = "litellm.image_generation" target_path = "image_generation" @@ -150,6 +164,8 @@ class LiteLLMArerankPatcher(FunctionWrapperPatcher): LiteLLMATextCompletionPatcher, LiteLLMResponsesPatcher, LiteLLMAresponsesPatcher, + LiteLLMDecisionsPatcher, + LiteLLMAdecisionsPatcher, LiteLLMImageGenerationPatcher, LiteLLMAimageGenerationPatcher, LiteLLMEmbeddingPatcher, diff --git a/py/src/braintrust/integrations/litellm/test_litellm.py b/py/src/braintrust/integrations/litellm/test_litellm.py index 43de185a5..cf390970f 100644 --- a/py/src/braintrust/integrations/litellm/test_litellm.py +++ b/py/src/braintrust/integrations/litellm/test_litellm.py @@ -8,10 +8,17 @@ from braintrust import Attachment, logger from braintrust.conftest import get_vcr_config from braintrust.integrations.litellm import patch_litellm +from braintrust.integrations.test_decisions import ( + DECISIONS_INPUT, + DECISIONS_MODEL, + DECISIONS_QUESTIONS, + assert_decisions_span, +) from braintrust.integrations.test_utils import ( assert_metrics_are_valid, verify_autoinstrument_script, ) +from braintrust.integrations.versioning import detect_module_version, version_satisfies from braintrust.test_helpers import assert_dict_matches, init_test_logger @@ -274,6 +281,36 @@ def test_litellm_completion_streaming(memory_logger, is_async): assert "24" in str(span["output"]) or "twenty-four" in str(span["output"]).lower() +@pytest.mark.vcr +@sync_async +def test_litellm_decisions_metrics(memory_logger, is_async): + if not version_satisfies(detect_module_version(litellm, ("litellm",)), ">=1.105.0"): + pytest.skip("Decisions requires LiteLLM >=1.105.0") + + model = f"openai/{DECISIONS_MODEL}" + start = time.time() + response = _run( + is_async, + litellm.decisions, + litellm.adecisions, + model, + DECISIONS_INPUT, + DECISIONS_QUESTIONS, + safety_identifier="do-not-log-this-identifier", + ) + end = time.time() + + assert_decisions_span( + memory_logger.pop(), + response, + name="Decision", + model=model, + instrumentation="litellm-auto", + start=start, + end=end, + ) + + @pytest.mark.vcr @sync_async def test_litellm_responses_metrics(memory_logger, is_async): diff --git a/py/src/braintrust/integrations/litellm/tracing.py b/py/src/braintrust/integrations/litellm/tracing.py index f26c7b76f..71e47f7d8 100644 --- a/py/src/braintrust/integrations/litellm/tracing.py +++ b/py/src/braintrust/integrations/litellm/tracing.py @@ -7,6 +7,7 @@ from types import TracebackType from typing import Any +from braintrust.integrations.openai.tracing import _decisions_input, _decisions_output from braintrust.integrations.utils import ( _extract_audio_output, _materialize_attachment, @@ -342,6 +343,53 @@ async def _aresponses_wrapper_async(wrapped, instance, args, kwargs): span.end() +def _decisions_span_payload(args, kwargs): + # Decisions accepts model, input and questions positionally as well as by keyword. + params = dict(zip(("model", "input", "questions"), args)) + params.update(kwargs) + return merge_dicts( + params.get("span_info", {}), + { + "input": _decisions_input(params.get("state", params.get("input")), params.get("questions") or []), + "metadata": _build_span_metadata({}, params.get("model")), + }, + ) + + +def _decisions_wrapper(wrapped, instance, args, kwargs): + with start_span( + **merge_dicts( + dict(name="Decision", span_attributes={"type": SpanTypeAttribute.QUESTION}), + _decisions_span_payload(args, kwargs), + ) + ) as span: + response = wrapped(*args, **kwargs) + span.log( + output=_decisions_output( + _get_field(response, "answers"), kwargs.get("questions", args[2] if len(args) > 2 else []) + ), + metrics=_parse_metrics_from_usage(_get_field(response, "usage")), + ) + return response + + +async def _adecisions_wrapper_async(wrapped, instance, args, kwargs): + with start_span( + **merge_dicts( + dict(name="Decision", span_attributes={"type": SpanTypeAttribute.QUESTION}), + _decisions_span_payload(args, kwargs), + ) + ) as span: + response = await wrapped(*args, **kwargs) + span.log( + output=_decisions_output( + _get_field(response, "answers"), kwargs.get("questions", args[2] if len(args) > 2 else []) + ), + metrics=_parse_metrics_from_usage(_get_field(response, "usage")), + ) + return response + + def _image_attachment_from_base64( data: Any, *, output_format: Any, index: int ) -> tuple[_ResolvedAttachment | None, int | None]: diff --git a/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_iterable_input[async].yaml b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_iterable_input[async].yaml new file mode 100644 index 000000000..f476cab8e --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_iterable_input[async].yaml @@ -0,0 +1,97 @@ +interactions: +- request: + body: '{"model":"gpt-6-luna","input":[{"role":"user","content":[{"type":"input_text","text":"The + customer says: I love this product!"}]}],"questions":[{"type":"predicate","name":"positive","instructions":"Is + the sentiment positive?"}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '228' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 3.29.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 3.29.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/decisions + response: + body: + string: '{"model":"gpt-6-luna","answers":[{"type":"predicate","name":"positive","probability":1.0}],"usage":{"input_tokens":162,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":0,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":162}}' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - x-request-id,retry-after,server-timing-total + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a49f633acdcf39ff-YYZ + connection: + - keep-alive + content-length: + - '270' + content-type: + - application/json + date: + - Tue, 13 Oct 2026 15:30:48 GMT + openai-processing-ms: + - '74' + server: + - cloudflare + server-timing-total: + - '74.548' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - origin, access-control-request-method, access-control-request-headers + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '180000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '179999838' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_3c64e5071d234548970eb73bb276fa58 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_iterable_input[sync].yaml b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_iterable_input[sync].yaml new file mode 100644 index 000000000..27f83bb9a --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_iterable_input[sync].yaml @@ -0,0 +1,97 @@ +interactions: +- request: + body: '{"model":"gpt-6-luna","input":[{"role":"user","content":[{"type":"input_text","text":"The + customer says: I love this product!"}]}],"questions":[{"type":"predicate","name":"positive","instructions":"Is + the sentiment positive?"}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '228' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 3.29.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 3.29.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/decisions + response: + body: + string: '{"model":"gpt-6-luna","answers":[{"type":"predicate","name":"positive","probability":1.0}],"usage":{"input_tokens":162,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":0,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":162}}' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - x-request-id,retry-after,server-timing-total + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a49f6335c841ac75-YYZ + connection: + - keep-alive + content-length: + - '270' + content-type: + - application/json + date: + - Tue, 13 Oct 2026 15:30:47 GMT + openai-processing-ms: + - '103' + server: + - cloudflare + server-timing-total: + - '103.148' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - origin, access-control-request-method, access-control-request-headers + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '180000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '179999838' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_858d6124ba054d3bb167a9d381a15c37 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_metrics[async].yaml b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_metrics[async].yaml new file mode 100644 index 000000000..7e71826d7 --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_metrics[async].yaml @@ -0,0 +1,88 @@ +interactions: +- request: + body: '{"model":"gpt-6-luna","input":"The customer says: I love this product!","questions":[{"type":"predicate","name":"positive","instructions":"Is + the sentiment positive?"},{"type":"choice","name":"sentiment","instructions":"What + is the sentiment?","choices":[{"value":"positive","description":"Happy feedback"},{"value":"negative"}]},{"type":"score","name":"enthusiasm","instructions":"How + enthusiastic is the customer?","levels":[{"label":"low"},{"label":"high","description":"Very + enthusiastic"}]},{"type":"predicate","instructions":"Does the customer like + the product?"}],"safety_identifier":"do-not-log-this-identifier"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '620' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 3.29.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 3.29.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/decisions + response: + body: + string: '{"model":"gpt-6-luna","answers":[{"type":"predicate","name":"positive","probability":1.0},{"type":"choice","name":"sentiment","choice":"positive","probabilities":[{"value":"positive","probability":1.0},{"value":"negative","probability":0.0}],"confidence":1.0},{"type":"score","name":"enthusiasm","score":1.0,"probabilities":[{"value":0,"label":"low","probability":0.0},{"value":1,"label":"high","probability":1.0}],"confidence":1.0},{"type":"predicate","name":null,"probability":1.0}],"usage":{"input_tokens":524,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":0,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":524}}' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - x-request-id,retry-after,server-timing-total + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a49f0eb61b21a1db-YYZ + connection: + - keep-alive + content-length: + - '664' + content-type: + - application/json + date: + - Tue, 13 Oct 2026 14:33:06 GMT + openai-processing-ms: + - '80' + server: + - cloudflare + server-timing-total: + - '80.087' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - origin, access-control-request-method, access-control-request-headers + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-request-id: + - req_3134eed4b83b4f4ea2114a24ce9aaf77 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_metrics[sync].yaml b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_metrics[sync].yaml new file mode 100644 index 000000000..15aa7b821 --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_metrics[sync].yaml @@ -0,0 +1,100 @@ +interactions: +- request: + body: '{"model":"gpt-6-luna","input":"The customer says: I love this product!","questions":[{"type":"predicate","name":"positive","instructions":"Is + the sentiment positive?"},{"type":"choice","name":"sentiment","instructions":"What + is the sentiment?","choices":[{"value":"positive","description":"Happy feedback"},{"value":"negative"}]},{"type":"score","name":"enthusiasm","instructions":"How + enthusiastic is the customer?","levels":[{"label":"low"},{"label":"high","description":"Very + enthusiastic"}]},{"type":"predicate","instructions":"Does the customer like + the product?"}],"safety_identifier":"do-not-log-this-identifier"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '620' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 3.29.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 3.29.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/decisions + response: + body: + string: '{"model":"gpt-6-luna","answers":[{"type":"predicate","name":"positive","probability":1.0},{"type":"choice","name":"sentiment","choice":"positive","probabilities":[{"value":"positive","probability":1.0},{"value":"negative","probability":0.0}],"confidence":1.0},{"type":"score","name":"enthusiasm","score":1.0,"probabilities":[{"value":0,"label":"low","probability":0.0},{"value":1,"label":"high","probability":1.0}],"confidence":1.0},{"type":"predicate","name":null,"probability":1.0}],"usage":{"input_tokens":524,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":0,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":524}}' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - x-request-id,retry-after,server-timing-total + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a49f0eb42cfca06e-YYZ + connection: + - keep-alive + content-length: + - '664' + content-type: + - application/json + date: + - Tue, 13 Oct 2026 14:33:06 GMT + openai-processing-ms: + - '112' + server: + - cloudflare + server-timing-total: + - '112.779' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - origin, access-control-request-method, access-control-request-headers + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '180000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '179999475' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_02f6c91e11bd48d299ddf7359b841be7 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/integration.py b/py/src/braintrust/integrations/openai/integration.py index f5f1fd06d..dd8763197 100644 --- a/py/src/braintrust/integrations/openai/integration.py +++ b/py/src/braintrust/integrations/openai/integration.py @@ -8,6 +8,7 @@ AudioTranscriptionsPatcher, AudioTranslationsPatcher, ChatCompletionsPatcher, + DecisionsPatcher, EmbeddingsPatcher, ImagesPatcher, ModerationsPatcher, @@ -30,4 +31,5 @@ class OpenAIIntegration(BaseIntegration): AudioTranslationsPatcher, ImagesPatcher, ResponsesPatcher, + DecisionsPatcher, ) diff --git a/py/src/braintrust/integrations/openai/patchers.py b/py/src/braintrust/integrations/openai/patchers.py index 0bc69fa4e..b924d7e77 100644 --- a/py/src/braintrust/integrations/openai/patchers.py +++ b/py/src/braintrust/integrations/openai/patchers.py @@ -13,6 +13,7 @@ _audio_translation_create_wrapper, _chat_completion_create_wrapper, _chat_completion_parse_wrapper, + _decisions_create_wrapper, _embedding_create_wrapper, _image_create_variation_wrapper, _image_edit_wrapper, @@ -436,6 +437,31 @@ class _WrapResponsesRaw(CompositeFunctionWrapperPatcher): sub_patchers = (_wrap_resp_raw_create, _wrap_resp_raw_parse) +# --------------------------------------------------------------------------- +# Decisions (OpenAI >= 3.26.0) +# --------------------------------------------------------------------------- + +_dec_create_sync, _dec_create_async, _wrap_dec_create = _make_method_patchers( + name_prefix="openai.decisions.create", + target_module="openai.resources.decisions", + sync_class="Decisions", + async_class="AsyncDecisions", + method="create", + wrapper=_decisions_create_wrapper, + wrap_name="openai.wrap.decisions.create", +) + + +class DecisionsPatcher(CompositeFunctionWrapperPatcher): + name = "openai.decisions" + sub_patchers = (_dec_create_sync, _dec_create_async) + + +class _WrapDecisions(CompositeFunctionWrapperPatcher): + name = "openai.wrap.decisions" + sub_patchers = (_wrap_dec_create,) + + # --------------------------------------------------------------------------- # Resource mapping — single source of truth for wrap_openai. # @@ -458,6 +484,7 @@ class _WrapResponsesRaw(CompositeFunctionWrapperPatcher): ("audio.translations", _WrapAudioTranslations), ("images", _WrapImages), ("responses", _WrapResponses), + ("decisions", _WrapDecisions), ("responses.with_raw_response", _WrapResponsesRaw), ("beta.chat.completions", _WrapChatCompletions), ) diff --git a/py/src/braintrust/integrations/openai/test_openai.py b/py/src/braintrust/integrations/openai/test_openai.py index 0b5641b20..a83bc918c 100644 --- a/py/src/braintrust/integrations/openai/test_openai.py +++ b/py/src/braintrust/integrations/openai/test_openai.py @@ -17,8 +17,15 @@ _materialize_logged_file_input, _process_attachments_in_chat_output, ) +from braintrust.integrations.test_decisions import ( + DECISIONS_INPUT, + DECISIONS_MODEL, + DECISIONS_QUESTIONS, + assert_decisions_span, +) from braintrust.integrations.test_utils import assert_metrics_are_valid, verify_autoinstrument_script from braintrust.integrations.utils import _try_to_dict +from braintrust.integrations.versioning import version_satisfies from braintrust.span_types import SpanTypeAttribute from braintrust.test_helpers import assert_dict_matches, init_test_logger from openai import AsyncOpenAI @@ -185,6 +192,73 @@ async def test_unwrapped_client_emits_no_spans(memory_logger, is_async, vcr_cass assert not memory_logger.pop() +@sync_async +@pytest.mark.asyncio +@pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path", "query", "body"]) +async def test_openai_decisions_metrics(memory_logger, is_async): + if not version_satisfies(openai.__version__, ">=3.26.0"): + pytest.skip("Decisions requires OpenAI >=3.26.0") + + client = wrap_openai(wrap_openai(_client(is_async))) + questions = [dict(question) for question in DECISIONS_QUESTIONS] + choices = iter(DECISIONS_QUESTIONS[1]["choices"]) + levels = iter(DECISIONS_QUESTIONS[2]["levels"]) + questions[1]["choices"] = choices + questions[2]["levels"] = levels + start = time.time() + response = await _call( + is_async, + client.decisions.create, + model=DECISIONS_MODEL, + input=DECISIONS_INPUT, + questions=iter(questions), + safety_identifier="do-not-log-this-identifier", + ) + end = time.time() + + assert questions[1]["choices"] is choices + assert questions[2]["levels"] is levels + assert_decisions_span( + memory_logger.pop(), + response, + name="openai.decisions.create", + model=DECISIONS_MODEL, + instrumentation="openai-auto", + start=start, + end=end, + ) + with pytest.raises(TypeError, match="required keyword-only arguments"): + await _call(is_async, client.decisions.create, model=DECISIONS_MODEL) + + +@sync_async +@pytest.mark.asyncio +@pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path", "query", "body"]) +async def test_openai_decisions_iterable_input(memory_logger, is_async): + if not version_satisfies(openai.__version__, ">=3.26.0"): + pytest.skip("Decisions requires OpenAI >=3.26.0") + + messages = [{"role": "user", "content": [{"type": "input_text", "text": DECISIONS_INPUT}]}] + content = iter(messages[0]["content"]) + request_messages = [{**messages[0], "content": content}] + client = wrap_openai(_client(is_async)) + response = await _call( + is_async, + client.decisions.create, + model=DECISIONS_MODEL, + input=iter(request_messages), + questions=DECISIONS_QUESTIONS[:1], + ) + + spans = memory_logger.pop() + assert len(spans) == 1 + assert request_messages[0]["content"] is content + assert spans[0]["input"]["state"] == messages + assert spans[0]["output"]["answers"] == [ + {"id": "positive", "type": "boolean", "probability": response.answers[0].probability} + ] + + @sync_async @pytest.mark.asyncio @pytest.mark.vcr diff --git a/py/src/braintrust/integrations/openai/tracing.py b/py/src/braintrust/integrations/openai/tracing.py index 2a479dcf1..193bc017c 100644 --- a/py/src/braintrust/integrations/openai/tracing.py +++ b/py/src/braintrust/integrations/openai/tracing.py @@ -5,10 +5,12 @@ import binascii import inspect import json +import logging import time from collections.abc import Callable, Iterable, Mapping from typing import Any +from braintrust.integrations.question import _question_span_input, _question_span_output from braintrust.integrations.utils import ( _extract_audio_output, _infer_audio_mime_type, @@ -26,6 +28,7 @@ _INSTRUMENTATION = "openai-auto" +_log = logging.getLogger(__name__) def start_span(*args, **kwargs): @@ -2038,6 +2041,174 @@ def process_output(self, response: dict[str, Any], span: Span): ) +def _decision_question_ids(questions: list[dict[str, Any]]) -> list[str]: + names = {question.get("name") for question in questions if isinstance(question.get("name"), str)} + used: set[str] = set() + ids = [] + for index, question in enumerate(questions): + identifier = question.get("name") + if not isinstance(identifier, str) or not identifier or identifier in used: + identifier = f"question_{index}" + while identifier in names or identifier in used: + identifier += "_" + used.add(identifier) + ids.append(identifier) + return ids + + +def _decision_choice_keys(question: dict[str, Any]) -> dict[tuple[type, Any], str]: + choices = question.get("choices", []) + string_values = {choice["value"] for choice in choices if isinstance(choice.get("value"), str)} + keys = {} + for choice in choices: + value = choice["value"] + key = value + if isinstance(value, bool): + key = str(value).lower() + while key in string_values: + key = f"boolean:{key}" + keys[type(value), value] = key + return keys + + +def _decisions_input(state: Any, questions: Any) -> dict[str, Any]: + try: + if isinstance(questions, Mapping): + return _question_span_input(state, questions) + questions = [_try_to_dict(question) for question in questions] + normalized = [] + for identifier, question in zip(_decision_question_ids(questions), questions): + kind = question.get("type") + payload = { + "id": identifier, + "type": "boolean" if kind == "predicate" else kind, + "instructions": question.get("instructions"), + } + if kind == "choice": + keys = _decision_choice_keys(question) + payload["criteria"] = { + keys[type(choice["value"]), choice["value"]]: choice.get("description") + for choice in question.get("choices", []) + } + elif kind == "score": + payload["criteria"] = [ + {key: level[key] for key in ("label", "description") if key in level} + for level in question.get("levels", []) + ] + normalized.append(payload) + return _question_span_input(state, normalized) + except Exception: + _log.debug("Failed to normalize Decisions questions", exc_info=True) + return {"state": state, "questions": []} + + +def _decisions_output(answers: Any, questions: Any) -> dict[str, Any]: + try: + if isinstance(answers, Mapping): + return _question_span_output(answers) + questions = [_try_to_dict(question) for question in questions] + normalized = [] + # OpenAI guarantees answer order, including refusals and unnamed questions. + # Use the request's IDs so all answers remain linked to their questions. + for identifier, question, answer in zip(_decision_question_ids(questions), questions, answers or []): + answer = _try_to_dict(answer) + if not isinstance(answer, dict): + continue + kind = answer.get("type") + payload = {"id": identifier, "type": "boolean" if kind == "predicate" else kind} + for key in ("probability", "score", "confidence"): + if key in answer: + payload[key] = answer[key] + if kind == "choice": + keys = _decision_choice_keys(question) + if "choice" in answer: + value = answer["choice"] + payload["choice"] = keys.get((type(value), value), str(value)) + if "probabilities" in answer: + payload["probabilities"] = { + keys.get((type(item["value"]), item["value"]), str(item["value"])): item["probability"] + for item in answer["probabilities"] + } + elif kind == "score" and "probabilities" in answer: + payload["probabilities"] = { + str(item["value"]): item["probability"] for item in answer["probabilities"] + } + payload["legend"] = {str(item["value"]): item["label"] for item in answer["probabilities"]} + normalized.append(payload) + return _question_span_output(normalized) + except Exception: + _log.debug("Failed to normalize Decisions answers", exc_info=True) + return {"answers": []} + + +def _materialize_decision_items(values: Any, nested_fields: tuple[str, ...]) -> Any: + if not isinstance(values, Iterable) or isinstance(values, (str, bytes, Mapping)): + return values + # Logging and the SDK must see the same values without consuming a caller's + # iterators twice or replacing fields on their objects. + items = [] + for item in values: + if isinstance(item, Mapping): + item = dict(item) + for field in nested_fields: + nested = item.get(field) + if isinstance(nested, Iterable) and not isinstance(nested, (list, str, bytes, Mapping)): + item[field] = list(nested) + items.append(item) + return items + + +def _decisions_span_params(params: dict[str, Any]) -> dict[str, Any]: + ret = params.pop("span_info", {}) + for field, nested_fields in (("questions", ("choices", "levels")), ("input", ("content",))): + if field in params: + params[field] = _materialize_decision_items(params[field], nested_fields) + return merge_dicts( + ret, + { + "input": _decisions_input( + _process_attachments_in_input(params.get("input")), params.get("questions") or [] + ), + "metadata": _filter_metadata(params, ("model",)), + }, + ) + + +def _log_decisions_response(span: Span, response: Any, questions: Any): + response = _try_to_dict(response) + if isinstance(response, dict): + span.log( + output=_decisions_output(response.get("answers"), questions or []), + metrics=_parse_metrics_from_usage(response.get("usage")), + ) + + +def _decisions_create_wrapper(wrapped, instance, args, kwargs): + if _is_async_callable(wrapped): + + async def call(): + with start_span( + **merge_dicts( + dict(name="openai.decisions.create", span_attributes={"type": SpanTypeAttribute.QUESTION}), + _decisions_span_params(kwargs), + ) + ) as span: + response = await wrapped(*args, **kwargs) + _log_decisions_response(span, response, kwargs.get("questions")) + return response + + return call() + with start_span( + **merge_dicts( + dict(name="openai.decisions.create", span_attributes={"type": SpanTypeAttribute.QUESTION}), + _decisions_span_params(kwargs), + ) + ) as span: + response = wrapped(*args, **kwargs) + _log_decisions_response(span, response, kwargs.get("questions")) + return response + + class ModerationWrapper(BaseWrapper): _metadata_params = _MODERATION_METADATA_PARAMS @@ -2308,11 +2479,15 @@ def process_output(self, response: Any, span: Span): def _parse_metrics_from_usage(usage: Any) -> dict[str, Any]: - return _parse_openai_usage_metrics( + metrics = _parse_openai_usage_metrics( usage, token_name_map=TOKEN_NAME_MAP, token_prefix_map=TOKEN_PREFIX_MAP, ) + cache_write_tokens = metrics.pop("prompt_cache_write_tokens", None) + if cache_write_tokens is not None: + metrics.setdefault("prompt_cache_creation_tokens", cache_write_tokens) + return metrics def prettify_params(params: dict[str, Any]) -> dict[str, Any]: diff --git a/py/src/braintrust/integrations/question.py b/py/src/braintrust/integrations/question.py new file mode 100644 index 000000000..71bf19423 --- /dev/null +++ b/py/src/braintrust/integrations/question.py @@ -0,0 +1,33 @@ +"""Shared question span structure from braintrust/typespecs/src/question.ts. + +Provider adapters normalize their question and answer types before using these +helpers. Named maps become arrays linked by ID; arrays already carry their IDs. +""" + +from collections.abc import Mapping +from typing import Any + +from .utils import _try_to_dict + + +_QUESTION_FIELDS = ("type", "instructions", "criteria") +_ANSWER_FIELDS = ("type", "noul", "probability", "choice", "score", "confidence", "legend", "probabilities") + + +def _question_items(values: Any, fields: tuple[str, ...]) -> Any: + if not isinstance(values, Mapping): + return values + items = [] + for identifier, value in values.items(): + value = _try_to_dict(value) + if isinstance(value, dict): + items.append({**{key: value[key] for key in fields if key in value}, "id": identifier}) + return items + + +def _question_span_input(state: Any, questions: Any) -> dict[str, Any]: + return {"state": state, "questions": _question_items(questions, _QUESTION_FIELDS)} + + +def _question_span_output(answers: Any) -> dict[str, Any]: + return {"answers": _question_items(answers, _ANSWER_FIELDS)} diff --git a/py/src/braintrust/integrations/test_decisions.py b/py/src/braintrust/integrations/test_decisions.py new file mode 100644 index 000000000..ad3b10c26 --- /dev/null +++ b/py/src/braintrust/integrations/test_decisions.py @@ -0,0 +1,179 @@ +import pytest +from braintrust.integrations.litellm.tracing import _parse_metrics_from_usage as parse_litellm_usage +from braintrust.integrations.openai.tracing import _decisions_input, _decisions_output +from braintrust.integrations.openai.tracing import _parse_metrics_from_usage as parse_openai_usage +from braintrust.test_helpers import assert_dict_matches + + +DECISIONS_MODEL = "gpt-6-luna" +DECISIONS_INPUT = "The customer says: I love this product!" +DECISIONS_QUESTIONS = [ + {"type": "predicate", "name": "positive", "instructions": "Is the sentiment positive?"}, + { + "type": "choice", + "name": "sentiment", + "instructions": "What is the sentiment?", + "choices": [{"value": "positive", "description": "Happy feedback"}, {"value": "negative"}], + }, + { + "type": "score", + "name": "enthusiasm", + "instructions": "How enthusiastic is the customer?", + "levels": [{"label": "low"}, {"label": "high", "description": "Very enthusiastic"}], + }, + {"type": "predicate", "instructions": "Does the customer like the product?"}, +] + + +def assert_decisions_span(spans, response, *, name, model, instrumentation, start=None, end=None): + assert len(spans) == 1 + span = spans[0] + assert_dict_matches(span["span_attributes"], {"name": name, "type": "question"}) + assert span["input"] == { + "state": DECISIONS_INPUT, + "questions": [ + {"id": "positive", "type": "boolean", "instructions": "Is the sentiment positive?"}, + { + "id": "sentiment", + "type": "choice", + "instructions": "What is the sentiment?", + "criteria": {"positive": "Happy feedback", "negative": None}, + }, + { + "id": "enthusiasm", + "type": "score", + "instructions": "How enthusiastic is the customer?", + "criteria": [{"label": "low"}, {"label": "high", "description": "Very enthusiastic"}], + }, + {"id": "question_3", "type": "boolean", "instructions": "Does the customer like the product?"}, + ], + } + assert [answer["id"] for answer in span["output"]["answers"]] == [ + "positive", + "sentiment", + "enthusiasm", + "question_3", + ] + assert span["output"]["answers"][0] == { + "id": "positive", + "type": "boolean", + "probability": response.answers[0].probability, + } + assert span["output"]["answers"][1]["choice"] == response.answers[1].choice + assert span["output"]["answers"][1]["probabilities"] == { + item.value: item.probability for item in response.answers[1].probabilities + } + assert span["output"]["answers"][1]["confidence"] == response.answers[1].confidence + assert span["output"]["answers"][2]["score"] == response.answers[2].score + assert span["output"]["answers"][2]["confidence"] == response.answers[2].confidence + assert span["output"]["answers"][2]["probabilities"] == { + str(item.value): item.probability for item in response.answers[2].probabilities + } + assert span["output"]["answers"][2]["legend"] == { + str(item.value): item.label for item in response.answers[2].probabilities + } + assert span["output"]["answers"][3] == { + "id": "question_3", + "type": "boolean", + "probability": response.answers[3].probability, + } + assert span["metadata"] == {"model": model, "provider": "openai"} + assert span["context"]["span_origin"]["instrumentation"]["name"] == instrumentation + metrics = span["metrics"] + assert metrics["start"] <= metrics["end"] + if start is not None and end is not None: + assert start <= metrics["start"] <= metrics["end"] <= end + assert all(metrics[key] >= 0 for key in ("tokens", "prompt_tokens", "completion_tokens")) + assert metrics["tokens"] == response.usage.total_tokens + assert metrics["prompt_tokens"] == response.usage.input_tokens + assert metrics["completion_tokens"] == response.usage.output_tokens + assert metrics["prompt_cached_tokens"] == response.usage.input_tokens_details.cached_tokens + assert metrics["prompt_cache_creation_tokens"] == response.usage.input_tokens_details.cache_write_tokens + assert metrics["completion_reasoning_tokens"] == response.usage.output_tokens_details.reasoning_tokens + assert "prompt_cache_write_tokens" not in metrics + + +def test_decisions_ids_are_unique_and_match_answers(): + questions = [ + {"type": "predicate", "name": "question_1", "instructions": "First"}, + {"type": "predicate", "instructions": "Second"}, + {"type": "predicate", "name": "question_1", "instructions": "Third"}, + ] + span_input = _decisions_input("state", questions) + span_output = _decisions_output([{"type": "predicate", "probability": 0.5}] * 3, questions) + ids = [question["id"] for question in span_input["questions"]] + assert ids == ["question_1", "question_1_", "question_2"] + assert [answer["id"] for answer in span_output["answers"]] == ids + + +def test_decisions_preserves_typed_choices(): + questions = [ + { + "type": "choice", + "name": "typed", + "instructions": "Choose", + "choices": [{"value": True}, {"value": "true"}], + } + ] + span_input = _decisions_input("state", questions) + span_output = _decisions_output( + [ + { + "type": "choice", + "choice": True, + "probabilities": [{"value": True, "probability": 0.8}, {"value": "true", "probability": 0.2}], + } + ], + questions, + ) + assert span_input["questions"][0]["criteria"] == {"boolean:true": None, "true": None} + assert span_output["answers"][0] == { + "id": "typed", + "type": "choice", + "choice": "boolean:true", + "probabilities": {"boolean:true": 0.8, "true": 0.2}, + } + + +def test_decisions_refusals_and_allowlists(): + questions = [{"type": "predicate", "name": "refused", "instructions": "Question", "secret": "omit"}] + span_input = _decisions_input({"message": "state"}, questions) + span_output = _decisions_output([{"type": "refusal", "name": "refused", "secret": "omit"}], questions) + assert span_input == { + "state": {"message": "state"}, + "questions": [{"id": "refused", "type": "boolean", "instructions": "Question"}], + } + assert span_output == {"answers": [{"id": "refused", "type": "refusal"}]} + + +def test_decisions_system_one_maps_match_typesafe_structure(): + questions = {"positive": {"type": "noul", "instructions": "Is this positive?", "secret": "omit"}} + answers = {"positive": {"type": "noul", "noul": 0.9, "secret": "omit"}} + assert _decisions_input("state", questions) == { + "state": "state", + "questions": [{"id": "positive", "type": "noul", "instructions": "Is this positive?"}], + } + assert _decisions_output(answers, questions) == {"answers": [{"id": "positive", "type": "noul", "noul": 0.9}]} + + +@pytest.mark.parametrize("parse_usage", (parse_openai_usage, parse_litellm_usage), ids=("openai", "litellm")) +def test_decisions_usage_with_nonzero_token_details(parse_usage): + # The live Decisions recordings report zero for these optional details. + # Exercise nonzero values directly through the local usage parser too. + metrics = parse_usage( + { + "input_tokens": 100, + "output_tokens": 20, + "total_tokens": 120, + "input_tokens_details": {"cached_tokens": 32, "cache_write_tokens": 16}, + "output_tokens_details": {"reasoning_tokens": 6}, + } + ) + assert metrics == { + "prompt_tokens": 100, + "completion_tokens": 20, + "tokens": 120, + "prompt_cached_tokens": 32, + "prompt_cache_creation_tokens": 16, + "completion_reasoning_tokens": 6, + } diff --git a/py/src/braintrust/integrations/typesafe/tracing.py b/py/src/braintrust/integrations/typesafe/tracing.py index 247a21719..e39225c10 100644 --- a/py/src/braintrust/integrations/typesafe/tracing.py +++ b/py/src/braintrust/integrations/typesafe/tracing.py @@ -3,6 +3,12 @@ from collections.abc import Mapping from typing import Any +from braintrust.integrations.question import ( + _ANSWER_FIELDS, + _QUESTION_FIELDS, + _question_span_input, + _question_span_output, +) from braintrust.integrations.utils import _log_and_end_span, _log_error_and_end_span from braintrust.logger import start_span as _bt_start_span from braintrust.span_types import SpanTypeAttribute @@ -10,8 +16,6 @@ _INSTRUMENTATION = "typesafe" _MISSING = object() -_QUESTION_FIELDS = ("type", "instructions", "criteria") -_ANSWER_FIELDS = ("type", "noul", "choice", "score", "confidence", "legend", "probabilities") def start_span(*args, **kwargs): @@ -44,12 +48,6 @@ def _answer_payload(value: Any) -> dict[str, Any]: return _tagged_payload(value, _ANSWER_FIELDS, type_suffix="Answer") -def _items_with_ids(values: Any, payload_fn) -> Any: - if not isinstance(values, Mapping): - return values - return [{**payload_fn(value), "id": identifier} for identifier, value in values.items()] - - def _call_value(args: Any, kwargs: dict[str, Any], name: str, position: int) -> Any: if name in kwargs: return kwargs[name] @@ -74,10 +72,9 @@ def _request_parts(instance: Any, args: Any, kwargs: dict[str, Any]) -> tuple[An def _span_input(state: Any, questions: Any) -> dict[str, Any]: - return { - "state": state, - "questions": _items_with_ids(questions, _question_payload), - } + if isinstance(questions, Mapping): + questions = {identifier: _question_payload(value) for identifier, value in questions.items()} + return _question_span_input(state, questions) def _token_metrics(response: Any) -> dict[str, int]: @@ -96,10 +93,12 @@ def _token_metrics(response: Any) -> dict[str, int]: def _log_response(span: Any, response: Any) -> None: answers = getattr(response, "answers", None) + if isinstance(answers, Mapping): + answers = {identifier: _answer_payload(value) for identifier, value in answers.items()} response_model = getattr(response, "model", None) _log_and_end_span( span, - output={"answers": _items_with_ids(answers, _answer_payload)}, + output=_question_span_output(answers), metrics=_token_metrics(response), metadata={"model": response_model} if isinstance(response_model, str) else None, ) From 97210fa72aa6c0f9c5c421e65b1542a71ed2b82f Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Tue, 13 Oct 2026 13:01:20 -0400 Subject: [PATCH 2/3] fix(integrations): emit decision spans Use the decision span type for OpenAI, LiteLLM, and TypeSafe according to the updated instrumentation spec. Keep the legacy question enum for compatibility and update VCR and type-test assertions. --- .../integrations/auto_test_scripts/test_auto_typesafe.py | 1 + py/src/braintrust/integrations/litellm/tracing.py | 4 ++-- py/src/braintrust/integrations/openai/tracing.py | 4 ++-- py/src/braintrust/integrations/question.py | 2 +- py/src/braintrust/integrations/test_decisions.py | 2 +- py/src/braintrust/integrations/typesafe/test_typesafe.py | 2 +- py/src/braintrust/integrations/typesafe/tracing.py | 2 +- py/src/braintrust/span_types.py | 1 + py/src/braintrust/type_tests/test_span_types.py | 3 ++- 9 files changed, 12 insertions(+), 9 deletions(-) diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_typesafe.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_typesafe.py index bc4ca78d3..058b3fb60 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_typesafe.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_typesafe.py @@ -22,6 +22,7 @@ spans = memory_logger.pop() assert len(spans) == 1 span = spans[0] + assert span["span_attributes"]["type"] == "decision" assert span["metadata"]["provider"] == "typesafe" assert span["metadata"]["model"].startswith("jev-") assert span["input"]["questions"][0]["id"] == "positive" diff --git a/py/src/braintrust/integrations/litellm/tracing.py b/py/src/braintrust/integrations/litellm/tracing.py index 71e47f7d8..d01a9c2b3 100644 --- a/py/src/braintrust/integrations/litellm/tracing.py +++ b/py/src/braintrust/integrations/litellm/tracing.py @@ -359,7 +359,7 @@ def _decisions_span_payload(args, kwargs): def _decisions_wrapper(wrapped, instance, args, kwargs): with start_span( **merge_dicts( - dict(name="Decision", span_attributes={"type": SpanTypeAttribute.QUESTION}), + dict(name="Decision", span_attributes={"type": SpanTypeAttribute.DECISION}), _decisions_span_payload(args, kwargs), ) ) as span: @@ -376,7 +376,7 @@ def _decisions_wrapper(wrapped, instance, args, kwargs): async def _adecisions_wrapper_async(wrapped, instance, args, kwargs): with start_span( **merge_dicts( - dict(name="Decision", span_attributes={"type": SpanTypeAttribute.QUESTION}), + dict(name="Decision", span_attributes={"type": SpanTypeAttribute.DECISION}), _decisions_span_payload(args, kwargs), ) ) as span: diff --git a/py/src/braintrust/integrations/openai/tracing.py b/py/src/braintrust/integrations/openai/tracing.py index 193bc017c..1b5a147b1 100644 --- a/py/src/braintrust/integrations/openai/tracing.py +++ b/py/src/braintrust/integrations/openai/tracing.py @@ -2189,7 +2189,7 @@ def _decisions_create_wrapper(wrapped, instance, args, kwargs): async def call(): with start_span( **merge_dicts( - dict(name="openai.decisions.create", span_attributes={"type": SpanTypeAttribute.QUESTION}), + dict(name="openai.decisions.create", span_attributes={"type": SpanTypeAttribute.DECISION}), _decisions_span_params(kwargs), ) ) as span: @@ -2200,7 +2200,7 @@ async def call(): return call() with start_span( **merge_dicts( - dict(name="openai.decisions.create", span_attributes={"type": SpanTypeAttribute.QUESTION}), + dict(name="openai.decisions.create", span_attributes={"type": SpanTypeAttribute.DECISION}), _decisions_span_params(kwargs), ) ) as span: diff --git a/py/src/braintrust/integrations/question.py b/py/src/braintrust/integrations/question.py index 71bf19423..4376f41d7 100644 --- a/py/src/braintrust/integrations/question.py +++ b/py/src/braintrust/integrations/question.py @@ -1,4 +1,4 @@ -"""Shared question span structure from braintrust/typespecs/src/question.ts. +"""Shared decision span structure from braintrust/typespecs/src/question.ts. Provider adapters normalize their question and answer types before using these helpers. Named maps become arrays linked by ID; arrays already carry their IDs. diff --git a/py/src/braintrust/integrations/test_decisions.py b/py/src/braintrust/integrations/test_decisions.py index ad3b10c26..2854820c3 100644 --- a/py/src/braintrust/integrations/test_decisions.py +++ b/py/src/braintrust/integrations/test_decisions.py @@ -28,7 +28,7 @@ def assert_decisions_span(spans, response, *, name, model, instrumentation, start=None, end=None): assert len(spans) == 1 span = spans[0] - assert_dict_matches(span["span_attributes"], {"name": name, "type": "question"}) + assert_dict_matches(span["span_attributes"], {"name": name, "type": "decision"}) assert span["input"] == { "state": DECISIONS_INPUT, "questions": [ diff --git a/py/src/braintrust/integrations/typesafe/test_typesafe.py b/py/src/braintrust/integrations/typesafe/test_typesafe.py index f00ad2f00..b12816527 100644 --- a/py/src/braintrust/integrations/typesafe/test_typesafe.py +++ b/py/src/braintrust/integrations/typesafe/test_typesafe.py @@ -21,7 +21,7 @@ def memory_logger(): def _assert_span(span, *, question_ids): assert span["span_attributes"]["name"] == "typesafe.systemOne" - assert span["span_attributes"]["type"] == SpanTypeAttribute.QUESTION + assert span["span_attributes"]["type"] == SpanTypeAttribute.DECISION assert span["context"]["span_origin"]["instrumentation"]["name"] == "typesafe" assert span["metadata"]["provider"] == "typesafe" assert span["metadata"]["model"].startswith("jev-") diff --git a/py/src/braintrust/integrations/typesafe/tracing.py b/py/src/braintrust/integrations/typesafe/tracing.py index e39225c10..f628bda25 100644 --- a/py/src/braintrust/integrations/typesafe/tracing.py +++ b/py/src/braintrust/integrations/typesafe/tracing.py @@ -111,7 +111,7 @@ def _start_system_one_span(instance: Any, args: Any, kwargs: dict[str, Any]): metadata["model"] = model return start_span( name="typesafe.systemOne", - type=SpanTypeAttribute.QUESTION, + type=SpanTypeAttribute.DECISION, input=_span_input(state, questions), metadata=metadata, ) diff --git a/py/src/braintrust/span_types.py b/py/src/braintrust/span_types.py index 59ef74d6c..dafc4c0ac 100644 --- a/py/src/braintrust/span_types.py +++ b/py/src/braintrust/span_types.py @@ -18,6 +18,7 @@ class SpanTypeAttribute(str, Enum): PREPROCESSOR = "preprocessor" CLASSIFIER = "classifier" REVIEW = "review" + DECISION = "decision" QUESTION = "question" LOG = "log" diff --git a/py/src/braintrust/type_tests/test_span_types.py b/py/src/braintrust/type_tests/test_span_types.py index 6650dfb13..73e9381e9 100644 --- a/py/src/braintrust/type_tests/test_span_types.py +++ b/py/src/braintrust/type_tests/test_span_types.py @@ -1,10 +1,11 @@ import braintrust from braintrust.logger import Span +from braintrust.span_types import SpanTypeAttribute from typing_extensions import assert_type def span_context_manager_keeps_span_type() -> None: - with braintrust.start_span(name="typed") as span: + with braintrust.start_span(name="typed", type=SpanTypeAttribute.DECISION) as span: assert_type(span, Span) span.log(output="ok") From 45b39c2cae470a3b2421abc4625ab540545116d6 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Tue, 13 Oct 2026 15:54:26 -0400 Subject: [PATCH 3/3] fix(integrations): materialize decisions image attachments --- .../test_litellm_decisions_image[async].yaml | 79 +++++++++++++++ .../test_litellm_decisions_image[sync].yaml | 79 +++++++++++++++ .../integrations/litellm/test_litellm.py | 20 ++++ .../test_openai_decisions_image[async].yaml | 97 +++++++++++++++++++ .../test_openai_decisions_image[sync].yaml | 97 +++++++++++++++++++ .../integrations/openai/test_openai.py | 21 ++++ .../braintrust/integrations/openai/tracing.py | 16 ++- .../braintrust/integrations/test_decisions.py | 48 +++++++++ 8 files changed, 454 insertions(+), 3 deletions(-) create mode 100644 py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_decisions_image[async].yaml create mode 100644 py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_decisions_image[sync].yaml create mode 100644 py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_image[async].yaml create mode 100644 py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_image[sync].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_decisions_image[async].yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_decisions_image[async].yaml new file mode 100644 index 000000000..4c66eb9ae --- /dev/null +++ b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_decisions_image[async].yaml @@ -0,0 +1,79 @@ +interactions: +- request: + body: '{"model":"gpt-6-luna","input":[{"role":"user","type":"message","content":[{"type":"input_text","text":"The + customer says: I love this product!"},{"type":"input_image","image_url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFElEQVR42mP4z8DwnxjMMKqQvgoBksPHOXvuG4oAAAAASUVORK5CYII=","detail":"low"}]}],"questions":[{"type":"predicate","name":"positive","instructions":"Is + the sentiment positive?"}]}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '424' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - litellm/1.105.0 + method: POST + uri: https://api.openai.com/v1/decisions + response: + body: + string: '{"model":"gpt-6-luna","answers":[{"type":"predicate","name":"positive","probability":1.0}],"usage":{"input_tokens":164,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":0,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":164}}' + headers: + Access-Control-Allow-Origin: + - '*' + Access-Control-Expose-Headers: + - x-request-id,retry-after,server-timing-total + - CF-Ray + Alt-Svc: + - h3=":443"; ma=86400 + CF-RAY: + - a4a0e088bd4136d8-YYZ + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Tue, 13 Oct 2026 19:51:06 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + Vary: + - origin, access-control-request-method, access-control-request-headers + X-Content-Type-Options: + - nosniff + X-Request-ID: + - req_8bd95865f1cd4299aa68ce9614eb8730 + cf-cache-status: + - DYNAMIC + content-length: + - '270' + openai-processing-ms: + - '55' + server-timing-total: + - '55.028' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '180000000' + x-ratelimit-remaining-requests: + - '29998' + x-ratelimit-remaining-tokens: + - '179999670' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_decisions_image[sync].yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_decisions_image[sync].yaml new file mode 100644 index 000000000..2cfb6d6d9 --- /dev/null +++ b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_decisions_image[sync].yaml @@ -0,0 +1,79 @@ +interactions: +- request: + body: '{"model":"gpt-6-luna","input":[{"role":"user","type":"message","content":[{"type":"input_text","text":"The + customer says: I love this product!"},{"type":"input_image","image_url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFElEQVR42mP4z8DwnxjMMKqQvgoBksPHOXvuG4oAAAAASUVORK5CYII=","detail":"low"}]}],"questions":[{"type":"predicate","name":"positive","instructions":"Is + the sentiment positive?"}]}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '424' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - litellm/1.105.0 + method: POST + uri: https://api.openai.com/v1/decisions + response: + body: + string: '{"model":"gpt-6-luna","answers":[{"type":"predicate","name":"positive","probability":1.0}],"usage":{"input_tokens":164,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":0,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":164}}' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - x-request-id,retry-after,server-timing-total + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a4a0e0868aa48ea1-YYZ + connection: + - keep-alive + content-length: + - '270' + content-type: + - application/json + date: + - Tue, 13 Oct 2026 19:51:06 GMT + openai-processing-ms: + - '60' + server: + - cloudflare + server-timing-total: + - '60.500' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - origin, access-control-request-method, access-control-request-headers + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '180000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '179999835' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_068b966cce534bdf911aa85c410110f2 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/litellm/test_litellm.py b/py/src/braintrust/integrations/litellm/test_litellm.py index cf390970f..ead3681c8 100644 --- a/py/src/braintrust/integrations/litellm/test_litellm.py +++ b/py/src/braintrust/integrations/litellm/test_litellm.py @@ -12,7 +12,9 @@ DECISIONS_INPUT, DECISIONS_MODEL, DECISIONS_QUESTIONS, + assert_decisions_image_span, assert_decisions_span, + decisions_image_input, ) from braintrust.integrations.test_utils import ( assert_metrics_are_valid, @@ -311,6 +313,24 @@ def test_litellm_decisions_metrics(memory_logger, is_async): ) +@pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path", "query", "body"]) +@sync_async +def test_litellm_decisions_image(memory_logger, is_async): + if not version_satisfies(detect_module_version(litellm, ("litellm",)), ">=1.105.0"): + pytest.skip("Decisions requires LiteLLM >=1.105.0") + + messages = decisions_image_input() + response = _run( + is_async, + litellm.decisions, + litellm.adecisions, + model=f"openai/{DECISIONS_MODEL}", + input=messages, + questions=DECISIONS_QUESTIONS[:1], + ) + assert_decisions_image_span(memory_logger.pop(), response, messages) + + @pytest.mark.vcr @sync_async def test_litellm_responses_metrics(memory_logger, is_async): diff --git a/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_image[async].yaml b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_image[async].yaml new file mode 100644 index 000000000..7c5a50b59 --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_image[async].yaml @@ -0,0 +1,97 @@ +interactions: +- request: + body: '{"model":"gpt-6-luna","input":[{"role":"user","content":[{"type":"input_text","text":"The + customer says: I love this product!"},{"type":"input_image","image_url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFElEQVR42mP4z8DwnxjMMKqQvgoBksPHOXvuG4oAAAAASUVORK5CYII=","detail":"low"}]}],"questions":[{"type":"predicate","name":"positive","instructions":"Is + the sentiment positive?"}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '407' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 3.29.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 3.29.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/decisions + response: + body: + string: '{"model":"gpt-6-luna","answers":[{"type":"predicate","name":"positive","probability":1.0}],"usage":{"input_tokens":164,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":0,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":164}}' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - x-request-id,retry-after,server-timing-total + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a4a0e06fa84da2d9-YYZ + connection: + - keep-alive + content-length: + - '270' + content-type: + - application/json + date: + - Tue, 13 Oct 2026 19:51:02 GMT + openai-processing-ms: + - '95' + server: + - cloudflare + server-timing-total: + - '95.131' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - origin, access-control-request-method, access-control-request-headers + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '180000000' + x-ratelimit-remaining-requests: + - '29998' + x-ratelimit-remaining-tokens: + - '179999670' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_225e76ef1aa54761afc4da1208c70d50 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_image[sync].yaml b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_image[sync].yaml new file mode 100644 index 000000000..770f4815b --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_image[sync].yaml @@ -0,0 +1,97 @@ +interactions: +- request: + body: '{"model":"gpt-6-luna","input":[{"role":"user","content":[{"type":"input_text","text":"The + customer says: I love this product!"},{"type":"input_image","image_url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFElEQVR42mP4z8DwnxjMMKqQvgoBksPHOXvuG4oAAAAASUVORK5CYII=","detail":"low"}]}],"questions":[{"type":"predicate","name":"positive","instructions":"Is + the sentiment positive?"}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '407' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 3.29.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 3.29.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/decisions + response: + body: + string: '{"model":"gpt-6-luna","answers":[{"type":"predicate","name":"positive","probability":1.0}],"usage":{"input_tokens":164,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":0,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":164}}' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - x-request-id,retry-after,server-timing-total + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a4a0e06a5f73fcd9-YYZ + connection: + - keep-alive + content-length: + - '270' + content-type: + - application/json + date: + - Tue, 13 Oct 2026 19:51:02 GMT + openai-processing-ms: + - '95' + server: + - cloudflare + server-timing-total: + - '95.586' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - origin, access-control-request-method, access-control-request-headers + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '180000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '179999835' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_89672dba3b884abfb5d42a66a7f06ffc + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/test_openai.py b/py/src/braintrust/integrations/openai/test_openai.py index a83bc918c..1a1992835 100644 --- a/py/src/braintrust/integrations/openai/test_openai.py +++ b/py/src/braintrust/integrations/openai/test_openai.py @@ -21,7 +21,9 @@ DECISIONS_INPUT, DECISIONS_MODEL, DECISIONS_QUESTIONS, + assert_decisions_image_span, assert_decisions_span, + decisions_image_input, ) from braintrust.integrations.test_utils import assert_metrics_are_valid, verify_autoinstrument_script from braintrust.integrations.utils import _try_to_dict @@ -259,6 +261,25 @@ async def test_openai_decisions_iterable_input(memory_logger, is_async): ] +@sync_async +@pytest.mark.asyncio +@pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path", "query", "body"]) +async def test_openai_decisions_image(memory_logger, is_async): + if not version_satisfies(openai.__version__, ">=3.26.0"): + pytest.skip("Decisions requires OpenAI >=3.26.0") + + messages = decisions_image_input() + client = wrap_openai(_client(is_async)) + response = await _call( + is_async, + client.decisions.create, + model=DECISIONS_MODEL, + input=messages, + questions=DECISIONS_QUESTIONS[:1], + ) + assert_decisions_image_span(memory_logger.pop(), response, messages) + + @sync_async @pytest.mark.asyncio @pytest.mark.vcr diff --git a/py/src/braintrust/integrations/openai/tracing.py b/py/src/braintrust/integrations/openai/tracing.py index 1b5a147b1..37ee6f5a5 100644 --- a/py/src/braintrust/integrations/openai/tracing.py +++ b/py/src/braintrust/integrations/openai/tracing.py @@ -279,6 +279,17 @@ def _process_attachments_in_input(input_data: Any) -> Any: return [_process_attachments_in_input(item) for item in input_data] if isinstance(input_data, dict): + if input_data.get("type") == "input_image" and isinstance(input_data.get("image_url"), str): + resolved = _materialize_attachment(input_data["image_url"]) + if resolved is None: + return input_data + image_part = dict(input_data) + image_part["type"] = "image_url" + image_part["image_url"] = {"url": resolved.attachment} + if "detail" in image_part: + image_part["image_url"]["detail"] = image_part.pop("detail") + return image_part + if ( input_data.get("type") == "image_url" and isinstance(input_data.get("image_url"), dict) @@ -2073,6 +2084,7 @@ def _decision_choice_keys(question: dict[str, Any]) -> dict[tuple[type, Any], st def _decisions_input(state: Any, questions: Any) -> dict[str, Any]: try: + state = _process_attachments_in_input(state) if isinstance(questions, Mapping): return _question_span_input(state, questions) questions = [_try_to_dict(question) for question in questions] @@ -2166,9 +2178,7 @@ def _decisions_span_params(params: dict[str, Any]) -> dict[str, Any]: return merge_dicts( ret, { - "input": _decisions_input( - _process_attachments_in_input(params.get("input")), params.get("questions") or [] - ), + "input": _decisions_input(params.get("input"), params.get("questions") or []), "metadata": _filter_metadata(params, ("model",)), }, ) diff --git a/py/src/braintrust/integrations/test_decisions.py b/py/src/braintrust/integrations/test_decisions.py index 2854820c3..4742cf95a 100644 --- a/py/src/braintrust/integrations/test_decisions.py +++ b/py/src/braintrust/integrations/test_decisions.py @@ -1,4 +1,8 @@ +import base64 +from pathlib import Path + import pytest +from braintrust import Attachment from braintrust.integrations.litellm.tracing import _parse_metrics_from_usage as parse_litellm_usage from braintrust.integrations.openai.tracing import _decisions_input, _decisions_output from braintrust.integrations.openai.tracing import _parse_metrics_from_usage as parse_openai_usage @@ -25,6 +29,50 @@ ] +def decisions_image_input(): + image = (Path(__file__).parent.parent / "fixtures" / "test-image.png").read_bytes() + return [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": DECISIONS_INPUT}, + { + "type": "input_image", + "image_url": "data:image/png;base64," + base64.b64encode(image).decode(), + "detail": "low", + }, + ], + } + ] + + +def assert_decisions_image_span(spans, response, messages): + assert len(spans) == 1 + span = spans[0] + assert span["span_attributes"]["type"] == "decision" + image = span["input"]["state"][0]["content"][1] + attachment = image["image_url"]["url"] + assert isinstance(attachment, Attachment) + original = messages[0]["content"][1] + assert original["type"] == "input_image" + assert original["detail"] == "low" + assert attachment.data == base64.b64decode(original["image_url"].split(",", 1)[1]) + assert attachment.reference["content_type"] == "image/png" + assert attachment.reference["filename"] == "image.png" + assert span["input"]["state"] == [ + { + **messages[0], + "content": [ + messages[0]["content"][0], + {"type": "image_url", "image_url": {"url": attachment, "detail": "low"}}, + ], + } + ] + assert span["output"]["answers"] == [ + {"id": "positive", "type": "boolean", "probability": response.answers[0].probability} + ] + + def assert_decisions_span(spans, response, *, name, model, instrumentation, start=None, end=None): assert len(spans) == 1 span = spans[0]