diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_litellm.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_litellm.py index d5cdd667e..8dd7f529a 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_litellm.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_litellm.py @@ -1,9 +1,18 @@ """Test auto_instrument for LiteLLM.""" +import asyncio + import litellm from braintrust.auto import auto_instrument from braintrust.integrations.litellm import LiteLLMIntegration +from braintrust.integrations.test_decisions import ( + DECISIONS_INPUT, + DECISIONS_MODEL, + DECISIONS_QUESTIONS, + assert_decisions_span, +) from braintrust.integrations.test_utils import autoinstrument_test_context +from braintrust.integrations.versioning import detect_module_version, version_satisfies # 1. Verify not patched initially @@ -34,4 +43,22 @@ span = spans[0] assert span["metadata"]["provider"] == "openai" +if version_satisfies(detect_module_version(litellm, ("litellm",)), ">=1.105.0"): + with autoinstrument_test_context("test_litellm_decisions_metrics[async]", integration="litellm") as memory_logger: + response = asyncio.run( + litellm.adecisions( + model=f"openai/{DECISIONS_MODEL}", + input=DECISIONS_INPUT, + questions=DECISIONS_QUESTIONS, + safety_identifier="do-not-log-this-identifier", + ) + ) + assert_decisions_span( + memory_logger.pop(), + response, + name="Decision", + model=f"openai/{DECISIONS_MODEL}", + instrumentation="litellm-auto", + ) + print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_openai.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_openai.py index b988640a6..ac9e63710 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_openai.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_openai.py @@ -1,11 +1,19 @@ """Test auto_instrument for OpenAI.""" +import asyncio import inspect import openai from braintrust import wrap_openai from braintrust.auto import auto_instrument +from braintrust.integrations.test_decisions import ( + DECISIONS_INPUT, + DECISIONS_MODEL, + DECISIONS_QUESTIONS, + assert_decisions_span, +) from braintrust.integrations.test_utils import autoinstrument_test_context +from braintrust.integrations.versioning import version_satisfies from wrapt import FunctionWrapper @@ -15,6 +23,10 @@ def _is_braintrust_wrapped(resource) -> bool: resources = [openai.resources.chat.completions.Completions] +if version_satisfies(openai.__version__, ">=3.26.0"): + from openai.resources.decisions import AsyncDecisions, Decisions + + resources.extend((Decisions, AsyncDecisions)) try: from openai.resources.beta.agents.sessions.sessions import Sessions @@ -66,4 +78,23 @@ def _is_braintrust_wrapped(resource) -> bool: assert spans[0]["metrics"]["prompt_tokens"] == response.usage.input_tokens assert spans[0]["metrics"]["completion_tokens"] == response.usage.output_tokens +if version_satisfies(openai.__version__, ">=3.26.0"): + with autoinstrument_test_context("test_openai_decisions_metrics[async]", integration="openai") as memory_logger: + client = wrap_openai(wrap_openai(openai.AsyncOpenAI())) + response = asyncio.run( + client.decisions.create( + model=DECISIONS_MODEL, + input=DECISIONS_INPUT, + questions=DECISIONS_QUESTIONS, + safety_identifier="do-not-log-this-identifier", + ) + ) + assert_decisions_span( + memory_logger.pop(), + response, + name="openai.decisions.create", + model=DECISIONS_MODEL, + instrumentation="openai-auto", + ) + print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_typesafe.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_typesafe.py index bc4ca78d3..058b3fb60 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_typesafe.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_typesafe.py @@ -22,6 +22,7 @@ spans = memory_logger.pop() assert len(spans) == 1 span = spans[0] + assert span["span_attributes"]["type"] == "decision" assert span["metadata"]["provider"] == "typesafe" assert span["metadata"]["model"].startswith("jev-") assert span["input"]["questions"][0]["id"] == "positive" diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_decisions_image[async].yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_decisions_image[async].yaml new file mode 100644 index 000000000..4c66eb9ae --- /dev/null +++ b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_decisions_image[async].yaml @@ -0,0 +1,79 @@ +interactions: +- request: + body: '{"model":"gpt-6-luna","input":[{"role":"user","type":"message","content":[{"type":"input_text","text":"The + customer says: I love this product!"},{"type":"input_image","image_url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFElEQVR42mP4z8DwnxjMMKqQvgoBksPHOXvuG4oAAAAASUVORK5CYII=","detail":"low"}]}],"questions":[{"type":"predicate","name":"positive","instructions":"Is + the sentiment positive?"}]}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '424' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - litellm/1.105.0 + method: POST + uri: https://api.openai.com/v1/decisions + response: + body: + string: '{"model":"gpt-6-luna","answers":[{"type":"predicate","name":"positive","probability":1.0}],"usage":{"input_tokens":164,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":0,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":164}}' + headers: + Access-Control-Allow-Origin: + - '*' + Access-Control-Expose-Headers: + - x-request-id,retry-after,server-timing-total + - CF-Ray + Alt-Svc: + - h3=":443"; ma=86400 + CF-RAY: + - a4a0e088bd4136d8-YYZ + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Tue, 13 Oct 2026 19:51:06 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + Vary: + - origin, access-control-request-method, access-control-request-headers + X-Content-Type-Options: + - nosniff + X-Request-ID: + - req_8bd95865f1cd4299aa68ce9614eb8730 + cf-cache-status: + - DYNAMIC + content-length: + - '270' + openai-processing-ms: + - '55' + server-timing-total: + - '55.028' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '180000000' + x-ratelimit-remaining-requests: + - '29998' + x-ratelimit-remaining-tokens: + - '179999670' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_decisions_image[sync].yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_decisions_image[sync].yaml new file mode 100644 index 000000000..2cfb6d6d9 --- /dev/null +++ b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_decisions_image[sync].yaml @@ -0,0 +1,79 @@ +interactions: +- request: + body: '{"model":"gpt-6-luna","input":[{"role":"user","type":"message","content":[{"type":"input_text","text":"The + customer says: I love this product!"},{"type":"input_image","image_url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFElEQVR42mP4z8DwnxjMMKqQvgoBksPHOXvuG4oAAAAASUVORK5CYII=","detail":"low"}]}],"questions":[{"type":"predicate","name":"positive","instructions":"Is + the sentiment positive?"}]}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '424' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - litellm/1.105.0 + method: POST + uri: https://api.openai.com/v1/decisions + response: + body: + string: '{"model":"gpt-6-luna","answers":[{"type":"predicate","name":"positive","probability":1.0}],"usage":{"input_tokens":164,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":0,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":164}}' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - x-request-id,retry-after,server-timing-total + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a4a0e0868aa48ea1-YYZ + connection: + - keep-alive + content-length: + - '270' + content-type: + - application/json + date: + - Tue, 13 Oct 2026 19:51:06 GMT + openai-processing-ms: + - '60' + server: + - cloudflare + server-timing-total: + - '60.500' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - origin, access-control-request-method, access-control-request-headers + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '180000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '179999835' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_068b966cce534bdf911aa85c410110f2 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_decisions_metrics[async].yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_decisions_metrics[async].yaml new file mode 100644 index 000000000..49366214f --- /dev/null +++ b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_decisions_metrics[async].yaml @@ -0,0 +1,82 @@ +interactions: +- request: + body: '{"model":"gpt-6-luna","input":"The customer says: I love this product!","questions":[{"type":"predicate","name":"positive","instructions":"Is + the sentiment positive?"},{"type":"choice","name":"sentiment","instructions":"What + is the sentiment?","choices":[{"value":"positive","description":"Happy feedback"},{"value":"negative"}]},{"type":"score","name":"enthusiasm","instructions":"How + enthusiastic is the customer?","levels":[{"label":"low"},{"label":"high","description":"Very + enthusiastic"}]},{"type":"predicate","instructions":"Does the customer like + the product?"}],"safety_identifier":"do-not-log-this-identifier"}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '620' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - litellm/1.105.0 + method: POST + uri: https://api.openai.com/v1/decisions + response: + body: + string: '{"model":"gpt-6-luna","answers":[{"type":"predicate","name":"positive","probability":1.0},{"type":"choice","name":"sentiment","choice":"positive","probabilities":[{"value":"positive","probability":1.0},{"value":"negative","probability":0.0}],"confidence":1.0},{"type":"score","name":"enthusiasm","score":1.0,"probabilities":[{"value":0,"label":"low","probability":0.0},{"value":1,"label":"high","probability":1.0}],"confidence":1.0},{"type":"predicate","name":null,"probability":1.0}],"usage":{"input_tokens":524,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":0,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":524}}' + headers: + Access-Control-Allow-Origin: + - '*' + Access-Control-Expose-Headers: + - x-request-id,retry-after,server-timing-total + - CF-Ray + Alt-Svc: + - h3=":443"; ma=86400 + CF-RAY: + - a49f0ec66fc3180e-YYZ + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Tue, 13 Oct 2026 14:33:09 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + Vary: + - origin, access-control-request-method, access-control-request-headers + X-Content-Type-Options: + - nosniff + X-Request-ID: + - req_ad2cbcdd424f4f08adde160d303c9c40 + cf-cache-status: + - DYNAMIC + content-length: + - '664' + openai-processing-ms: + - '134' + server-timing-total: + - '134.317' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '180000000' + x-ratelimit-remaining-requests: + - '29996' + x-ratelimit-remaining-tokens: + - '179997900' + x-ratelimit-reset-requests: + - 8ms + x-ratelimit-reset-tokens: + - 0s + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_decisions_metrics[sync].yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_decisions_metrics[sync].yaml new file mode 100644 index 000000000..b1c1d63d2 --- /dev/null +++ b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_decisions_metrics[sync].yaml @@ -0,0 +1,82 @@ +interactions: +- request: + body: '{"model":"gpt-6-luna","input":"The customer says: I love this product!","questions":[{"type":"predicate","name":"positive","instructions":"Is + the sentiment positive?"},{"type":"choice","name":"sentiment","instructions":"What + is the sentiment?","choices":[{"value":"positive","description":"Happy feedback"},{"value":"negative"}]},{"type":"score","name":"enthusiasm","instructions":"How + enthusiastic is the customer?","levels":[{"label":"low"},{"label":"high","description":"Very + enthusiastic"}]},{"type":"predicate","instructions":"Does the customer like + the product?"}],"safety_identifier":"do-not-log-this-identifier"}' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '620' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - litellm/1.105.0 + method: POST + uri: https://api.openai.com/v1/decisions + response: + body: + string: '{"model":"gpt-6-luna","answers":[{"type":"predicate","name":"positive","probability":1.0},{"type":"choice","name":"sentiment","choice":"positive","probabilities":[{"value":"positive","probability":1.0},{"value":"negative","probability":0.0}],"confidence":1.0},{"type":"score","name":"enthusiasm","score":1.0,"probabilities":[{"value":0,"label":"low","probability":0.0},{"value":1,"label":"high","probability":1.0}],"confidence":1.0},{"type":"predicate","name":null,"probability":1.0}],"usage":{"input_tokens":524,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":0,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":524}}' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - x-request-id,retry-after,server-timing-total + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a49f0ec4b8af592d-YYZ + connection: + - keep-alive + content-length: + - '664' + content-type: + - application/json + date: + - Tue, 13 Oct 2026 14:33:08 GMT + openai-processing-ms: + - '62' + server: + - cloudflare + server-timing-total: + - '62.746' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - origin, access-control-request-method, access-control-request-headers + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '180000000' + x-ratelimit-remaining-requests: + - '29997' + x-ratelimit-remaining-tokens: + - '179998425' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_a88e743b0474432c9b7a5898edd89a6c + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/litellm/patchers.py b/py/src/braintrust/integrations/litellm/patchers.py index 2c1d487ed..a05b18881 100644 --- a/py/src/braintrust/integrations/litellm/patchers.py +++ b/py/src/braintrust/integrations/litellm/patchers.py @@ -6,6 +6,7 @@ from .tracing import ( _acompletion_wrapper_async, + _adecisions_wrapper_async, _aembedding_wrapper_async, _aimage_generation_wrapper_async, _amoderation_wrapper_async, @@ -15,6 +16,7 @@ _atext_completion_wrapper_async, _atranscription_wrapper_async, _completion_wrapper, + _decisions_wrapper, _embedding_wrapper, _image_generation_wrapper, _moderation_wrapper, @@ -67,6 +69,18 @@ class LiteLLMAresponsesPatcher(FunctionWrapperPatcher): wrapper = _aresponses_wrapper_async +class LiteLLMDecisionsPatcher(FunctionWrapperPatcher): + name = "litellm.decisions" + target_path = "decisions" + wrapper = _decisions_wrapper + + +class LiteLLMAdecisionsPatcher(FunctionWrapperPatcher): + name = "litellm.adecisions" + target_path = "adecisions" + wrapper = _adecisions_wrapper_async + + class LiteLLMImageGenerationPatcher(FunctionWrapperPatcher): name = "litellm.image_generation" target_path = "image_generation" @@ -150,6 +164,8 @@ class LiteLLMArerankPatcher(FunctionWrapperPatcher): LiteLLMATextCompletionPatcher, LiteLLMResponsesPatcher, LiteLLMAresponsesPatcher, + LiteLLMDecisionsPatcher, + LiteLLMAdecisionsPatcher, LiteLLMImageGenerationPatcher, LiteLLMAimageGenerationPatcher, LiteLLMEmbeddingPatcher, diff --git a/py/src/braintrust/integrations/litellm/test_litellm.py b/py/src/braintrust/integrations/litellm/test_litellm.py index 43de185a5..ead3681c8 100644 --- a/py/src/braintrust/integrations/litellm/test_litellm.py +++ b/py/src/braintrust/integrations/litellm/test_litellm.py @@ -8,10 +8,19 @@ from braintrust import Attachment, logger from braintrust.conftest import get_vcr_config from braintrust.integrations.litellm import patch_litellm +from braintrust.integrations.test_decisions import ( + DECISIONS_INPUT, + DECISIONS_MODEL, + DECISIONS_QUESTIONS, + assert_decisions_image_span, + assert_decisions_span, + decisions_image_input, +) from braintrust.integrations.test_utils import ( assert_metrics_are_valid, verify_autoinstrument_script, ) +from braintrust.integrations.versioning import detect_module_version, version_satisfies from braintrust.test_helpers import assert_dict_matches, init_test_logger @@ -274,6 +283,54 @@ def test_litellm_completion_streaming(memory_logger, is_async): assert "24" in str(span["output"]) or "twenty-four" in str(span["output"]).lower() +@pytest.mark.vcr +@sync_async +def test_litellm_decisions_metrics(memory_logger, is_async): + if not version_satisfies(detect_module_version(litellm, ("litellm",)), ">=1.105.0"): + pytest.skip("Decisions requires LiteLLM >=1.105.0") + + model = f"openai/{DECISIONS_MODEL}" + start = time.time() + response = _run( + is_async, + litellm.decisions, + litellm.adecisions, + model, + DECISIONS_INPUT, + DECISIONS_QUESTIONS, + safety_identifier="do-not-log-this-identifier", + ) + end = time.time() + + assert_decisions_span( + memory_logger.pop(), + response, + name="Decision", + model=model, + instrumentation="litellm-auto", + start=start, + end=end, + ) + + +@pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path", "query", "body"]) +@sync_async +def test_litellm_decisions_image(memory_logger, is_async): + if not version_satisfies(detect_module_version(litellm, ("litellm",)), ">=1.105.0"): + pytest.skip("Decisions requires LiteLLM >=1.105.0") + + messages = decisions_image_input() + response = _run( + is_async, + litellm.decisions, + litellm.adecisions, + model=f"openai/{DECISIONS_MODEL}", + input=messages, + questions=DECISIONS_QUESTIONS[:1], + ) + assert_decisions_image_span(memory_logger.pop(), response, messages) + + @pytest.mark.vcr @sync_async def test_litellm_responses_metrics(memory_logger, is_async): diff --git a/py/src/braintrust/integrations/litellm/tracing.py b/py/src/braintrust/integrations/litellm/tracing.py index f26c7b76f..d01a9c2b3 100644 --- a/py/src/braintrust/integrations/litellm/tracing.py +++ b/py/src/braintrust/integrations/litellm/tracing.py @@ -7,6 +7,7 @@ from types import TracebackType from typing import Any +from braintrust.integrations.openai.tracing import _decisions_input, _decisions_output from braintrust.integrations.utils import ( _extract_audio_output, _materialize_attachment, @@ -342,6 +343,53 @@ async def _aresponses_wrapper_async(wrapped, instance, args, kwargs): span.end() +def _decisions_span_payload(args, kwargs): + # Decisions accepts model, input and questions positionally as well as by keyword. + params = dict(zip(("model", "input", "questions"), args)) + params.update(kwargs) + return merge_dicts( + params.get("span_info", {}), + { + "input": _decisions_input(params.get("state", params.get("input")), params.get("questions") or []), + "metadata": _build_span_metadata({}, params.get("model")), + }, + ) + + +def _decisions_wrapper(wrapped, instance, args, kwargs): + with start_span( + **merge_dicts( + dict(name="Decision", span_attributes={"type": SpanTypeAttribute.DECISION}), + _decisions_span_payload(args, kwargs), + ) + ) as span: + response = wrapped(*args, **kwargs) + span.log( + output=_decisions_output( + _get_field(response, "answers"), kwargs.get("questions", args[2] if len(args) > 2 else []) + ), + metrics=_parse_metrics_from_usage(_get_field(response, "usage")), + ) + return response + + +async def _adecisions_wrapper_async(wrapped, instance, args, kwargs): + with start_span( + **merge_dicts( + dict(name="Decision", span_attributes={"type": SpanTypeAttribute.DECISION}), + _decisions_span_payload(args, kwargs), + ) + ) as span: + response = await wrapped(*args, **kwargs) + span.log( + output=_decisions_output( + _get_field(response, "answers"), kwargs.get("questions", args[2] if len(args) > 2 else []) + ), + metrics=_parse_metrics_from_usage(_get_field(response, "usage")), + ) + return response + + def _image_attachment_from_base64( data: Any, *, output_format: Any, index: int ) -> tuple[_ResolvedAttachment | None, int | None]: diff --git a/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_image[async].yaml b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_image[async].yaml new file mode 100644 index 000000000..7c5a50b59 --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_image[async].yaml @@ -0,0 +1,97 @@ +interactions: +- request: + body: '{"model":"gpt-6-luna","input":[{"role":"user","content":[{"type":"input_text","text":"The + customer says: I love this product!"},{"type":"input_image","image_url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFElEQVR42mP4z8DwnxjMMKqQvgoBksPHOXvuG4oAAAAASUVORK5CYII=","detail":"low"}]}],"questions":[{"type":"predicate","name":"positive","instructions":"Is + the sentiment positive?"}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '407' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 3.29.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 3.29.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/decisions + response: + body: + string: '{"model":"gpt-6-luna","answers":[{"type":"predicate","name":"positive","probability":1.0}],"usage":{"input_tokens":164,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":0,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":164}}' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - x-request-id,retry-after,server-timing-total + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a4a0e06fa84da2d9-YYZ + connection: + - keep-alive + content-length: + - '270' + content-type: + - application/json + date: + - Tue, 13 Oct 2026 19:51:02 GMT + openai-processing-ms: + - '95' + server: + - cloudflare + server-timing-total: + - '95.131' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - origin, access-control-request-method, access-control-request-headers + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '180000000' + x-ratelimit-remaining-requests: + - '29998' + x-ratelimit-remaining-tokens: + - '179999670' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_225e76ef1aa54761afc4da1208c70d50 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_image[sync].yaml b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_image[sync].yaml new file mode 100644 index 000000000..770f4815b --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_image[sync].yaml @@ -0,0 +1,97 @@ +interactions: +- request: + body: '{"model":"gpt-6-luna","input":[{"role":"user","content":[{"type":"input_text","text":"The + customer says: I love this product!"},{"type":"input_image","image_url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFElEQVR42mP4z8DwnxjMMKqQvgoBksPHOXvuG4oAAAAASUVORK5CYII=","detail":"low"}]}],"questions":[{"type":"predicate","name":"positive","instructions":"Is + the sentiment positive?"}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '407' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 3.29.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 3.29.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/decisions + response: + body: + string: '{"model":"gpt-6-luna","answers":[{"type":"predicate","name":"positive","probability":1.0}],"usage":{"input_tokens":164,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":0,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":164}}' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - x-request-id,retry-after,server-timing-total + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a4a0e06a5f73fcd9-YYZ + connection: + - keep-alive + content-length: + - '270' + content-type: + - application/json + date: + - Tue, 13 Oct 2026 19:51:02 GMT + openai-processing-ms: + - '95' + server: + - cloudflare + server-timing-total: + - '95.586' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - origin, access-control-request-method, access-control-request-headers + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '180000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '179999835' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_89672dba3b884abfb5d42a66a7f06ffc + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_iterable_input[async].yaml b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_iterable_input[async].yaml new file mode 100644 index 000000000..f476cab8e --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_iterable_input[async].yaml @@ -0,0 +1,97 @@ +interactions: +- request: + body: '{"model":"gpt-6-luna","input":[{"role":"user","content":[{"type":"input_text","text":"The + customer says: I love this product!"}]}],"questions":[{"type":"predicate","name":"positive","instructions":"Is + the sentiment positive?"}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '228' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 3.29.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 3.29.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/decisions + response: + body: + string: '{"model":"gpt-6-luna","answers":[{"type":"predicate","name":"positive","probability":1.0}],"usage":{"input_tokens":162,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":0,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":162}}' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - x-request-id,retry-after,server-timing-total + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a49f633acdcf39ff-YYZ + connection: + - keep-alive + content-length: + - '270' + content-type: + - application/json + date: + - Tue, 13 Oct 2026 15:30:48 GMT + openai-processing-ms: + - '74' + server: + - cloudflare + server-timing-total: + - '74.548' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - origin, access-control-request-method, access-control-request-headers + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '180000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '179999838' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_3c64e5071d234548970eb73bb276fa58 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_iterable_input[sync].yaml b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_iterable_input[sync].yaml new file mode 100644 index 000000000..27f83bb9a --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_iterable_input[sync].yaml @@ -0,0 +1,97 @@ +interactions: +- request: + body: '{"model":"gpt-6-luna","input":[{"role":"user","content":[{"type":"input_text","text":"The + customer says: I love this product!"}]}],"questions":[{"type":"predicate","name":"positive","instructions":"Is + the sentiment positive?"}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '228' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 3.29.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 3.29.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/decisions + response: + body: + string: '{"model":"gpt-6-luna","answers":[{"type":"predicate","name":"positive","probability":1.0}],"usage":{"input_tokens":162,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":0,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":162}}' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - x-request-id,retry-after,server-timing-total + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a49f6335c841ac75-YYZ + connection: + - keep-alive + content-length: + - '270' + content-type: + - application/json + date: + - Tue, 13 Oct 2026 15:30:47 GMT + openai-processing-ms: + - '103' + server: + - cloudflare + server-timing-total: + - '103.148' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - origin, access-control-request-method, access-control-request-headers + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '180000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '179999838' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_858d6124ba054d3bb167a9d381a15c37 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_metrics[async].yaml b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_metrics[async].yaml new file mode 100644 index 000000000..7e71826d7 --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_metrics[async].yaml @@ -0,0 +1,88 @@ +interactions: +- request: + body: '{"model":"gpt-6-luna","input":"The customer says: I love this product!","questions":[{"type":"predicate","name":"positive","instructions":"Is + the sentiment positive?"},{"type":"choice","name":"sentiment","instructions":"What + is the sentiment?","choices":[{"value":"positive","description":"Happy feedback"},{"value":"negative"}]},{"type":"score","name":"enthusiasm","instructions":"How + enthusiastic is the customer?","levels":[{"label":"low"},{"label":"high","description":"Very + enthusiastic"}]},{"type":"predicate","instructions":"Does the customer like + the product?"}],"safety_identifier":"do-not-log-this-identifier"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '620' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 3.29.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 3.29.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/decisions + response: + body: + string: '{"model":"gpt-6-luna","answers":[{"type":"predicate","name":"positive","probability":1.0},{"type":"choice","name":"sentiment","choice":"positive","probabilities":[{"value":"positive","probability":1.0},{"value":"negative","probability":0.0}],"confidence":1.0},{"type":"score","name":"enthusiasm","score":1.0,"probabilities":[{"value":0,"label":"low","probability":0.0},{"value":1,"label":"high","probability":1.0}],"confidence":1.0},{"type":"predicate","name":null,"probability":1.0}],"usage":{"input_tokens":524,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":0,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":524}}' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - x-request-id,retry-after,server-timing-total + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a49f0eb61b21a1db-YYZ + connection: + - keep-alive + content-length: + - '664' + content-type: + - application/json + date: + - Tue, 13 Oct 2026 14:33:06 GMT + openai-processing-ms: + - '80' + server: + - cloudflare + server-timing-total: + - '80.087' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - origin, access-control-request-method, access-control-request-headers + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-request-id: + - req_3134eed4b83b4f4ea2114a24ce9aaf77 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_metrics[sync].yaml b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_metrics[sync].yaml new file mode 100644 index 000000000..15aa7b821 --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_decisions_metrics[sync].yaml @@ -0,0 +1,100 @@ +interactions: +- request: + body: '{"model":"gpt-6-luna","input":"The customer says: I love this product!","questions":[{"type":"predicate","name":"positive","instructions":"Is + the sentiment positive?"},{"type":"choice","name":"sentiment","instructions":"What + is the sentiment?","choices":[{"value":"positive","description":"Happy feedback"},{"value":"negative"}]},{"type":"score","name":"enthusiasm","instructions":"How + enthusiastic is the customer?","levels":[{"label":"low"},{"label":"high","description":"Very + enthusiastic"}]},{"type":"predicate","instructions":"Does the customer like + the product?"}],"safety_identifier":"do-not-log-this-identifier"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '620' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 3.29.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 3.29.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/decisions + response: + body: + string: '{"model":"gpt-6-luna","answers":[{"type":"predicate","name":"positive","probability":1.0},{"type":"choice","name":"sentiment","choice":"positive","probabilities":[{"value":"positive","probability":1.0},{"value":"negative","probability":0.0}],"confidence":1.0},{"type":"score","name":"enthusiasm","score":1.0,"probabilities":[{"value":0,"label":"low","probability":0.0},{"value":1,"label":"high","probability":1.0}],"confidence":1.0},{"type":"predicate","name":null,"probability":1.0}],"usage":{"input_tokens":524,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":0,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":524}}' + headers: + access-control-allow-origin: + - '*' + access-control-expose-headers: + - x-request-id,retry-after,server-timing-total + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a49f0eb42cfca06e-YYZ + connection: + - keep-alive + content-length: + - '664' + content-type: + - application/json + date: + - Tue, 13 Oct 2026 14:33:06 GMT + openai-processing-ms: + - '112' + server: + - cloudflare + server-timing-total: + - '112.779' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + vary: + - origin, access-control-request-method, access-control-request-headers + x-content-type-options: + - nosniff + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '180000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '179999475' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_02f6c91e11bd48d299ddf7359b841be7 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/integration.py b/py/src/braintrust/integrations/openai/integration.py index f5f1fd06d..dd8763197 100644 --- a/py/src/braintrust/integrations/openai/integration.py +++ b/py/src/braintrust/integrations/openai/integration.py @@ -8,6 +8,7 @@ AudioTranscriptionsPatcher, AudioTranslationsPatcher, ChatCompletionsPatcher, + DecisionsPatcher, EmbeddingsPatcher, ImagesPatcher, ModerationsPatcher, @@ -30,4 +31,5 @@ class OpenAIIntegration(BaseIntegration): AudioTranslationsPatcher, ImagesPatcher, ResponsesPatcher, + DecisionsPatcher, ) diff --git a/py/src/braintrust/integrations/openai/patchers.py b/py/src/braintrust/integrations/openai/patchers.py index 0bc69fa4e..b924d7e77 100644 --- a/py/src/braintrust/integrations/openai/patchers.py +++ b/py/src/braintrust/integrations/openai/patchers.py @@ -13,6 +13,7 @@ _audio_translation_create_wrapper, _chat_completion_create_wrapper, _chat_completion_parse_wrapper, + _decisions_create_wrapper, _embedding_create_wrapper, _image_create_variation_wrapper, _image_edit_wrapper, @@ -436,6 +437,31 @@ class _WrapResponsesRaw(CompositeFunctionWrapperPatcher): sub_patchers = (_wrap_resp_raw_create, _wrap_resp_raw_parse) +# --------------------------------------------------------------------------- +# Decisions (OpenAI >= 3.26.0) +# --------------------------------------------------------------------------- + +_dec_create_sync, _dec_create_async, _wrap_dec_create = _make_method_patchers( + name_prefix="openai.decisions.create", + target_module="openai.resources.decisions", + sync_class="Decisions", + async_class="AsyncDecisions", + method="create", + wrapper=_decisions_create_wrapper, + wrap_name="openai.wrap.decisions.create", +) + + +class DecisionsPatcher(CompositeFunctionWrapperPatcher): + name = "openai.decisions" + sub_patchers = (_dec_create_sync, _dec_create_async) + + +class _WrapDecisions(CompositeFunctionWrapperPatcher): + name = "openai.wrap.decisions" + sub_patchers = (_wrap_dec_create,) + + # --------------------------------------------------------------------------- # Resource mapping — single source of truth for wrap_openai. # @@ -458,6 +484,7 @@ class _WrapResponsesRaw(CompositeFunctionWrapperPatcher): ("audio.translations", _WrapAudioTranslations), ("images", _WrapImages), ("responses", _WrapResponses), + ("decisions", _WrapDecisions), ("responses.with_raw_response", _WrapResponsesRaw), ("beta.chat.completions", _WrapChatCompletions), ) diff --git a/py/src/braintrust/integrations/openai/test_openai.py b/py/src/braintrust/integrations/openai/test_openai.py index 0b5641b20..1a1992835 100644 --- a/py/src/braintrust/integrations/openai/test_openai.py +++ b/py/src/braintrust/integrations/openai/test_openai.py @@ -17,8 +17,17 @@ _materialize_logged_file_input, _process_attachments_in_chat_output, ) +from braintrust.integrations.test_decisions import ( + DECISIONS_INPUT, + DECISIONS_MODEL, + DECISIONS_QUESTIONS, + assert_decisions_image_span, + assert_decisions_span, + decisions_image_input, +) from braintrust.integrations.test_utils import assert_metrics_are_valid, verify_autoinstrument_script from braintrust.integrations.utils import _try_to_dict +from braintrust.integrations.versioning import version_satisfies from braintrust.span_types import SpanTypeAttribute from braintrust.test_helpers import assert_dict_matches, init_test_logger from openai import AsyncOpenAI @@ -185,6 +194,92 @@ async def test_unwrapped_client_emits_no_spans(memory_logger, is_async, vcr_cass assert not memory_logger.pop() +@sync_async +@pytest.mark.asyncio +@pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path", "query", "body"]) +async def test_openai_decisions_metrics(memory_logger, is_async): + if not version_satisfies(openai.__version__, ">=3.26.0"): + pytest.skip("Decisions requires OpenAI >=3.26.0") + + client = wrap_openai(wrap_openai(_client(is_async))) + questions = [dict(question) for question in DECISIONS_QUESTIONS] + choices = iter(DECISIONS_QUESTIONS[1]["choices"]) + levels = iter(DECISIONS_QUESTIONS[2]["levels"]) + questions[1]["choices"] = choices + questions[2]["levels"] = levels + start = time.time() + response = await _call( + is_async, + client.decisions.create, + model=DECISIONS_MODEL, + input=DECISIONS_INPUT, + questions=iter(questions), + safety_identifier="do-not-log-this-identifier", + ) + end = time.time() + + assert questions[1]["choices"] is choices + assert questions[2]["levels"] is levels + assert_decisions_span( + memory_logger.pop(), + response, + name="openai.decisions.create", + model=DECISIONS_MODEL, + instrumentation="openai-auto", + start=start, + end=end, + ) + with pytest.raises(TypeError, match="required keyword-only arguments"): + await _call(is_async, client.decisions.create, model=DECISIONS_MODEL) + + +@sync_async +@pytest.mark.asyncio +@pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path", "query", "body"]) +async def test_openai_decisions_iterable_input(memory_logger, is_async): + if not version_satisfies(openai.__version__, ">=3.26.0"): + pytest.skip("Decisions requires OpenAI >=3.26.0") + + messages = [{"role": "user", "content": [{"type": "input_text", "text": DECISIONS_INPUT}]}] + content = iter(messages[0]["content"]) + request_messages = [{**messages[0], "content": content}] + client = wrap_openai(_client(is_async)) + response = await _call( + is_async, + client.decisions.create, + model=DECISIONS_MODEL, + input=iter(request_messages), + questions=DECISIONS_QUESTIONS[:1], + ) + + spans = memory_logger.pop() + assert len(spans) == 1 + assert request_messages[0]["content"] is content + assert spans[0]["input"]["state"] == messages + assert spans[0]["output"]["answers"] == [ + {"id": "positive", "type": "boolean", "probability": response.answers[0].probability} + ] + + +@sync_async +@pytest.mark.asyncio +@pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path", "query", "body"]) +async def test_openai_decisions_image(memory_logger, is_async): + if not version_satisfies(openai.__version__, ">=3.26.0"): + pytest.skip("Decisions requires OpenAI >=3.26.0") + + messages = decisions_image_input() + client = wrap_openai(_client(is_async)) + response = await _call( + is_async, + client.decisions.create, + model=DECISIONS_MODEL, + input=messages, + questions=DECISIONS_QUESTIONS[:1], + ) + assert_decisions_image_span(memory_logger.pop(), response, messages) + + @sync_async @pytest.mark.asyncio @pytest.mark.vcr diff --git a/py/src/braintrust/integrations/openai/tracing.py b/py/src/braintrust/integrations/openai/tracing.py index 2a479dcf1..37ee6f5a5 100644 --- a/py/src/braintrust/integrations/openai/tracing.py +++ b/py/src/braintrust/integrations/openai/tracing.py @@ -5,10 +5,12 @@ import binascii import inspect import json +import logging import time from collections.abc import Callable, Iterable, Mapping from typing import Any +from braintrust.integrations.question import _question_span_input, _question_span_output from braintrust.integrations.utils import ( _extract_audio_output, _infer_audio_mime_type, @@ -26,6 +28,7 @@ _INSTRUMENTATION = "openai-auto" +_log = logging.getLogger(__name__) def start_span(*args, **kwargs): @@ -276,6 +279,17 @@ def _process_attachments_in_input(input_data: Any) -> Any: return [_process_attachments_in_input(item) for item in input_data] if isinstance(input_data, dict): + if input_data.get("type") == "input_image" and isinstance(input_data.get("image_url"), str): + resolved = _materialize_attachment(input_data["image_url"]) + if resolved is None: + return input_data + image_part = dict(input_data) + image_part["type"] = "image_url" + image_part["image_url"] = {"url": resolved.attachment} + if "detail" in image_part: + image_part["image_url"]["detail"] = image_part.pop("detail") + return image_part + if ( input_data.get("type") == "image_url" and isinstance(input_data.get("image_url"), dict) @@ -2038,6 +2052,173 @@ def process_output(self, response: dict[str, Any], span: Span): ) +def _decision_question_ids(questions: list[dict[str, Any]]) -> list[str]: + names = {question.get("name") for question in questions if isinstance(question.get("name"), str)} + used: set[str] = set() + ids = [] + for index, question in enumerate(questions): + identifier = question.get("name") + if not isinstance(identifier, str) or not identifier or identifier in used: + identifier = f"question_{index}" + while identifier in names or identifier in used: + identifier += "_" + used.add(identifier) + ids.append(identifier) + return ids + + +def _decision_choice_keys(question: dict[str, Any]) -> dict[tuple[type, Any], str]: + choices = question.get("choices", []) + string_values = {choice["value"] for choice in choices if isinstance(choice.get("value"), str)} + keys = {} + for choice in choices: + value = choice["value"] + key = value + if isinstance(value, bool): + key = str(value).lower() + while key in string_values: + key = f"boolean:{key}" + keys[type(value), value] = key + return keys + + +def _decisions_input(state: Any, questions: Any) -> dict[str, Any]: + try: + state = _process_attachments_in_input(state) + if isinstance(questions, Mapping): + return _question_span_input(state, questions) + questions = [_try_to_dict(question) for question in questions] + normalized = [] + for identifier, question in zip(_decision_question_ids(questions), questions): + kind = question.get("type") + payload = { + "id": identifier, + "type": "boolean" if kind == "predicate" else kind, + "instructions": question.get("instructions"), + } + if kind == "choice": + keys = _decision_choice_keys(question) + payload["criteria"] = { + keys[type(choice["value"]), choice["value"]]: choice.get("description") + for choice in question.get("choices", []) + } + elif kind == "score": + payload["criteria"] = [ + {key: level[key] for key in ("label", "description") if key in level} + for level in question.get("levels", []) + ] + normalized.append(payload) + return _question_span_input(state, normalized) + except Exception: + _log.debug("Failed to normalize Decisions questions", exc_info=True) + return {"state": state, "questions": []} + + +def _decisions_output(answers: Any, questions: Any) -> dict[str, Any]: + try: + if isinstance(answers, Mapping): + return _question_span_output(answers) + questions = [_try_to_dict(question) for question in questions] + normalized = [] + # OpenAI guarantees answer order, including refusals and unnamed questions. + # Use the request's IDs so all answers remain linked to their questions. + for identifier, question, answer in zip(_decision_question_ids(questions), questions, answers or []): + answer = _try_to_dict(answer) + if not isinstance(answer, dict): + continue + kind = answer.get("type") + payload = {"id": identifier, "type": "boolean" if kind == "predicate" else kind} + for key in ("probability", "score", "confidence"): + if key in answer: + payload[key] = answer[key] + if kind == "choice": + keys = _decision_choice_keys(question) + if "choice" in answer: + value = answer["choice"] + payload["choice"] = keys.get((type(value), value), str(value)) + if "probabilities" in answer: + payload["probabilities"] = { + keys.get((type(item["value"]), item["value"]), str(item["value"])): item["probability"] + for item in answer["probabilities"] + } + elif kind == "score" and "probabilities" in answer: + payload["probabilities"] = { + str(item["value"]): item["probability"] for item in answer["probabilities"] + } + payload["legend"] = {str(item["value"]): item["label"] for item in answer["probabilities"]} + normalized.append(payload) + return _question_span_output(normalized) + except Exception: + _log.debug("Failed to normalize Decisions answers", exc_info=True) + return {"answers": []} + + +def _materialize_decision_items(values: Any, nested_fields: tuple[str, ...]) -> Any: + if not isinstance(values, Iterable) or isinstance(values, (str, bytes, Mapping)): + return values + # Logging and the SDK must see the same values without consuming a caller's + # iterators twice or replacing fields on their objects. + items = [] + for item in values: + if isinstance(item, Mapping): + item = dict(item) + for field in nested_fields: + nested = item.get(field) + if isinstance(nested, Iterable) and not isinstance(nested, (list, str, bytes, Mapping)): + item[field] = list(nested) + items.append(item) + return items + + +def _decisions_span_params(params: dict[str, Any]) -> dict[str, Any]: + ret = params.pop("span_info", {}) + for field, nested_fields in (("questions", ("choices", "levels")), ("input", ("content",))): + if field in params: + params[field] = _materialize_decision_items(params[field], nested_fields) + return merge_dicts( + ret, + { + "input": _decisions_input(params.get("input"), params.get("questions") or []), + "metadata": _filter_metadata(params, ("model",)), + }, + ) + + +def _log_decisions_response(span: Span, response: Any, questions: Any): + response = _try_to_dict(response) + if isinstance(response, dict): + span.log( + output=_decisions_output(response.get("answers"), questions or []), + metrics=_parse_metrics_from_usage(response.get("usage")), + ) + + +def _decisions_create_wrapper(wrapped, instance, args, kwargs): + if _is_async_callable(wrapped): + + async def call(): + with start_span( + **merge_dicts( + dict(name="openai.decisions.create", span_attributes={"type": SpanTypeAttribute.DECISION}), + _decisions_span_params(kwargs), + ) + ) as span: + response = await wrapped(*args, **kwargs) + _log_decisions_response(span, response, kwargs.get("questions")) + return response + + return call() + with start_span( + **merge_dicts( + dict(name="openai.decisions.create", span_attributes={"type": SpanTypeAttribute.DECISION}), + _decisions_span_params(kwargs), + ) + ) as span: + response = wrapped(*args, **kwargs) + _log_decisions_response(span, response, kwargs.get("questions")) + return response + + class ModerationWrapper(BaseWrapper): _metadata_params = _MODERATION_METADATA_PARAMS @@ -2308,11 +2489,15 @@ def process_output(self, response: Any, span: Span): def _parse_metrics_from_usage(usage: Any) -> dict[str, Any]: - return _parse_openai_usage_metrics( + metrics = _parse_openai_usage_metrics( usage, token_name_map=TOKEN_NAME_MAP, token_prefix_map=TOKEN_PREFIX_MAP, ) + cache_write_tokens = metrics.pop("prompt_cache_write_tokens", None) + if cache_write_tokens is not None: + metrics.setdefault("prompt_cache_creation_tokens", cache_write_tokens) + return metrics def prettify_params(params: dict[str, Any]) -> dict[str, Any]: diff --git a/py/src/braintrust/integrations/question.py b/py/src/braintrust/integrations/question.py new file mode 100644 index 000000000..4376f41d7 --- /dev/null +++ b/py/src/braintrust/integrations/question.py @@ -0,0 +1,33 @@ +"""Shared decision span structure from braintrust/typespecs/src/question.ts. + +Provider adapters normalize their question and answer types before using these +helpers. Named maps become arrays linked by ID; arrays already carry their IDs. +""" + +from collections.abc import Mapping +from typing import Any + +from .utils import _try_to_dict + + +_QUESTION_FIELDS = ("type", "instructions", "criteria") +_ANSWER_FIELDS = ("type", "noul", "probability", "choice", "score", "confidence", "legend", "probabilities") + + +def _question_items(values: Any, fields: tuple[str, ...]) -> Any: + if not isinstance(values, Mapping): + return values + items = [] + for identifier, value in values.items(): + value = _try_to_dict(value) + if isinstance(value, dict): + items.append({**{key: value[key] for key in fields if key in value}, "id": identifier}) + return items + + +def _question_span_input(state: Any, questions: Any) -> dict[str, Any]: + return {"state": state, "questions": _question_items(questions, _QUESTION_FIELDS)} + + +def _question_span_output(answers: Any) -> dict[str, Any]: + return {"answers": _question_items(answers, _ANSWER_FIELDS)} diff --git a/py/src/braintrust/integrations/test_decisions.py b/py/src/braintrust/integrations/test_decisions.py new file mode 100644 index 000000000..4742cf95a --- /dev/null +++ b/py/src/braintrust/integrations/test_decisions.py @@ -0,0 +1,227 @@ +import base64 +from pathlib import Path + +import pytest +from braintrust import Attachment +from braintrust.integrations.litellm.tracing import _parse_metrics_from_usage as parse_litellm_usage +from braintrust.integrations.openai.tracing import _decisions_input, _decisions_output +from braintrust.integrations.openai.tracing import _parse_metrics_from_usage as parse_openai_usage +from braintrust.test_helpers import assert_dict_matches + + +DECISIONS_MODEL = "gpt-6-luna" +DECISIONS_INPUT = "The customer says: I love this product!" +DECISIONS_QUESTIONS = [ + {"type": "predicate", "name": "positive", "instructions": "Is the sentiment positive?"}, + { + "type": "choice", + "name": "sentiment", + "instructions": "What is the sentiment?", + "choices": [{"value": "positive", "description": "Happy feedback"}, {"value": "negative"}], + }, + { + "type": "score", + "name": "enthusiasm", + "instructions": "How enthusiastic is the customer?", + "levels": [{"label": "low"}, {"label": "high", "description": "Very enthusiastic"}], + }, + {"type": "predicate", "instructions": "Does the customer like the product?"}, +] + + +def decisions_image_input(): + image = (Path(__file__).parent.parent / "fixtures" / "test-image.png").read_bytes() + return [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": DECISIONS_INPUT}, + { + "type": "input_image", + "image_url": "data:image/png;base64," + base64.b64encode(image).decode(), + "detail": "low", + }, + ], + } + ] + + +def assert_decisions_image_span(spans, response, messages): + assert len(spans) == 1 + span = spans[0] + assert span["span_attributes"]["type"] == "decision" + image = span["input"]["state"][0]["content"][1] + attachment = image["image_url"]["url"] + assert isinstance(attachment, Attachment) + original = messages[0]["content"][1] + assert original["type"] == "input_image" + assert original["detail"] == "low" + assert attachment.data == base64.b64decode(original["image_url"].split(",", 1)[1]) + assert attachment.reference["content_type"] == "image/png" + assert attachment.reference["filename"] == "image.png" + assert span["input"]["state"] == [ + { + **messages[0], + "content": [ + messages[0]["content"][0], + {"type": "image_url", "image_url": {"url": attachment, "detail": "low"}}, + ], + } + ] + assert span["output"]["answers"] == [ + {"id": "positive", "type": "boolean", "probability": response.answers[0].probability} + ] + + +def assert_decisions_span(spans, response, *, name, model, instrumentation, start=None, end=None): + assert len(spans) == 1 + span = spans[0] + assert_dict_matches(span["span_attributes"], {"name": name, "type": "decision"}) + assert span["input"] == { + "state": DECISIONS_INPUT, + "questions": [ + {"id": "positive", "type": "boolean", "instructions": "Is the sentiment positive?"}, + { + "id": "sentiment", + "type": "choice", + "instructions": "What is the sentiment?", + "criteria": {"positive": "Happy feedback", "negative": None}, + }, + { + "id": "enthusiasm", + "type": "score", + "instructions": "How enthusiastic is the customer?", + "criteria": [{"label": "low"}, {"label": "high", "description": "Very enthusiastic"}], + }, + {"id": "question_3", "type": "boolean", "instructions": "Does the customer like the product?"}, + ], + } + assert [answer["id"] for answer in span["output"]["answers"]] == [ + "positive", + "sentiment", + "enthusiasm", + "question_3", + ] + assert span["output"]["answers"][0] == { + "id": "positive", + "type": "boolean", + "probability": response.answers[0].probability, + } + assert span["output"]["answers"][1]["choice"] == response.answers[1].choice + assert span["output"]["answers"][1]["probabilities"] == { + item.value: item.probability for item in response.answers[1].probabilities + } + assert span["output"]["answers"][1]["confidence"] == response.answers[1].confidence + assert span["output"]["answers"][2]["score"] == response.answers[2].score + assert span["output"]["answers"][2]["confidence"] == response.answers[2].confidence + assert span["output"]["answers"][2]["probabilities"] == { + str(item.value): item.probability for item in response.answers[2].probabilities + } + assert span["output"]["answers"][2]["legend"] == { + str(item.value): item.label for item in response.answers[2].probabilities + } + assert span["output"]["answers"][3] == { + "id": "question_3", + "type": "boolean", + "probability": response.answers[3].probability, + } + assert span["metadata"] == {"model": model, "provider": "openai"} + assert span["context"]["span_origin"]["instrumentation"]["name"] == instrumentation + metrics = span["metrics"] + assert metrics["start"] <= metrics["end"] + if start is not None and end is not None: + assert start <= metrics["start"] <= metrics["end"] <= end + assert all(metrics[key] >= 0 for key in ("tokens", "prompt_tokens", "completion_tokens")) + assert metrics["tokens"] == response.usage.total_tokens + assert metrics["prompt_tokens"] == response.usage.input_tokens + assert metrics["completion_tokens"] == response.usage.output_tokens + assert metrics["prompt_cached_tokens"] == response.usage.input_tokens_details.cached_tokens + assert metrics["prompt_cache_creation_tokens"] == response.usage.input_tokens_details.cache_write_tokens + assert metrics["completion_reasoning_tokens"] == response.usage.output_tokens_details.reasoning_tokens + assert "prompt_cache_write_tokens" not in metrics + + +def test_decisions_ids_are_unique_and_match_answers(): + questions = [ + {"type": "predicate", "name": "question_1", "instructions": "First"}, + {"type": "predicate", "instructions": "Second"}, + {"type": "predicate", "name": "question_1", "instructions": "Third"}, + ] + span_input = _decisions_input("state", questions) + span_output = _decisions_output([{"type": "predicate", "probability": 0.5}] * 3, questions) + ids = [question["id"] for question in span_input["questions"]] + assert ids == ["question_1", "question_1_", "question_2"] + assert [answer["id"] for answer in span_output["answers"]] == ids + + +def test_decisions_preserves_typed_choices(): + questions = [ + { + "type": "choice", + "name": "typed", + "instructions": "Choose", + "choices": [{"value": True}, {"value": "true"}], + } + ] + span_input = _decisions_input("state", questions) + span_output = _decisions_output( + [ + { + "type": "choice", + "choice": True, + "probabilities": [{"value": True, "probability": 0.8}, {"value": "true", "probability": 0.2}], + } + ], + questions, + ) + assert span_input["questions"][0]["criteria"] == {"boolean:true": None, "true": None} + assert span_output["answers"][0] == { + "id": "typed", + "type": "choice", + "choice": "boolean:true", + "probabilities": {"boolean:true": 0.8, "true": 0.2}, + } + + +def test_decisions_refusals_and_allowlists(): + questions = [{"type": "predicate", "name": "refused", "instructions": "Question", "secret": "omit"}] + span_input = _decisions_input({"message": "state"}, questions) + span_output = _decisions_output([{"type": "refusal", "name": "refused", "secret": "omit"}], questions) + assert span_input == { + "state": {"message": "state"}, + "questions": [{"id": "refused", "type": "boolean", "instructions": "Question"}], + } + assert span_output == {"answers": [{"id": "refused", "type": "refusal"}]} + + +def test_decisions_system_one_maps_match_typesafe_structure(): + questions = {"positive": {"type": "noul", "instructions": "Is this positive?", "secret": "omit"}} + answers = {"positive": {"type": "noul", "noul": 0.9, "secret": "omit"}} + assert _decisions_input("state", questions) == { + "state": "state", + "questions": [{"id": "positive", "type": "noul", "instructions": "Is this positive?"}], + } + assert _decisions_output(answers, questions) == {"answers": [{"id": "positive", "type": "noul", "noul": 0.9}]} + + +@pytest.mark.parametrize("parse_usage", (parse_openai_usage, parse_litellm_usage), ids=("openai", "litellm")) +def test_decisions_usage_with_nonzero_token_details(parse_usage): + # The live Decisions recordings report zero for these optional details. + # Exercise nonzero values directly through the local usage parser too. + metrics = parse_usage( + { + "input_tokens": 100, + "output_tokens": 20, + "total_tokens": 120, + "input_tokens_details": {"cached_tokens": 32, "cache_write_tokens": 16}, + "output_tokens_details": {"reasoning_tokens": 6}, + } + ) + assert metrics == { + "prompt_tokens": 100, + "completion_tokens": 20, + "tokens": 120, + "prompt_cached_tokens": 32, + "prompt_cache_creation_tokens": 16, + "completion_reasoning_tokens": 6, + } diff --git a/py/src/braintrust/integrations/typesafe/test_typesafe.py b/py/src/braintrust/integrations/typesafe/test_typesafe.py index f00ad2f00..b12816527 100644 --- a/py/src/braintrust/integrations/typesafe/test_typesafe.py +++ b/py/src/braintrust/integrations/typesafe/test_typesafe.py @@ -21,7 +21,7 @@ def memory_logger(): def _assert_span(span, *, question_ids): assert span["span_attributes"]["name"] == "typesafe.systemOne" - assert span["span_attributes"]["type"] == SpanTypeAttribute.QUESTION + assert span["span_attributes"]["type"] == SpanTypeAttribute.DECISION assert span["context"]["span_origin"]["instrumentation"]["name"] == "typesafe" assert span["metadata"]["provider"] == "typesafe" assert span["metadata"]["model"].startswith("jev-") diff --git a/py/src/braintrust/integrations/typesafe/tracing.py b/py/src/braintrust/integrations/typesafe/tracing.py index 247a21719..f628bda25 100644 --- a/py/src/braintrust/integrations/typesafe/tracing.py +++ b/py/src/braintrust/integrations/typesafe/tracing.py @@ -3,6 +3,12 @@ from collections.abc import Mapping from typing import Any +from braintrust.integrations.question import ( + _ANSWER_FIELDS, + _QUESTION_FIELDS, + _question_span_input, + _question_span_output, +) from braintrust.integrations.utils import _log_and_end_span, _log_error_and_end_span from braintrust.logger import start_span as _bt_start_span from braintrust.span_types import SpanTypeAttribute @@ -10,8 +16,6 @@ _INSTRUMENTATION = "typesafe" _MISSING = object() -_QUESTION_FIELDS = ("type", "instructions", "criteria") -_ANSWER_FIELDS = ("type", "noul", "choice", "score", "confidence", "legend", "probabilities") def start_span(*args, **kwargs): @@ -44,12 +48,6 @@ def _answer_payload(value: Any) -> dict[str, Any]: return _tagged_payload(value, _ANSWER_FIELDS, type_suffix="Answer") -def _items_with_ids(values: Any, payload_fn) -> Any: - if not isinstance(values, Mapping): - return values - return [{**payload_fn(value), "id": identifier} for identifier, value in values.items()] - - def _call_value(args: Any, kwargs: dict[str, Any], name: str, position: int) -> Any: if name in kwargs: return kwargs[name] @@ -74,10 +72,9 @@ def _request_parts(instance: Any, args: Any, kwargs: dict[str, Any]) -> tuple[An def _span_input(state: Any, questions: Any) -> dict[str, Any]: - return { - "state": state, - "questions": _items_with_ids(questions, _question_payload), - } + if isinstance(questions, Mapping): + questions = {identifier: _question_payload(value) for identifier, value in questions.items()} + return _question_span_input(state, questions) def _token_metrics(response: Any) -> dict[str, int]: @@ -96,10 +93,12 @@ def _token_metrics(response: Any) -> dict[str, int]: def _log_response(span: Any, response: Any) -> None: answers = getattr(response, "answers", None) + if isinstance(answers, Mapping): + answers = {identifier: _answer_payload(value) for identifier, value in answers.items()} response_model = getattr(response, "model", None) _log_and_end_span( span, - output={"answers": _items_with_ids(answers, _answer_payload)}, + output=_question_span_output(answers), metrics=_token_metrics(response), metadata={"model": response_model} if isinstance(response_model, str) else None, ) @@ -112,7 +111,7 @@ def _start_system_one_span(instance: Any, args: Any, kwargs: dict[str, Any]): metadata["model"] = model return start_span( name="typesafe.systemOne", - type=SpanTypeAttribute.QUESTION, + type=SpanTypeAttribute.DECISION, input=_span_input(state, questions), metadata=metadata, ) diff --git a/py/src/braintrust/span_types.py b/py/src/braintrust/span_types.py index 59ef74d6c..dafc4c0ac 100644 --- a/py/src/braintrust/span_types.py +++ b/py/src/braintrust/span_types.py @@ -18,6 +18,7 @@ class SpanTypeAttribute(str, Enum): PREPROCESSOR = "preprocessor" CLASSIFIER = "classifier" REVIEW = "review" + DECISION = "decision" QUESTION = "question" LOG = "log" diff --git a/py/src/braintrust/type_tests/test_span_types.py b/py/src/braintrust/type_tests/test_span_types.py index 6650dfb13..73e9381e9 100644 --- a/py/src/braintrust/type_tests/test_span_types.py +++ b/py/src/braintrust/type_tests/test_span_types.py @@ -1,10 +1,11 @@ import braintrust from braintrust.logger import Span +from braintrust.span_types import SpanTypeAttribute from typing_extensions import assert_type def span_context_manager_keeps_span_type() -> None: - with braintrust.start_span(name="typed") as span: + with braintrust.start_span(name="typed", type=SpanTypeAttribute.DECISION) as span: assert_type(span, Span) span.log(output="ok")