feat(integrations): trace openai and litellm decisions API

py/src/braintrust/integrations/auto_test_scripts/test_auto_litellm.py

@@ -1,9 +1,18 @@
 """Test auto_instrument for LiteLLM."""
 
+import asyncio
+
 import litellm
 from braintrust.auto import auto_instrument
 from braintrust.integrations.litellm import LiteLLMIntegration
+from braintrust.integrations.test_decisions import (
+    DECISIONS_INPUT,
+    DECISIONS_MODEL,
+    DECISIONS_QUESTIONS,
+    assert_decisions_span,
+)
 from braintrust.integrations.test_utils import autoinstrument_test_context
+from braintrust.integrations.versioning import detect_module_version, version_satisfies
 
 
 # 1. Verify not patched initially
@@ -34,4 +43,22 @@
     span = spans[0]
     assert span["metadata"]["provider"] == "openai"
 
+if version_satisfies(detect_module_version(litellm, ("litellm",)), ">=1.105.0"):
+    with autoinstrument_test_context("test_litellm_decisions_metrics[async]", integration="litellm") as memory_logger:
+        response = asyncio.run(
+            litellm.adecisions(
+                model=f"openai/{DECISIONS_MODEL}",
+                input=DECISIONS_INPUT,
+                questions=DECISIONS_QUESTIONS,
+                safety_identifier="do-not-log-this-identifier",
+            )
+        )
+        assert_decisions_span(
+            memory_logger.pop(),
+            response,
+            name="Decision",
+            model=f"openai/{DECISIONS_MODEL}",
+            instrumentation="litellm-auto",
+        )
+
 print("SUCCESS")

py/src/braintrust/integrations/auto_test_scripts/test_auto_openai.py

@@ -1,11 +1,19 @@
 """Test auto_instrument for OpenAI."""
 
+import asyncio
 import inspect
 
 import openai
 from braintrust import wrap_openai
 from braintrust.auto import auto_instrument
+from braintrust.integrations.test_decisions import (
+    DECISIONS_INPUT,
+    DECISIONS_MODEL,
+    DECISIONS_QUESTIONS,
+    assert_decisions_span,
+)
 from braintrust.integrations.test_utils import autoinstrument_test_context
+from braintrust.integrations.versioning import version_satisfies
 from wrapt import FunctionWrapper
 
 
@@ -15,6 +23,10 @@ def _is_braintrust_wrapped(resource) -> bool:
 
 
 resources = [openai.resources.chat.completions.Completions]
+if version_satisfies(openai.__version__, ">=3.26.0"):
+    from openai.resources.decisions import AsyncDecisions, Decisions
+
+    resources.extend((Decisions, AsyncDecisions))
 try:
     from openai.resources.beta.agents.sessions.sessions import Sessions
 
@@ -66,4 +78,23 @@ def _is_braintrust_wrapped(resource) -> bool:
     assert spans[0]["metrics"]["prompt_tokens"] == response.usage.input_tokens
     assert spans[0]["metrics"]["completion_tokens"] == response.usage.output_tokens
 
+if version_satisfies(openai.__version__, ">=3.26.0"):
+    with autoinstrument_test_context("test_openai_decisions_metrics[async]", integration="openai") as memory_logger:
+        client = wrap_openai(wrap_openai(openai.AsyncOpenAI()))
+        response = asyncio.run(
+            client.decisions.create(
+                model=DECISIONS_MODEL,
+                input=DECISIONS_INPUT,
+                questions=DECISIONS_QUESTIONS,
+                safety_identifier="do-not-log-this-identifier",
+            )
+        )
+        assert_decisions_span(
+            memory_logger.pop(),
+            response,
+            name="openai.decisions.create",
+            model=DECISIONS_MODEL,
+            instrumentation="openai-auto",
+        )
+
 print("SUCCESS")

py/src/braintrust/integrations/auto_test_scripts/test_auto_typesafe.py

@@ -22,6 +22,7 @@
     spans = memory_logger.pop()
     assert len(spans) == 1
     span = spans[0]
+    assert span["span_attributes"]["type"] == "decision"
     assert span["metadata"]["provider"] == "typesafe"
     assert span["metadata"]["model"].startswith("jev-")
     assert span["input"]["questions"][0]["id"] == "positive"