diff --git a/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_responses_access_programs.yaml b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_responses_access_programs.yaml new file mode 100644 index 000000000..8db53cdae --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/latest/test_openai_responses_access_programs.yaml @@ -0,0 +1,119 @@ +interactions: +- request: + body: '{"model":"gpt-4o-mini","access_programs":{"cyber":"standard"},"input":"Say + hello in one word."}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '95' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 3.24.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 3.24.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: "{\n \"id\": \"resp_0adaa81152676f24006ac3c5961e8087d2ae1eddf31568ae9b\",\n + \ \"object\": \"response\",\n \"created_at\": 1791214998,\n \"status\": + \"completed\",\n \"access_programs\": {\n \"cyber\": \"standard\"\n },\n + \ \"background\": false,\n \"billing\": {\n \"payer\": \"developer\"\n + \ },\n \"completed_at\": 1791214998,\n \"error\": null,\n \"frequency_penalty\": + 0.0,\n \"incomplete_details\": null,\n \"instructions\": null,\n \"max_output_tokens\": + null,\n \"max_tool_calls\": null,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"moderation\": null,\n \"output\": [\n {\n \"id\": \"msg_0adaa81152676f24006ac3c5967e8087d2958cb4a5f681894e\",\n + \ \"type\": \"message\",\n \"status\": \"completed\",\n \"content\": + [\n {\n \"type\": \"output_text\",\n \"annotations\": + [],\n \"logprobs\": [],\n \"text\": \"Hello!\"\n }\n + \ ],\n \"role\": \"assistant\"\n }\n ],\n \"parallel_tool_calls\": + true,\n \"presence_penalty\": 0.0,\n \"previous_response_id\": null,\n \"prompt_cache_key\": + null,\n \"prompt_cache_retention\": \"in_memory\",\n \"reasoning\": {\n + \ \"context\": null,\n \"effort\": null,\n \"summary\": null\n },\n + \ \"safety_identifier\": null,\n \"service_tier\": \"default\",\n \"store\": + true,\n \"temperature\": 1.0,\n \"text\": {\n \"format\": {\n \"type\": + \"text\"\n },\n \"verbosity\": \"medium\"\n },\n \"tool_choice\": + \"auto\",\n \"tool_usage\": {\n \"image_gen\": {\n \"input_tokens\": + 0,\n \"input_tokens_details\": {\n \"image_tokens\": 0,\n \"text_tokens\": + 0\n },\n \"output_tokens\": 0,\n \"output_tokens_details\": + {\n \"image_tokens\": 0,\n \"text_tokens\": 0\n },\n \"total_tokens\": + 0\n },\n \"web_search\": {\n \"num_requests\": 0\n }\n },\n + \ \"tools\": [],\n \"top_logprobs\": 0,\n \"top_p\": 1.0,\n \"truncation\": + \"disabled\",\n \"usage\": {\n \"input_tokens\": 13,\n \"input_tokens_details\": + {\n \"cache_write_tokens\": 0,\n \"cached_tokens\": 0\n },\n + \ \"output_tokens\": 3,\n \"output_tokens_details\": {\n \"reasoning_tokens\": + 0\n },\n \"total_tokens\": 16\n },\n \"user\": null,\n \"metadata\": + {}\n}" + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a45d8a875e5555b7-YYZ + connection: + - keep-alive + content-length: + - '2027' + content-type: + - application/json + date: + - Mon, 05 Oct 2026 15:43:18 GMT + openai-processing-ms: + - '575' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999970' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_4916278519c84882bae494e7ef39875b + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/test_openai.py b/py/src/braintrust/integrations/openai/test_openai.py index b0165fcc2..e9ff40b6f 100644 --- a/py/src/braintrust/integrations/openai/test_openai.py +++ b/py/src/braintrust/integrations/openai/test_openai.py @@ -93,6 +93,15 @@ def _supports_agents_api() -> bool: return True +def _supports_responses_access_programs() -> bool: + try: + from openai.resources.responses import Responses + + return "access_programs" in inspect.signature(Responses.create).parameters + except (ImportError, AttributeError, ValueError): + return False + + @pytest.mark.vcr def test_openai_chat_metrics(memory_logger): assert not memory_logger.pop() @@ -189,6 +198,11 @@ def test_openai_responses_metrics(memory_logger): assert TEST_MODEL in span["metadata"]["model"] assert span["metadata"]["provider"] == "openai" assert span["metadata"]["instructions"] == "Just the number please" + if hasattr(response, "access_programs"): + # Current OpenAI cassettes include the field with a null value. Older + # provider pins do not expose it; neither shape should add null metadata. + assert response.access_programs is None + assert "access_programs" not in span["metadata"] assert TEST_PROMPT in str(span["input"]) assert len(span["output"]) > 0 span_output_text = span["output"][0]["content"][0]["text"] @@ -239,6 +253,25 @@ class NumberAnswer(BaseModel): assert span["output"][0]["content"][0]["parsed"]["reasoning"] == parse_response.output_parsed.reasoning +@pytest.mark.vcr +def test_openai_responses_access_programs(memory_logger): + if not _supports_responses_access_programs(): + pytest.skip("Responses access_programs is not available in this SDK version") + + access_programs = {"cyber": "standard"} + client = wrap_openai(openai.OpenAI()) + response = client.responses.create( + model=TEST_MODEL, + input="Say hello in one word.", + access_programs=access_programs, + ) + + spans = memory_logger.pop() + assert len(spans) == 1 + assert _try_to_dict(response.access_programs) == access_programs + assert _try_to_dict(spans[0]["metadata"]["access_programs"]) == access_programs + + @pytest.mark.vcr def test_openai_chat_completion_inline_moderation_metadata(memory_logger): if not _supports_inline_moderation(): diff --git a/py/src/braintrust/integrations/openai/tracing.py b/py/src/braintrust/integrations/openai/tracing.py index 554203b7b..2a479dcf1 100644 --- a/py/src/braintrust/integrations/openai/tracing.py +++ b/py/src/braintrust/integrations/openai/tracing.py @@ -105,6 +105,7 @@ def start_span(*args, **kwargs): "conversation", "verbosity", "moderation", + "access_programs", ) _RESPONSES_RESULT_METADATA_KEYS = ( @@ -114,6 +115,7 @@ def start_span(*args, **kwargs): "created_at", "moderation", "service_tier", + "access_programs", "object", "background", "incomplete_details", @@ -1183,6 +1185,10 @@ def _log_response_tool_spans(output: Any, *, parent_export: str | None) -> None: "command_execution": ("command", "cwd"), "mcp_call": ("arguments",), "web_search_call": ("action",), + "computer_use_call": ("title",), + "browser_authentication_request": ("request_id",), + "computer_use_approval_request": ("request_id", "request"), + "computer_use_approval_request_result": ("request_id",), "create_subagent_call": ("content", "model", "reasoning_effort"), "send_subagent_input_call": ("content", "recipient_agent_id"), "resume_subagent_call": ("recipient_agent_id",), @@ -1194,6 +1200,7 @@ def _log_response_tool_spans(output: Any, *, parent_export: str | None) -> None: _AGENT_TOOL_ITEM_OUTPUT_KEYS = { "command_execution": ("output", "exit_code", "duration_ms"), "mcp_call": ("output",), + "computer_use_approval_request_result": ("response",), } _AGENT_TURN_EVENTS = { @@ -1272,7 +1279,13 @@ def _agent_tool_span_name(item: Any) -> str: def _agent_tool_span_data(item: Any, keys: tuple[str, ...]) -> Any: - values = clean_nones({key: getattr(item, key, None) for key in keys}) + values = clean_nones( + { + key: _try_to_dict(value) if key in {"request", "response"} else value + for key in keys + if (value := getattr(item, key, None)) is not None + } + ) if not values: return None if keys == ("arguments",): @@ -1286,7 +1299,7 @@ def _agent_tool_span_metadata(item: Any) -> dict[str, Any]: "tool_type": item.type, "tool_id": item.id, "call_id": getattr(item, "call_id", None), - "status": item.status, + "status": getattr(item, "status", None), "turn_id": item.turn_id, "server_label": getattr(item, "server_label", None), "agent_id": getattr(item, "agent_id", None), @@ -1299,7 +1312,7 @@ def _agent_tool_span_error(item: Any) -> Any: error = getattr(item, "error", None) if error is not None: return error - if item.status == "failed": + if getattr(item, "status", None) == "failed": return "Agent tool call failed" return None @@ -1383,7 +1396,7 @@ def _observe_completed_item(self, item: Any) -> None: error = _agent_tool_span_error(item) if error is not None: tool_span.log(error=error) - output = _agent_tool_span_data(item, _AGENT_TOOL_ITEM_OUTPUT_KEYS.get(item.type, ())) + output = _agent_tool_span_output(item) if output is not None: tool_span.log(output=output) @@ -1412,6 +1425,27 @@ def finish(self) -> None: self.span.end() +def _agent_tool_span_output(item: Any) -> Any: + if item.type == "computer_use_call": + output = getattr(item, "output", None) + screenshot = getattr(output, "image_url", None) + if screenshot is None and isinstance(output, Mapping): + screenshot = output.get("image_url") + if screenshot is not None: + resolved = _materialize_attachment(screenshot, prefix="computer_use_screenshot") + if resolved is not None: + output_data = _try_to_dict(output) + if isinstance(output_data, dict): + return { + **output_data, + "image_url": resolved.multimodal_part_payload["image_url"], + } + return resolved.multimodal_part_payload + return output + + return _agent_tool_span_data(item, _AGENT_TOOL_ITEM_OUTPUT_KEYS.get(item.type, ())) + + class AgentSessionWrapper: def __init__(self, create_fn: Callable[..., Any] | None, acreate_fn: Callable[..., Any] | None) -> None: self.create_fn = create_fn