From a877d9295309cb1c5cb189989bee36cbd0e53987 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Tue, 13 Oct 2026 11:25:59 -0400 Subject: [PATCH 1/2] fix(pydantic-ai): capture conversation input on agent spans Normalize Conversation messages and id into the existing message_history and conversation_id input fields so trace normalizers retain prior turns. Add version-gated VCR coverage for all five agent run methods and consolidate overlapping pydantic-ai tests and recordings. Fixes #923 --- ...s_nested_chat_span_without_class_scan.yaml | 109 ----- .../latest/test_agent_with_conversation.yaml | 237 ++++++++++ .../test_agent_with_conversation_stream.yaml | 257 +++++++++++ ...s_nested_chat_span_without_class_scan.yaml | 119 ----- .../test_pydantic_ai_integration.py | 427 ++++++++---------- .../integrations/pydantic_ai/tracing.py | 18 + 6 files changed, 688 insertions(+), 479 deletions(-) delete mode 100644 py/src/braintrust/integrations/pydantic_ai/cassettes/1.10.0/test_direct_model_request_creates_nested_chat_span_without_class_scan.yaml create mode 100644 py/src/braintrust/integrations/pydantic_ai/cassettes/latest/test_agent_with_conversation.yaml create mode 100644 py/src/braintrust/integrations/pydantic_ai/cassettes/latest/test_agent_with_conversation_stream.yaml delete mode 100644 py/src/braintrust/integrations/pydantic_ai/cassettes/latest/test_direct_model_request_creates_nested_chat_span_without_class_scan.yaml diff --git a/py/src/braintrust/integrations/pydantic_ai/cassettes/1.10.0/test_direct_model_request_creates_nested_chat_span_without_class_scan.yaml b/py/src/braintrust/integrations/pydantic_ai/cassettes/1.10.0/test_direct_model_request_creates_nested_chat_span_without_class_scan.yaml deleted file mode 100644 index d4ff5c2ee..000000000 --- a/py/src/braintrust/integrations/pydantic_ai/cassettes/1.10.0/test_direct_model_request_creates_nested_chat_span_without_class_scan.yaml +++ /dev/null @@ -1,109 +0,0 @@ -interactions: -- request: - body: '{"messages":[{"role":"user","content":"What is 2+2? Answer with just the - number."}],"model":"gpt-4o-mini","stream":false}' - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate, zstd - Connection: - - keep-alive - Content-Length: - - '121' - Content-Type: - - application/json - Host: - - api.openai.com - User-Agent: - - pydantic-ai/1.66.0 - X-Stainless-Arch: - - arm64 - X-Stainless-Async: - - async:asyncio - X-Stainless-Lang: - - python - X-Stainless-OS: - - MacOS - X-Stainless-Package-Version: - - 2.24.0 - X-Stainless-Runtime: - - CPython - X-Stainless-Runtime-Version: - - 3.13.3 - x-stainless-read-timeout: - - '600' - x-stainless-retry-count: - - '0' - method: POST - uri: https://api.openai.com/v1/chat/completions - response: - body: - string: "{\n \"id\": \"chatcmpl-DPVzHBFp5i6BHHGWuTsYqBGubpikG\",\n \"object\": - \"chat.completion\",\n \"created\": 1774973531,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n - \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": - \"assistant\",\n \"content\": \"4\",\n \"refusal\": null,\n - \ \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": - \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 20,\n \"completion_tokens\": - 1,\n \"total_tokens\": 21,\n \"prompt_tokens_details\": {\n \"cached_tokens\": - 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": - {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": - 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": - \"default\",\n \"system_fingerprint\": \"fp_ca3e7d71bf\"\n}\n" - headers: - CF-Cache-Status: - - DYNAMIC - CF-Ray: - - 9e50a256ab1f178c-YYZ - Connection: - - keep-alive - Content-Type: - - application/json - Date: - - Tue, 31 Mar 2026 16:12:11 GMT - Server: - - cloudflare - Strict-Transport-Security: - - max-age=31536000; includeSubDomains; preload - Transfer-Encoding: - - chunked - X-Content-Type-Options: - - nosniff - access-control-expose-headers: - - X-Request-ID - alt-svc: - - h3=":443"; ma=86400 - content-length: - - '807' - openai-organization: - - braintrust-data - openai-processing-ms: - - '392' - openai-project: - - proj_vsCSXafhhByzWOThMrJcZiw9 - openai-version: - - '2020-10-01' - set-cookie: - - __cf_bm=zLt6TNwkPTB2O.Bp3ceD4Ijrj3biQuLYigEMbvthfOs-1774973530.6689873-1.0.1.1-PL.I8XK.cSEDkAeAeA.SAH05Z2qjjTzvfUfqd4fC3.j0tktm6p0Tqb1.Zwy0776VHR2qgoUpogSKSon8qg2QYY179MO.33I7RVvoQMPznp3dxHHWwIBuPqEqxwJZqbZ2; - HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Tue, 31 Mar 2026 - 16:42:11 GMT - x-openai-proxy-wasm: - - v0.1 - x-ratelimit-limit-requests: - - '30000' - x-ratelimit-limit-tokens: - - '150000000' - x-ratelimit-remaining-requests: - - '29999' - x-ratelimit-remaining-tokens: - - '149999987' - x-ratelimit-reset-requests: - - 2ms - x-ratelimit-reset-tokens: - - 0s - x-request-id: - - req_c9480ee8e49349b7a68c3a2767779fd3 - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/pydantic_ai/cassettes/latest/test_agent_with_conversation.yaml b/py/src/braintrust/integrations/pydantic_ai/cassettes/latest/test_agent_with_conversation.yaml new file mode 100644 index 000000000..23829f3d4 --- /dev/null +++ b/py/src/braintrust/integrations/pydantic_ai/cassettes/latest/test_agent_with_conversation.yaml @@ -0,0 +1,237 @@ +interactions: +- request: + body: '{"model":"gpt-4o-mini","stream":false,"input":[{"role":"user","content":"My + name is Ada. Reply with just my name."}],"max_output_tokens":50}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '140' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - pydantic-ai/2.55.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 3.26.1 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: "{\n \"id\": \"resp_0f661341203e8301006ace3f4baa4087d1979db29673a070b7\",\n + \ \"object\": \"response\",\n \"created_at\": 1791901515,\n \"status\": + \"completed\",\n \"access_programs\": null,\n \"background\": false,\n \"billing\": + {\n \"payer\": \"developer\"\n },\n \"completed_at\": 1791901516,\n \"error\": + null,\n \"frequency_penalty\": 0.0,\n \"incomplete_details\": null,\n \"instructions\": + null,\n \"max_output_tokens\": 50,\n \"max_tool_calls\": null,\n \"model\": + \"gpt-4o-mini-2024-07-18\",\n \"moderation\": null,\n \"output\": [\n {\n + \ \"id\": \"msg_0f661341203e8301006ace3f4c2f7c87d197a1c50aed736ab8\",\n + \ \"type\": \"message\",\n \"status\": \"completed\",\n \"content\": + [\n {\n \"type\": \"output_text\",\n \"annotations\": + [],\n \"logprobs\": [],\n \"text\": \"Ada.\"\n }\n + \ ],\n \"role\": \"assistant\"\n }\n ],\n \"parallel_tool_calls\": + true,\n \"presence_penalty\": 0.0,\n \"previous_response_id\": null,\n \"prompt_cache_key\": + null,\n \"prompt_cache_retention\": \"in_memory\",\n \"reasoning\": {\n + \ \"context\": null,\n \"effort\": null,\n \"summary\": null\n },\n + \ \"safety_identifier\": null,\n \"service_tier\": \"default\",\n \"store\": + true,\n \"temperature\": 1.0,\n \"text\": {\n \"format\": {\n \"type\": + \"text\"\n },\n \"verbosity\": \"medium\"\n },\n \"tool_choice\": + \"auto\",\n \"tool_usage\": {\n \"image_gen\": {\n \"input_tokens\": + 0,\n \"input_tokens_details\": {\n \"image_tokens\": 0,\n \"text_tokens\": + 0\n },\n \"output_tokens\": 0,\n \"output_tokens_details\": + {\n \"image_tokens\": 0,\n \"text_tokens\": 0\n },\n \"total_tokens\": + 0\n },\n \"web_search\": {\n \"num_requests\": 0\n }\n },\n + \ \"tools\": [],\n \"top_logprobs\": 0,\n \"top_p\": 1.0,\n \"truncation\": + \"disabled\",\n \"usage\": {\n \"input_tokens\": 18,\n \"input_tokens_details\": + {\n \"cache_write_tokens\": 0,\n \"cached_tokens\": 0\n },\n + \ \"output_tokens\": 3,\n \"output_tokens_details\": {\n \"reasoning_tokens\": + 0\n },\n \"total_tokens\": 21\n },\n \"user\": null,\n \"metadata\": + {}\n}" + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a49f0337aa91ab90-YYZ + connection: + - keep-alive + content-length: + - '1998' + content-type: + - application/json + date: + - Tue, 13 Oct 2026 14:25:16 GMT + openai-processing-ms: + - '754' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999965' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_08b23a6c4ef94bd5939db6997a1e7bd5 + status: + code: 200 + message: OK +- request: + body: '{"model":"gpt-4o-mini","stream":false,"input":[{"role":"user","content":"My + name is Ada. Reply with just my name."},{"role":"assistant","content":"Ada."},{"role":"user","content":"What + is my name? Answer with just the name."}],"max_output_tokens":50}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '250' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - pydantic-ai/2.55.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 3.26.1 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: "{\n \"id\": \"resp_0702fff2292364b8006ace3f4c96e487d198a69a8b2d392e21\",\n + \ \"object\": \"response\",\n \"created_at\": 1791901516,\n \"status\": + \"completed\",\n \"access_programs\": null,\n \"background\": false,\n \"billing\": + {\n \"payer\": \"developer\"\n },\n \"completed_at\": 1791901516,\n \"error\": + null,\n \"frequency_penalty\": 0.0,\n \"incomplete_details\": null,\n \"instructions\": + null,\n \"max_output_tokens\": 50,\n \"max_tool_calls\": null,\n \"model\": + \"gpt-4o-mini-2024-07-18\",\n \"moderation\": null,\n \"output\": [\n {\n + \ \"id\": \"msg_0702fff2292364b8006ace3f4ce76c87d1bb9496f798f3e20a\",\n + \ \"type\": \"message\",\n \"status\": \"completed\",\n \"content\": + [\n {\n \"type\": \"output_text\",\n \"annotations\": + [],\n \"logprobs\": [],\n \"text\": \"Ada.\"\n }\n + \ ],\n \"role\": \"assistant\"\n }\n ],\n \"parallel_tool_calls\": + true,\n \"presence_penalty\": 0.0,\n \"previous_response_id\": null,\n \"prompt_cache_key\": + null,\n \"prompt_cache_retention\": \"in_memory\",\n \"reasoning\": {\n + \ \"context\": null,\n \"effort\": null,\n \"summary\": null\n },\n + \ \"safety_identifier\": null,\n \"service_tier\": \"default\",\n \"store\": + true,\n \"temperature\": 1.0,\n \"text\": {\n \"format\": {\n \"type\": + \"text\"\n },\n \"verbosity\": \"medium\"\n },\n \"tool_choice\": + \"auto\",\n \"tool_usage\": {\n \"image_gen\": {\n \"input_tokens\": + 0,\n \"input_tokens_details\": {\n \"image_tokens\": 0,\n \"text_tokens\": + 0\n },\n \"output_tokens\": 0,\n \"output_tokens_details\": + {\n \"image_tokens\": 0,\n \"text_tokens\": 0\n },\n \"total_tokens\": + 0\n },\n \"web_search\": {\n \"num_requests\": 0\n }\n },\n + \ \"tools\": [],\n \"top_logprobs\": 0,\n \"top_p\": 1.0,\n \"truncation\": + \"disabled\",\n \"usage\": {\n \"input_tokens\": 39,\n \"input_tokens_details\": + {\n \"cache_write_tokens\": 0,\n \"cached_tokens\": 0\n },\n + \ \"output_tokens\": 3,\n \"output_tokens_details\": {\n \"reasoning_tokens\": + 0\n },\n \"total_tokens\": 42\n },\n \"user\": null,\n \"metadata\": + {}\n}" + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a49f033e19abab90-YYZ + connection: + - keep-alive + content-length: + - '1998' + content-type: + - application/json + date: + - Tue, 13 Oct 2026 14:25:17 GMT + openai-processing-ms: + - '555' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29998' + x-ratelimit-remaining-tokens: + - '149999905' + x-ratelimit-reset-requests: + - 4ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_9c2af03b9f984b4ea01dfe4fe988a5c3 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/pydantic_ai/cassettes/latest/test_agent_with_conversation_stream.yaml b/py/src/braintrust/integrations/pydantic_ai/cassettes/latest/test_agent_with_conversation_stream.yaml new file mode 100644 index 000000000..1c6112ecc --- /dev/null +++ b/py/src/braintrust/integrations/pydantic_ai/cassettes/latest/test_agent_with_conversation_stream.yaml @@ -0,0 +1,257 @@ +interactions: +- request: + body: '{"model":"gpt-4o-mini","stream":false,"input":[{"role":"user","content":"My + name is Ada. Reply with just my name."}],"max_output_tokens":50}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '140' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - pydantic-ai/2.55.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 3.26.1 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: "{\n \"id\": \"resp_0d79d6c6d2e39809006ace3f4d6c9c87d19c6c49f42ddaae79\",\n + \ \"object\": \"response\",\n \"created_at\": 1791901517,\n \"status\": + \"completed\",\n \"access_programs\": null,\n \"background\": false,\n \"billing\": + {\n \"payer\": \"developer\"\n },\n \"completed_at\": 1791901517,\n \"error\": + null,\n \"frequency_penalty\": 0.0,\n \"incomplete_details\": null,\n \"instructions\": + null,\n \"max_output_tokens\": 50,\n \"max_tool_calls\": null,\n \"model\": + \"gpt-4o-mini-2024-07-18\",\n \"moderation\": null,\n \"output\": [\n {\n + \ \"id\": \"msg_0d79d6c6d2e39809006ace3f4de3bc87d1b152c682afca492a\",\n + \ \"type\": \"message\",\n \"status\": \"completed\",\n \"content\": + [\n {\n \"type\": \"output_text\",\n \"annotations\": + [],\n \"logprobs\": [],\n \"text\": \"Ada.\"\n }\n + \ ],\n \"role\": \"assistant\"\n }\n ],\n \"parallel_tool_calls\": + true,\n \"presence_penalty\": 0.0,\n \"previous_response_id\": null,\n \"prompt_cache_key\": + null,\n \"prompt_cache_retention\": \"in_memory\",\n \"reasoning\": {\n + \ \"context\": null,\n \"effort\": null,\n \"summary\": null\n },\n + \ \"safety_identifier\": null,\n \"service_tier\": \"default\",\n \"store\": + true,\n \"temperature\": 1.0,\n \"text\": {\n \"format\": {\n \"type\": + \"text\"\n },\n \"verbosity\": \"medium\"\n },\n \"tool_choice\": + \"auto\",\n \"tool_usage\": {\n \"image_gen\": {\n \"input_tokens\": + 0,\n \"input_tokens_details\": {\n \"image_tokens\": 0,\n \"text_tokens\": + 0\n },\n \"output_tokens\": 0,\n \"output_tokens_details\": + {\n \"image_tokens\": 0,\n \"text_tokens\": 0\n },\n \"total_tokens\": + 0\n },\n \"web_search\": {\n \"num_requests\": 0\n }\n },\n + \ \"tools\": [],\n \"top_logprobs\": 0,\n \"top_p\": 1.0,\n \"truncation\": + \"disabled\",\n \"usage\": {\n \"input_tokens\": 18,\n \"input_tokens_details\": + {\n \"cache_write_tokens\": 0,\n \"cached_tokens\": 0\n },\n + \ \"output_tokens\": 3,\n \"output_tokens_details\": {\n \"reasoning_tokens\": + 0\n },\n \"total_tokens\": 21\n },\n \"user\": null,\n \"metadata\": + {}\n}" + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a49f03435e9dab18-YYZ + connection: + - keep-alive + content-length: + - '1998' + content-type: + - application/json + date: + - Tue, 13 Oct 2026 14:25:18 GMT + openai-processing-ms: + - '681' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29997' + x-ratelimit-remaining-tokens: + - '149999865' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_f3daa239faed4a4999cb38c923e98dc9 + status: + code: 200 + message: OK +- request: + body: '{"model":"gpt-4o-mini","stream":true,"input":[{"role":"user","content":"My + name is Ada. Reply with just my name."},{"role":"assistant","content":"Ada."},{"role":"user","content":"What + is my name? Answer with just the name."}],"max_output_tokens":50}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '249' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - pydantic-ai/2.55.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 3.26.1 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: 'event: response.created + + data: {"type":"response.created","response":{"id":"resp_0e571353cd0c1bd4006ace3f4e411087d1b18a59c0f1a213fd","object":"response","created_at":1791901518,"status":"in_progress","access_programs":null,"background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":50,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + + + event: response.in_progress + + data: {"type":"response.in_progress","response":{"id":"resp_0e571353cd0c1bd4006ace3f4e411087d1b18a59c0f1a213fd","object":"response","created_at":1791901518,"status":"in_progress","access_programs":null,"background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":50,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + + + event: response.output_item.added + + data: {"type":"response.output_item.added","item":{"id":"msg_0e571353cd0c1bd4006ace3f4eed4487d1b372bfc420d41a31","type":"message","status":"in_progress","content":[],"role":"assistant"},"output_index":0,"sequence_number":2} + + + event: response.content_part.added + + data: {"type":"response.content_part.added","content_index":0,"item_id":"msg_0e571353cd0c1bd4006ace3f4eed4487d1b372bfc420d41a31","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":3} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":"Ada","item_id":"msg_0e571353cd0c1bd4006ace3f4eed4487d1b372bfc420d41a31","logprobs":[],"obfuscation":"8rSBYwRLhnKMf","output_index":0,"sequence_number":4} + + + event: response.output_text.delta + + data: {"type":"response.output_text.delta","content_index":0,"delta":".","item_id":"msg_0e571353cd0c1bd4006ace3f4eed4487d1b372bfc420d41a31","logprobs":[],"obfuscation":"oPr6RZhAffMnfmB","output_index":0,"sequence_number":5} + + + event: response.output_text.done + + data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_0e571353cd0c1bd4006ace3f4eed4487d1b372bfc420d41a31","logprobs":[],"output_index":0,"sequence_number":6,"text":"Ada."} + + + event: response.content_part.done + + data: {"type":"response.content_part.done","content_index":0,"item_id":"msg_0e571353cd0c1bd4006ace3f4eed4487d1b372bfc420d41a31","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"Ada."},"sequence_number":7} + + + event: response.output_item.done + + data: {"type":"response.output_item.done","item":{"id":"msg_0e571353cd0c1bd4006ace3f4eed4487d1b372bfc420d41a31","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"Ada."}],"role":"assistant"},"output_index":0,"sequence_number":8} + + + event: response.completed + + data: {"type":"response.completed","response":{"id":"resp_0e571353cd0c1bd4006ace3f4e411087d1b18a59c0f1a213fd","object":"response","created_at":1791901518,"status":"completed","access_programs":null,"background":false,"completed_at":1791901518,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":50,"max_tool_calls":null,"model":"gpt-4o-mini-2024-07-18","moderation":null,"output":[{"id":"msg_0e571353cd0c1bd4006ace3f4eed4487d1b372bfc420d41a31","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"Ada."}],"role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":39,"input_tokens_details":{"cache_write_tokens":0,"cached_tokens":0},"output_tokens":3,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":42},"user":null,"metadata":{}},"sequence_number":9} + + + ' + headers: + access-control-expose-headers: + - X-Request-ID + - CF-Ray + - CF-Ray + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - a49f03488b6fab18-YYZ + connection: + - keep-alive + content-type: + - text/event-stream; charset=utf-8 + date: + - Tue, 13 Oct 2026 14:25:19 GMT + openai-processing-ms: + - '773' + openai-version: + - '2020-10-01' + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + transfer-encoding: + - chunked + x-content-type-options: + - nosniff + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999942' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_7dc83f7d018644639180ad407bda5867 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/pydantic_ai/cassettes/latest/test_direct_model_request_creates_nested_chat_span_without_class_scan.yaml b/py/src/braintrust/integrations/pydantic_ai/cassettes/latest/test_direct_model_request_creates_nested_chat_span_without_class_scan.yaml deleted file mode 100644 index 996942e8d..000000000 --- a/py/src/braintrust/integrations/pydantic_ai/cassettes/latest/test_direct_model_request_creates_nested_chat_span_without_class_scan.yaml +++ /dev/null @@ -1,119 +0,0 @@ -interactions: -- request: - body: '{"model":"gpt-4o-mini","stream":false,"input":[{"role":"user","content":"What - is 2+2? Answer with just the number."}]}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate, zstd - connection: - - keep-alive - content-length: - - '118' - content-type: - - application/json - host: - - api.openai.com - user-agent: - - pydantic-ai/2.55.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - async:asyncio - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 3.29.0 - x-stainless-read-timeout: - - '600' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.14.7 - method: POST - uri: https://api.openai.com/v1/responses - response: - body: - string: "{\n \"id\": \"resp_06423c6db136de33006ace3bb1af4487d1bf0e96e92913dcda\",\n - \ \"object\": \"response\",\n \"created_at\": 1791900593,\n \"status\": - \"completed\",\n \"access_programs\": null,\n \"background\": false,\n \"billing\": - {\n \"payer\": \"developer\"\n },\n \"completed_at\": 1791900594,\n \"error\": - null,\n \"frequency_penalty\": 0.0,\n \"incomplete_details\": null,\n \"instructions\": - null,\n \"max_output_tokens\": null,\n \"max_tool_calls\": null,\n \"model\": - \"gpt-4o-mini-2024-07-18\",\n \"moderation\": null,\n \"output\": [\n {\n - \ \"id\": \"msg_06423c6db136de33006ace3bb2a80887d1874245a925235a31\",\n - \ \"type\": \"message\",\n \"status\": \"completed\",\n \"content\": - [\n {\n \"type\": \"output_text\",\n \"annotations\": - [],\n \"logprobs\": [],\n \"text\": \"4\"\n }\n ],\n - \ \"role\": \"assistant\"\n }\n ],\n \"parallel_tool_calls\": true,\n - \ \"presence_penalty\": 0.0,\n \"previous_response_id\": null,\n \"prompt_cache_key\": - null,\n \"prompt_cache_retention\": \"in_memory\",\n \"reasoning\": {\n - \ \"context\": null,\n \"effort\": null,\n \"summary\": null\n },\n - \ \"safety_identifier\": null,\n \"service_tier\": \"default\",\n \"store\": - true,\n \"temperature\": 1.0,\n \"text\": {\n \"format\": {\n \"type\": - \"text\"\n },\n \"verbosity\": \"medium\"\n },\n \"tool_choice\": - \"auto\",\n \"tool_usage\": {\n \"image_gen\": {\n \"input_tokens\": - 0,\n \"input_tokens_details\": {\n \"image_tokens\": 0,\n \"text_tokens\": - 0\n },\n \"output_tokens\": 0,\n \"output_tokens_details\": - {\n \"image_tokens\": 0,\n \"text_tokens\": 0\n },\n \"total_tokens\": - 0\n },\n \"web_search\": {\n \"num_requests\": 0\n }\n },\n - \ \"tools\": [],\n \"top_logprobs\": 0,\n \"top_p\": 1.0,\n \"truncation\": - \"disabled\",\n \"usage\": {\n \"input_tokens\": 20,\n \"input_tokens_details\": - {\n \"cache_write_tokens\": 0,\n \"cached_tokens\": 0\n },\n - \ \"output_tokens\": 2,\n \"output_tokens_details\": {\n \"reasoning_tokens\": - 0\n },\n \"total_tokens\": 22\n },\n \"user\": null,\n \"metadata\": - {}\n}" - headers: - access-control-expose-headers: - - X-Request-ID - - CF-Ray - - CF-Ray - alt-svc: - - h3=":443"; ma=86400 - cf-cache-status: - - DYNAMIC - cf-ray: - - a49eecb27e8c5383-YYZ - connection: - - keep-alive - content-length: - - '1997' - content-type: - - application/json - date: - - Tue, 13 Oct 2026 14:09:54 GMT - openai-processing-ms: - - '1155' - openai-version: - - '2020-10-01' - server: - - cloudflare - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - transfer-encoding: - - chunked - x-content-type-options: - - nosniff - x-ratelimit-limit-requests: - - '30000' - x-ratelimit-limit-tokens: - - '150000000' - x-ratelimit-remaining-requests: - - '29999' - x-ratelimit-remaining-tokens: - - '149999962' - x-ratelimit-reset-requests: - - 2ms - x-ratelimit-reset-tokens: - - 0s - x-request-id: - - req_a3b3b32974ce404fb22a426d738112a7 - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py b/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py index 1e36720a1..c71535c7d 100644 --- a/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py +++ b/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py @@ -6,11 +6,13 @@ import os import time +import pydantic_ai import pytest from braintrust import logger, setup_pydantic_ai, traced from braintrust.integrations.test_utils import verify_autoinstrument_script from braintrust.span_types import SpanTypeAttribute from braintrust.test_helpers import init_test_logger +from packaging.version import Version from pydantic import BaseModel from pydantic_ai import Agent, ModelSettings from pydantic_ai.messages import ModelRequest, UserPromptPart @@ -21,6 +23,7 @@ MODEL = "openai:gpt-4o-mini" # Use cheaper model for tests TEST_PROMPT = "What is 2+2? Answer with just the number." MAX_REASONABLE_TTFT_SECONDS = 30.0 +CONVERSATION_SUPPORTED = Version(pydantic_ai.__version__) >= Version("2.55.0") @pytest.fixture(scope="module", autouse=True) @@ -209,129 +212,63 @@ def test_audio_seconds_is_recorded_as_metadata(): # pylint: enable=unsupported-membership-test,unsubscriptable-object -@pytest.mark.vcr @pytest.mark.asyncio -async def test_direct_model_request_creates_nested_chat_span_without_class_scan(memory_logger, direct): - """Direct calls should resolve and wrap models via _prepare_model, not class scanning.""" - assert not memory_logger.pop() - - messages = [ModelRequest(parts=[UserPromptPart(content=TEST_PROMPT)])] - - start = time.time() - response = await direct.model_request(model=MODEL, messages=messages) - end = time.time() - - assert response.parts - assert "4" in str(response.parts[0].content) - - spans = memory_logger.pop() - assert len(spans) >= 2, f"Expected at least 2 spans (model_request + chat), got {len(spans)}" - - direct_span = next((s for s in spans if s["span_attributes"]["name"] == "model_request"), None) - chat_span = next((s for s in spans if "chat" in s["span_attributes"]["name"]), None) - - assert direct_span is not None, "model_request span not found" - assert chat_span is not None, "chat span not found" - assert direct_span["context"]["span_origin"]["instrumentation"]["name"] == "pydantic-ai-auto" - assert chat_span["span_parents"] == [direct_span["span_id"]] - assert chat_span["metadata"]["model"] == "gpt-4o-mini" - assert chat_span["metadata"]["provider"] == "openai" - _assert_metrics_are_valid(direct_span["metrics"], start, end) - _assert_metrics_are_valid(chat_span["metrics"], start, end) - +@pytest.mark.parametrize( + "agent_kind", + [ + pytest.param("configured", marks=pytest.mark.vcr(cassette_name="test_agent_run_async")), + pytest.param("runtime_model", marks=pytest.mark.vcr(cassette_name="test_no_model_agent_run")), + pytest.param("wrapper", marks=pytest.mark.vcr(cassette_name="test_wrapper_agent_run_is_traced")), + ], +) +async def test_agent_run_async(memory_logger, agent_kind): + """Configured, runtime-selected, and wrapped agents emit the same span tree. -@pytest.mark.vcr -@pytest.mark.asyncio -async def test_agent_run_async(memory_logger): - """Test Agent.run() async method.""" + Runtime model selection covers #1324; WrapperAgent exercises tracing inherited + from AbstractAgent rather than patched directly on Agent. + """ assert not memory_logger.pop() - - agent = Agent(MODEL, model_settings=ModelSettings(max_tokens=50)) + run_kwargs = {} + if agent_kind == "runtime_model": + agent = Agent(model_settings=ModelSettings(max_tokens=50)) + run_kwargs["model"] = MODEL + elif agent_kind == "wrapper": + from pydantic_ai.agent.wrapper import WrapperAgent + + agent = WrapperAgent(Agent(MODEL, name="wrapped-agent", model_settings=ModelSettings(max_tokens=50))) + else: + agent = Agent(MODEL, model_settings=ModelSettings(max_tokens=50)) start = time.time() - result = await agent.run(TEST_PROMPT) + result = await agent.run(TEST_PROMPT, **run_kwargs) end = time.time() - - # Verify the result - assert result.output assert "4" in str(result.output) - # Check spans - should now have parent agent_run + nested chat span spans = memory_logger.pop() - assert len(spans) == 2, f"Expected 2 spans (agent_run + chat), got {len(spans)}" - - # Find agent_run and chat spans + assert len(spans) == 2 agent_span = _find_agent_span(spans) - chat_span = next((s for s in spans if "chat" in s["span_attributes"]["name"]), None) - - assert agent_span is not None, "agent_run span not found" - assert chat_span is not None, "chat span not found" - - # Check agent span + chat_span = next(span for span in spans if span["span_attributes"]["type"] == SpanTypeAttribute.LLM) + assert agent_span is not None assert agent_span["span_attributes"]["type"] == SpanTypeAttribute.TASK - assert agent_span["metadata"]["model"] == "gpt-4o-mini" - assert agent_span["metadata"]["provider"] == "openai" - assert TEST_PROMPT in str(agent_span["input"]) + if agent_kind == "wrapper": + assert agent_span["span_attributes"]["name"] == "agent_run [wrapped-agent]" + assert agent_span["input"]["user_prompt"] == TEST_PROMPT assert "4" in str(agent_span["output"]) - _assert_metrics_are_valid(agent_span["metrics"], start, end) - - # Check chat span is nested under agent span (use span_id, not id which is the row ID) - assert chat_span["span_parents"] == [agent_span["span_id"]], "chat span should be nested under agent_run" - assert chat_span["span_attributes"]["type"] == SpanTypeAttribute.LLM - assert chat_span["span_attributes"]["name"].startswith("chat ") - assert "gpt-4o-mini" in chat_span["span_attributes"]["name"] - assert chat_span["metadata"]["model"] == "gpt-4o-mini" - assert chat_span["metadata"]["provider"] == "openai" - _assert_metrics_are_valid(chat_span["metrics"], start, end) - - # Regression: wrapper agent_run span must NOT log token metrics. The leaf chat - # span already logs them, and trace-tree rollup (self + descendants) plus any - # unfiltered sum over metrics would otherwise double-count tokens/cost at the - # parent regardless of span type. + assert chat_span["span_parents"] == [agent_span["span_id"]] + assert chat_span["span_attributes"]["name"].startswith("chat gpt-4o-mini") + for span in (agent_span, chat_span): + assert span["metadata"]["model"] == "gpt-4o-mini" + assert span["metadata"]["provider"] == "openai" + assert span["context"]["span_origin"]["instrumentation"]["name"] == "pydantic-ai-auto" + _assert_metrics_are_valid(span["metrics"], start, end) + + # Only the leaf LLM owns token metrics; parents would double-count rollups. for token_key in ("prompt_tokens", "completion_tokens", "tokens", "prompt_cached_tokens"): - assert token_key not in agent_span["metrics"], ( - f"wrapper span must not log {token_key}; it duplicates the leaf chat span" - ) - - # Only the leaf chat span should be type=LLM, and it must own the token totals. - llm_spans = [s for s in spans if s["span_attributes"]["type"] == SpanTypeAttribute.LLM] - assert len(llm_spans) == 1, f"expected exactly one LLM-typed span, got {len(llm_spans)}" - assert llm_spans[0]["span_id"] == chat_span["span_id"] + assert token_key not in agent_span["metrics"] assert chat_span["metrics"]["prompt_tokens"] > 0 assert chat_span["metrics"]["completion_tokens"] > 0 -@pytest.mark.vcr -@pytest.mark.asyncio -async def test_wrapper_agent_run_is_traced(memory_logger): - """WrapperAgent inherits AbstractAgent methods and should be traced by setup().""" - from pydantic_ai.agent.wrapper import WrapperAgent - - assert not memory_logger.pop() - - wrapped = WrapperAgent(Agent(MODEL, name="wrapped-agent", model_settings=ModelSettings(max_tokens=50))) - - start = time.time() - result = await wrapped.run(TEST_PROMPT) - end = time.time() - - assert result.output - assert "4" in str(result.output) - - spans = memory_logger.pop() - assert len(spans) >= 2, f"Expected at least 2 spans (agent_run + chat), got {len(spans)}" - - agent_span = _find_agent_span(spans) - chat_span = next((s for s in spans if "chat" in s["span_attributes"]["name"]), None) - - assert agent_span is not None, "agent_run span not found" - assert chat_span is not None, "chat span not found" - assert agent_span["span_attributes"]["name"] == "agent_run [wrapped-agent]" - assert chat_span["span_parents"] == [agent_span["span_id"]] - _assert_metrics_are_valid(agent_span["metrics"], start, end) - _assert_metrics_are_valid(chat_span["metrics"], start, end) - - @pytest.mark.vcr def test_agent_run_sync(memory_logger): """Test Agent.run_sync() synchronous method.""" @@ -605,12 +542,10 @@ async def test_direct_model_request(memory_logger, direct): # Check spans spans = memory_logger.pop() - # Direct API calls may create 1 or 2 spans depending on model wrapping - assert len(spans) >= 1 - - # Find the direct API span - direct_span = next((s for s in spans if s["span_attributes"]["name"] == "model_request"), None) - assert direct_span is not None + # A custom subclass can emit both its own span and its base model's span. + assert len(spans) >= 2 + direct_span = next(span for span in spans if span["span_attributes"]["name"] == "model_request") + assert direct_span["context"]["span_origin"]["instrumentation"]["name"] == "pydantic-ai-auto" assert direct_span["span_attributes"]["type"] == SpanTypeAttribute.TASK assert direct_span["metadata"]["model"] == "gpt-4o-mini" @@ -688,6 +623,12 @@ async def test_direct_model_request_with_settings(memory_logger, direct): # Security invariant: `instrument` is telemetry-routing config, not user input. assert "instrument" not in direct_span["input"] + chat_span = next(span for span in spans if span["span_attributes"]["type"] == SpanTypeAttribute.LLM) + assert direct_span["context"]["span_origin"]["instrumentation"]["name"] == "pydantic-ai-auto" + assert chat_span["span_parents"] == [direct_span["span_id"]] + assert chat_span["metadata"]["model"] == "gpt-4o-mini" + assert chat_span["metadata"]["provider"] == "openai" + _assert_metrics_are_valid(chat_span["metrics"], start, end) assert direct_span["metadata"]["model"] == "gpt-4o-mini" assert direct_span["metadata"]["provider"] == "openai" @@ -837,112 +778,57 @@ class MathAnswer(BaseModel): assert chat_span["metrics"]["completion_tokens"] > 0 -@pytest.mark.vcr @pytest.mark.asyncio -async def test_agent_with_model_settings_in_metadata(memory_logger): - """Test that model_settings from agent config appears in metadata, not input.""" - assert not memory_logger.pop() - - custom_settings = ModelSettings(max_tokens=100, temperature=0.5) - agent = Agent(MODEL, model_settings=custom_settings) - - start = time.time() - result = await agent.run("Say hello") - end = time.time() - - assert result.output - - # Check spans - spans = memory_logger.pop() - assert len(spans) == 2, f"Expected 2 spans (agent_run + chat), got {len(spans)}" - - # Find agent_run and chat spans - agent_span = _find_agent_span(spans) - chat_span = next((s for s in spans if "chat" in s["span_attributes"]["name"]), None) - - assert agent_span is not None, "agent_run span not found" - assert chat_span is not None, "chat span not found" - - # Verify model_settings is in agent METADATA (not input, since it's agent config) - assert "model_settings" in agent_span["metadata"], "model_settings should be in agent_run metadata" - agent_settings = agent_span["metadata"]["model_settings"] - assert agent_settings["max_tokens"] == 100 - assert agent_settings["temperature"] == 0.5 - - # Verify model_settings is NOT in agent input (it wasn't passed to run()) - assert "model_settings" not in agent_span["input"], ( - "model_settings should NOT be in agent_run input when not passed to run()" - ) - - # Verify model_settings is in chat input (passed to the model) - assert "model_settings" in chat_span["input"], "model_settings should be in chat span input" - chat_settings = chat_span["input"]["model_settings"] - assert chat_settings["max_tokens"] == 100 - assert chat_settings["temperature"] == 0.5 - - assert chat_span["span_attributes"]["type"] == SpanTypeAttribute.LLM - invocation_params = chat_span["metadata"]["invocation_params"] - assert invocation_params["max_tokens"] == 100 - assert invocation_params["temperature"] == 0.5 - - assert "model_settings" not in chat_span["metadata"], "model_settings should NOT be in chat span metadata" - - assert chat_span["metadata"]["model"] == "gpt-4o-mini" - assert chat_span["metadata"]["provider"] == "openai" - - -@pytest.mark.vcr -@pytest.mark.asyncio -async def test_agent_with_model_settings_override_in_input(memory_logger): - """Test that model_settings passed to run() appears in input, not metadata, and - that non-allowlisted kwargs (`infer_name`, `usage`) do not leak into span input.""" +@pytest.mark.parametrize( + "settings_source", + [ + pytest.param("agent", marks=pytest.mark.vcr(cassette_name="test_agent_with_model_settings_in_metadata")), + pytest.param("run", marks=pytest.mark.vcr(cassette_name="test_agent_with_model_settings_override_in_input")), + ], +) +async def test_agent_model_settings(memory_logger, settings_source): + """Agent defaults go in metadata; run overrides go in input and reach the model.""" from pydantic_ai.usage import RunUsage assert not memory_logger.pop() - - # Agent has default settings - default_settings = ModelSettings(max_tokens=50) - agent = Agent(MODEL, model_settings=default_settings) - - # Override with different settings in run() call - override_settings = ModelSettings(max_tokens=200, temperature=0.9) + run_kwargs = {} + if settings_source == "run": + agent = Agent(MODEL, model_settings=ModelSettings(max_tokens=50)) + settings = ModelSettings(max_tokens=200, temperature=0.9) + prompt = "Tell me a story" + run_kwargs = {"model_settings": settings, "infer_name": False, "usage": RunUsage()} + else: + settings = ModelSettings(max_tokens=100, temperature=0.5) + agent = Agent(MODEL, model_settings=settings) + prompt = "Say hello" start = time.time() - result = await agent.run( - "Tell me a story", - model_settings=override_settings, - # Neither affects the outbound HTTP request; both must be dropped from span input. - infer_name=False, - usage=RunUsage(), - ) + result = await agent.run(prompt, **run_kwargs) end = time.time() - assert result.output - # Check spans spans = memory_logger.pop() - assert len(spans) == 2, f"Expected 2 spans (agent_run + chat), got {len(spans)}" - - # Find agent_run span + assert len(spans) == 2 agent_span = _find_agent_span(spans) - assert agent_span is not None, "agent_run span not found" - - # Verify override settings are in agent INPUT (because they were passed to run()) - assert "model_settings" in agent_span["input"], "model_settings should be in agent_run input when passed to run()" - input_settings = agent_span["input"]["model_settings"] - assert input_settings["max_tokens"] == 200, "Should use override settings from run() call" - assert input_settings["temperature"] == 0.9 - - # Verify agent default settings are NOT in metadata (when overridden in input, we don't duplicate in metadata) - assert "model_settings" not in agent_span["metadata"], ( - "model_settings should NOT be in metadata when explicitly passed to run()" - ) - - # Security invariant: non-allowlisted kwargs must not leak into span input. + chat_span = next(span for span in spans if span["span_attributes"]["type"] == SpanTypeAttribute.LLM) + assert agent_span is not None + if settings_source == "run": + assert agent_span["input"]["model_settings"] == settings + assert "model_settings" not in agent_span["metadata"] + else: + assert agent_span["metadata"]["model_settings"] == settings + assert "model_settings" not in agent_span["input"] assert "infer_name" not in agent_span["input"] assert "usage" not in agent_span["input"] - _assert_metrics_are_valid(agent_span["metrics"], start, end) + assert chat_span["input"]["model_settings"] == settings + assert chat_span["metadata"]["invocation_params"] == settings + assert "model_settings" not in chat_span["metadata"] + assert chat_span["span_parents"] == [agent_span["span_id"]] + for span in (agent_span, chat_span): + assert span["metadata"]["model"] == "gpt-4o-mini" + assert span["metadata"]["provider"] == "openai" + _assert_metrics_are_valid(span["metrics"], start, end) @pytest.mark.vcr @@ -1289,14 +1175,10 @@ async def test_agent_stream_early_break(memory_logger): spans = memory_logger.pop() assert len(spans) >= 1, f"Expected at least 1 span, got {len(spans)}" - # Find agent_run_stream span (if created) agent_span = next((s for s in spans if "agent_run_stream" in s["span_attributes"]["name"]), None) - - # Verify at least agent_run_stream span exists and has basic structure - if agent_span: - assert agent_span["span_attributes"]["type"] == SpanTypeAttribute.TASK - # Metrics may be incomplete due to early break - assert "start" in agent_span["metrics"] + assert agent_span is not None, "agent_run_stream span should survive an early break" + assert agent_span["span_attributes"]["type"] == SpanTypeAttribute.TASK + _assert_metrics_are_valid(agent_span["metrics"], start, end) @pytest.mark.vcr @@ -2130,46 +2012,6 @@ def test_setup_pydantic_ai_is_idempotent_across_new_patch_points(): assert agent_graph_module.ToolManager.__dict__[tool_method_name] is tool_method -@pytest.mark.vcr -@pytest.mark.asyncio -async def test_no_model_agent_run(memory_logger): - """Agent created without a model should work when model is passed at runtime. - - Reproduces https://github.com/braintrustdata/braintrust-sdk-python/issues/1324: - _ensure_model_wrapped() calls type(instance._model) without checking for None, - crashing with: AttributeError: type object 'NoneType' has no attribute 'request' - """ - assert not memory_logger.pop() - - agent = Agent(model_settings=ModelSettings(max_tokens=50)) - - start = time.time() - result = await agent.run(TEST_PROMPT, model=MODEL) - end = time.time() - - assert result.output - assert "4" in str(result.output) - - spans = memory_logger.pop() - assert len(spans) == 2, f"Expected 2 spans (agent_run + chat), got {len(spans)}" - - agent_span = _find_agent_span(spans) - chat_span = next((s for s in spans if "chat" in s["span_attributes"]["name"]), None) - - assert agent_span is not None, "agent_run span not found" - assert chat_span is not None, "chat span not found" - - assert agent_span["span_attributes"]["type"] == SpanTypeAttribute.TASK - assert agent_span["metadata"]["model"] == "gpt-4o-mini" - assert agent_span["metadata"]["provider"] == "openai" - assert TEST_PROMPT in str(agent_span["input"]) - assert "4" in str(agent_span["output"]) - - assert chat_span["span_parents"] == [agent_span["span_id"]] - assert chat_span["span_attributes"]["type"] == SpanTypeAttribute.LLM - assert chat_span["metadata"]["model"] == "gpt-4o-mini" - - class TestAutoInstrumentPydanticAI: """Tests for auto_instrument() with Pydantic AI.""" @@ -2282,6 +2124,89 @@ def wrapped(): assert instance.call_count == 1 +def _assert_conversation_input(spans, conversation, output, method): + agent_spans = [span for span in spans if span["span_attributes"]["name"].startswith("agent_run")] + assert any(span["span_attributes"]["name"] == f"agent_{method} [conversation-agent]" for span in agent_spans) + for span in agent_spans: + assert span["span_attributes"]["type"] == SpanTypeAttribute.TASK + assert span["context"]["span_origin"]["instrumentation"]["name"] == "pydantic-ai-auto" + assert span["input"]["user_prompt"] == "What is my name? Answer with just the name." + captured = span["input"] + assert "conversation" not in captured + assert captured["conversation_id"] == conversation.conversation_id + assert len(captured["message_history"]) == len(conversation.messages) == 2 + assert captured["message_history"][0]["parts"][0]["content"] == "My name is Ada. Reply with just my name." + assert captured["message_history"][1]["parts"][0]["content"] == conversation.messages[1].parts[0].content + + assert "Ada" in output + chat_span = next(span for span in spans if span["span_attributes"]["name"].startswith("chat ")) + assert chat_span["metadata"]["model"] == "gpt-4o-mini" + assert chat_span["metadata"]["provider"] == "openai" + assert chat_span["metrics"]["prompt_tokens"] > 0 + assert chat_span["metrics"]["completion_tokens"] > 0 + span_by_id = {span["span_id"]: span for span in spans} + assert all(_is_descendant(span_by_id, chat_span, span["span_id"]) for span in agent_spans) + + +@pytest.mark.skipif(not CONVERSATION_SUPPORTED, reason="conversation= requires pydantic-ai >= 2.55.0") +@pytest.mark.asyncio +@pytest.mark.parametrize( + "method", + [ + pytest.param("run", marks=pytest.mark.vcr(cassette_name="test_agent_with_conversation")), + pytest.param("run_stream", marks=pytest.mark.vcr(cassette_name="test_agent_with_conversation_stream")), + pytest.param("run_stream_events", marks=pytest.mark.vcr(cassette_name="test_agent_with_conversation_stream")), + ], +) +async def test_agent_with_conversation(memory_logger, method): + """Conversation history is captured on every async agent wrapper span.""" + agent = Agent(MODEL, name="conversation-agent", model_settings=ModelSettings(max_tokens=50)) + first_result = await agent.run("My name is Ada. Reply with just my name.") + conversation = first_result.conversation + memory_logger.pop() + prompt = "What is my name? Answer with just the name." + + if method == "run": + result = await agent.run(prompt, conversation=conversation) + output = result.output + elif method == "run_stream": + async with agent.run_stream(prompt, conversation=conversation) as result: + output = await result.get_output() + else: + async with agent.run_stream_events(prompt, conversation=conversation) as events: + async for event in events: + pass + output = events.result.output + + _assert_conversation_input(memory_logger.pop(), conversation, output, method) + + +@pytest.mark.skipif(not CONVERSATION_SUPPORTED, reason="conversation= requires pydantic-ai >= 2.55.0") +@pytest.mark.parametrize( + "method", + [ + pytest.param("run_sync", marks=pytest.mark.vcr(cassette_name="test_agent_with_conversation")), + pytest.param("run_stream_sync", marks=pytest.mark.vcr(cassette_name="test_agent_with_conversation_stream")), + ], +) +def test_agent_with_conversation_sync(memory_logger, method): + """Conversation history is captured on every sync agent wrapper span.""" + agent = Agent(MODEL, name="conversation-agent", model_settings=ModelSettings(max_tokens=50)) + first_result = agent.run_sync("My name is Ada. Reply with just my name.") + conversation = first_result.conversation + memory_logger.pop() + prompt = "What is my name? Answer with just the name." + + if method == "run_sync": + result = agent.run_sync(prompt, conversation=conversation) + output = result.output + else: + result = agent.run_stream_sync(prompt, conversation=conversation) + output = "".join(result.stream_text(delta=True)) + + _assert_conversation_input(memory_logger.pop(), conversation, output, method) + + @pytest.mark.vcr @pytest.mark.asyncio async def test_agent_with_prefill(memory_logger): diff --git a/py/src/braintrust/integrations/pydantic_ai/tracing.py b/py/src/braintrust/integrations/pydantic_ai/tracing.py index 61256c4f2..1003fd35f 100644 --- a/py/src/braintrust/integrations/pydantic_ai/tracing.py +++ b/py/src/braintrust/integrations/pydantic_ai/tracing.py @@ -1570,6 +1570,24 @@ def _build_agent_input_and_metadata(args: Any, kwargs: Any, instance: Any) -> tu else: input_data[key] = value + conversation = kwargs.get("conversation") + if conversation is not None: + try: + from pydantic_ai.messages import ModelMessagesTypeAdapter + + # Nested message parts otherwise serialize as repr strings. + messages = ModelMessagesTypeAdapter.dump_python(_field_value(conversation, "messages")) + # Use the existing normalizer fields after delegated kwargs have logged their None defaults. + input_data.update( + _shape_object( + conversation, + fields=("conversation_id",), + overrides={"message_history": _shape_messages(messages)}, + ) + ) + except Exception as e: + logger.debug(f"Failed to extract conversation for tracing: {e}") + if "model" in kwargs: model_name, provider = _parse_model_string(kwargs["model"]) else: From 8029196447848930a2636fbabed2fa629eb71b0b Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Tue, 13 Oct 2026 11:39:38 -0400 Subject: [PATCH 2/2] test(pydantic-ai): handle older SDK signatures in lint Scope unexpected-keyword-arg suppression to the conversation tests, which require pydantic-ai >= 2.55.0. Python 3.10 lint uses an older compatible SDK whose agent run signatures do not accept conversation yet. --- .../integrations/pydantic_ai/test_pydantic_ai_integration.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py b/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py index c71535c7d..cd3da225d 100644 --- a/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py +++ b/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py @@ -2160,6 +2160,8 @@ def _assert_conversation_input(spans, conversation, output, method): ) async def test_agent_with_conversation(memory_logger, method): """Conversation history is captured on every async agent wrapper span.""" + # Python 3.10 lint uses an older SDK; the version gate excludes it at runtime. + # pylint: disable=unexpected-keyword-arg agent = Agent(MODEL, name="conversation-agent", model_settings=ModelSettings(max_tokens=50)) first_result = await agent.run("My name is Ada. Reply with just my name.") conversation = first_result.conversation @@ -2191,6 +2193,8 @@ async def test_agent_with_conversation(memory_logger, method): ) def test_agent_with_conversation_sync(memory_logger, method): """Conversation history is captured on every sync agent wrapper span.""" + # Python 3.10 lint uses an older SDK; the version gate excludes it at runtime. + # pylint: disable=unexpected-keyword-arg agent = Agent(MODEL, name="conversation-agent", model_settings=ModelSettings(max_tokens=50)) first_result = agent.run_sync("My name is Ada. Reply with just my name.") conversation = first_result.conversation