11"""Tests for BraintrustPlugin combined with Pydantic AI's Temporal durability integration."""
22
3+ import asyncio
34import uuid
45import warnings
6+ from copy import deepcopy
57
68import pytest
79
1416import temporalio .workflow
1517from braintrust .integrations .pydantic_ai import PydanticAIIntegration
1618from braintrust .integrations .temporal import BraintrustPlugin
17- from braintrust .integrations .temporal .test_temporal import _spans_named , memory_logger , temporal_env # noqa: F401
19+ from braintrust .integrations .temporal .test_temporal import ( # noqa: F401
20+ _spans_named ,
21+ _wait_for_workflow_state ,
22+ memory_logger ,
23+ temporal_env ,
24+ )
25+ from braintrust .merge_row_batch import merge_row_batch
1826from pydantic_ai import Agent
1927from pydantic_ai .durable_exec .temporal import AgentPlugin , PydanticAIPlugin
2028from pydantic_ai .models .test import TestModel
2331
2432
2533_HAS_DURABILITY_CAPABILITY = hasattr (pydantic_ai_temporal , "TemporalDurability" )
34+ _DURABILITY_APIS = [
35+ "wrapper" ,
36+ pytest .param (
37+ "capability" ,
38+ marks = pytest .mark .skipif (
39+ not _HAS_DURABILITY_CAPABILITY , reason = "TemporalDurability needs a newer pydantic-ai"
40+ ),
41+ ),
42+ ]
2643
2744
2845def _api_for_current_workflow () -> str :
@@ -51,6 +68,9 @@ async def _delegate(task: str) -> str:
5168 _wrapper_parent = pydantic_ai_temporal .TemporalAgent (
5269 _wrapper_parent_agent , tool_activity_config = {"<agent>" : {"delegate_task" : False }}
5370 )
71+ _wrapper_chat = pydantic_ai_temporal .TemporalAgent (
72+ Agent (TestModel (custom_output_text = "chat done" ), name = "wrapper_chat_agent" )
73+ )
5474
5575if _HAS_DURABILITY_CAPABILITY :
5676 _capability_sub = Agent (
@@ -65,11 +85,20 @@ async def _delegate(task: str) -> str:
6585 )
6686 # `metadata={"temporal": False}` replaces `tool_activity_config` on the capability path.
6787 _capability_parent .tool_plain (name = "delegate_task" , metadata = {"temporal" : False })(_delegate )
88+ _capability_chat = Agent (
89+ TestModel (custom_output_text = "chat done" ),
90+ name = "capability_chat_agent" ,
91+ capabilities = [pydantic_ai_temporal .TemporalDurability ()],
92+ )
6893
6994_AGENTS = {
7095 "wrapper" : (_wrapper_parent , _wrapper_sub ),
7196 ** ({"capability" : (_capability_parent , _capability_sub )} if _HAS_DURABILITY_CAPABILITY else {}),
7297}
98+ _CHAT_AGENTS = {
99+ "wrapper" : _wrapper_chat ,
100+ ** ({"capability" : _capability_chat } if _HAS_DURABILITY_CAPABILITY else {}),
101+ }
73102
74103
75104@temporalio .workflow .defn
@@ -88,21 +117,40 @@ async def run(self, prompt: str) -> str:
88117 return (await parent_agent .run (prompt )).output
89118
90119
120+ @temporalio .workflow .defn
121+ class ChatWorkflow :
122+ """Keep a chat span open between agent runs while waiting for a later message."""
123+
124+ def __init__ (self ) -> None :
125+ self ._message : str | None = None
126+ self ._state = "starting"
127+
128+ @temporalio .workflow .run
129+ async def run (self , prompt : str ) -> list [str ]:
130+ agent = _CHAT_AGENTS [_api_for_current_workflow ()]
131+ with braintrust .start_span (name = "chat_session" , type = "task" ) as span :
132+ first = await agent .run (prompt )
133+ span .log (input = prompt , metadata = {"first_turn_complete" : True })
134+ self ._state = "waiting"
135+ await temporalio .workflow .wait_condition (lambda : self ._message is not None )
136+ second = await agent .run (self ._message , message_history = first .all_messages ())
137+ output = [first .output , second .output ]
138+ span .log (output = output )
139+ return output
140+
141+ @temporalio .workflow .signal
142+ def send_message (self , message : str ) -> None :
143+ self ._message = message
144+
145+ @temporalio .workflow .query
146+ def state (self ) -> str :
147+ return self ._state
148+
149+
91150# With no workflow cache, every workflow task replays the run from the start, as after a worker
92151# restart or cache eviction. Replays must not log the in-workflow Pydantic AI spans again.
93152@pytest .mark .parametrize ("max_cached_workflows" , [1000 , 0 ], ids = ["cached" , "replay" ])
94- @pytest .mark .parametrize (
95- "api" ,
96- [
97- "wrapper" ,
98- pytest .param (
99- "capability" ,
100- marks = pytest .mark .skipif (
101- not _HAS_DURABILITY_CAPABILITY , reason = "TemporalDurability needs a newer pydantic-ai"
102- ),
103- ),
104- ],
105- )
153+ @pytest .mark .parametrize ("api" , _DURABILITY_APIS )
106154@pytest .mark .asyncio
107155async def test_subagent_child_workflow_nests_under_delegate_task_span (
108156 temporal_env , # noqa: F811
@@ -158,3 +206,71 @@ async def test_subagent_child_workflow_nests_under_delegate_task_span(
158206 if span ["span_parents" ][0 ] in {other ["span_id" ] for other in chat_spans + model_activity_spans }
159207 }
160208 assert len (pairs ) == 3
209+
210+
211+ @pytest .mark .parametrize ("api" , _DURABILITY_APIS )
212+ @pytest .mark .asyncio
213+ async def test_chat_spans_survive_worker_restart (temporal_env , memory_logger , api ): # noqa: F811
214+ assert PydanticAIIntegration .setup ()
215+ agent = _CHAT_AGENTS [api ]
216+ client = await Client .connect (
217+ temporal_env .client .service_client .config .target_host ,
218+ namespace = temporal_env .client .namespace ,
219+ plugins = [PydanticAIPlugin ()],
220+ )
221+ task_queue = f"test-chat-restart-{ uuid .uuid4 ()} "
222+
223+ def worker ():
224+ return Worker (
225+ client ,
226+ task_queue = task_queue ,
227+ workflows = [ChatWorkflow ],
228+ max_cached_workflows = 0 ,
229+ plugins = [AgentPlugin (agent ), BraintrustPlugin (logger = memory_logger )],
230+ )
231+
232+ async with worker ():
233+ handle = await client .start_workflow (
234+ ChatWorkflow .run ,
235+ "first message" ,
236+ id = f"{ api } :{ uuid .uuid4 ()} " ,
237+ task_queue = task_queue ,
238+ )
239+ await _wait_for_workflow_state (handle , "waiting" )
240+
241+ # Shutdown evicts any cached workflow. Capture what was actually exported before replay:
242+ # a still-running chat must remain open, including after its coroutine has been unwound.
243+ braintrust .flush ()
244+ before_restart = memory_logger .pop ()
245+ (original_workflow ,) = _spans_named (before_restart , "temporal.workflow.ChatWorkflow" )
246+ (original_chat ,) = _spans_named (before_restart , "chat_session" )
247+ for span in (original_workflow , original_chat ):
248+ assert "end" not in span ["metrics" ]
249+ assert "error" not in span
250+
251+ # A new worker must rebuild the context from history, then append the next turn to the
252+ # original trace. No in-memory span or interceptor state is carried across workers.
253+ async with worker ():
254+ await _wait_for_workflow_state (handle , "waiting" )
255+ await handle .signal (ChatWorkflow .send_message , "second message" )
256+ assert await asyncio .wait_for (handle .result (), timeout = 15 ) == ["chat done" , "chat done" ]
257+
258+ braintrust .flush ()
259+ spans = merge_row_batch (deepcopy (before_restart ) + memory_logger .pop ())
260+ (workflow_span ,) = _spans_named (spans , "temporal.workflow.ChatWorkflow" )
261+ (chat_span ,) = _spans_named (spans , "chat_session" )
262+ for original , resumed in ((original_workflow , workflow_span ), (original_chat , chat_span )):
263+ for field in ("id" , "span_id" , "root_span_id" ):
264+ assert resumed [field ] == original [field ]
265+ assert resumed ["metrics" ]["start" ] == original ["metrics" ]["start" ]
266+ assert resumed ["metrics" ]["end" ] > original ["metrics" ]["start" ]
267+ assert "error" not in resumed
268+ assert chat_span ["input" ] == "first message"
269+ assert chat_span ["metadata" ]["first_turn_complete" ] is True
270+ assert chat_span ["output" ] == ["chat done" , "chat done" ]
271+ assert chat_span ["span_parents" ] == [workflow_span ["span_id" ]]
272+ agent_spans = _spans_named (spans , f"agent_run [{ agent .name } ]" )
273+ assert len (agent_spans ) == 2
274+ assert {span ["root_span_id" ] for span in spans } == {workflow_span ["root_span_id" ]}
275+ assert all (span ["span_parents" ] == [chat_span ["span_id" ]] for span in agent_spans )
276+ assert all (chat_span ["metrics" ]["end" ] >= span ["metrics" ]["end" ] for span in agent_spans )
0 commit comments