Skip to content

Commit e5df89b

Browse files
committed
test(temporal): cover chat spans across cache eviction
> 🤖 **This text was AI-generated.** Cache eviction could lose the original Braintrust span for a long-running Pydantic AI chat. Before the fix in PR #1054, replay created multiple workflow spans for the same chat: ```text first message -> cache eviction -> replay -> duplicate workflow spans ``` Add a two-turn chat regression for both Temporal durability APIs. Force cache eviction, stop the worker while the chat waits, and resume it on a fresh worker: ```text workflow span └─ chat_session (same IDs and start time after restart) ├─ first agent run └─ second agent run ``` Check that open spans remain unfinished during eviction and retain their input, metadata, and parent relationships when the chat completes. Share the durability API cases with the existing subagent regression. Refs: https://linear.app/braintrustdata/issue/SDK-781 Refs: #1054
1 parent cf69e61 commit e5df89b

1 file changed

Lines changed: 129 additions & 13 deletions

File tree

‎py/src/braintrust/integrations/temporal/test_temporal_pydantic_ai.py‎

Lines changed: 129 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
"""Tests for BraintrustPlugin combined with Pydantic AI's Temporal durability integration."""
22

3+
import asyncio
34
import uuid
45
import warnings
6+
from copy import deepcopy
57

68
import pytest
79

@@ -14,7 +16,13 @@
1416
import temporalio.workflow
1517
from braintrust.integrations.pydantic_ai import PydanticAIIntegration
1618
from braintrust.integrations.temporal import BraintrustPlugin
17-
from braintrust.integrations.temporal.test_temporal import _spans_named, memory_logger, temporal_env # noqa: F401
19+
from braintrust.integrations.temporal.test_temporal import ( # noqa: F401
20+
_spans_named,
21+
_wait_for_workflow_state,
22+
memory_logger,
23+
temporal_env,
24+
)
25+
from braintrust.merge_row_batch import merge_row_batch
1826
from pydantic_ai import Agent
1927
from pydantic_ai.durable_exec.temporal import AgentPlugin, PydanticAIPlugin
2028
from pydantic_ai.models.test import TestModel
@@ -23,6 +31,15 @@
2331

2432

2533
_HAS_DURABILITY_CAPABILITY = hasattr(pydantic_ai_temporal, "TemporalDurability")
34+
_DURABILITY_APIS = [
35+
"wrapper",
36+
pytest.param(
37+
"capability",
38+
marks=pytest.mark.skipif(
39+
not _HAS_DURABILITY_CAPABILITY, reason="TemporalDurability needs a newer pydantic-ai"
40+
),
41+
),
42+
]
2643

2744

2845
def _api_for_current_workflow() -> str:
@@ -51,6 +68,9 @@ async def _delegate(task: str) -> str:
5168
_wrapper_parent = pydantic_ai_temporal.TemporalAgent(
5269
_wrapper_parent_agent, tool_activity_config={"<agent>": {"delegate_task": False}}
5370
)
71+
_wrapper_chat = pydantic_ai_temporal.TemporalAgent(
72+
Agent(TestModel(custom_output_text="chat done"), name="wrapper_chat_agent")
73+
)
5474

5575
if _HAS_DURABILITY_CAPABILITY:
5676
_capability_sub = Agent(
@@ -65,11 +85,20 @@ async def _delegate(task: str) -> str:
6585
)
6686
# `metadata={"temporal": False}` replaces `tool_activity_config` on the capability path.
6787
_capability_parent.tool_plain(name="delegate_task", metadata={"temporal": False})(_delegate)
88+
_capability_chat = Agent(
89+
TestModel(custom_output_text="chat done"),
90+
name="capability_chat_agent",
91+
capabilities=[pydantic_ai_temporal.TemporalDurability()],
92+
)
6893

6994
_AGENTS = {
7095
"wrapper": (_wrapper_parent, _wrapper_sub),
7196
**({"capability": (_capability_parent, _capability_sub)} if _HAS_DURABILITY_CAPABILITY else {}),
7297
}
98+
_CHAT_AGENTS = {
99+
"wrapper": _wrapper_chat,
100+
**({"capability": _capability_chat} if _HAS_DURABILITY_CAPABILITY else {}),
101+
}
73102

74103

75104
@temporalio.workflow.defn
@@ -88,21 +117,40 @@ async def run(self, prompt: str) -> str:
88117
return (await parent_agent.run(prompt)).output
89118

90119

120+
@temporalio.workflow.defn
121+
class ChatWorkflow:
122+
"""Keep a chat span open between agent runs while waiting for a later message."""
123+
124+
def __init__(self) -> None:
125+
self._message: str | None = None
126+
self._state = "starting"
127+
128+
@temporalio.workflow.run
129+
async def run(self, prompt: str) -> list[str]:
130+
agent = _CHAT_AGENTS[_api_for_current_workflow()]
131+
with braintrust.start_span(name="chat_session", type="task") as span:
132+
first = await agent.run(prompt)
133+
span.log(input=prompt, metadata={"first_turn_complete": True})
134+
self._state = "waiting"
135+
await temporalio.workflow.wait_condition(lambda: self._message is not None)
136+
second = await agent.run(self._message, message_history=first.all_messages())
137+
output = [first.output, second.output]
138+
span.log(output=output)
139+
return output
140+
141+
@temporalio.workflow.signal
142+
def send_message(self, message: str) -> None:
143+
self._message = message
144+
145+
@temporalio.workflow.query
146+
def state(self) -> str:
147+
return self._state
148+
149+
91150
# With no workflow cache, every workflow task replays the run from the start, as after a worker
92151
# restart or cache eviction. Replays must not log the in-workflow Pydantic AI spans again.
93152
@pytest.mark.parametrize("max_cached_workflows", [1000, 0], ids=["cached", "replay"])
94-
@pytest.mark.parametrize(
95-
"api",
96-
[
97-
"wrapper",
98-
pytest.param(
99-
"capability",
100-
marks=pytest.mark.skipif(
101-
not _HAS_DURABILITY_CAPABILITY, reason="TemporalDurability needs a newer pydantic-ai"
102-
),
103-
),
104-
],
105-
)
153+
@pytest.mark.parametrize("api", _DURABILITY_APIS)
106154
@pytest.mark.asyncio
107155
async def test_subagent_child_workflow_nests_under_delegate_task_span(
108156
temporal_env, # noqa: F811
@@ -158,3 +206,71 @@ async def test_subagent_child_workflow_nests_under_delegate_task_span(
158206
if span["span_parents"][0] in {other["span_id"] for other in chat_spans + model_activity_spans}
159207
}
160208
assert len(pairs) == 3
209+
210+
211+
@pytest.mark.parametrize("api", _DURABILITY_APIS)
212+
@pytest.mark.asyncio
213+
async def test_chat_spans_survive_worker_restart(temporal_env, memory_logger, api): # noqa: F811
214+
assert PydanticAIIntegration.setup()
215+
agent = _CHAT_AGENTS[api]
216+
client = await Client.connect(
217+
temporal_env.client.service_client.config.target_host,
218+
namespace=temporal_env.client.namespace,
219+
plugins=[PydanticAIPlugin()],
220+
)
221+
task_queue = f"test-chat-restart-{uuid.uuid4()}"
222+
223+
def worker():
224+
return Worker(
225+
client,
226+
task_queue=task_queue,
227+
workflows=[ChatWorkflow],
228+
max_cached_workflows=0,
229+
plugins=[AgentPlugin(agent), BraintrustPlugin(logger=memory_logger)],
230+
)
231+
232+
async with worker():
233+
handle = await client.start_workflow(
234+
ChatWorkflow.run,
235+
"first message",
236+
id=f"{api}:{uuid.uuid4()}",
237+
task_queue=task_queue,
238+
)
239+
await _wait_for_workflow_state(handle, "waiting")
240+
241+
# Shutdown evicts any cached workflow. Capture what was actually exported before replay:
242+
# a still-running chat must remain open, including after its coroutine has been unwound.
243+
braintrust.flush()
244+
before_restart = memory_logger.pop()
245+
(original_workflow,) = _spans_named(before_restart, "temporal.workflow.ChatWorkflow")
246+
(original_chat,) = _spans_named(before_restart, "chat_session")
247+
for span in (original_workflow, original_chat):
248+
assert "end" not in span["metrics"]
249+
assert "error" not in span
250+
251+
# A new worker must rebuild the context from history, then append the next turn to the
252+
# original trace. No in-memory span or interceptor state is carried across workers.
253+
async with worker():
254+
await _wait_for_workflow_state(handle, "waiting")
255+
await handle.signal(ChatWorkflow.send_message, "second message")
256+
assert await asyncio.wait_for(handle.result(), timeout=15) == ["chat done", "chat done"]
257+
258+
braintrust.flush()
259+
spans = merge_row_batch(deepcopy(before_restart) + memory_logger.pop())
260+
(workflow_span,) = _spans_named(spans, "temporal.workflow.ChatWorkflow")
261+
(chat_span,) = _spans_named(spans, "chat_session")
262+
for original, resumed in ((original_workflow, workflow_span), (original_chat, chat_span)):
263+
for field in ("id", "span_id", "root_span_id"):
264+
assert resumed[field] == original[field]
265+
assert resumed["metrics"]["start"] == original["metrics"]["start"]
266+
assert resumed["metrics"]["end"] > original["metrics"]["start"]
267+
assert "error" not in resumed
268+
assert chat_span["input"] == "first message"
269+
assert chat_span["metadata"]["first_turn_complete"] is True
270+
assert chat_span["output"] == ["chat done", "chat done"]
271+
assert chat_span["span_parents"] == [workflow_span["span_id"]]
272+
agent_spans = _spans_named(spans, f"agent_run [{agent.name}]")
273+
assert len(agent_spans) == 2
274+
assert {span["root_span_id"] for span in spans} == {workflow_span["root_span_id"]}
275+
assert all(span["span_parents"] == [chat_span["span_id"]] for span in agent_spans)
276+
assert all(chat_span["metrics"]["end"] >= span["metrics"]["end"] for span in agent_spans)

0 commit comments

Comments
 (0)