From 8e6ed63654d0965aa4eb4d5e56fe619af276295e Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 15 Sep 2026 14:33:26 +0200 Subject: [PATCH] ref(langgraph): Stop setting gen_ai.response.model on Invoke Agent spans --- sentry_sdk/integrations/langgraph.py | 17 - .../integrations/langgraph/test_langgraph.py | 336 ------------------ 2 files changed, 353 deletions(-) diff --git a/sentry_sdk/integrations/langgraph.py b/sentry_sdk/integrations/langgraph.py index 32f9f32a10..2aa472a30f 100644 --- a/sentry_sdk/integrations/langgraph.py +++ b/sentry_sdk/integrations/langgraph.py @@ -299,22 +299,6 @@ def _set_usage_data(span: "Span", messages: "Any") -> None: ) -def _set_response_model_name(span: "Span", messages: "Any") -> None: - if len(messages) == 0: - return - - last_message = messages[-1] - response_metadata = last_message.get("response_metadata") - if response_metadata is None: - return - - model_name = response_metadata.get("model_name") - if model_name is None: - return - - set_data_normalized(span, SPANDATA.GEN_AI_RESPONSE_MODEL, model_name) - - def _set_response_attributes( span: "Any", input_messages: "Optional[List[Any]]", @@ -328,7 +312,6 @@ def _set_response_attributes( return _set_usage_data(span, new_messages) - _set_response_model_name(span, new_messages) if _should_record_outputs(integration): llm_response_text = _extract_llm_response_text(new_messages) diff --git a/tests/integrations/langgraph/test_langgraph.py b/tests/integrations/langgraph/test_langgraph.py index c0d6c182ca..a79c5cae7a 100644 --- a/tests/integrations/langgraph/test_langgraph.py +++ b/tests/integrations/langgraph/test_langgraph.py @@ -809,340 +809,6 @@ async def run_test(): assert invoke_agent_span["attributes"]["gen_ai.usage.total_tokens"] == 50 -def test_pregel_invoke_span_includes_response_model( - sentry_init, - capture_items, -): - """ - Test that invoke_agent spans include the response model. - When an agent makes multiple LLM calls, it should report the last model used. - """ - sentry_init( - integrations=[LanggraphIntegration()], - traces_sample_rate=1.0, - ) - - test_state = { - "messages": [ - MockMessage("Hello, can you help me?", name="user"), - MockMessage("Of course! How can I assist you?", name="assistant"), - ] - } - - pregel = MockPregelInstance("test_graph") - - expected_assistant_response = "I'll help you with that task!" - expected_tool_calls = [ - { - "id": "call_test_123", - "type": "function", - "function": {"name": "search_tool", "arguments": '{"query": "help"}'}, - } - ] - - def original_invoke(self, *args, **kwargs): - input_messages = args[0].get("messages", []) - new_messages = input_messages + [ - MockMessage( - content=expected_assistant_response, - name="assistant", - tool_calls=expected_tool_calls, - response_metadata={ - "token_usage": { - "total_tokens": 30, - "prompt_tokens": 10, - "completion_tokens": 20, - }, - "model_name": "gpt-4.1-2025-04-14", - }, - ) - ] - return {"messages": new_messages} - - items = capture_items("span") - - wrapped_invoke = _wrap_pregel_invoke(original_invoke) - result = wrapped_invoke(pregel, test_state) - - assert result is not None - - sentry_sdk.flush() - spans = [item.payload for item in items] - invoke_spans = [ - span - for span in spans - if span["attributes"]["sentry.op"] == OP.GEN_AI_INVOKE_AGENT - ] - assert len(invoke_spans) == 1 - - invoke_agent_span = invoke_spans[0] - - # Verify invoke_agent span has response model - assert invoke_agent_span["name"] == "invoke_agent test_graph" - assert "gen_ai.response.model" in invoke_agent_span["attributes"] - assert ( - invoke_agent_span["attributes"]["gen_ai.response.model"] == "gpt-4.1-2025-04-14" - ) - - -def test_pregel_ainvoke_span_includes_response_model( - sentry_init, - capture_items, -): - """ - Test that invoke_agent spans include the response model. - When an agent makes multiple LLM calls, it should report the last model used. - """ - sentry_init( - integrations=[LanggraphIntegration()], - traces_sample_rate=1.0, - ) - - test_state = { - "messages": [ - MockMessage("Hello, can you help me?", name="user"), - MockMessage("Of course! How can I assist you?", name="assistant"), - ] - } - - pregel = MockPregelInstance("test_graph") - - expected_assistant_response = "I'll help you with that task!" - expected_tool_calls = [ - { - "id": "call_test_123", - "type": "function", - "function": {"name": "search_tool", "arguments": '{"query": "help"}'}, - } - ] - - async def original_ainvoke(self, *args, **kwargs): - input_messages = args[0].get("messages", []) - new_messages = input_messages + [ - MockMessage( - content=expected_assistant_response, - name="assistant", - tool_calls=expected_tool_calls, - response_metadata={ - "token_usage": { - "total_tokens": 30, - "prompt_tokens": 10, - "completion_tokens": 20, - }, - "model_name": "gpt-4.1-2025-04-14", - }, - ) - ] - return {"messages": new_messages} - - async def run_test(): - wrapped_ainvoke = _wrap_pregel_ainvoke(original_ainvoke) - result = await wrapped_ainvoke(pregel, test_state) - return result - - items = capture_items("span") - - result = asyncio.run(run_test()) - assert result is not None - - sentry_sdk.flush() - spans = [item.payload for item in items] - invoke_spans = [ - span - for span in spans - if span["attributes"]["sentry.op"] == OP.GEN_AI_INVOKE_AGENT - ] - assert len(invoke_spans) == 1 - - invoke_agent_span = invoke_spans[0] - - # Verify invoke_agent span has response model - assert invoke_agent_span["name"] == "invoke_agent test_graph" - assert "gen_ai.response.model" in invoke_agent_span["attributes"] - assert ( - invoke_agent_span["attributes"]["gen_ai.response.model"] == "gpt-4.1-2025-04-14" - ) - - -def test_pregel_invoke_span_uses_last_response_model( - sentry_init, - capture_items, -): - """ - Test that when an agent makes multiple LLM calls (e.g., with tools), - the invoke_agent span reports the last response model used. - """ - sentry_init( - integrations=[LanggraphIntegration()], - traces_sample_rate=1.0, - ) - - test_state = { - "messages": [ - MockMessage("Hello, can you help me?", name="user"), - MockMessage("Of course! How can I assist you?", name="assistant"), - ] - } - - pregel = MockPregelInstance("test_graph") - - expected_assistant_response = "I'll help you with that task!" - expected_tool_calls = [ - { - "id": "call_test_123", - "type": "function", - "function": {"name": "search_tool", "arguments": '{"query": "help"}'}, - } - ] - - def original_invoke(self, *args, **kwargs): - input_messages = args[0].get("messages", []) - new_messages = input_messages + [ - MockMessage( - content=expected_assistant_response, - name="assistant", - tool_calls=expected_tool_calls, - response_metadata={ - "token_usage": { - "total_tokens": 15, - "prompt_tokens": 10, - "completion_tokens": 5, - }, - "model_name": "gpt-4-0613", - }, - ), - MockMessage( - content=expected_assistant_response, - name="assistant", - tool_calls=expected_tool_calls, - response_metadata={ - "token_usage": { - "total_tokens": 35, - "prompt_tokens": 20, - "completion_tokens": 15, - }, - "model_name": "gpt-4.1-2025-04-14", - }, - ), - ] - return {"messages": new_messages} - - items = capture_items("span") - - wrapped_invoke = _wrap_pregel_invoke(original_invoke) - result = wrapped_invoke(pregel, test_state) - - assert result is not None - - sentry_sdk.flush() - spans = [item.payload for item in items] - invoke_spans = [ - span - for span in spans - if span["attributes"]["sentry.op"] == OP.GEN_AI_INVOKE_AGENT - ] - assert len(invoke_spans) == 1 - - invoke_agent_span = invoke_spans[0] - - # Verify invoke_agent span uses the LAST response model - assert "gen_ai.response.model" in invoke_agent_span["attributes"] - assert ( - invoke_agent_span["attributes"]["gen_ai.response.model"] == "gpt-4.1-2025-04-14" - ) - - -def test_pregel_ainvoke_span_uses_last_response_model( - sentry_init, - capture_items, -): - """ - Test that when an agent makes multiple LLM calls (e.g., with tools), - the invoke_agent span reports the last response model used. - """ - sentry_init( - integrations=[LanggraphIntegration()], - traces_sample_rate=1.0, - ) - - test_state = { - "messages": [ - MockMessage("Hello, can you help me?", name="user"), - MockMessage("Of course! How can I assist you?", name="assistant"), - ] - } - - pregel = MockPregelInstance("test_graph") - - expected_assistant_response = "I'll help you with that task!" - expected_tool_calls = [ - { - "id": "call_test_123", - "type": "function", - "function": {"name": "search_tool", "arguments": '{"query": "help"}'}, - } - ] - - async def original_ainvoke(self, *args, **kwargs): - input_messages = args[0].get("messages", []) - new_messages = input_messages + [ - MockMessage( - content=expected_assistant_response, - name="assistant", - tool_calls=expected_tool_calls, - response_metadata={ - "token_usage": { - "total_tokens": 15, - "prompt_tokens": 10, - "completion_tokens": 5, - }, - "model_name": "gpt-4-0613", - }, - ), - MockMessage( - content=expected_assistant_response, - name="assistant", - tool_calls=expected_tool_calls, - response_metadata={ - "token_usage": { - "total_tokens": 35, - "prompt_tokens": 20, - "completion_tokens": 15, - }, - "model_name": "gpt-4.1-2025-04-14", - }, - ), - ] - return {"messages": new_messages} - - async def run_test(): - wrapped_ainvoke = _wrap_pregel_ainvoke(original_ainvoke) - result = await wrapped_ainvoke(pregel, test_state) - return result - - items = capture_items("span") - - result = asyncio.run(run_test()) - assert result is not None - - sentry_sdk.flush() - spans = [item.payload for item in items] - invoke_spans = [ - span - for span in spans - if span["attributes"]["sentry.op"] == OP.GEN_AI_INVOKE_AGENT - ] - assert len(invoke_spans) == 1 - - invoke_agent_span = invoke_spans[0] - - # Verify invoke_agent span uses the LAST response model - assert "gen_ai.response.model" in invoke_agent_span["attributes"] - assert ( - invoke_agent_span["attributes"]["gen_ai.response.model"] == "gpt-4.1-2025-04-14" - ) - - def test_complex_message_parsing(): """Test message parsing with complex message structures.""" messages = [ @@ -1713,7 +1379,6 @@ def original_invoke(self, *args, **kwargs): assert data[SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert data[SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 assert data[SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert data[SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-4.1-2025-04-14" def test_pregel_ainvoke_message_delta_ignores_gen_ai_inputs_setting( @@ -1784,4 +1449,3 @@ async def run_test(): assert data[SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert data[SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 assert data[SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 - assert data[SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-4.1-2025-04-14"