diff --git a/sentry_sdk/integrations/openai_agents/patches/runner.py b/sentry_sdk/integrations/openai_agents/patches/runner.py index 66c3482695..363fc20b4c 100644 --- a/sentry_sdk/integrations/openai_agents/patches/runner.py +++ b/sentry_sdk/integrations/openai_agents/patches/runner.py @@ -4,10 +4,8 @@ import sentry_sdk from sentry_sdk.consts import SPANDATA from sentry_sdk.integrations import DidNotEnable -from sentry_sdk.scope import should_send_default_pii from sentry_sdk.utils import ( capture_internal_exceptions, - has_data_collection_enabled, reraise, ) @@ -56,10 +54,7 @@ async def on_tool_start( context._sentry_execute_tool_span = span client = sentry_sdk.get_client() - if has_data_collection_enabled(client.options): - if not client.options["data_collection"]["gen_ai"]["inputs"]: - return - elif not should_send_default_pii(): + if not client.options["data_collection"]["gen_ai"]["inputs"]: return span.set_attribute(SPANDATA.GEN_AI_TOOL_INPUT, context.tool_arguments) diff --git a/sentry_sdk/integrations/openai_agents/spans/ai_client.py b/sentry_sdk/integrations/openai_agents/spans/ai_client.py index 203e4efd74..5c095b6b6c 100644 --- a/sentry_sdk/integrations/openai_agents/spans/ai_client.py +++ b/sentry_sdk/integrations/openai_agents/spans/ai_client.py @@ -5,7 +5,6 @@ from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations import DidNotEnable from sentry_sdk.traces import Span, _AgentFrameworkChatGenerationContext -from sentry_sdk.utils import has_data_collection_enabled try: from agents import ( @@ -219,15 +218,7 @@ def ai_client_context( _set_agent_data(context.span, agent) - if has_data_collection_enabled(client_options): - if client_options["data_collection"]["gen_ai"]["inputs"]: - context.span.set_attribute( - SPANDATA.GEN_AI_TOOL_DEFINITIONS, - json.dumps(_transform_tool_definitions(agent.tools)), - ) - else: - # This is set unconditionally prior to data collection being introduced. - # Remove this block once data collection is fully rolled out + if client_options["data_collection"]["gen_ai"]["inputs"]: context.span.set_attribute( SPANDATA.GEN_AI_TOOL_DEFINITIONS, json.dumps(_transform_tool_definitions(agent.tools)), diff --git a/sentry_sdk/integrations/openai_agents/spans/execute_tool.py b/sentry_sdk/integrations/openai_agents/spans/execute_tool.py index 9cadea6fdb..b4ec65da0f 100644 --- a/sentry_sdk/integrations/openai_agents/spans/execute_tool.py +++ b/sentry_sdk/integrations/openai_agents/spans/execute_tool.py @@ -2,9 +2,7 @@ import sentry_sdk from sentry_sdk.consts import OP, SPANDATA -from sentry_sdk.scope import should_send_default_pii from sentry_sdk.traces import Span, SpanStatus -from sentry_sdk.utils import has_data_collection_enabled from ..consts import SPAN_ORIGIN from ..utils import _set_agent_data @@ -45,10 +43,7 @@ def update_execute_tool_span( ): span.status = SpanStatus.ERROR - if has_data_collection_enabled(client.options): - if client.options["data_collection"]["gen_ai"]["outputs"]: - span.set_attribute(SPANDATA.GEN_AI_TOOL_OUTPUT, result) - elif should_send_default_pii(): + if client.options["data_collection"]["gen_ai"]["outputs"]: span.set_attribute(SPANDATA.GEN_AI_TOOL_OUTPUT, result) # Add conversation ID from agent diff --git a/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py index ff306e4138..89d28ed3d7 100644 --- a/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py +++ b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py @@ -6,9 +6,8 @@ set_data_normalized, ) from sentry_sdk.consts import OP, SPANDATA -from sentry_sdk.scope import should_send_default_pii from sentry_sdk.traces import Span -from sentry_sdk.utils import has_data_collection_enabled, safe_serialize +from sentry_sdk.utils import safe_serialize from ..consts import SPAN_ORIGIN from ..utils import _set_agent_data @@ -33,14 +32,7 @@ def invoke_agent_span( }, ) - record_inputs = False - if has_data_collection_enabled(client_options): - if client_options["data_collection"]["gen_ai"]["inputs"]: - record_inputs = True - elif should_send_default_pii(): - record_inputs = True - - if record_inputs: + if client_options["data_collection"]["gen_ai"]["inputs"]: messages = [] if agent.instructions: message = ( @@ -89,12 +81,7 @@ def update_invoke_agent_span( output: "Any" = None, ) -> None: client = sentry_sdk.get_client() - if has_data_collection_enabled(client.options): - if client.options["data_collection"]["gen_ai"]["outputs"]: - set_data_normalized( - span, SPANDATA.GEN_AI_RESPONSE_TEXT, output, unpack=False - ) - elif should_send_default_pii(): + if client.options["data_collection"]["gen_ai"]["outputs"]: set_data_normalized(span, SPANDATA.GEN_AI_RESPONSE_TEXT, output, unpack=False) # Add conversation ID from agent diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index b104b2c042..7d631aff27 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -15,11 +15,9 @@ ) from sentry_sdk.consts import SPANDATA from sentry_sdk.integrations import DidNotEnable -from sentry_sdk.scope import should_send_default_pii from sentry_sdk.traces import Span from sentry_sdk.utils import ( event_from_exception, - has_data_collection_enabled, safe_serialize, ) @@ -109,11 +107,9 @@ def _set_input_data( get_response_kwargs: "dict[str, Any]", ) -> None: client = sentry_sdk.get_client() - if has_data_collection_enabled(client.options): - if not client.options["data_collection"]["gen_ai"]["inputs"]: - return - elif not should_send_default_pii(): + if not client.options["data_collection"]["gen_ai"]["inputs"]: return + request_messages = [] messages: "str | list[TResponseInputItem]" = get_response_kwargs.get("input", []) @@ -184,13 +180,8 @@ def _set_input_data( def _set_output_data(span: "Span", result: "Any") -> None: client = sentry_sdk.get_client() - record_outputs = False - if has_data_collection_enabled(client.options): - record_outputs = client.options["data_collection"]["gen_ai"]["outputs"] - elif should_send_default_pii(): - record_outputs = True - if not record_outputs: + if not client.options["data_collection"]["gen_ai"]["outputs"]: return output_messages: "dict[str, list[Any]]" = { @@ -209,13 +200,13 @@ def _set_output_data(span: "Span", result: "Any") -> None: # Unknown output message type, just return the json output_messages["response"].append(output_message.dict()) - if record_outputs and len(output_messages["tool"]) > 0: + if len(output_messages["tool"]) > 0: span.set_attribute( SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, safe_serialize(output_messages["tool"]), ) - if record_outputs and len(output_messages["response"]) > 0: + if len(output_messages["response"]) > 0: set_data_normalized( span, SPANDATA.GEN_AI_RESPONSE_TEXT, output_messages["response"] ) diff --git a/tests/integrations/openai_agents/test_openai_agents.py b/tests/integrations/openai_agents/test_openai_agents.py index 227001b3ec..4b3048dc3d 100644 --- a/tests/integrations/openai_agents/test_openai_agents.py +++ b/tests/integrations/openai_agents/test_openai_agents.py @@ -392,7 +392,11 @@ def some_function(a: str, b: list[int]) -> str: integrations=[OpenAIAgentsIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=False, + data_collection={ + "gen_ai": { + "inputs": True, + } + }, ) items = capture_items("span") @@ -419,7 +423,7 @@ def some_function(a: str, b: list[int]) -> str: @pytest.mark.asyncio -async def test_agent_invocation_span_no_pii( +async def test_agent_invocation_span_no_sensitive_data( sentry_init, capture_items, test_agent, @@ -442,7 +446,12 @@ async def test_agent_invocation_span_no_pii( integrations=[OpenAIAgentsIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=False, + data_collection={ + "gen_ai": { + "inputs": False, + "outputs": False, + } + }, ) items = capture_items("span") @@ -489,470 +498,6 @@ async def test_agent_invocation_span_no_pii( assert ai_client_span["attributes"]["gen_ai.request.top_p"] == 1.0 -@pytest.mark.parametrize( - "init_kwargs,expect_messages", - [ - pytest.param( - {"data_collection": {"gen_ai": {"inputs": True}}}, - True, - id="gen_ai_inputs_true", - ), - pytest.param( - {"data_collection": {"gen_ai": {"inputs": False}}}, - False, - id="gen_ai_inputs_false", - ), - pytest.param( - {"data_collection": {}}, - True, - id="data_collection_defaults_to_enabled", - ), - pytest.param( - { - "send_default_pii": True, - "data_collection": {"gen_ai": {"inputs": False}}, - }, - False, - id="data_collection_wins_over_send_default_pii_true", - ), - pytest.param( - { - "send_default_pii": False, - "data_collection": {"gen_ai": {"inputs": True}}, - }, - True, - id="data_collection_wins_over_send_default_pii_false", - ), - pytest.param( - {"send_default_pii": True}, - True, - id="legacy_send_default_pii_true", - ), - pytest.param( - {"send_default_pii": False}, - False, - id="legacy_send_default_pii_false", - ), - ], -) -@pytest.mark.asyncio -async def test_invoke_agent_span_data_collection_inputs( - sentry_init, - capture_items, - test_agent, - nonstreaming_responses_model_response, - get_model_response, - init_kwargs, - expect_messages, -): - client = AsyncOpenAI(api_key="test-key") - model = OpenAIResponsesModel(model="gpt-4", openai_client=client) - agent = test_agent.clone(model=model) - - response = get_model_response( - nonstreaming_responses_model_response, serialize_pydantic=True - ) - - system_message = { - "role": "system", - "content": [{"text": "You are a helpful test assistant.", "type": "text"}], - } - user_message = { - "role": "user", - "content": [{"text": "Test input", "type": "text"}], - } - expected_messages = [system_message, user_message] - with patch.object( - agent.model._client._client, - "send", - return_value=response, - ) as _: - sentry_init( - integrations=[OpenAIAgentsIntegration()], - disabled_integrations=[StdlibIntegration], - traces_sample_rate=1.0, - **init_kwargs, - ) - - items = capture_items("span") - - result = await agents.Runner.run( - agent, "Test input", run_config=test_run_config - ) - - assert result is not None - - sentry_sdk.flush() - spans = [item.payload for item in items] - invoke_agent_span = next( - span - for span in spans - if span["attributes"]["sentry.op"] == OP.GEN_AI_INVOKE_AGENT - ) - span_data = invoke_agent_span["attributes"] - - if expect_messages: - assert ( - json.loads(span_data[SPANDATA.GEN_AI_REQUEST_MESSAGES]) == expected_messages - ) - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span_data - - -@pytest.mark.parametrize( - "init_kwargs,expect_response_text", - [ - pytest.param( - {"data_collection": {"gen_ai": {"outputs": True}}}, - True, - id="gen_ai_outputs_true", - ), - pytest.param( - {"data_collection": {"gen_ai": {"outputs": False}}}, - False, - id="gen_ai_outputs_false", - ), - pytest.param( - {"data_collection": {}}, - True, - id="data_collection_defaults_to_enabled", - ), - pytest.param( - { - "send_default_pii": True, - "data_collection": {"gen_ai": {"outputs": False}}, - }, - False, - id="data_collection_wins_over_send_default_pii_true", - ), - pytest.param( - { - "send_default_pii": False, - "data_collection": {"gen_ai": {"outputs": True}}, - }, - True, - id="data_collection_wins_over_send_default_pii_false", - ), - pytest.param( - {"send_default_pii": True}, - True, - id="legacy_send_default_pii_true", - ), - pytest.param( - {"send_default_pii": False}, - False, - id="legacy_send_default_pii_false", - ), - ], -) -@pytest.mark.asyncio -async def test_invoke_agent_span_data_collection_outputs( - sentry_init, - capture_items, - test_agent, - nonstreaming_responses_model_response, - get_model_response, - init_kwargs, - expect_response_text, -): - client = AsyncOpenAI(api_key="test-key") - model = OpenAIResponsesModel(model="gpt-4", openai_client=client) - agent = test_agent.clone(model=model) - - response = get_model_response( - nonstreaming_responses_model_response, serialize_pydantic=True - ) - with patch.object( - agent.model._client._client, - "send", - return_value=response, - ) as _: - sentry_init( - integrations=[OpenAIAgentsIntegration()], - disabled_integrations=[StdlibIntegration], - traces_sample_rate=1.0, - **init_kwargs, - ) - - items = capture_items("span") - - result = await agents.Runner.run( - agent, "Test input", run_config=test_run_config - ) - - assert result is not None - - sentry_sdk.flush() - spans = [item.payload for item in items] - invoke_agent_span = next( - span - for span in spans - if span["attributes"]["sentry.op"] == OP.GEN_AI_INVOKE_AGENT - ) - span_data = invoke_agent_span["attributes"] - - if expect_response_text: - assert span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] == "Hello, how can I help you?" - else: - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data - - -@pytest.mark.parametrize( - "data_collection,send_default_pii,expect_input", - [ - pytest.param( - {"gen_ai": {"inputs": True}}, - False, - True, - id="gen-ai-inputs-enabled-overrides-pii-disabled", - ), - pytest.param( - {"gen_ai": {"inputs": False}}, - True, - False, - id="gen-ai-inputs-disabled-overrides-pii-enabled", - ), - pytest.param( - {}, - False, - True, - id="gen-ai-omitted-defaults-to-enabled", - ), - pytest.param( - {"gen_ai": {"inputs": False}}, - False, - False, - id="gen-ai-inputs-disabled-and-pii-disabled", - ), - pytest.param( - {"gen_ai": {"outputs": False}}, - False, - True, - id="gen-ai-outputs-disabled-does-not-affect-inputs", - ), - pytest.param( - None, - False, - False, - id="no-data-collection-falls-back-to-send-default-pii", - ), - pytest.param( - None, - True, - True, - id="no-data-collection-pii-enabled-collects", - ), - ], -) -@pytest.mark.asyncio -async def test_data_collection_inputs( - sentry_init, - capture_items, - test_agent, - simple_test_tool, - nonstreaming_responses_model_response, - get_model_response, - data_collection, - send_default_pii, - expect_input, -): - client = AsyncOpenAI(api_key="test-key") - model = OpenAIResponsesModel(model="gpt-4", openai_client=client) - agent = test_agent.clone(model=model, tools=[simple_test_tool]) - - response = get_model_response( - nonstreaming_responses_model_response, serialize_pydantic=True - ) - - init_kwargs = { - "integrations": [OpenAIAgentsIntegration()], - "traces_sample_rate": 1.0, - "send_default_pii": send_default_pii, - } - init_kwargs["disabled_integrations"] = [StdlibIntegration] - if data_collection is not None: - init_kwargs["data_collection"] = data_collection - - with patch.object( - agent.model._client._client, - "send", - return_value=response, - ) as _: - sentry_init(**init_kwargs) - items = capture_items("span") - - result = await agents.Runner.run( - agent, "Test input", run_config=test_run_config - ) - - assert result is not None - sentry_sdk.flush() - spans = [item.payload for item in items] - ai_client_span = next( - span for span in spans if span["attributes"]["sentry.op"] == OP.GEN_AI_CHAT - ) - span_data = ai_client_span["attributes"] - - if expect_input: - assert "Test input" in span_data[SPANDATA.GEN_AI_REQUEST_MESSAGES] - assert ( - "helpful test assistant" in span_data[SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span_data - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span_data - - # Tool definitions are only gated once data_collection is configured; without it - # they are set unconditionally, as they were before data_collection existed. - if expect_input or data_collection is None: - assert "simple_test_tool" in span_data[SPANDATA.GEN_AI_TOOL_DEFINITIONS] - else: - assert SPANDATA.GEN_AI_TOOL_DEFINITIONS not in span_data - - # Non-PII data is unaffected by the gate - assert span_data[SPANDATA.GEN_AI_REQUEST_MODEL] == "gpt-4" - assert span_data[SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 - - -@pytest.mark.parametrize( - "data_collection,send_default_pii,expect_output", - [ - pytest.param( - {"gen_ai": {"outputs": True}}, - False, - True, - id="gen-ai-outputs-enabled-overrides-pii-disabled", - ), - pytest.param( - {"gen_ai": {"outputs": False}}, - True, - False, - id="gen-ai-outputs-disabled-overrides-pii-enabled", - ), - pytest.param( - {}, - False, - True, - id="gen-ai-omitted-defaults-to-enabled", - ), - pytest.param( - {"gen_ai": {"inputs": False, "outputs": False}}, - False, - False, - id="gen-ai-inputs-and-outputs-disabled-and-pii-disabled", - ), - pytest.param( - {"gen_ai": {"inputs": False}}, - False, - True, - id="gen-ai-inputs-disabled-keeps-outputs-and-tool-calls", - ), - pytest.param( - None, - False, - False, - id="no-data-collection-falls-back-to-send-default-pii", - ), - pytest.param( - None, - True, - True, - id="no-data-collection-pii-enabled-collects", - ), - ], -) -@pytest.mark.asyncio -async def test_data_collection_outputs( - sentry_init, - capture_items, - test_agent, - simple_test_tool, - get_model_response, - nonstreaming_responses_tool_call_model_responses, - data_collection, - send_default_pii, - expect_output, -): - client = AsyncOpenAI(api_key="test-key") - model = OpenAIResponsesModel(model="gpt-4", openai_client=client) - agent_with_tool = test_agent.clone(tools=[simple_test_tool], model=model) - - usage = ResponseUsage( - input_tokens=10, - input_tokens_details=InputTokensDetails( - cached_tokens=0, - cache_write_tokens=0, - ), - output_tokens=5, - output_tokens_details=OutputTokensDetails( - reasoning_tokens=0, - ), - total_tokens=15, - ) - responses = nonstreaming_responses_tool_call_model_responses( - tool_name="simple_test_tool", - arguments='{"message": "hello"}', - response_model="gpt-4", - response_text="Task completed using the tool", - response_ids=iter(["resp_tool_123", "resp_final_123"]), - usages=iter([usage, usage]), - ) - tool_response = get_model_response(next(responses), serialize_pydantic=True) - final_response = get_model_response(next(responses), serialize_pydantic=True) - - init_kwargs = { - "integrations": [OpenAIAgentsIntegration()], - "traces_sample_rate": 1.0, - "send_default_pii": send_default_pii, - } - init_kwargs["disabled_integrations"] = [StdlibIntegration] - if data_collection is not None: - init_kwargs["data_collection"] = data_collection - - with patch.object( - agent_with_tool.model._client._client, - "send", - side_effect=[tool_response, final_response], - ) as _: - sentry_init(**init_kwargs) - items = capture_items("span") - - await agents.Runner.run( - agent_with_tool, - "Please use the simple test tool", - run_config=test_run_config, - ) - sentry_sdk.flush() - spans = [item.payload for item in items] - chat_span_data = [ - span["attributes"] - for span in spans - if span["attributes"].get("sentry.op") == OP.GEN_AI_CHAT - ] - - assert len(chat_span_data) == 2 - - if expect_output: - assert any( - "simple_test_tool" in data.get(SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, "") - for data in chat_span_data - ) - assert any( - "Task completed using the tool" - in str(data.get(SPANDATA.GEN_AI_RESPONSE_TEXT, "")) - for data in chat_span_data - ) - else: - for data in chat_span_data: - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in data - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in data - - # Non-PII data is unaffected by the gate - for data in chat_span_data: - assert data[SPANDATA.GEN_AI_REQUEST_MODEL] == "gpt-4" - assert data[SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 5 - - @pytest.mark.asyncio @pytest.mark.parametrize( "instructions,input,expected_system_instructions,expected_request_messages", @@ -1174,7 +719,12 @@ async def test_agent_invocation_span( integrations=[OpenAIAgentsIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=True, + data_collection={ + "gen_ai": { + "inputs": True, + "outputs": True, + } + }, ) items = capture_items("span") @@ -1257,6 +807,7 @@ async def test_client_span_custom_model( integrations=[OpenAIAgentsIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, + data_collection={}, ) items = capture_items("span") @@ -1278,7 +829,7 @@ async def test_client_span_custom_model( assert ai_client_span["attributes"]["gen_ai.request.model"] == "my-custom-model" -def test_agent_invocation_span_sync_no_pii( +def test_agent_invocation_span_sync_no_sensitive_data( sentry_init, capture_items, test_agent, @@ -1305,7 +856,12 @@ def test_agent_invocation_span_sync_no_pii( integrations=[OpenAIAgentsIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=False, + data_collection={ + "gen_ai": { + "inputs": False, + "outputs": False, + } + }, ) items = capture_items("span") @@ -1569,7 +1125,12 @@ def test_agent_invocation_span_sync( integrations=[OpenAIAgentsIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=True, + data_collection={ + "gen_ai": { + "inputs": True, + "outputs": True, + } + }, ) items = capture_items("span") @@ -1690,7 +1251,12 @@ async def test_tool_execution_span( integrations=[OpenAIAgentsIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=True, + data_collection={ + "gen_ai": { + "inputs": True, + "outputs": True, + } + }, ) items = capture_items("span") @@ -1957,7 +1523,12 @@ async def test_run_streamed_tool_execution_span( integrations=[OpenAIAgentsIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=True, + data_collection={ + "gen_ai": { + "inputs": True, + "outputs": True, + } + }, ) items = capture_items("span") @@ -2120,25 +1691,10 @@ def simple_test_tool(message: str) -> str: @pytest.mark.parametrize( - "data_collection,send_default_pii,expect_input,expect_output", + "data_collection,expect_input,expect_output", [ - pytest.param( - {"gen_ai": {"inputs": True, "outputs": True}}, - False, - True, - True, - id="gen-ai-inputs-and-outputs-enabled-overrides-pii-disabled", - ), - pytest.param( - {"gen_ai": {"inputs": False, "outputs": False}}, - True, - False, - False, - id="gen-ai-inputs-and-outputs-disabled-overrides-pii-enabled", - ), pytest.param( {"gen_ai": {"inputs": True, "outputs": False}}, - False, True, False, id="gen-ai-only-inputs-enabled", @@ -2146,38 +1702,9 @@ def simple_test_tool(message: str) -> str: pytest.param( {"gen_ai": {"inputs": False, "outputs": True}}, False, - False, True, id="gen-ai-only-outputs-enabled", ), - pytest.param( - {}, - False, - True, - True, - id="gen-ai-omitted-defaults-to-enabled", - ), - pytest.param( - {"gen_ai": {"inputs": False, "outputs": False}}, - False, - False, - False, - id="gen-ai-inputs-and-outputs-disabled-and-pii-disabled", - ), - pytest.param( - None, - False, - False, - False, - id="no-gen-ai-data-collection-falls-back-to-send-default-pii", - ), - pytest.param( - None, - True, - True, - True, - id="no-gen-ai-data-collection-pii-enabled-collects", - ), ], ) @pytest.mark.asyncio @@ -2185,13 +1712,10 @@ async def test_tool_execution_span_data_collection( run_tool_agent, simple_test_tool, data_collection, - send_default_pii, expect_input, expect_output, ): - init_kwargs = { - "send_default_pii": send_default_pii, - } + init_kwargs = {} if data_collection is not None: init_kwargs["data_collection"] = data_collection @@ -2214,33 +1738,13 @@ async def test_tool_execution_span_data_collection( @pytest.mark.asyncio -async def test_tool_execution_error_data_collection( - run_tool_agent, -): - @agents.function_tool - def failing_tool(message: str) -> str: - """A tool that fails""" - raise ValueError("Tool execution failed") - - tool_span, tool_span_data = await run_tool_agent( - failing_tool, - data_collection={"gen_ai": {"outputs": False}}, - ) - - assert tool_span_data[SPANDATA.GEN_AI_TOOL_NAME] == "failing_tool" - assert tool_span["status"] == "error" - assert SPANDATA.GEN_AI_TOOL_OUTPUT not in tool_span_data - - -@pytest.mark.asyncio -async def test_tool_execution_span_non_pii_data_always_set( +async def test_tool_execution_span_non_sensitive_data_always_set( run_tool_agent, simple_test_tool, ): _, tool_span_data = await run_tool_agent( simple_test_tool, run_kwargs={"conversation_id": "conv_tool_test_456"}, - data_collection={"gen_ai": {"inputs": False, "outputs": False}}, ) assert tool_span_data[SPANDATA.GEN_AI_TOOL_NAME] == "simple_test_tool" @@ -2285,6 +1789,7 @@ async def test_hosted_mcp_tool_propagation_header_streamed( integrations=[OpenAIAgentsIntegration()], traces_sample_rate=1.0, release="d08ebdb9309e1b004c6f52202de58a09c2268e42", + data_collection={}, ) request_headers = {"X-Stainless-Raw-Response": "stream"} @@ -2452,6 +1957,7 @@ async def test_hosted_mcp_tool_propagation_headers( integrations=[OpenAIAgentsIntegration()], traces_sample_rate=1.0, release="d08ebdb9309e1b004c6f52202de58a09c2268e42", + data_collection={}, ) response = get_model_response(EXAMPLE_RESPONSE, serialize_pydantic=True) @@ -2555,7 +2061,12 @@ async def test_model_behavior_error( integrations=[OpenAIAgentsIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=True, + data_collection={ + "gen_ai": { + "inputs": True, + "outputs": True, + } + }, ) items = capture_items("span") @@ -2603,6 +2114,7 @@ async def test_run_error_handling( ], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, + data_collection={}, ) items = capture_items("event", "span") @@ -2654,6 +2166,7 @@ async def test_run_streamed_error_handling( ], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, + data_collection={}, ) items = capture_items("event", "span") @@ -2716,7 +2229,12 @@ async def test_error_captures_input_data( ], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=True, + data_collection={ + "gen_ai": { + "inputs": True, + "outputs": True, + } + }, ) items = capture_items("event", "span") @@ -2765,6 +2283,7 @@ async def test_span_status_error( ], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, + data_collection={}, ) items = capture_items("event", "span") @@ -2802,7 +2321,12 @@ def test_openai_agents_message_role_mapping(sentry_init, test_message, expected_ sentry_init( integrations=[OpenAIAgentsIntegration()], traces_sample_rate=1.0, - send_default_pii=True, + data_collection={ + "gen_ai": { + "inputs": True, + "outputs": True, + } + }, ) get_response_kwargs = {"input": [test_message]} @@ -2900,7 +2424,12 @@ def failing_tool(message: str) -> str: integrations=[OpenAIAgentsIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=True, + data_collection={ + "gen_ai": { + "inputs": True, + "outputs": True, + } + }, ) items = capture_items("span") @@ -2998,7 +2527,12 @@ async def test_ai_client_span_includes_response_model( integrations=[OpenAIAgentsIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=True, + data_collection={ + "gen_ai": { + "inputs": True, + "outputs": True, + } + }, ) items = capture_items("span") @@ -3088,6 +2622,7 @@ async def test_ai_client_span_response_model_with_chat_completions( integrations=[OpenAIAgentsIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, + data_collection={}, ) items = capture_items("span") @@ -3174,7 +2709,12 @@ async def test_invoke_agent_span_includes_response_model( integrations=[OpenAIAgentsIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=True, + data_collection={ + "gen_ai": { + "inputs": True, + "outputs": True, + } + }, ) items = capture_items("span") @@ -3299,7 +2839,12 @@ def calculator(a: int, b: int) -> int: integrations=[OpenAIAgentsIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=True, + data_collection={ + "gen_ai": { + "inputs": True, + "outputs": True, + } + }, ) items = capture_items("span") @@ -3341,7 +2886,12 @@ async def test_streaming_span_update_captures_response_data( sentry_init( integrations=[OpenAIAgentsIntegration()], traces_sample_rate=1.0, - send_default_pii=True, + data_collection={ + "gen_ai": { + "inputs": True, + "outputs": True, + } + }, ) # Create a mock streaming response object (similar to what we'd get from ResponseCompletedEvent) @@ -3410,6 +2960,7 @@ async def test_streaming_ttft_on_chat_span( sentry_init( integrations=[OpenAIAgentsIntegration()], traces_sample_rate=1.0, + data_collection={}, ) request_headers = {"X-Stainless-Raw-Response": "stream"} @@ -3552,6 +3103,7 @@ async def test_conversation_id_on_all_spans( integrations=[OpenAIAgentsIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, + data_collection={}, ) items = capture_items("span") @@ -3685,6 +3237,7 @@ def simple_tool(message: str) -> str: integrations=[OpenAIAgentsIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, + data_collection={}, ) items = capture_items("span")