diff --git a/sentry_sdk/integrations/langchain.py b/sentry_sdk/integrations/langchain.py index 8ab01d7801..b188dc811d 100644 --- a/sentry_sdk/integrations/langchain.py +++ b/sentry_sdk/integrations/langchain.py @@ -14,7 +14,6 @@ ) from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations import DidNotEnable, Integration, _check_minimum_version -from sentry_sdk.scope import should_send_default_pii from sentry_sdk.traces import Span, _AgentFrameworkChatGenerationContext from sentry_sdk.tracing_utils import ( _get_value, @@ -22,7 +21,6 @@ from sentry_sdk.utils import ( capture_internal_exceptions, event_from_exception, - has_data_collection_enabled, logger, parse_version, ) @@ -94,19 +92,6 @@ def _get_ai_system(all_params: "Dict[str, Any]") -> "Optional[str]": return ai_type -DATA_FIELDS = { - "frequency_penalty": SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY, - # "function_call" is an OpenAI convention for the now-legacy Chat Completions API field - "function_call": SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, - "max_tokens": SPANDATA.GEN_AI_REQUEST_MAX_TOKENS, - "presence_penalty": SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY, - "temperature": SPANDATA.GEN_AI_REQUEST_TEMPERATURE, - "tool_calls": SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, - "top_k": SPANDATA.GEN_AI_REQUEST_TOP_K, - "top_p": SPANDATA.GEN_AI_REQUEST_TOP_P, -} - - def _transform_langchain_content_block( content_block: "Dict[str, Any]", ) -> "Dict[str, Any]": @@ -369,47 +354,85 @@ def on_llm_start( client = sentry_sdk.get_client() - for key, attribute in DATA_FIELDS.items(): - if key in all_params and all_params[key] is not None: - # This is correctly gated on "inputs" at the moment because the - # "on_llm_start" method is the start of a request. - # - # TODO: GEN_AI_RESPONSE_TOOL_CALLS will need to be - # transitioned to non-deprecated tool call attributes - - if ( - attribute == SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS - and has_data_collection_enabled(client.options) - and not client.options["data_collection"]["gen_ai"]["inputs"] - ): - continue + frequency_penalty = all_params.get("frequency_penalty") + if frequency_penalty is not None: + set_data_normalized( + span, + SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY, + frequency_penalty, + unpack=False, + ) - set_data_normalized(span, attribute, all_params[key], unpack=False) + max_tokens = all_params.get("max_tokens") + if max_tokens is not None: + set_data_normalized( + span, SPANDATA.GEN_AI_REQUEST_MAX_TOKENS, max_tokens, unpack=False + ) + + presence_penalty = all_params.get("presence_penalty") + if presence_penalty is not None: + set_data_normalized( + span, + SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY, + presence_penalty, + unpack=False, + ) + + temperature = all_params.get("temperature") + if temperature is not None: + set_data_normalized( + span, SPANDATA.GEN_AI_REQUEST_TEMPERATURE, temperature, unpack=False + ) + + top_k = all_params.get("top_k") + if top_k is not None: + set_data_normalized( + span, SPANDATA.GEN_AI_REQUEST_TOP_K, top_k, unpack=False + ) + + top_p = all_params.get("top_p") + if top_p is not None: + set_data_normalized( + span, SPANDATA.GEN_AI_REQUEST_TOP_P, top_p, unpack=False + ) _set_tools_on_span(span, all_params.get("tools")) - record_inputs = False - if has_data_collection_enabled(client.options): - record_inputs = client.options["data_collection"]["gen_ai"]["inputs"] - elif should_send_default_pii(): - # TODO: Remove this branch once `send_default_pii` is deprecated - record_inputs = True + if not client.options["data_collection"]["gen_ai"]["inputs"]: + return - if record_inputs: - normalized_messages = [ - { - "role": GEN_AI_ALLOWED_MESSAGE_ROLES.USER, - "content": {"type": "text", "text": prompt}, - } - for prompt in prompts - ] + function_call = all_params.get("function_call") + if function_call is not None: set_data_normalized( span, - SPANDATA.GEN_AI_REQUEST_MESSAGES, - normalized_messages, + SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, + function_call, + unpack=False, + ) + + tool_calls = all_params.get("tool_calls") + if tool_calls is not None: + set_data_normalized( + span, + SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, + tool_calls, unpack=False, ) + normalized_messages = [ + { + "role": GEN_AI_ALLOWED_MESSAGE_ROLES.USER, + "content": {"type": "text", "text": prompt}, + } + for prompt in prompts + ] + set_data_normalized( + span, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + normalized_messages, + unpack=False, + ) + def on_chat_model_start( self: "SentryLangchainCallback", serialized: "Dict[str, Any]", @@ -465,56 +488,95 @@ def on_chat_model_start( client = sentry_sdk.get_client() - for key, attribute in DATA_FIELDS.items(): - if key in all_params and all_params[key] is not None: - if ( - # This is correctly gated on "inputs" at the moment because the - # "on_chat_model_start" method is the start of a request. - # - # TODO: GEN_AI_RESPONSE_TOOL_CALLS will need to be - # transitioned to non-deprecated tool call attributes - attribute == SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS - and has_data_collection_enabled(client.options) - and not client.options["data_collection"]["gen_ai"]["inputs"] - ): - continue + frequency_penalty = all_params.get("frequency_penalty") + if frequency_penalty is not None: + set_data_normalized( + span, + SPANDATA.GEN_AI_REQUEST_FREQUENCY_PENALTY, + frequency_penalty, + unpack=False, + ) - set_data_normalized(span, attribute, all_params[key], unpack=False) + max_tokens = all_params.get("max_tokens") + if max_tokens is not None: + set_data_normalized( + span, SPANDATA.GEN_AI_REQUEST_MAX_TOKENS, max_tokens, unpack=False + ) - _set_tools_on_span(span, all_params.get("tools")) + presence_penalty = all_params.get("presence_penalty") + if presence_penalty is not None: + set_data_normalized( + span, + SPANDATA.GEN_AI_REQUEST_PRESENCE_PENALTY, + presence_penalty, + unpack=False, + ) - record_inputs = False - if has_data_collection_enabled(client.options): - record_inputs = client.options["data_collection"]["gen_ai"]["inputs"] - elif should_send_default_pii(): - # TODO: Remove this branch once `send_default_pii` is deprecated - record_inputs = True + temperature = all_params.get("temperature") + if temperature is not None: + set_data_normalized( + span, SPANDATA.GEN_AI_REQUEST_TEMPERATURE, temperature, unpack=False + ) - if record_inputs: - system_instructions = _get_system_instructions(messages) - if len(system_instructions) > 0: - span.set_attribute( - SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, - json.dumps(_transform_system_instructions(system_instructions)), - ) + top_k = all_params.get("top_k") + if top_k is not None: + set_data_normalized( + span, SPANDATA.GEN_AI_REQUEST_TOP_K, top_k, unpack=False + ) - normalized_messages = [] - for list_ in messages: - for message in list_: - if message.type == "system": - continue + top_p = all_params.get("top_p") + if top_p is not None: + set_data_normalized( + span, SPANDATA.GEN_AI_REQUEST_TOP_P, top_p, unpack=False + ) - normalized_messages.append( - self._normalize_langchain_message(message) - ) - normalized_messages = normalize_message_roles(normalized_messages) + _set_tools_on_span(span, all_params.get("tools")) + + if not client.options["data_collection"]["gen_ai"]["inputs"]: + return + + function_call = all_params.get("function_call") + if function_call is not None: set_data_normalized( span, - SPANDATA.GEN_AI_REQUEST_MESSAGES, - normalized_messages, + SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, + function_call, + unpack=False, + ) + + tool_calls = all_params.get("tool_calls") + if tool_calls is not None: + set_data_normalized( + span, + SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, + tool_calls, unpack=False, ) + system_instructions = _get_system_instructions(messages) + if len(system_instructions) > 0: + span.set_attribute( + SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, + json.dumps(_transform_system_instructions(system_instructions)), + ) + + normalized_messages = [] + for list_ in messages: + for message in list_: + if message.type == "system": + continue + + normalized_messages.append( + self._normalize_langchain_message(message) + ) + normalized_messages = normalize_message_roles(normalized_messages) + set_data_normalized( + span, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + normalized_messages, + unpack=False, + ) + def on_chat_model_end( self: "SentryLangchainCallback", response: "LLMResult", @@ -534,16 +596,7 @@ def on_chat_model_end( else context ) - client = sentry_sdk.get_client() - - record_outputs = False - if has_data_collection_enabled(client.options): - record_outputs = client.options["data_collection"]["gen_ai"]["outputs"] - elif should_send_default_pii(): - # TODO: Remove this branch once `send_default_pii` is deprecated - record_outputs = True - - if record_outputs: + if sentry_sdk.get_client().options["data_collection"]["gen_ai"]["outputs"]: set_data_normalized( span, SPANDATA.GEN_AI_RESPONSE_TEXT, @@ -574,13 +627,6 @@ def on_llm_end( client = sentry_sdk.get_client() - record_outputs = False - if has_data_collection_enabled(client.options): - record_outputs = client.options["data_collection"]["gen_ai"]["outputs"] - elif should_send_default_pii(): - # TODO: Remove this branch once `send_default_pii` is deprecated - record_outputs = True - try: generation = response.generations[0][0] except IndexError: @@ -599,7 +645,7 @@ def on_llm_end( if response_model is not None: span.set_attribute(SPANDATA.GEN_AI_RESPONSE_MODEL, response_model) - if record_outputs: + if client.options["data_collection"]["gen_ai"]["outputs"]: tool_calls = getattr(generation.message, "tool_calls", None) if tool_calls is not None and tool_calls != []: set_data_normalized( @@ -609,7 +655,7 @@ def on_llm_end( unpack=False, ) - if record_outputs: + if client.options["data_collection"]["gen_ai"]["outputs"]: set_data_normalized( span, SPANDATA.GEN_AI_RESPONSE_TEXT, @@ -684,19 +730,14 @@ def on_tool_start( client = sentry_sdk.get_client() - record_inputs = False - if has_data_collection_enabled(client.options): - record_inputs = client.options["data_collection"]["gen_ai"]["inputs"] - elif should_send_default_pii(): - # TODO: Remove this branch once `send_default_pii` is deprecated - record_inputs = True + if not client.options["data_collection"]["gen_ai"]["inputs"]: + return - if record_inputs: - set_data_normalized( - span, - SPANDATA.GEN_AI_TOOL_INPUT, - kwargs.get("inputs", [input_str]), - ) + set_data_normalized( + span, + SPANDATA.GEN_AI_TOOL_INPUT, + kwargs.get("inputs", [input_str]), + ) def on_tool_end( self: "SentryLangchainCallback", output: str, *, run_id: "UUID", **kwargs: "Any" @@ -713,16 +754,7 @@ def on_tool_end( else context ) - client = sentry_sdk.get_client() - - record_outputs = False - if has_data_collection_enabled(client.options): - record_outputs = client.options["data_collection"]["gen_ai"]["outputs"] - elif should_send_default_pii(): - # TODO: Remove this branch once `send_default_pii` is deprecated - record_outputs = True - - if record_outputs: + if sentry_sdk.get_client().options["data_collection"]["gen_ai"]["outputs"]: set_data_normalized(span, SPANDATA.GEN_AI_TOOL_OUTPUT, output) self._exit_span(span, run_id) @@ -999,19 +1031,14 @@ def _set_tools_on_span(span: "Span", tools: "Any") -> None: return client = sentry_sdk.get_client() - attribute_name = SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS - if has_data_collection_enabled(client.options): - if not client.options["data_collection"]["gen_ai"]["inputs"]: - return - else: - attribute_name = SPANDATA.GEN_AI_TOOL_DEFINITIONS - # Before data collection was introduced this was set unconditionally, so it - # stays that way when data collection is not configured. + if not client.options["data_collection"]["gen_ai"]["inputs"]: + return + simplified_tools = _simplify_langchain_tools(tools) if simplified_tools: set_data_normalized( span, - attribute_name, + SPANDATA.GEN_AI_TOOL_DEFINITIONS, simplified_tools, unpack=False, ) @@ -1104,16 +1131,6 @@ def new_invoke(self: "Any", *args: "Any", **kwargs: "Any") -> "Any": run_name, tools = _get_request_data(self, args, kwargs) - record_inputs = False - record_outputs = False - if has_data_collection_enabled(client.options): - record_inputs = client.options["data_collection"]["gen_ai"]["inputs"] - record_outputs = client.options["data_collection"]["gen_ai"]["outputs"] - elif should_send_default_pii(): - # TODO: Remove this branch once `send_default_pii` is deprecated - record_inputs = True - record_outputs = True - with sentry_sdk.start_span( name=f"invoke_agent {run_name}" if run_name else "invoke_agent", attributes={ @@ -1129,7 +1146,10 @@ def new_invoke(self: "Any", *args: "Any", **kwargs: "Any") -> "Any": result = f(self, *args, **kwargs) input = result.get("input") - if input is not None and record_inputs: + if ( + input is not None + and client.options["data_collection"]["gen_ai"]["inputs"] + ): normalized_messages = normalize_message_roles([input]) set_data_normalized( span, @@ -1139,7 +1159,10 @@ def new_invoke(self: "Any", *args: "Any", **kwargs: "Any") -> "Any": ) output = result.get("output") - if output is not None and record_outputs: + if ( + output is not None + and client.options["data_collection"]["gen_ai"]["outputs"] + ): set_data_normalized(span, SPANDATA.GEN_AI_RESPONSE_TEXT, output) return result @@ -1157,16 +1180,6 @@ def new_stream(self: "Any", *args: "Any", **kwargs: "Any") -> "Any": run_name, tools = _get_request_data(self, args, kwargs) - record_inputs = False - record_outputs = False - if has_data_collection_enabled(client.options): - record_inputs = client.options["data_collection"]["gen_ai"]["inputs"] - record_outputs = client.options["data_collection"]["gen_ai"]["outputs"] - elif should_send_default_pii(): - # TODO: Remove this branch once `send_default_pii` is deprecated - record_inputs = True - record_outputs = True - span = sentry_sdk.start_span( name=f"invoke_agent {run_name}" if run_name else "invoke_agent", attributes={ @@ -1180,7 +1193,7 @@ def new_stream(self: "Any", *args: "Any", **kwargs: "Any") -> "Any": span.set_attribute(SPANDATA.GEN_AI_FUNCTION_ID, run_name) input = args[0].get("input") if len(args) >= 1 else None - if input is not None and record_inputs: + if input is not None and client.options["data_collection"]["gen_ai"]["inputs"]: normalized_messages = normalize_message_roles([input]) set_data_normalized( span, @@ -1205,7 +1218,10 @@ def new_iterator() -> "Iterator[Any]": except Exception: output = None - if output is not None and record_outputs: + if ( + output is not None + and client.options["data_collection"]["gen_ai"]["outputs"] + ): set_data_normalized(span, SPANDATA.GEN_AI_RESPONSE_TEXT, output) span.__exit__(None, None, None) @@ -1226,7 +1242,10 @@ async def new_iterator_async() -> "AsyncIterator[Any]": except Exception: output = None - if output is not None and record_outputs: + if ( + output is not None + and client.options["data_collection"]["gen_ai"]["outputs"] + ): set_data_normalized(span, SPANDATA.GEN_AI_RESPONSE_TEXT, output) span.__exit__(None, None, None) diff --git a/tests/integrations/langchain/test_langchain.py b/tests/integrations/langchain/test_langchain.py index 9cf2360fd2..f777d73ef7 100644 --- a/tests/integrations/langchain/test_langchain.py +++ b/tests/integrations/langchain/test_langchain.py @@ -291,7 +291,12 @@ def test_langchain_text_completion( integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=True, + data_collection={ + "gen_ai": { + "inputs": True, + "outputs": True, + } + }, ) model_response = get_model_response( @@ -365,7 +370,12 @@ def test_langchain_chat_with_run_name( integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=True, + data_collection={ + "gen_ai": { + "inputs": True, + "outputs": True, + } + }, ) request_headers = {} @@ -429,7 +439,12 @@ def test_langchain_multi_choice_response( integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=True, + data_collection={ + "gen_ai": { + "inputs": True, + "outputs": True, + } + }, ) model_response = get_model_response( @@ -475,7 +490,12 @@ def test_langchain_tool_call_with_run_name( integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=True, + data_collection={ + "gen_ai": { + "inputs": True, + "outputs": True, + } + }, ) items = capture_items("span") @@ -499,10 +519,6 @@ def test_langchain_tool_call_with_run_name( LANGCHAIN_VERSION < (1,), reason="LangChain 1.0+ required (ONE AGENT refactor)", ) -@pytest.mark.parametrize( - "send_default_pii", - [True, False], -) @pytest.mark.parametrize( "system_instructions_content,expected_system_instructions", [ @@ -536,7 +552,6 @@ def test_langchain_tool_call_with_run_name( def test_langchain_create_agent( sentry_init, capture_items, - send_default_pii, system_instructions_content, expected_system_instructions, get_model_response, @@ -546,7 +561,12 @@ def test_langchain_create_agent( integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=send_default_pii, + data_collection={ + "gen_ai": { + "inputs": True, + "outputs": True, + } + }, ) model_response = get_model_response( @@ -617,50 +637,132 @@ def test_langchain_create_agent( if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): assert chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-4" - if send_default_pii: - assert ( - chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == "Hello, how can I help you?" - ) + assert ( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] + == "Hello, how can I help you?" + ) - assert json.loads( - chat_spans[0]["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] - ) == [ - { - "role": "user", - "content": "Message demonstrating the absence of truncation.", - }, + assert json.loads( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] + ) == [ + { + "role": "user", + "content": "Message demonstrating the absence of truncation.", + }, + { + "role": "user", + "content": "How many letters in the word eudca", + }, + ] + + assert expected_system_instructions == json.loads( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] + ) + + +@pytest.mark.skipif( + LANGCHAIN_VERSION < (1,), + reason="LangChain 1.0+ required (ONE AGENT refactor)", +) +def test_langchain_create_agent_no_sensitive_data( + sentry_init, + capture_items, + get_model_response, + nonstreaming_responses_model_response, +): + sentry_init( + integrations=[LangchainIntegration()], + disabled_integrations=[StdlibIntegration], + traces_sample_rate=1.0, + data_collection={ + "gen_ai": { + "inputs": False, + "outputs": False, + } + }, + ) + + model_response = get_model_response( + nonstreaming_responses_model_response, + serialize_pydantic=True, + request_headers={ + "X-Stainless-Raw-Response": "True", + }, + ) + + llm = ChatOpenAI( + model_name="gpt-3.5-turbo", + temperature=0, + openai_api_key="badkey", + use_responses_api=True, + ) + agent = create_agent( + model=llm, + tools=[get_word_length], + name="word_length_agent", + ) + items = capture_items("span") + + with patch.object( + llm.client._client._client, + "send", + return_value=model_response, + ): + agent.invoke( { - "role": "user", - "content": "How many letters in the word eudca", + "messages": [ + HumanMessage( + content="Message demonstrating the absence of truncation." + ), + HumanMessage(content="How many letters in the word eudca"), + ], }, - ] - - assert expected_system_instructions == json.loads( - chat_spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] - ) - else: - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[0].get( - "attributes", {} - ) - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get( - "attributes", {} ) - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("attributes", {}) + + sentry_sdk.flush() + spans = [item.payload for item in items] + chat_spans = list( + x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.chat" + ) + assert len(chat_spans) == 1 + assert chat_spans[0]["attributes"]["sentry.origin"] == "auto.ai.langchain" + + assert chat_spans[0]["attributes"][SPANDATA.GEN_AI_PROVIDER_NAME] == "openai-chat" + assert chat_spans[0]["attributes"]["gen_ai.agent.name"] == "word_length_agent" + + assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 10 + assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 20 + assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 30 + + assert ( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] == 4 + ) + assert ( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS] + == 6 + ) + + assert ( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] == 5 + ) + + if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): + assert chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-4" + + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[0].get( + "attributes", {} + ) + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get("attributes", {}) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("attributes", {}) @pytest.mark.skipif( LANGCHAIN_VERSION < (1,), reason="LangChain 1.0+ required (ONE AGENT refactor)", ) -@pytest.mark.parametrize( - "send_default_pii", - [True, False], -) def test_tool_execution_span( sentry_init, capture_items, - send_default_pii, get_model_response, nonstreaming_responses_tool_call_model_responses, ): @@ -668,7 +770,12 @@ def test_tool_execution_span( integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=send_default_pii, + data_collection={ + "gen_ai": { + "inputs": True, + "outputs": True, + } + }, ) responses = nonstreaming_responses_tool_call_model_responses( @@ -806,95 +913,103 @@ def test_tool_execution_span( chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-4-0613" ) - if send_default_pii: - assert "word" in tool_exec_span["attributes"][SPANDATA.GEN_AI_TOOL_INPUT] + assert "word" in tool_exec_span["attributes"][SPANDATA.GEN_AI_TOOL_INPUT] - assert "5" in chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] + assert "5" in chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] + + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in chat_spans[0].get("attributes", {}) + tool_calls_data = chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] + assert isinstance(tool_calls_data, str) + assert "get_word_length" in tool_calls_data - # Verify tool calls are recorded when PII is enabled - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in chat_spans[0].get( - "attributes", {} - ), "Tool calls should be recorded when send_default_pii=True" - tool_calls_data = chat_spans[0]["attributes"][ - SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS - ] - assert isinstance(tool_calls_data, str) - assert "get_word_length" in tool_calls_data - else: - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get( - "attributes", {} - ) - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("attributes", {}) - assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[1].get( - "attributes", {} - ) - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[1].get("attributes", {}) - assert SPANDATA.GEN_AI_TOOL_INPUT not in tool_exec_span.get("attributes", {}) - assert SPANDATA.GEN_AI_TOOL_OUTPUT not in tool_exec_span.get("attributes", {}) - - # Verify tool calls are NOT recorded when PII is disabled - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[0].get( - "attributes", {} - ), f"Tool calls should NOT be recorded when send_default_pii={send_default_pii}" - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[1].get( - "attributes", {} - ), f"Tool calls should NOT be recorded when send_default_pii={send_default_pii}" - - # Verify that available tools are always recorded regardless of PII settings for chat_span in chat_spans: - tools_data = chat_span["attributes"][SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS] + tools_data = chat_span["attributes"][SPANDATA.GEN_AI_TOOL_DEFINITIONS] assert "get_word_length" in tools_data -def test_langchain_openai_tools_agent_no_sensitive_data( +@pytest.mark.skipif( + LANGCHAIN_VERSION < (1,), + reason="LangChain 1.0+ required (ONE AGENT refactor)", +) +def test_tool_execution_span_no_sensitive_data( sentry_init, capture_items, get_model_response, - server_side_event_chunks, - streaming_chat_completions_model_responses, + nonstreaming_responses_tool_call_model_responses, ): sentry_init( integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=False, + data_collection={ + "gen_ai": { + "inputs": False, + "outputs": False, + } + }, ) - prompt = ChatPromptTemplate.from_messages( - [ - ( - "system", - "You are very powerful assistant, but don't know current events", - ), - ("user", "{input}"), - MessagesPlaceholder(variable_name="agent_scratchpad"), - ] + responses = nonstreaming_responses_tool_call_model_responses( + tool_name="get_word_length", + arguments='{"word": "eudca"}', + response_model="gpt-4-0613", + response_text="The word eudca has 5 letters.", + response_ids=iter(["resp_1", "resp_2"]), + usages=iter( + [ + ResponseUsage( + input_tokens=142, + input_tokens_details=InputTokensDetails( + cached_tokens=69, + cache_write_tokens=31, + ), + output_tokens=50, + output_tokens_details=OutputTokensDetails( + reasoning_tokens=10, + ), + total_tokens=192, + ), + ResponseUsage( + input_tokens=89, + input_tokens_details=InputTokensDetails( + cached_tokens=69, + cache_write_tokens=10, + ), + output_tokens=28, + output_tokens_details=OutputTokensDetails( + reasoning_tokens=11, + ), + total_tokens=117, + ), + ] + ), ) - - model_responses = streaming_chat_completions_model_responses() - tool_response = get_model_response( - server_side_event_chunks( - next(model_responses), - include_event_type=False, - ) + next(responses), + serialize_pydantic=True, + request_headers={ + "X-Stainless-Raw-Response": "True", + }, ) - final_response = get_model_response( - server_side_event_chunks( - next(model_responses), - include_event_type=False, - ) + next(responses), + serialize_pydantic=True, + request_headers={ + "X-Stainless-Raw-Response": "True", + }, ) llm = ChatOpenAI( - model_name="gpt-3.5-turbo", + model_name="gpt-4", temperature=0, openai_api_key="badkey", + use_responses_api=True, + ) + agent = create_agent( + model=llm, + tools=[get_word_length], + name="word_length_agent", ) - agent = create_openai_tools_agent(llm, [get_word_length], prompt) - - agent_executor = AgentExecutor(agent=agent, tools=[get_word_length], verbose=True) items = capture_items("span") with patch.object( @@ -902,98 +1017,86 @@ def test_langchain_openai_tools_agent_no_sensitive_data( "send", side_effect=[tool_response, final_response], ): - list( - agent_executor.invoke( - {"input": "How many letters in the word eudca"}, - {"run_name": "my-snazzy-pipeline"}, - ) + agent.invoke( + { + "messages": [ + HumanMessage(content="How many letters in the word eudca"), + ], + }, ) sentry_sdk.flush() spans = [item.payload for item in items] - invoke_agent_span = next( - x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.invoke_agent" - ) chat_spans = list( x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.chat" ) - tool_exec_span = next( + assert len(chat_spans) == 2 + + tool_exec_spans = list( x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.execute_tool" ) + assert len(tool_exec_spans) == 1 + tool_exec_span = tool_exec_spans[0] - assert len(chat_spans) == 2 - - assert invoke_agent_span["attributes"]["sentry.origin"] == "auto.ai.langchain" assert chat_spans[0]["attributes"]["sentry.origin"] == "auto.ai.langchain" assert chat_spans[1]["attributes"]["sentry.origin"] == "auto.ai.langchain" assert tool_exec_span["attributes"]["sentry.origin"] == "auto.ai.langchain" - assert invoke_agent_span["attributes"]["gen_ai.function_id"] == "my-snazzy-pipeline" + assert chat_spans[0]["attributes"]["gen_ai.agent.name"] == "word_length_agent" + assert chat_spans[1]["attributes"]["gen_ai.agent.name"] == "word_length_agent" + assert tool_exec_span["attributes"]["gen_ai.agent.name"] == "word_length_agent" - # We can't guarantee anything about the "shape" of the langchain execution graph + assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 142 + assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 50 + assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 192 assert ( - len(list(x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.chat")) - > 0 + chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] == 69 ) + assert ( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS] + == 31 + ) + assert ( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] == 10 + ) + assert chat_spans[0]["attributes"][SPANDATA.GEN_AI_PROVIDER_NAME] == "openai-chat" - # Token usage is only available in newer versions of langchain (v0.2+) - # where usage_metadata is supported on AIMessageChunk - if "gen_ai.usage.input_tokens" in chat_spans[0]["attributes"]: - assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 142 - assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 50 - assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 192 - - if "gen_ai.usage.input_tokens" in chat_spans[1]["attributes"]: - assert chat_spans[1]["attributes"]["gen_ai.usage.input_tokens"] == 89 - assert chat_spans[1]["attributes"]["gen_ai.usage.output_tokens"] == 28 - assert chat_spans[1]["attributes"]["gen_ai.usage.total_tokens"] == 117 + assert chat_spans[1]["attributes"]["gen_ai.usage.input_tokens"] == 89 + assert chat_spans[1]["attributes"]["gen_ai.usage.output_tokens"] == 28 + assert chat_spans[1]["attributes"]["gen_ai.usage.total_tokens"] == 117 + assert ( + chat_spans[1]["attributes"][SPANDATA.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS] == 69 + ) + assert ( + chat_spans[1]["attributes"][SPANDATA.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS] + == 10 + ) + assert ( + chat_spans[1]["attributes"][SPANDATA.GEN_AI_USAGE_REASONING_OUTPUT_TOKENS] == 11 + ) + assert chat_spans[1]["attributes"][SPANDATA.GEN_AI_PROVIDER_NAME] == "openai-chat" if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): assert ( - chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] - == "gpt-3.5-turbo" + chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-4-0613" ) assert ( - chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] - == "gpt-3.5-turbo" + chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] == "gpt-4-0613" ) - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[0].get( - "attributes", {} - ) assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get("attributes", {}) assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("attributes", {}) - assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[1].get( - "attributes", {} - ) assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[1].get("attributes", {}) assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[1].get("attributes", {}) assert SPANDATA.GEN_AI_TOOL_INPUT not in tool_exec_span.get("attributes", {}) assert SPANDATA.GEN_AI_TOOL_OUTPUT not in tool_exec_span.get("attributes", {}) - # Verify tool calls are NOT recorded when PII is disabled assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[0].get( "attributes", {} - ), "Tool calls should NOT be recorded when send_default_pii=False" + ) assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[1].get( "attributes", {} - ), "Tool calls should NOT be recorded when send_default_pii=False" - - # Verify finish_reasons is always an array of strings - assert chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "function_call" - ] - assert chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ - "stop" - ] - - # Verify that available tools are always recorded regardless of PII settings - for chat_span in chat_spans: - tools_data = chat_span["attributes"][SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS] - assert tools_data is not None, ( - "Available tools should always be recorded regardless of PII settings" - ) - assert "get_word_length" in tools_data + ) @pytest.mark.parametrize( @@ -1052,7 +1155,12 @@ def test_langchain_openai_tools_agent( integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=True, + data_collection={ + "gen_ai": { + "inputs": True, + "outputs": True, + } + }, ) prompt = ChatPromptTemplate.from_messages( @@ -1172,21 +1280,169 @@ def test_langchain_openai_tools_agent( chat_spans[0]["attributes"][SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS] ) - assert "5" in chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - - # Verify tool calls are recorded when PII is enabled - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in chat_spans[0].get("attributes", {}), ( - "Tool calls should be recorded when send_default_pii=True" + assert "5" in chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] + + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in chat_spans[0].get("attributes", {}) + tool_calls_data = chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] + + assert isinstance(tool_calls_data, (list, str)) # Could be serialized + if isinstance(tool_calls_data, str): + assert "get_word_length" in tool_calls_data + elif isinstance(tool_calls_data, list) and len(tool_calls_data) > 0: + # Check if tool calls contain expected function name + tool_call_str = str(tool_calls_data) + assert "get_word_length" in tool_call_str + + # Verify finish_reasons is always an array of strings + assert chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ + "function_call" + ] + assert chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ + "stop" + ] + + for chat_span in chat_spans: + tools_data = chat_span["attributes"][SPANDATA.GEN_AI_TOOL_DEFINITIONS] + assert tools_data is not None + assert "get_word_length" in tools_data + + +def test_langchain_openai_tools_agent_no_sensitive_data( + sentry_init, + capture_items, + get_model_response, + server_side_event_chunks, + streaming_chat_completions_model_responses, +): + sentry_init( + integrations=[LangchainIntegration()], + disabled_integrations=[StdlibIntegration], + traces_sample_rate=1.0, + data_collection={ + "gen_ai": { + "inputs": False, + "outputs": False, + } + }, + ) + + prompt = ChatPromptTemplate.from_messages( + [ + ( + "system", + "You are very powerful assistant, but don't know current events", + ), + ("user", "{input}"), + MessagesPlaceholder(variable_name="agent_scratchpad"), + ] + ) + + model_responses = streaming_chat_completions_model_responses() + + tool_response = get_model_response( + server_side_event_chunks( + next(model_responses), + include_event_type=False, + ) + ) + + final_response = get_model_response( + server_side_event_chunks( + next(model_responses), + include_event_type=False, + ) + ) + + llm = ChatOpenAI( + model_name="gpt-3.5-turbo", + temperature=0, + openai_api_key="badkey", + ) + agent = create_openai_tools_agent(llm, [get_word_length], prompt) + + agent_executor = AgentExecutor(agent=agent, tools=[get_word_length], verbose=True) + items = capture_items("span") + + with patch.object( + llm.client._client._client, + "send", + side_effect=[tool_response, final_response], + ): + list( + agent_executor.invoke( + {"input": "How many letters in the word eudca"}, + {"run_name": "my-snazzy-pipeline"}, + ) + ) + + sentry_sdk.flush() + spans = [item.payload for item in items] + invoke_agent_span = next( + x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.invoke_agent" + ) + chat_spans = list( + x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.chat" + ) + tool_exec_span = next( + x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.execute_tool" + ) + + assert len(chat_spans) == 2 + + assert invoke_agent_span["attributes"]["sentry.origin"] == "auto.ai.langchain" + assert chat_spans[0]["attributes"]["sentry.origin"] == "auto.ai.langchain" + assert chat_spans[1]["attributes"]["sentry.origin"] == "auto.ai.langchain" + assert tool_exec_span["attributes"]["sentry.origin"] == "auto.ai.langchain" + + assert invoke_agent_span["attributes"]["gen_ai.function_id"] == "my-snazzy-pipeline" + + # We can't guarantee anything about the "shape" of the langchain execution graph + assert ( + len(list(x for x in spans if x["attributes"].get("sentry.op") == "gen_ai.chat")) + > 0 + ) + + # Token usage is only available in newer versions of langchain (v0.2+) + # where usage_metadata is supported on AIMessageChunk + if "gen_ai.usage.input_tokens" in chat_spans[0]["attributes"]: + assert chat_spans[0]["attributes"]["gen_ai.usage.input_tokens"] == 142 + assert chat_spans[0]["attributes"]["gen_ai.usage.output_tokens"] == 50 + assert chat_spans[0]["attributes"]["gen_ai.usage.total_tokens"] == 192 + + if "gen_ai.usage.input_tokens" in chat_spans[1]["attributes"]: + assert chat_spans[1]["attributes"]["gen_ai.usage.input_tokens"] == 89 + assert chat_spans[1]["attributes"]["gen_ai.usage.output_tokens"] == 28 + assert chat_spans[1]["attributes"]["gen_ai.usage.total_tokens"] == 117 + + if LANGCHAIN_OPENAI_VERSION >= (0, 3, 13): + assert ( + chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] + == "gpt-3.5-turbo" + ) + assert ( + chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_MODEL] + == "gpt-3.5-turbo" + ) + + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[0].get( + "attributes", {} + ) + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[0].get("attributes", {}) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[0].get("attributes", {}) + assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in chat_spans[1].get( + "attributes", {} + ) + assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in chat_spans[1].get("attributes", {}) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in chat_spans[1].get("attributes", {}) + assert SPANDATA.GEN_AI_TOOL_INPUT not in tool_exec_span.get("attributes", {}) + assert SPANDATA.GEN_AI_TOOL_OUTPUT not in tool_exec_span.get("attributes", {}) + + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[0].get( + "attributes", {} + ) + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[1].get( + "attributes", {} ) - tool_calls_data = chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] - - assert isinstance(tool_calls_data, (list, str)) # Could be serialized - if isinstance(tool_calls_data, str): - assert "get_word_length" in tool_calls_data - elif isinstance(tool_calls_data, list) and len(tool_calls_data) > 0: - # Check if tool calls contain expected function name - tool_call_str = str(tool_calls_data) - assert "get_word_length" in tool_call_str # Verify finish_reasons is always an array of strings assert chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ @@ -1196,14 +1452,6 @@ def test_langchain_openai_tools_agent( "stop" ] - # Verify that available tools are always recorded regardless of PII settings - for chat_span in chat_spans: - tools_data = chat_span["attributes"][SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS] - assert tools_data is not None, ( - "Available tools should always be recorded regardless of PII settings" - ) - assert "get_word_length" in tools_data - def test_langchain_openai_tools_agent_with_config( sentry_init, @@ -1216,7 +1464,12 @@ def test_langchain_openai_tools_agent_with_config( integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=True, + data_collection={ + "gen_ai": { + "inputs": True, + "outputs": True, + } + }, ) prompt = ChatPromptTemplate.from_messages( @@ -1288,7 +1541,12 @@ def test_langchain_openai_tools_agent_stream_no_sensitive_data( integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=False, + data_collection={ + "gen_ai": { + "inputs": False, + "outputs": False, + } + }, ) prompt = ChatPromptTemplate.from_messages( @@ -1404,13 +1662,12 @@ def test_langchain_openai_tools_agent_stream_no_sensitive_data( assert SPANDATA.GEN_AI_TOOL_INPUT not in tool_exec_span.get("attributes", {}) assert SPANDATA.GEN_AI_TOOL_OUTPUT not in tool_exec_span.get("attributes", {}) - # Verify tool calls are NOT recorded when PII is disabled assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[0].get( "attributes", {} - ), "Tool calls should NOT be recorded when send_default_pii=False" + ) assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in chat_spans[1].get( "attributes", {} - ), "Tool calls should NOT be recorded when send_default_pii=False" + ) # Verify finish_reasons is always an array of strings assert chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_FINISH_REASONS] == [ @@ -1420,15 +1677,6 @@ def test_langchain_openai_tools_agent_stream_no_sensitive_data( "stop" ] - # Verify that available tools are always recorded regardless of PII settings - for chat_span in chat_spans: - tools_data = chat_span["attributes"][SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS] - - assert tools_data is not None, ( - "Available tools should always be recorded regardless of PII settings" - ) - assert "get_word_length" in tools_data - @pytest.mark.parametrize( "system_instructions_content,expected_system_instructions", @@ -1487,7 +1735,12 @@ def test_langchain_openai_tools_agent_stream( integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=True, + data_collection={ + "gen_ai": { + "inputs": True, + "outputs": True, + } + }, ) prompt = ChatPromptTemplate.from_messages( @@ -1612,10 +1865,7 @@ def test_langchain_openai_tools_agent_stream( assert "5" in chat_spans[1]["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - # Verify tool calls are recorded when PII is enabled - assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in chat_spans[0].get("attributes", {}), ( - "Tool calls should be recorded when send_default_pii=True" - ) + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS in chat_spans[0].get("attributes", {}) tool_calls_data = chat_spans[0]["attributes"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] assert isinstance(tool_calls_data, (list, str)) # Could be serialized @@ -1634,12 +1884,9 @@ def test_langchain_openai_tools_agent_stream( "stop" ] - # Verify that available tools are always recorded regardless of PII settings for chat_span in chat_spans: - tools_data = chat_span["attributes"][SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS] - assert tools_data is not None, ( - "Available tools should always be recorded regardless of PII settings" - ) + tools_data = chat_span["attributes"][SPANDATA.GEN_AI_TOOL_DEFINITIONS] + assert tools_data is not None assert "get_word_length" in tools_data @@ -1654,7 +1901,12 @@ def test_langchain_openai_tools_agent_stream_with_config( integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=True, + data_collection={ + "gen_ai": { + "inputs": True, + "outputs": True, + } + }, ) prompt = ChatPromptTemplate.from_messages( @@ -1740,7 +1992,12 @@ def _llm_type(self) -> str: integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=True, + data_collection={ + "gen_ai": { + "inputs": True, + "outputs": True, + } + }, ) prompt = ChatPromptTemplate.from_messages( @@ -1797,6 +2054,7 @@ def _llm_type(self) -> str: integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, + data_collection={}, ) items = capture_items("event", "span") @@ -1851,6 +2109,7 @@ def test_langchain_tool_error( integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, + data_collection={}, ) responses = nonstreaming_responses_tool_call_model_responses( @@ -1953,7 +2212,7 @@ def _identifying_params(self): sentry_init( integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], - _experiments={"gen_ai_as_v2_spans": True}, + data_collection={}, ) # Create a manual SentryLangchainCallback @@ -1994,6 +2253,7 @@ def test_langchain_callback_manager(sentry_init): integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, + data_collection={}, ) local_manager = BaseCallbackManager(handlers=[]) @@ -2027,6 +2287,7 @@ def test_langchain_callback_manager_with_sentry_callback(sentry_init): integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, + data_collection={}, ) sentry_callback = SentryLangchainCallback() local_manager = BaseCallbackManager(handlers=[sentry_callback]) @@ -2060,6 +2321,7 @@ def test_langchain_callback_list(sentry_init): integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, + data_collection={}, ) local_callbacks = [] @@ -2093,6 +2355,7 @@ def test_langchain_callback_list_existing_callback(sentry_init): integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, + data_collection={}, ) sentry_callback = SentryLangchainCallback() local_callbacks = [sentry_callback] @@ -2157,7 +2420,12 @@ def _llm_type(self) -> str: integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=True, + data_collection={ + "gen_ai": { + "inputs": True, + "outputs": True, + } + }, ) prompt = ChatPromptTemplate.from_messages( @@ -2541,6 +2809,7 @@ def test_langchain_ai_system_detection( integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, + data_collection={}, ) callback = SentryLangchainCallback() @@ -2695,37 +2964,10 @@ def test_transform_list_with_legacy_image_url(self): @pytest.mark.parametrize( - "data_collection,send_default_pii,expected_present,expected_absent", + "data_collection,expected_present,expected_absent", [ - pytest.param( - {"gen_ai": {"inputs": True, "outputs": True}}, - False, - { - SPANDATA.GEN_AI_REQUEST_MESSAGES: [ - {"role": "user", "content": "How many letters in the word eudca"} - ], - SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS: [ - {"type": "text", "content": "You are a helpful assistant."} - ], - SPANDATA.GEN_AI_RESPONSE_TEXT: "the model response", - }, - [], - id="gen-ai-inputs-and-outputs-enabled-override-legacy-off", - ), - pytest.param( - {"gen_ai": {"inputs": False, "outputs": False}}, - True, - {}, - [ - SPANDATA.GEN_AI_REQUEST_MESSAGES, - SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, - SPANDATA.GEN_AI_RESPONSE_TEXT, - ], - id="gen-ai-inputs-and-outputs-disabled-override-legacy-on", - ), pytest.param( {"gen_ai": {"inputs": True, "outputs": False}}, - False, { SPANDATA.GEN_AI_REQUEST_MESSAGES: [ {"role": "user", "content": "How many letters in the word eudca"} @@ -2739,7 +2981,6 @@ def test_transform_list_with_legacy_image_url(self): ), pytest.param( {"gen_ai": {"inputs": False, "outputs": True}}, - False, {SPANDATA.GEN_AI_RESPONSE_TEXT: "the model response"}, [ SPANDATA.GEN_AI_REQUEST_MESSAGES, @@ -2747,47 +2988,6 @@ def test_transform_list_with_legacy_image_url(self): ], id="gen-ai-outputs-enabled-inputs-disabled", ), - pytest.param( - {"gen_ai": {}}, - False, - { - SPANDATA.GEN_AI_REQUEST_MESSAGES: [ - {"role": "user", "content": "How many letters in the word eudca"} - ], - SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS: [ - {"type": "text", "content": "You are a helpful assistant."} - ], - SPANDATA.GEN_AI_RESPONSE_TEXT: "the model response", - }, - [], - id="gen-ai-inputs-and-outputs-omitted-default-to-enabled", - ), - pytest.param( - None, - True, - { - SPANDATA.GEN_AI_REQUEST_MESSAGES: [ - {"role": "user", "content": "How many letters in the word eudca"} - ], - SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS: [ - {"type": "text", "content": "You are a helpful assistant."} - ], - SPANDATA.GEN_AI_RESPONSE_TEXT: "the model response", - }, - [], - id="no-gen-ai-config-legacy-pii-and-include-prompts-enabled", - ), - pytest.param( - None, - False, - {}, - [ - SPANDATA.GEN_AI_REQUEST_MESSAGES, - SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS, - SPANDATA.GEN_AI_RESPONSE_TEXT, - ], - id="no-gen-ai-config-legacy-pii-disabled", - ), ], ) def test_langchain_chat_data_collection( @@ -2797,7 +2997,6 @@ def test_langchain_chat_data_collection( get_model_response, nonstreaming_chat_completions_model_response, data_collection, - send_default_pii, expected_present, expected_absent, ): @@ -2805,7 +3004,6 @@ def test_langchain_chat_data_collection( integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=send_default_pii, ) if data_collection is not None: sentry_init_kwargs["data_collection"] = data_collection @@ -2868,46 +3066,16 @@ def test_langchain_chat_data_collection( for key in expected_absent: assert key not in span_data, f"{key} should not have been collected" - # Data collection never gates non-PII attributes assert span_data[SPANDATA.GEN_AI_REQUEST_MODEL] == "gpt-3.5-turbo" assert span_data[SPANDATA.GEN_AI_PROVIDER_NAME] == "openai-chat" assert span_data[SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30 @pytest.mark.parametrize( - "data_collection,send_default_pii,expected_present,expected_absent", + "data_collection,expected_present,expected_absent", [ - pytest.param( - {"gen_ai": {"inputs": True, "outputs": True}}, - False, - { - SPANDATA.GEN_AI_REQUEST_MESSAGES: [ - { - "role": "user", - "content": { - "type": "text", - "text": "What is the capital of France?", - }, - } - ], - SPANDATA.GEN_AI_RESPONSE_TEXT: "The capital of France is Paris.", - }, - [], - id="gen-ai-inputs-and-outputs-enabled-override-legacy-off", - ), - pytest.param( - {"gen_ai": {"inputs": False, "outputs": False}}, - True, - {}, - [ - SPANDATA.GEN_AI_REQUEST_MESSAGES, - SPANDATA.GEN_AI_RESPONSE_TEXT, - ], - id="gen-ai-inputs-and-outputs-disabled-override-legacy-on", - ), pytest.param( {"gen_ai": {"inputs": True, "outputs": False}}, - False, { SPANDATA.GEN_AI_REQUEST_MESSAGES: [ { @@ -2924,57 +3092,10 @@ def test_langchain_chat_data_collection( ), pytest.param( {"gen_ai": {"inputs": False, "outputs": True}}, - False, {SPANDATA.GEN_AI_RESPONSE_TEXT: "The capital of France is Paris."}, [SPANDATA.GEN_AI_REQUEST_MESSAGES], id="gen-ai-outputs-enabled-inputs-disabled", ), - pytest.param( - {"gen_ai": {}}, - False, - { - SPANDATA.GEN_AI_REQUEST_MESSAGES: [ - { - "role": "user", - "content": { - "type": "text", - "text": "What is the capital of France?", - }, - } - ], - SPANDATA.GEN_AI_RESPONSE_TEXT: "The capital of France is Paris.", - }, - [], - id="gen-ai-inputs-and-outputs-omitted-default-to-enabled", - ), - pytest.param( - None, - True, - { - SPANDATA.GEN_AI_REQUEST_MESSAGES: [ - { - "role": "user", - "content": { - "type": "text", - "text": "What is the capital of France?", - }, - } - ], - SPANDATA.GEN_AI_RESPONSE_TEXT: "The capital of France is Paris.", - }, - [], - id="no-gen-ai-config-legacy-pii-and-include-prompts-enabled", - ), - pytest.param( - None, - False, - {}, - [ - SPANDATA.GEN_AI_REQUEST_MESSAGES, - SPANDATA.GEN_AI_RESPONSE_TEXT, - ], - id="no-gen-ai-config-legacy-pii-disabled", - ), ], ) def test_langchain_text_completion_data_collection( @@ -2983,7 +3104,6 @@ def test_langchain_text_completion_data_collection( capture_items, get_model_response, data_collection, - send_default_pii, expected_present, expected_absent, ): @@ -2991,7 +3111,6 @@ def test_langchain_text_completion_data_collection( integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=send_default_pii, ) if data_collection is not None: sentry_init_kwargs["data_collection"] = data_collection @@ -3053,7 +3172,6 @@ def test_langchain_text_completion_data_collection( for key in expected_absent: assert key not in span_data, f"{key} should not have been collected" - # Data collection never gates non-PII attributes assert span_data[SPANDATA.GEN_AI_REQUEST_MODEL] == "gpt-3.5-turbo" assert span_data[SPANDATA.GEN_AI_PROVIDER_NAME] == "openai" assert span_data[SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7 @@ -3061,32 +3179,10 @@ def test_langchain_text_completion_data_collection( @pytest.mark.parametrize( - "data_collection,send_default_pii,expected_present,expected_absent", + "data_collection,expected_present,expected_absent", [ - pytest.param( - {"gen_ai": {"inputs": True, "outputs": True}}, - False, - [ - SPANDATA.GEN_AI_TOOL_DEFINITIONS, - SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, - ], - [SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS], - id="gen-ai-inputs-and-outputs-enabled-tools-collected", - ), - pytest.param( - {"gen_ai": {"inputs": False, "outputs": False}}, - True, - [], - [ - SPANDATA.GEN_AI_TOOL_DEFINITIONS, - SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, - SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, - ], - id="gen-ai-inputs-and-outputs-disabled-tools-not-collected", - ), pytest.param( {"gen_ai": {"inputs": True, "outputs": False}}, - False, [SPANDATA.GEN_AI_TOOL_DEFINITIONS], [ # REQUEST_AVAILABLE_TOOLS is the legacy value set when data collection is not enabled @@ -3097,7 +3193,6 @@ def test_langchain_text_completion_data_collection( ), pytest.param( {"gen_ai": {"inputs": False, "outputs": True}}, - True, [SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS], [ SPANDATA.GEN_AI_TOOL_DEFINITIONS, @@ -3105,36 +3200,6 @@ def test_langchain_text_completion_data_collection( ], id="gen-ai-outputs-enabled-inputs-disabled-only-response-tool-calls-collected", ), - pytest.param( - {"gen_ai": {}}, - False, - [ - SPANDATA.GEN_AI_TOOL_DEFINITIONS, - SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, - ], - [SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS], - id="gen-ai-inputs-and-outputs-omitted-default-to-enabled", - ), - pytest.param( - None, - True, - [ - SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS, - SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, - ], - [SPANDATA.GEN_AI_TOOL_DEFINITIONS], - id="no-gen-ai-config-legacy-pii-and-include-prompts-enabled", - ), - pytest.param( - None, - False, - [SPANDATA.GEN_AI_REQUEST_AVAILABLE_TOOLS], - [ - SPANDATA.GEN_AI_TOOL_DEFINITIONS, - SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, - ], - id="no-gen-ai-config-available-tools-collected-regardless-of-pii", - ), ], ) def test_langchain_data_collection_tools( @@ -3145,7 +3210,6 @@ def test_langchain_data_collection_tools( server_side_event_chunks, streaming_chat_completions_model_responses, data_collection, - send_default_pii, expected_present, expected_absent, ): @@ -3153,7 +3217,6 @@ def test_langchain_data_collection_tools( integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=send_default_pii, ) if data_collection is not None: sentry_init_kwargs["data_collection"] = data_collection @@ -3224,7 +3287,6 @@ def test_langchain_data_collection_tools( assert (key in chat_spans[1]) is collected -@pytest.mark.parametrize("send_default_pii", [True, False]) @pytest.mark.parametrize( "data_collection,tool_calls_collected", [ @@ -3238,16 +3300,6 @@ def test_langchain_data_collection_tools( False, id="gen-ai-inputs-disabled-request-tool-calls-not-collected", ), - pytest.param( - {"gen_ai": {}}, - True, - id="gen-ai-inputs-omitted-default-to-enabled", - ), - pytest.param( - None, - True, - id="no-gen-ai-config-request-tool-calls-collected-regardless-of-pii", - ), ], ) def test_langchain_data_collection_request_tool_call_params( @@ -3255,14 +3307,12 @@ def test_langchain_data_collection_request_tool_call_params( capture_events, capture_items, data_collection, - send_default_pii, tool_calls_collected, ): sentry_init_kwargs = dict( integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=send_default_pii, ) if data_collection is not None: sentry_init_kwargs["data_collection"] = data_collection @@ -3300,71 +3350,24 @@ def test_langchain_data_collection_request_tool_call_params( else: assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span_data - # Data collection never gates non-PII attributes assert span_data[SPANDATA.GEN_AI_REQUEST_MODEL] == "gpt-3.5-turbo" @pytest.mark.parametrize( - "data_collection,send_default_pii,expected_present,expected_absent", + "data_collection,expected_present,expected_absent", [ - pytest.param( - {"gen_ai": {"inputs": True, "outputs": True}}, - False, - { - SPANDATA.GEN_AI_TOOL_INPUT: {"word": "eudca"}, - SPANDATA.GEN_AI_TOOL_OUTPUT: 5, - }, - [], - id="gen-ai-inputs-and-outputs-enabled-override-legacy-off", - ), - pytest.param( - {"gen_ai": {"inputs": False, "outputs": False}}, - True, - {}, - [SPANDATA.GEN_AI_TOOL_INPUT, SPANDATA.GEN_AI_TOOL_OUTPUT], - id="gen-ai-inputs-and-outputs-disabled-override-legacy-on", - ), pytest.param( {"gen_ai": {"inputs": True, "outputs": False}}, - False, {SPANDATA.GEN_AI_TOOL_INPUT: {"word": "eudca"}}, [SPANDATA.GEN_AI_TOOL_OUTPUT], id="gen-ai-inputs-enabled-outputs-disabled", ), pytest.param( {"gen_ai": {"inputs": False, "outputs": True}}, - False, {SPANDATA.GEN_AI_TOOL_OUTPUT: 5}, [SPANDATA.GEN_AI_TOOL_INPUT], id="gen-ai-outputs-enabled-inputs-disabled", ), - pytest.param( - {"gen_ai": {}}, - False, - { - SPANDATA.GEN_AI_TOOL_INPUT: {"word": "eudca"}, - SPANDATA.GEN_AI_TOOL_OUTPUT: 5, - }, - [], - id="gen-ai-inputs-and-outputs-omitted-default-to-enabled", - ), - pytest.param( - None, - True, - { - SPANDATA.GEN_AI_TOOL_INPUT: {"word": "eudca"}, - SPANDATA.GEN_AI_TOOL_OUTPUT: 5, - }, - [], - id="no-gen-ai-config-legacy-pii-and-include-prompts-enabled", - ), - pytest.param( - None, - False, - {}, - [SPANDATA.GEN_AI_TOOL_INPUT, SPANDATA.GEN_AI_TOOL_OUTPUT], - id="no-gen-ai-config-legacy-pii-disabled", - ), ], ) def test_langchain_tool_execution_data_collection( @@ -3375,7 +3378,6 @@ def test_langchain_tool_execution_data_collection( server_side_event_chunks, streaming_chat_completions_model_responses, data_collection, - send_default_pii, expected_present, expected_absent, ): @@ -3383,7 +3385,6 @@ def test_langchain_tool_execution_data_collection( integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=send_default_pii, ) if data_collection is not None: sentry_init_kwargs["data_collection"] = data_collection @@ -3445,79 +3446,26 @@ def test_langchain_tool_execution_data_collection( for key in expected_absent: assert key not in span_data, f"{key} should not have been collected" - # Data collection never gates non-PII attributes assert span_data[SPANDATA.GEN_AI_TOOL_NAME] == "get_word_length" assert span_data[SPANDATA.GEN_AI_OPERATION_NAME] == "execute_tool" @pytest.mark.parametrize("agent_method", ["invoke", "stream"]) @pytest.mark.parametrize( - "data_collection,send_default_pii,expected_present,expected_absent", + "data_collection,expected_present,expected_absent", [ - pytest.param( - {"gen_ai": {"inputs": True, "outputs": True}}, - False, - { - SPANDATA.GEN_AI_REQUEST_MESSAGES: [ - "How many letters in the word eudca" - ], - SPANDATA.GEN_AI_RESPONSE_TEXT: "The word eudca has 5 letters.", - }, - [], - id="gen-ai-inputs-and-outputs-enabled-override-legacy-off", - ), - pytest.param( - {"gen_ai": {"inputs": False, "outputs": False}}, - True, - {}, - [SPANDATA.GEN_AI_REQUEST_MESSAGES, SPANDATA.GEN_AI_RESPONSE_TEXT], - id="gen-ai-inputs-and-outputs-disabled-override-legacy-on", - ), pytest.param( {"gen_ai": {"inputs": True, "outputs": False}}, - False, {SPANDATA.GEN_AI_REQUEST_MESSAGES: ["How many letters in the word eudca"]}, [SPANDATA.GEN_AI_RESPONSE_TEXT], id="gen-ai-inputs-enabled-outputs-disabled", ), pytest.param( {"gen_ai": {"inputs": False, "outputs": True}}, - False, {SPANDATA.GEN_AI_RESPONSE_TEXT: "The word eudca has 5 letters."}, [SPANDATA.GEN_AI_REQUEST_MESSAGES], id="gen-ai-outputs-enabled-inputs-disabled", ), - pytest.param( - {"gen_ai": {}}, - False, - { - SPANDATA.GEN_AI_REQUEST_MESSAGES: [ - "How many letters in the word eudca" - ], - SPANDATA.GEN_AI_RESPONSE_TEXT: "The word eudca has 5 letters.", - }, - [], - id="gen-ai-inputs-and-outputs-omitted-default-to-enabled", - ), - pytest.param( - None, - True, - { - SPANDATA.GEN_AI_REQUEST_MESSAGES: [ - "How many letters in the word eudca" - ], - SPANDATA.GEN_AI_RESPONSE_TEXT: "The word eudca has 5 letters.", - }, - [], - id="no-gen-ai-config-legacy-pii-and-include-prompts-enabled", - ), - pytest.param( - None, - False, - {}, - [SPANDATA.GEN_AI_REQUEST_MESSAGES, SPANDATA.GEN_AI_RESPONSE_TEXT], - id="no-gen-ai-config-legacy-pii-disabled", - ), ], ) def test_langchain_agent_executor_data_collection( @@ -3528,7 +3476,6 @@ def test_langchain_agent_executor_data_collection( server_side_event_chunks, streaming_chat_completions_model_responses, data_collection, - send_default_pii, expected_present, expected_absent, agent_method, @@ -3537,7 +3484,6 @@ def test_langchain_agent_executor_data_collection( integrations=[LangchainIntegration()], disabled_integrations=[StdlibIntegration], traces_sample_rate=1.0, - send_default_pii=send_default_pii, ) if data_collection is not None: sentry_init_kwargs["data_collection"] = data_collection @@ -3602,5 +3548,4 @@ def test_langchain_agent_executor_data_collection( for key in expected_absent: assert key not in span_data, f"{key} should not have been collected" - # Data collection never gates non-PII attributes assert span_data[SPANDATA.GEN_AI_OPERATION_NAME] == "invoke_agent"