diff --git a/sentry_sdk/integrations/clickhouse_driver.py b/sentry_sdk/integrations/clickhouse_driver.py index 9a1018f52e..457ec3d626 100644 --- a/sentry_sdk/integrations/clickhouse_driver.py +++ b/sentry_sdk/integrations/clickhouse_driver.py @@ -4,9 +4,7 @@ import sentry_sdk from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations import DidNotEnable, Integration, _check_minimum_version -from sentry_sdk.scope import should_send_default_pii from sentry_sdk.traces import Span -from sentry_sdk.utils import has_data_collection_enabled # Hack to get new Python features working in older versions # without introducing a hard dependency on `typing_extensions` @@ -125,13 +123,7 @@ def _inner_end(*args: "P.args", **kwargs: "P.kwargs") -> "T": if query is not None and breadcrumb_data is not None: client_options = sentry_sdk.get_client().options - if ( - has_data_collection_enabled(client_options) - and client_options["data_collection"]["database_query_data"] - ) or ( - not has_data_collection_enabled(client_options) - and should_send_default_pii() - ): + if client_options["data_collection"]["database_query_data"]: breadcrumb_data = {"db.result": res, **breadcrumb_data} sentry_sdk.get_isolation_scope().add_breadcrumb( diff --git a/tests/integrations/clickhouse_driver/test_clickhouse_driver.py b/tests/integrations/clickhouse_driver/test_clickhouse_driver.py index 4e5b5c1615..5ddf9b1ab0 100644 --- a/tests/integrations/clickhouse_driver/test_clickhouse_driver.py +++ b/tests/integrations/clickhouse_driver/test_clickhouse_driver.py @@ -5,8 +5,6 @@ ``` """ -from unittest import mock - import clickhouse_driver from clickhouse_driver import Client, connect @@ -23,6 +21,7 @@ def test_clickhouse_client_breadcrumbs(sentry_init, capture_events) -> None: sentry_init( integrations=[ClickhouseDriverIntegration()], + data_collection={}, _experiments={"record_sql_params": True}, ) events = capture_events() @@ -123,107 +122,6 @@ def test_clickhouse_client_breadcrumbs(sentry_init, capture_events) -> None: assert actual_query_breadcrumbs == expected_breadcrumbs -def test_clickhouse_client_breadcrumbs_with_pii(sentry_init, capture_events) -> None: - sentry_init( - integrations=[ClickhouseDriverIntegration()], - send_default_pii=True, - _experiments={"record_sql_params": True}, - ) - events = capture_events() - - client = Client("localhost") - client.execute("DROP TABLE IF EXISTS test") - client.execute("CREATE TABLE test (x Int32) ENGINE = Memory") - client.execute("INSERT INTO test (x) VALUES", [{"x": 100}]) - client.execute("INSERT INTO test (x) VALUES", [[170], [200]]) - - res = client.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) - assert res[0][0] == 370 - - capture_message("hi") - - (event,) = events - - expected_breadcrumbs = [ - { - "category": "query", - "data": { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - "db.result": [], - }, - "message": "DROP TABLE IF EXISTS test", - "type": "default", - }, - { - "category": "query", - "data": { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - "db.result": [], - }, - "message": "CREATE TABLE test (x Int32) ENGINE = Memory", - "type": "default", - }, - { - "category": "query", - "data": { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - }, - "message": "INSERT INTO test (x) VALUES", - "type": "default", - }, - { - "category": "query", - "data": { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - }, - "message": "INSERT INTO test (x) VALUES", - "type": "default", - }, - { - "category": "query", - "data": { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - "db.result": [[370]], - }, - "message": "SELECT sum(x) FROM test WHERE x > 150", - "type": "default", - }, - ] - - for crumb in expected_breadcrumbs: - crumb["data"] = ApproxDict(crumb["data"]) - - for crumb in event["breadcrumbs"]["values"]: - crumb.pop("timestamp", None) - - assert event["breadcrumbs"]["values"] == expected_breadcrumbs - - def test_clickhouse_client_breadcrumbs_with_data_collection( sentry_init, capture_events ) -> None: @@ -425,106 +323,6 @@ def test_clickhouse_client_breadcrumbs_with_data_collection_disabled( assert event["breadcrumbs"]["values"] == expected_breadcrumbs -def test_clickhouse_client_breadcrumbs_data_collection_overrides_pii( - sentry_init, capture_events -) -> None: - sentry_init( - integrations=[ClickhouseDriverIntegration()], - send_default_pii=True, - data_collection={"database_query_data": False}, - ) - events = capture_events() - - client = Client("localhost") - client.execute("DROP TABLE IF EXISTS test") - client.execute("CREATE TABLE test (x Int32) ENGINE = Memory") - client.execute("INSERT INTO test (x) VALUES", [{"x": 100}]) - client.execute("INSERT INTO test (x) VALUES", [[170], [200]]) - - res = client.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) - assert res[0][0] == 370 - - capture_message("hi") - - (event,) = events - - expected_breadcrumbs = [ - { - "category": "query", - "data": { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - }, - "message": "DROP TABLE IF EXISTS test", - "type": "default", - }, - { - "category": "query", - "data": { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - }, - "message": "CREATE TABLE test (x Int32) ENGINE = Memory", - "type": "default", - }, - { - "category": "query", - "data": { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - }, - "message": "INSERT INTO test (x) VALUES", - "type": "default", - }, - { - "category": "query", - "data": { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - }, - "message": "INSERT INTO test (x) VALUES", - "type": "default", - }, - { - "category": "query", - "data": { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - }, - "message": "SELECT sum(x) FROM test WHERE x > 150", - "type": "default", - }, - ] - - for crumb in expected_breadcrumbs: - crumb["data"] = ApproxDict(crumb["data"]) - - for crumb in event["breadcrumbs"]["values"]: - crumb.pop("timestamp", None) - - assert event["breadcrumbs"]["values"] == expected_breadcrumbs - - def test_clickhouse_client_breadcrumbs_with_data_collection_default( sentry_init, capture_events ) -> None: @@ -628,8 +426,6 @@ def test_clickhouse_client_breadcrumbs_with_data_collection_default( def test_clickhouse_client_with_data_collection(sentry_init, capture_items) -> None: - # Streamed spans never carry db.params/db.result, regardless of the - # data_collection configuration sentry_init( integrations=[ClickhouseDriverIntegration()], traces_sample_rate=1.0, @@ -730,6 +526,7 @@ def test_clickhouse_client_spans( "record_sql_params": True, }, traces_sample_rate=1.0, + data_collection={}, ) items = capture_items("span") @@ -855,168 +652,41 @@ def test_clickhouse_client_spans( assert spans == expected_spans -def test_clickhouse_client_spans_with_pii( - sentry_init, - capture_items, - capture_envelopes, -): +def test_clickhouse_dbapi_breadcrumbs(sentry_init, capture_events) -> None: sentry_init( integrations=[ClickhouseDriverIntegration()], - _experiments={ - "record_sql_params": True, - }, - traces_sample_rate=1.0, - send_default_pii=True, + data_collection={}, ) - items = capture_items("span") - - trace_id = None - span_id = None + events = capture_events() - with sentry_sdk.start_span(name="custom parent") as span: - trace_id = span.trace_id - span_id = span.span_id + conn = connect("clickhouse://localhost") + cursor = conn.cursor() + cursor.execute("DROP TABLE IF EXISTS test") + cursor.execute("CREATE TABLE test (x Int32) ENGINE = Memory") + cursor.executemany("INSERT INTO test (x) VALUES", [{"x": 100}]) + cursor.executemany("INSERT INTO test (x) VALUES", [[170], [200]]) + cursor.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) + res = cursor.fetchall() - client = Client("localhost") - client.execute("DROP TABLE IF EXISTS test") - client.execute("CREATE TABLE test (x Int32) ENGINE = Memory") - client.execute("INSERT INTO test (x) VALUES", [{"x": 100}]) - client.execute("INSERT INTO test (x) VALUES", [[170], [200]]) + assert res[0][0] == 370 - res = client.execute( - "SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150} - ) - assert res[0][0] == 370 + capture_message("hi") - sentry_sdk.flush() - spans = [item.payload for item in items] + (event,) = events - expected_spans = [ + expected_breadcrumbs = [ { - "name": "DROP TABLE IF EXISTS test", - "attributes": { - "db.system.name": "clickhouse", - "db.namespace": "", + "category": "query", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", "db.user": "default", - "db.query.text": "DROP TABLE IF EXISTS test", "server.address": "localhost", "server.port": 9000, - "thread.id": mock.ANY, - "thread.name": mock.ANY, }, - "trace_id": trace_id, - "parent_span_id": span_id, - }, - { - "name": "CREATE TABLE test (x Int32) ENGINE = Memory", - "attributes": { - "db.system.name": "clickhouse", - "db.namespace": "", - "db.user": "default", - "db.query.text": "CREATE TABLE test (x Int32) ENGINE = Memory", - "server.address": "localhost", - "server.port": 9000, - "thread.id": mock.ANY, - "thread.name": mock.ANY, - }, - "trace_id": trace_id, - "parent_span_id": span_id, - }, - { - "name": "INSERT INTO test (x) VALUES", - "attributes": { - "db.system.name": "clickhouse", - "db.namespace": "", - "db.user": "default", - "db.query.text": "INSERT INTO test (x) VALUES", - "server.address": "localhost", - "server.port": 9000, - "thread.id": mock.ANY, - "thread.name": mock.ANY, - }, - "trace_id": trace_id, - "parent_span_id": span_id, - }, - { - "name": "INSERT INTO test (x) VALUES", - "attributes": { - "db.system.name": "clickhouse", - "db.namespace": "", - "db.user": "default", - "db.query.text": "INSERT INTO test (x) VALUES", - "server.address": "localhost", - "server.port": 9000, - }, - "trace_id": trace_id, - "parent_span_id": span_id, - }, - { - "name": "SELECT sum(x) FROM test WHERE x > 150", - "attributes": { - "db.system.name": "clickhouse", - "db.namespace": "", - "db.user": "default", - "db.query.text": "SELECT sum(x) FROM test WHERE x > 150", - "server.address": "localhost", - "server.port": 9000, - }, - "trace_id": trace_id, - "parent_span_id": span_id, - }, - { - "name": "custom parent", - "attributes": [], - "trace_id": trace_id, - }, - ] - - for span in expected_spans: - span["attributes"] = ApproxDict(span["attributes"]) - - for span in spans: - span.pop("span_id", None) - span.pop("start_timestamp", None) - span.pop("end_timestamp", None) - span.pop("is_segment", None) - span.pop("status", None) - - assert spans == expected_spans - - -def test_clickhouse_dbapi_breadcrumbs(sentry_init, capture_events) -> None: - sentry_init( - integrations=[ClickhouseDriverIntegration()], - ) - events = capture_events() - - conn = connect("clickhouse://localhost") - cursor = conn.cursor() - cursor.execute("DROP TABLE IF EXISTS test") - cursor.execute("CREATE TABLE test (x Int32) ENGINE = Memory") - cursor.executemany("INSERT INTO test (x) VALUES", [{"x": 100}]) - cursor.executemany("INSERT INTO test (x) VALUES", [[170], [200]]) - cursor.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) - res = cursor.fetchall() - - assert res[0][0] == 370 - - capture_message("hi") - - (event,) = events - - expected_breadcrumbs = [ - { - "category": "query", - "data": { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - }, - "message": "DROP TABLE IF EXISTS test", - "type": "default", + "message": "DROP TABLE IF EXISTS test", + "type": "default", }, { "category": "query", @@ -1081,108 +751,6 @@ def test_clickhouse_dbapi_breadcrumbs(sentry_init, capture_events) -> None: assert event["breadcrumbs"]["values"] == expected_breadcrumbs -def test_clickhouse_dbapi_breadcrumbs_with_pii(sentry_init, capture_events) -> None: - sentry_init( - integrations=[ClickhouseDriverIntegration()], - send_default_pii=True, - ) - events = capture_events() - - conn = connect("clickhouse://localhost") - cursor = conn.cursor() - cursor.execute("DROP TABLE IF EXISTS test") - cursor.execute("CREATE TABLE test (x Int32) ENGINE = Memory") - cursor.executemany("INSERT INTO test (x) VALUES", [{"x": 100}]) - cursor.executemany("INSERT INTO test (x) VALUES", [[170], [200]]) - cursor.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) - res = cursor.fetchall() - - assert res[0][0] == 370 - - capture_message("hi") - - (event,) = events - - expected_breadcrumbs = [ - { - "category": "query", - "data": { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - "db.result": [[], []], - }, - "message": "DROP TABLE IF EXISTS test", - "type": "default", - }, - { - "category": "query", - "data": { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - "db.result": [[], []], - }, - "message": "CREATE TABLE test (x Int32) ENGINE = Memory", - "type": "default", - }, - { - "category": "query", - "data": { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - }, - "message": "INSERT INTO test (x) VALUES", - "type": "default", - }, - { - "category": "query", - "data": { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - }, - "message": "INSERT INTO test (x) VALUES", - "type": "default", - }, - { - "category": "query", - "data": { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - "db.result": [[["370"]], [["'sum(x)'", "'Int64'"]]], - }, - "message": "SELECT sum(x) FROM test WHERE x > 150", - "type": "default", - }, - ] - - for crumb in expected_breadcrumbs: - crumb["data"] = ApproxDict(crumb["data"]) - - for crumb in event["breadcrumbs"]["values"]: - crumb.pop("timestamp", None) - - assert event["breadcrumbs"]["values"] == expected_breadcrumbs - - def test_clickhouse_dbapi_spans( sentry_init, capture_items, @@ -1194,6 +762,7 @@ def test_clickhouse_dbapi_spans( "record_sql_params": True, }, traces_sample_rate=1.0, + data_collection={}, ) items = capture_items("span") @@ -1316,146 +885,11 @@ def test_clickhouse_dbapi_spans( assert spans == expected_spans -def test_clickhouse_dbapi_spans_with_pii( - sentry_init, - capture_items, - capture_envelopes, -): - sentry_init( - integrations=[ClickhouseDriverIntegration()], - _experiments={ - "record_sql_params": True, - }, - traces_sample_rate=1.0, - send_default_pii=True, - ) - items = capture_items("span") - - trace_id = None - span_id = None - - with sentry_sdk.start_span(name="custom parent") as span: - trace_id = span.trace_id - span_id = span.span_id - - conn = connect("clickhouse://localhost") - cursor = conn.cursor() - cursor.execute("DROP TABLE IF EXISTS test") - cursor.execute("CREATE TABLE test (x Int32) ENGINE = Memory") - cursor.executemany("INSERT INTO test (x) VALUES", [{"x": 100}]) - cursor.executemany("INSERT INTO test (x) VALUES", [[170], [200]]) - cursor.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) - res = cursor.fetchall() - - assert res[0][0] == 370 - - sentry_sdk.flush() - spans = [item.payload for item in items] - - expected_spans = [ - { - "name": "DROP TABLE IF EXISTS test", - "attributes": { - "db.system.name": "clickhouse", - "db.namespace": "", - "db.user": "default", - "db.query.text": "DROP TABLE IF EXISTS test", - "sentry.op": "db", - "sentry.origin": "auto.db.clickhouse_driver", - "server.address": "localhost", - "server.port": 9000, - }, - "trace_id": trace_id, - "parent_span_id": span_id, - }, - { - "name": "CREATE TABLE test (x Int32) ENGINE = Memory", - "attributes": { - "db.system.name": "clickhouse", - "db.namespace": "", - "db.user": "default", - "db.query.text": "CREATE TABLE test (x Int32) ENGINE = Memory", - "sentry.op": "db", - "sentry.origin": "auto.db.clickhouse_driver", - "server.address": "localhost", - "server.port": 9000, - }, - "trace_id": trace_id, - "parent_span_id": span_id, - }, - { - "name": "INSERT INTO test (x) VALUES", - "attributes": { - "db.system.name": "clickhouse", - "db.namespace": "", - "db.user": "default", - "db.query.text": "INSERT INTO test (x) VALUES", - "sentry.op": "db", - "sentry.origin": "auto.db.clickhouse_driver", - "server.address": "localhost", - "server.port": 9000, - }, - "trace_id": trace_id, - "parent_span_id": span_id, - }, - { - "name": "INSERT INTO test (x) VALUES", - "attributes": { - "db.system.name": "clickhouse", - "db.namespace": "", - "db.user": "default", - "db.query.text": "INSERT INTO test (x) VALUES", - "sentry.op": "db", - "sentry.origin": "auto.db.clickhouse_driver", - "server.address": "localhost", - "server.port": 9000, - }, - "trace_id": trace_id, - "parent_span_id": span_id, - }, - { - "name": "SELECT sum(x) FROM test WHERE x > 150", - "attributes": { - "db.system.name": "clickhouse", - "db.namespace": "", - "db.user": "default", - "db.query.text": "SELECT sum(x) FROM test WHERE x > 150", - "sentry.op": "db", - "sentry.origin": "auto.db.clickhouse_driver", - "server.address": "localhost", - "server.port": 9000, - }, - "trace_id": trace_id, - "parent_span_id": span_id, - }, - { - "name": "custom parent", - "attributes": [], - "trace_id": trace_id, - }, - ] - - for span in expected_spans: - span["attributes"] = ApproxDict(span["attributes"]) - - for span in spans: - span.pop("span_id", None) - span.pop("start_timestamp", None) - span.pop("end_timestamp", None) - span.pop("is_segment", None) - span.pop("status", None) - - assert spans == expected_spans - - -def test_span_origin( - sentry_init, - capture_items, - capture_envelopes, -): +def test_span_origin(sentry_init, capture_items, capture_envelopes): sentry_init( integrations=[ClickhouseDriverIntegration()], traces_sample_rate=1.0, + data_collection={}, ) items = capture_items("span")