From 9fa29e5f394a4bad7304347df60cd26afadc7840 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 1 Sep 2026 11:25:56 +0200 Subject: [PATCH 01/14] feat(clickhouse_driver): Set breadcrumbs in the streaming trace lifecycle --- sentry_sdk/integrations/clickhouse_driver.py | 13 +++++++++ .../test_clickhouse_driver.py | 28 +++++++++++++++---- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/sentry_sdk/integrations/clickhouse_driver.py b/sentry_sdk/integrations/clickhouse_driver.py index 4e9901acaf..59198fab20 100644 --- a/sentry_sdk/integrations/clickhouse_driver.py +++ b/sentry_sdk/integrations/clickhouse_driver.py @@ -95,6 +95,19 @@ def _inner(*args: "P.args", **kwargs: "P.kwargs") -> "T": SPANDATA.DB_QUERY_TEXT: str(query), }, ) + + sentry_sdk.get_isolation_scope().add_breadcrumb( + message=query, + category="query", + data={ + SPANDATA.DB_SYSTEM: "clickhouse", + SPANDATA.DB_NAME: connection.database, + SPANDATA.DB_DRIVER_NAME: "clickhouse-driver", + SPANDATA.SERVER_ADDRESS: connection.host, + SPANDATA.SERVER_PORT: connection.port, + SPANDATA.DB_USER: connection.user, + }, + ) else: span = sentry_sdk.start_span( op=OP.DB, diff --git a/tests/integrations/clickhouse_driver/test_clickhouse_driver.py b/tests/integrations/clickhouse_driver/test_clickhouse_driver.py index 1cec232548..b51f34856c 100644 --- a/tests/integrations/clickhouse_driver/test_clickhouse_driver.py +++ b/tests/integrations/clickhouse_driver/test_clickhouse_driver.py @@ -21,9 +21,13 @@ EXPECT_PARAMS_IN_SELECT = False -def test_clickhouse_client_breadcrumbs(sentry_init, capture_events) -> None: +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_clickhouse_client_breadcrumbs( + sentry_init, capture_events, span_streaming +) -> None: sentry_init( integrations=[ClickhouseDriverIntegration()], + trace_lifecycle="stream" if span_streaming else "static", _experiments={"record_sql_params": True}, ) events = capture_events() @@ -127,10 +131,14 @@ def test_clickhouse_client_breadcrumbs(sentry_init, capture_events) -> None: assert actual_query_breadcrumbs == expected_breadcrumbs -def test_clickhouse_client_breadcrumbs_with_pii(sentry_init, capture_events) -> None: +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_clickhouse_client_breadcrumbs_with_pii( + sentry_init, capture_events, span_streaming +) -> None: sentry_init( integrations=[ClickhouseDriverIntegration()], send_default_pii=True, + trace_lifecycle="stream" if span_streaming else "static", _experiments={"record_sql_params": True}, ) events = capture_events() @@ -229,11 +237,13 @@ def test_clickhouse_client_breadcrumbs_with_pii(sentry_init, capture_events) -> assert event["breadcrumbs"]["values"] == expected_breadcrumbs +@pytest.mark.parametrize("span_streaming", [True, False]) def test_clickhouse_client_breadcrumbs_with_data_collection( - sentry_init, capture_events + sentry_init, capture_events, span_streaming ) -> None: sentry_init( integrations=[ClickhouseDriverIntegration()], + trace_lifecycle="stream" if span_streaming else "static", _experiments={"data_collection": {"database_query_data": True}}, ) events = capture_events() @@ -332,11 +342,13 @@ def test_clickhouse_client_breadcrumbs_with_data_collection( assert event["breadcrumbs"]["values"] == expected_breadcrumbs +@pytest.mark.parametrize("span_streaming", [True, False]) def test_clickhouse_client_breadcrumbs_with_data_collection_disabled( - sentry_init, capture_events + sentry_init, capture_events, span_streaming ) -> None: sentry_init( integrations=[ClickhouseDriverIntegration()], + trace_lifecycle="stream" if span_streaming else "static", _experiments={"data_collection": {"database_query_data": False}}, ) events = capture_events() @@ -431,12 +443,14 @@ def test_clickhouse_client_breadcrumbs_with_data_collection_disabled( assert "db.result" not in crumb["data"] +@pytest.mark.parametrize("span_streaming", [True, False]) def test_clickhouse_client_breadcrumbs_data_collection_overrides_pii( - sentry_init, capture_events + sentry_init, capture_events, span_streaming ) -> None: sentry_init( integrations=[ClickhouseDriverIntegration()], send_default_pii=True, + trace_lifecycle="stream" if span_streaming else "static", _experiments={"data_collection": {"database_query_data": False}}, ) events = capture_events() @@ -531,11 +545,13 @@ def test_clickhouse_client_breadcrumbs_data_collection_overrides_pii( assert "db.result" not in crumb["data"] +@pytest.mark.parametrize("span_streaming", [True, False]) def test_clickhouse_client_breadcrumbs_with_data_collection_default( - sentry_init, capture_events + sentry_init, capture_events, span_streaming ) -> None: sentry_init( integrations=[ClickhouseDriverIntegration()], + trace_lifecycle="stream" if span_streaming else "static", _experiments={"data_collection": {}}, ) events = capture_events() From 206cb38d394b818e915948bec4a6ba60809e783a Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 1 Sep 2026 11:34:42 +0200 Subject: [PATCH 02/14] add db.result to breadcrumb data --- sentry_sdk/integrations/clickhouse_driver.py | 35 ++++++++++++-------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/sentry_sdk/integrations/clickhouse_driver.py b/sentry_sdk/integrations/clickhouse_driver.py index 59198fab20..19e4730f64 100644 --- a/sentry_sdk/integrations/clickhouse_driver.py +++ b/sentry_sdk/integrations/clickhouse_driver.py @@ -15,7 +15,7 @@ # from: https://stackoverflow.com/a/71944042/300572 if TYPE_CHECKING: from collections.abc import Iterator - from typing import Any, Callable, ParamSpec, Union + from typing import Any, Callable, Optional, ParamSpec, Union else: # Fake ParamSpec class ParamSpec: @@ -96,18 +96,15 @@ def _inner(*args: "P.args", **kwargs: "P.kwargs") -> "T": }, ) - sentry_sdk.get_isolation_scope().add_breadcrumb( - message=query, - category="query", - data={ - SPANDATA.DB_SYSTEM: "clickhouse", - SPANDATA.DB_NAME: connection.database, - SPANDATA.DB_DRIVER_NAME: "clickhouse-driver", - SPANDATA.SERVER_ADDRESS: connection.host, - SPANDATA.SERVER_PORT: connection.port, - SPANDATA.DB_USER: connection.user, - }, - ) + connection._query = query # type: ignore[attr-defined] + connection._breadcrumb_data = { # type: ignore[attr-defined] + SPANDATA.DB_SYSTEM: "clickhouse", + SPANDATA.DB_NAME: connection.database, + SPANDATA.DB_DRIVER_NAME: "clickhouse-driver", + SPANDATA.SERVER_ADDRESS: connection.host, + SPANDATA.SERVER_PORT: connection.port, + SPANDATA.DB_USER: connection.user, + } else: span = sentry_sdk.start_span( op=OP.DB, @@ -150,6 +147,18 @@ def _inner_end(*args: "P.args", **kwargs: "P.kwargs") -> "T": return res if isinstance(span, StreamedSpan): + query = getattr(instance.connection, "_query", None) # type: ignore[attr-defined] + breadcrumb_data: "Optional[dict[str, Any]]" = getattr( + instance.connection, "_breadcrumb_data", None + ) # type: ignore[attr-defined] + + if query is not None and breadcrumb_data is not None: + sentry_sdk.get_isolation_scope().add_breadcrumb( + message=query, + category="query", + data={"db.result": res, **breadcrumb_data}, + ) + span.end() else: if res is not None: From a2e30a8f31b817cb2bcda92917f536fe0be61bad Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 1 Sep 2026 11:36:56 +0200 Subject: [PATCH 03/14] mypy happy --- sentry_sdk/integrations/clickhouse_driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/clickhouse_driver.py b/sentry_sdk/integrations/clickhouse_driver.py index 19e4730f64..c5a77c6abb 100644 --- a/sentry_sdk/integrations/clickhouse_driver.py +++ b/sentry_sdk/integrations/clickhouse_driver.py @@ -79,7 +79,7 @@ def _inner(*args: "P.args", **kwargs: "P.kwargs") -> "T": if client.get_integration(ClickhouseDriverIntegration) is None: return f(*args, **kwargs) - connection = args[0] + connection: "Connection" = args[0] query = args[1] query_id = args[2] if len(args) > 2 else kwargs.get("query_id") params = args[3] if len(args) > 3 else kwargs.get("params") From 6abe1896442d14d70fd6302d946ebb3eb3a758ed Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 1 Sep 2026 11:40:23 +0200 Subject: [PATCH 04/14] mypy again --- sentry_sdk/integrations/clickhouse_driver.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sentry_sdk/integrations/clickhouse_driver.py b/sentry_sdk/integrations/clickhouse_driver.py index c5a77c6abb..266447d01d 100644 --- a/sentry_sdk/integrations/clickhouse_driver.py +++ b/sentry_sdk/integrations/clickhouse_driver.py @@ -96,8 +96,8 @@ def _inner(*args: "P.args", **kwargs: "P.kwargs") -> "T": }, ) - connection._query = query # type: ignore[attr-defined] - connection._breadcrumb_data = { # type: ignore[attr-defined] + connection._query = query + connection._breadcrumb_data = { SPANDATA.DB_SYSTEM: "clickhouse", SPANDATA.DB_NAME: connection.database, SPANDATA.DB_DRIVER_NAME: "clickhouse-driver", @@ -124,7 +124,7 @@ def _inner(*args: "P.args", **kwargs: "P.kwargs") -> "T": elif should_send_default_pii(): span.set_data("db.params", params) - connection._sentry_span = span # type: ignore[attr-defined] + connection._sentry_span = span if span is not None: _set_db_data(span, connection) @@ -140,17 +140,17 @@ def _inner(*args: "P.args", **kwargs: "P.kwargs") -> "T": def _wrap_end(f: "Callable[P, T]") -> "Callable[P, T]": def _inner_end(*args: "P.args", **kwargs: "P.kwargs") -> "T": res = f(*args, **kwargs) - instance = args[0] - span = getattr(instance.connection, "_sentry_span", None) # type: ignore[attr-defined] + instance: "Client" = args[0] + span = getattr(instance.connection, "_sentry_span", None) if span is None: return res if isinstance(span, StreamedSpan): - query = getattr(instance.connection, "_query", None) # type: ignore[attr-defined] + query = getattr(instance.connection, "_query", None) breadcrumb_data: "Optional[dict[str, Any]]" = getattr( instance.connection, "_breadcrumb_data", None - ) # type: ignore[attr-defined] + ) if query is not None and breadcrumb_data is not None: sentry_sdk.get_isolation_scope().add_breadcrumb( From 518af23a08abd91d66f56ac7bd0379171b801598 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 1 Sep 2026 11:50:43 +0200 Subject: [PATCH 05/14] set breadcrumb when span is None --- sentry_sdk/integrations/clickhouse_driver.py | 40 +++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/sentry_sdk/integrations/clickhouse_driver.py b/sentry_sdk/integrations/clickhouse_driver.py index 266447d01d..47b0da9c6b 100644 --- a/sentry_sdk/integrations/clickhouse_driver.py +++ b/sentry_sdk/integrations/clickhouse_driver.py @@ -8,7 +8,7 @@ from sentry_sdk.traces import StreamedSpan from sentry_sdk.tracing import Span from sentry_sdk.tracing_utils import has_span_streaming_enabled -from sentry_sdk.utils import capture_internal_exceptions, has_data_collection_enabled +from sentry_sdk.utils import has_data_collection_enabled # Hack to get new Python features working in older versions # without introducing a hard dependency on `typing_extensions` @@ -141,24 +141,33 @@ def _wrap_end(f: "Callable[P, T]") -> "Callable[P, T]": def _inner_end(*args: "P.args", **kwargs: "P.kwargs") -> "T": res = f(*args, **kwargs) instance: "Client" = args[0] + + query = getattr(instance.connection, "_query", None) + breadcrumb_data: "Optional[dict[str, Any]]" = getattr( + instance.connection, "_breadcrumb_data", None + ) + + if query is not None and breadcrumb_data is not None: + client_options = sentry_sdk.get_client().options + if ( + has_data_collection_enabled(client_options) + and client_options["data_collection"]["database_query_data"] + or should_send_default_pii() + ): + breadcrumb_data = {"db.result": res, **breadcrumb_data} + + sentry_sdk.get_isolation_scope().add_breadcrumb( + message=query, + category="query", + data={"db.result": res, **breadcrumb_data}, + ) + span = getattr(instance.connection, "_sentry_span", None) if span is None: return res if isinstance(span, StreamedSpan): - query = getattr(instance.connection, "_query", None) - breadcrumb_data: "Optional[dict[str, Any]]" = getattr( - instance.connection, "_breadcrumb_data", None - ) - - if query is not None and breadcrumb_data is not None: - sentry_sdk.get_isolation_scope().add_breadcrumb( - message=query, - category="query", - data={"db.result": res, **breadcrumb_data}, - ) - span.end() else: if res is not None: @@ -169,11 +178,6 @@ def _inner_end(*args: "P.args", **kwargs: "P.kwargs") -> "T": elif should_send_default_pii(): span.set_data("db.result", res) - with capture_internal_exceptions(): - span.scope.add_breadcrumb( - message=span._data.pop("query"), category="query", data=span._data - ) - span.finish() return res From 675ddae384a383620892a4574b1787aed9344f15 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 1 Sep 2026 11:52:03 +0200 Subject: [PATCH 06/14] . --- sentry_sdk/integrations/clickhouse_driver.py | 21 ++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/sentry_sdk/integrations/clickhouse_driver.py b/sentry_sdk/integrations/clickhouse_driver.py index 47b0da9c6b..ef1bacb895 100644 --- a/sentry_sdk/integrations/clickhouse_driver.py +++ b/sentry_sdk/integrations/clickhouse_driver.py @@ -84,6 +84,17 @@ def _inner(*args: "P.args", **kwargs: "P.kwargs") -> "T": query_id = args[2] if len(args) > 2 else kwargs.get("query_id") params = args[3] if len(args) > 3 else kwargs.get("params") + connection._query = query + connection._breadcrumb_data = { + SPANDATA.DB_SYSTEM: "clickhouse", + SPANDATA.DB_NAME: connection.database, + SPANDATA.DB_DRIVER_NAME: "clickhouse-driver", + SPANDATA.SERVER_ADDRESS: connection.host, + SPANDATA.SERVER_PORT: connection.port, + SPANDATA.DB_USER: connection.user, + "db.params": params, + } + if has_span_streaming_enabled(client.options): span = None if sentry_sdk.traces.get_current_span() is not None: @@ -95,16 +106,6 @@ def _inner(*args: "P.args", **kwargs: "P.kwargs") -> "T": SPANDATA.DB_QUERY_TEXT: str(query), }, ) - - connection._query = query - connection._breadcrumb_data = { - SPANDATA.DB_SYSTEM: "clickhouse", - SPANDATA.DB_NAME: connection.database, - SPANDATA.DB_DRIVER_NAME: "clickhouse-driver", - SPANDATA.SERVER_ADDRESS: connection.host, - SPANDATA.SERVER_PORT: connection.port, - SPANDATA.DB_USER: connection.user, - } else: span = sentry_sdk.start_span( op=OP.DB, From 6aefca59f4aea826b7b4f9a671f0e3c4f9645686 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 1 Sep 2026 11:57:39 +0200 Subject: [PATCH 07/14] . --- sentry_sdk/integrations/clickhouse_driver.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/sentry_sdk/integrations/clickhouse_driver.py b/sentry_sdk/integrations/clickhouse_driver.py index ef1bacb895..d0760aa5d7 100644 --- a/sentry_sdk/integrations/clickhouse_driver.py +++ b/sentry_sdk/integrations/clickhouse_driver.py @@ -85,15 +85,19 @@ def _inner(*args: "P.args", **kwargs: "P.kwargs") -> "T": params = args[3] if len(args) > 3 else kwargs.get("params") connection._query = query - connection._breadcrumb_data = { + + breadcrumb_data = { SPANDATA.DB_SYSTEM: "clickhouse", SPANDATA.DB_NAME: connection.database, SPANDATA.DB_DRIVER_NAME: "clickhouse-driver", SPANDATA.SERVER_ADDRESS: connection.host, SPANDATA.SERVER_PORT: connection.port, SPANDATA.DB_USER: connection.user, - "db.params": params, } + if params: + breadcrumb_data["db.params"] = params + + connection._breadcrumb_data = breadcrumb_data if has_span_streaming_enabled(client.options): span = None @@ -160,7 +164,7 @@ def _inner_end(*args: "P.args", **kwargs: "P.kwargs") -> "T": sentry_sdk.get_isolation_scope().add_breadcrumb( message=query, category="query", - data={"db.result": res, **breadcrumb_data}, + data=breadcrumb_data, ) span = getattr(instance.connection, "_sentry_span", None) From 100c587ff1c17eb22cde0342a3d8cc91b4e367aa Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 1 Sep 2026 12:55:09 +0200 Subject: [PATCH 08/14] . --- sentry_sdk/integrations/clickhouse_driver.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/sentry_sdk/integrations/clickhouse_driver.py b/sentry_sdk/integrations/clickhouse_driver.py index d0760aa5d7..26de528af8 100644 --- a/sentry_sdk/integrations/clickhouse_driver.py +++ b/sentry_sdk/integrations/clickhouse_driver.py @@ -89,12 +89,18 @@ def _inner(*args: "P.args", **kwargs: "P.kwargs") -> "T": breadcrumb_data = { SPANDATA.DB_SYSTEM: "clickhouse", SPANDATA.DB_NAME: connection.database, - SPANDATA.DB_DRIVER_NAME: "clickhouse-driver", SPANDATA.SERVER_ADDRESS: connection.host, SPANDATA.SERVER_PORT: connection.port, SPANDATA.DB_USER: connection.user, } - if params: + if params and ( + ( + has_data_collection_enabled(client.options) + and client.options["data_collection"]["database_query_data"] + ) + or not has_data_collection_enabled(client.options) + and should_send_default_pii() + ): breadcrumb_data["db.params"] = params connection._breadcrumb_data = breadcrumb_data @@ -157,7 +163,9 @@ def _inner_end(*args: "P.args", **kwargs: "P.kwargs") -> "T": if ( has_data_collection_enabled(client_options) and client_options["data_collection"]["database_query_data"] - or should_send_default_pii() + ) or ( + not has_data_collection_enabled(client_options) + and should_send_default_pii() ): breadcrumb_data = {"db.result": res, **breadcrumb_data} From 36b6f6ee65f0fa2850ff601a56339d4e2df1f752 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 1 Sep 2026 13:00:31 +0200 Subject: [PATCH 09/14] . --- sentry_sdk/integrations/clickhouse_driver.py | 21 ++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/sentry_sdk/integrations/clickhouse_driver.py b/sentry_sdk/integrations/clickhouse_driver.py index 26de528af8..7e7be8306c 100644 --- a/sentry_sdk/integrations/clickhouse_driver.py +++ b/sentry_sdk/integrations/clickhouse_driver.py @@ -160,12 +160,15 @@ def _inner_end(*args: "P.args", **kwargs: "P.kwargs") -> "T": if query is not None and breadcrumb_data is not None: client_options = sentry_sdk.get_client().options - if ( - has_data_collection_enabled(client_options) - and client_options["data_collection"]["database_query_data"] - ) or ( - not has_data_collection_enabled(client_options) - and should_send_default_pii() + if res is not None and ( + ( + has_data_collection_enabled(client_options) + and client_options["data_collection"]["database_query_data"] + ) + or ( + not has_data_collection_enabled(client_options) + and should_send_default_pii() + ) ): breadcrumb_data = {"db.result": res, **breadcrumb_data} @@ -237,6 +240,9 @@ def wrapped_generator() -> "Iterator[Any]": data = wrapped_generator() span.set_data("db.params", db_params) + breadcrumb_data = getattr(self.connection, "_breadcrumb_data", None) + if breadcrumb_data is not None: + breadcrumb_data["db.params"] = db_params elif should_send_default_pii(): db_params = span._data.get("db.params", []) @@ -258,6 +264,9 @@ def wrapped_generator() -> "Iterator[Any]": data = wrapped_generator() span.set_data("db.params", db_params) + breadcrumb_data = getattr(self.connection, "_breadcrumb_data", None) + if breadcrumb_data is not None: + breadcrumb_data["db.params"] = db_params return original_send_data( self, sample_block, data, types_check, columnar, *args, **kwargs From 18d1f1ff3e49c0e3a184c391986331520777f4c7 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 1 Sep 2026 13:04:54 +0200 Subject: [PATCH 10/14] . --- sentry_sdk/integrations/clickhouse_driver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sentry_sdk/integrations/clickhouse_driver.py b/sentry_sdk/integrations/clickhouse_driver.py index 7e7be8306c..9f5aa9dc0c 100644 --- a/sentry_sdk/integrations/clickhouse_driver.py +++ b/sentry_sdk/integrations/clickhouse_driver.py @@ -88,6 +88,7 @@ def _inner(*args: "P.args", **kwargs: "P.kwargs") -> "T": breadcrumb_data = { SPANDATA.DB_SYSTEM: "clickhouse", + SPANDATA.DB_DRIVER_NAME: "clickhouse-driver", SPANDATA.DB_NAME: connection.database, SPANDATA.SERVER_ADDRESS: connection.host, SPANDATA.SERVER_PORT: connection.port, From 7c5fda80ce77940060c9c6654b6f1a54d21f3781 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 1 Sep 2026 13:24:29 +0200 Subject: [PATCH 11/14] . --- sentry_sdk/integrations/clickhouse_driver.py | 55 +-- .../test_clickhouse_driver.py | 402 +++++++++++++++++- 2 files changed, 402 insertions(+), 55 deletions(-) diff --git a/sentry_sdk/integrations/clickhouse_driver.py b/sentry_sdk/integrations/clickhouse_driver.py index 9f5aa9dc0c..ebc456a55a 100644 --- a/sentry_sdk/integrations/clickhouse_driver.py +++ b/sentry_sdk/integrations/clickhouse_driver.py @@ -84,28 +84,6 @@ def _inner(*args: "P.args", **kwargs: "P.kwargs") -> "T": query_id = args[2] if len(args) > 2 else kwargs.get("query_id") params = args[3] if len(args) > 3 else kwargs.get("params") - connection._query = query - - breadcrumb_data = { - SPANDATA.DB_SYSTEM: "clickhouse", - SPANDATA.DB_DRIVER_NAME: "clickhouse-driver", - SPANDATA.DB_NAME: connection.database, - SPANDATA.SERVER_ADDRESS: connection.host, - SPANDATA.SERVER_PORT: connection.port, - SPANDATA.DB_USER: connection.user, - } - if params and ( - ( - has_data_collection_enabled(client.options) - and client.options["data_collection"]["database_query_data"] - ) - or not has_data_collection_enabled(client.options) - and should_send_default_pii() - ): - breadcrumb_data["db.params"] = params - - connection._breadcrumb_data = breadcrumb_data - if has_span_streaming_enabled(client.options): span = None if sentry_sdk.traces.get_current_span() is not None: @@ -117,6 +95,16 @@ def _inner(*args: "P.args", **kwargs: "P.kwargs") -> "T": SPANDATA.DB_QUERY_TEXT: str(query), }, ) + + connection._query = query + connection._breadcrumb_data = { + SPANDATA.DB_SYSTEM: "clickhouse", + SPANDATA.DB_NAME: connection.database, + SPANDATA.DB_DRIVER_NAME: "clickhouse-driver", + SPANDATA.SERVER_ADDRESS: connection.host, + SPANDATA.SERVER_PORT: connection.port, + SPANDATA.DB_USER: connection.user, + } else: span = sentry_sdk.start_span( op=OP.DB, @@ -161,22 +149,19 @@ def _inner_end(*args: "P.args", **kwargs: "P.kwargs") -> "T": if query is not None and breadcrumb_data is not None: client_options = sentry_sdk.get_client().options - if res is not None and ( - ( - has_data_collection_enabled(client_options) - and client_options["data_collection"]["database_query_data"] - ) - or ( - not has_data_collection_enabled(client_options) - and should_send_default_pii() - ) + if ( + has_data_collection_enabled(client_options) + and client_options["data_collection"]["database_query_data"] + ) or ( + not has_data_collection_enabled(client_options) + and should_send_default_pii() ): breadcrumb_data = {"db.result": res, **breadcrumb_data} sentry_sdk.get_isolation_scope().add_breadcrumb( message=query, category="query", - data=breadcrumb_data, + data={"db.result": res, **breadcrumb_data}, ) span = getattr(instance.connection, "_sentry_span", None) @@ -241,9 +226,6 @@ def wrapped_generator() -> "Iterator[Any]": data = wrapped_generator() span.set_data("db.params", db_params) - breadcrumb_data = getattr(self.connection, "_breadcrumb_data", None) - if breadcrumb_data is not None: - breadcrumb_data["db.params"] = db_params elif should_send_default_pii(): db_params = span._data.get("db.params", []) @@ -265,9 +247,6 @@ def wrapped_generator() -> "Iterator[Any]": data = wrapped_generator() span.set_data("db.params", db_params) - breadcrumb_data = getattr(self.connection, "_breadcrumb_data", None) - if breadcrumb_data is not None: - breadcrumb_data["db.params"] = db_params return original_send_data( self, sample_block, data, types_check, columnar, *args, **kwargs diff --git a/tests/integrations/clickhouse_driver/test_clickhouse_driver.py b/tests/integrations/clickhouse_driver/test_clickhouse_driver.py index b51f34856c..18ae961563 100644 --- a/tests/integrations/clickhouse_driver/test_clickhouse_driver.py +++ b/tests/integrations/clickhouse_driver/test_clickhouse_driver.py @@ -131,14 +131,10 @@ def test_clickhouse_client_breadcrumbs( assert actual_query_breadcrumbs == expected_breadcrumbs -@pytest.mark.parametrize("span_streaming", [True, False]) -def test_clickhouse_client_breadcrumbs_with_pii( - sentry_init, capture_events, span_streaming -) -> None: +def test_clickhouse_client_breadcrumbs_with_pii(sentry_init, capture_events) -> None: sentry_init( integrations=[ClickhouseDriverIntegration()], send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", _experiments={"record_sql_params": True}, ) events = capture_events() @@ -237,13 +233,11 @@ def test_clickhouse_client_breadcrumbs_with_pii( assert event["breadcrumbs"]["values"] == expected_breadcrumbs -@pytest.mark.parametrize("span_streaming", [True, False]) def test_clickhouse_client_breadcrumbs_with_data_collection( - sentry_init, capture_events, span_streaming + sentry_init, capture_events ) -> None: sentry_init( integrations=[ClickhouseDriverIntegration()], - trace_lifecycle="stream" if span_streaming else "static", _experiments={"data_collection": {"database_query_data": True}}, ) events = capture_events() @@ -342,13 +336,11 @@ def test_clickhouse_client_breadcrumbs_with_data_collection( assert event["breadcrumbs"]["values"] == expected_breadcrumbs -@pytest.mark.parametrize("span_streaming", [True, False]) def test_clickhouse_client_breadcrumbs_with_data_collection_disabled( - sentry_init, capture_events, span_streaming + sentry_init, capture_events ) -> None: sentry_init( integrations=[ClickhouseDriverIntegration()], - trace_lifecycle="stream" if span_streaming else "static", _experiments={"data_collection": {"database_query_data": False}}, ) events = capture_events() @@ -443,14 +435,12 @@ def test_clickhouse_client_breadcrumbs_with_data_collection_disabled( assert "db.result" not in crumb["data"] -@pytest.mark.parametrize("span_streaming", [True, False]) def test_clickhouse_client_breadcrumbs_data_collection_overrides_pii( - sentry_init, capture_events, span_streaming + sentry_init, capture_events ) -> None: sentry_init( integrations=[ClickhouseDriverIntegration()], send_default_pii=True, - trace_lifecycle="stream" if span_streaming else "static", _experiments={"data_collection": {"database_query_data": False}}, ) events = capture_events() @@ -545,13 +535,11 @@ def test_clickhouse_client_breadcrumbs_data_collection_overrides_pii( assert "db.result" not in crumb["data"] -@pytest.mark.parametrize("span_streaming", [True, False]) def test_clickhouse_client_breadcrumbs_with_data_collection_default( - sentry_init, capture_events, span_streaming + sentry_init, capture_events ) -> None: sentry_init( integrations=[ClickhouseDriverIntegration()], - trace_lifecycle="stream" if span_streaming else "static", _experiments={"data_collection": {}}, ) events = capture_events() @@ -2046,3 +2034,383 @@ def test_span_origin( assert event["contexts"]["trace"]["origin"] == "manual" assert event["spans"][0]["origin"] == "auto.db.clickhouse_driver" + + +# ---- Span-first (streaming) breadcrumb tests ---- +# These mirror the breadcrumb tests above but specifically target the +# span-first code path (trace_lifecycle="stream"). In span-first mode +# breadcrumbs are emitted from the dedicated breadcrumb_data dict set on +# the connection, NOT derived from the span. db.params and db.result are +# never attached to breadcrumbs in this mode. + + +def test_clickhouse_client_breadcrumbs_span_streaming( + sentry_init, capture_events +) -> None: + sentry_init( + integrations=[ClickhouseDriverIntegration()], + trace_lifecycle="stream", + ) + events = capture_events() + + client = Client("localhost") + client.execute("DROP TABLE IF EXISTS test") + client.execute("CREATE TABLE test (x Int32) ENGINE = Memory") + client.execute("INSERT INTO test (x) VALUES", [{"x": 100}]) + client.execute("INSERT INTO test (x) VALUES", [[170], [200]]) + + res = client.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) + assert res[0][0] == 370 + + capture_message("hi") + + (event,) = events + + expected_base = { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + } + + expected_breadcrumbs = [ + { + "category": "query", + "data": {**expected_base}, + "message": "DROP TABLE IF EXISTS test", + "type": "default", + }, + { + "category": "query", + "data": {**expected_base}, + "message": "CREATE TABLE test (x Int32) ENGINE = Memory", + "type": "default", + }, + { + "category": "query", + "data": {**expected_base}, + "message": "INSERT INTO test (x) VALUES", + "type": "default", + }, + { + "category": "query", + "data": {**expected_base}, + "message": "INSERT INTO test (x) VALUES", + "type": "default", + }, + { + "category": "query", + "data": {**expected_base}, + "message": "SELECT sum(x) FROM test WHERE x > 150", + "type": "default", + }, + ] + + for crumb in expected_breadcrumbs: + crumb["data"] = ApproxDict(crumb["data"]) + + for crumb in event["breadcrumbs"]["values"]: + crumb.pop("timestamp", None) + + actual_query_breadcrumbs = [ + breadcrumb + for breadcrumb in event["breadcrumbs"]["values"] + if breadcrumb["category"] == "query" + ] + + assert actual_query_breadcrumbs == expected_breadcrumbs + + # Span-first breadcrumbs never carry db.params + for crumb in actual_query_breadcrumbs: + assert "db.params" not in crumb["data"] + + +@pytest.mark.parametrize("send_default_pii", [True, False]) +def test_clickhouse_client_breadcrumbs_span_streaming_with_pii( + sentry_init, capture_events, send_default_pii +) -> None: + sentry_init( + integrations=[ClickhouseDriverIntegration()], + send_default_pii=send_default_pii, + trace_lifecycle="stream", + ) + events = capture_events() + + client = Client("localhost") + client.execute("DROP TABLE IF EXISTS test") + client.execute("CREATE TABLE test (x Int32) ENGINE = Memory") + client.execute("INSERT INTO test (x) VALUES", [{"x": 100}]) + client.execute("INSERT INTO test (x) VALUES", [[170], [200]]) + + res = client.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) + assert res[0][0] == 370 + + capture_message("hi") + + (event,) = events + + expected_base = { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + } + + actual_query_breadcrumbs = [ + crumb + for crumb in event["breadcrumbs"]["values"] + if crumb["category"] == "query" + ] + + assert len(actual_query_breadcrumbs) == 5 + + for crumb in actual_query_breadcrumbs: + crumb.pop("timestamp", None) + assert crumb["type"] == "default" + assert crumb["category"] == "query" + assert crumb["data"] == ApproxDict(expected_base) + + assert actual_query_breadcrumbs[0]["message"] == "DROP TABLE IF EXISTS test" + assert ( + actual_query_breadcrumbs[1]["message"] + == "CREATE TABLE test (x Int32) ENGINE = Memory" + ) + assert actual_query_breadcrumbs[2]["message"] == "INSERT INTO test (x) VALUES" + assert actual_query_breadcrumbs[3]["message"] == "INSERT INTO test (x) VALUES" + assert ( + actual_query_breadcrumbs[4]["message"] + == "SELECT sum(x) FROM test WHERE x > 150" + ) + + # Span-first breadcrumbs never carry db.params regardless of PII setting + for crumb in actual_query_breadcrumbs: + assert "db.params" not in crumb["data"] + + +@pytest.mark.parametrize( + "data_collection_setting", + [ + pytest.param({"database_query_data": True}, id="enabled"), + pytest.param({"database_query_data": False}, id="disabled"), + pytest.param({}, id="default"), + ], +) +def test_clickhouse_client_breadcrumbs_span_streaming_data_collection( + sentry_init, capture_events, data_collection_setting +) -> None: + sentry_init( + integrations=[ClickhouseDriverIntegration()], + trace_lifecycle="stream", + _experiments={"data_collection": data_collection_setting}, + ) + events = capture_events() + + client = Client("localhost") + client.execute("DROP TABLE IF EXISTS test") + client.execute("CREATE TABLE test (x Int32) ENGINE = Memory") + client.execute("INSERT INTO test (x) VALUES", [{"x": 100}]) + client.execute("INSERT INTO test (x) VALUES", [[170], [200]]) + + res = client.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) + assert res[0][0] == 370 + + capture_message("hi") + + (event,) = events + + expected_base = { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + } + + actual_query_breadcrumbs = [ + crumb + for crumb in event["breadcrumbs"]["values"] + if crumb["category"] == "query" + ] + + assert len(actual_query_breadcrumbs) == 5 + + for crumb in actual_query_breadcrumbs: + crumb.pop("timestamp", None) + assert crumb["data"] == ApproxDict(expected_base) + + # Span-first breadcrumbs never carry db.params regardless of data_collection + for crumb in actual_query_breadcrumbs: + assert "db.params" not in crumb["data"] + + +def test_clickhouse_client_breadcrumbs_span_streaming_data_collection_overrides_pii( + sentry_init, capture_events +) -> None: + """data_collection disabled takes precedence over send_default_pii=True.""" + sentry_init( + integrations=[ClickhouseDriverIntegration()], + send_default_pii=True, + trace_lifecycle="stream", + _experiments={"data_collection": {"database_query_data": False}}, + ) + events = capture_events() + + client = Client("localhost") + client.execute("DROP TABLE IF EXISTS test") + client.execute("CREATE TABLE test (x Int32) ENGINE = Memory") + client.execute("INSERT INTO test (x) VALUES", [{"x": 100}]) + client.execute("INSERT INTO test (x) VALUES", [[170], [200]]) + + res = client.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) + assert res[0][0] == 370 + + capture_message("hi") + + (event,) = events + + actual_query_breadcrumbs = [ + crumb + for crumb in event["breadcrumbs"]["values"] + if crumb["category"] == "query" + ] + + assert len(actual_query_breadcrumbs) == 5 + + for crumb in actual_query_breadcrumbs: + assert "db.params" not in crumb["data"] + + +def test_clickhouse_dbapi_breadcrumbs_span_streaming( + sentry_init, capture_events +) -> None: + sentry_init( + integrations=[ClickhouseDriverIntegration()], + trace_lifecycle="stream", + ) + events = capture_events() + + conn = connect("clickhouse://localhost") + cursor = conn.cursor() + cursor.execute("DROP TABLE IF EXISTS test") + cursor.execute("CREATE TABLE test (x Int32) ENGINE = Memory") + cursor.executemany("INSERT INTO test (x) VALUES", [{"x": 100}]) + cursor.executemany("INSERT INTO test (x) VALUES", [[170], [200]]) + cursor.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) + res = cursor.fetchall() + + assert res[0][0] == 370 + + capture_message("hi") + + (event,) = events + + expected_base = { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + } + + expected_breadcrumbs = [ + { + "category": "query", + "data": {**expected_base}, + "message": "DROP TABLE IF EXISTS test", + "type": "default", + }, + { + "category": "query", + "data": {**expected_base}, + "message": "CREATE TABLE test (x Int32) ENGINE = Memory", + "type": "default", + }, + { + "category": "query", + "data": {**expected_base}, + "message": "INSERT INTO test (x) VALUES", + "type": "default", + }, + { + "category": "query", + "data": {**expected_base}, + "message": "INSERT INTO test (x) VALUES", + "type": "default", + }, + { + "category": "query", + "data": {**expected_base}, + "message": "SELECT sum(x) FROM test WHERE x > 150", + "type": "default", + }, + ] + + for crumb in expected_breadcrumbs: + crumb["data"] = ApproxDict(crumb["data"]) + + for crumb in event["breadcrumbs"]["values"]: + crumb.pop("timestamp", None) + + assert event["breadcrumbs"]["values"] == expected_breadcrumbs + + # Span-first breadcrumbs never carry db.params + for crumb in event["breadcrumbs"]["values"]: + assert "db.params" not in crumb["data"] + + +@pytest.mark.parametrize("send_default_pii", [True, False]) +def test_clickhouse_dbapi_breadcrumbs_span_streaming_with_pii( + sentry_init, capture_events, send_default_pii +) -> None: + sentry_init( + integrations=[ClickhouseDriverIntegration()], + send_default_pii=send_default_pii, + trace_lifecycle="stream", + ) + events = capture_events() + + conn = connect("clickhouse://localhost") + cursor = conn.cursor() + cursor.execute("DROP TABLE IF EXISTS test") + cursor.execute("CREATE TABLE test (x Int32) ENGINE = Memory") + cursor.executemany("INSERT INTO test (x) VALUES", [{"x": 100}]) + cursor.executemany("INSERT INTO test (x) VALUES", [[170], [200]]) + cursor.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) + res = cursor.fetchall() + + assert res[0][0] == 370 + + capture_message("hi") + + (event,) = events + + expected_base = { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + } + + actual_query_breadcrumbs = [ + crumb + for crumb in event["breadcrumbs"]["values"] + if crumb["category"] == "query" + ] + + assert len(actual_query_breadcrumbs) == 5 + + for crumb in actual_query_breadcrumbs: + crumb.pop("timestamp", None) + assert crumb["data"] == ApproxDict(expected_base) + + # Span-first breadcrumbs never carry db.params regardless of PII setting + for crumb in actual_query_breadcrumbs: + assert "db.params" not in crumb["data"] From 53a624a5a5d72d1bfb6c2ad0aa9871888a714aff Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 1 Sep 2026 13:51:53 +0200 Subject: [PATCH 12/14] fix tests --- .../test_clickhouse_driver.py | 2749 ++++++++++------- 1 file changed, 1637 insertions(+), 1112 deletions(-) diff --git a/tests/integrations/clickhouse_driver/test_clickhouse_driver.py b/tests/integrations/clickhouse_driver/test_clickhouse_driver.py index 18ae961563..c521b8b79e 100644 --- a/tests/integrations/clickhouse_driver/test_clickhouse_driver.py +++ b/tests/integrations/clickhouse_driver/test_clickhouse_driver.py @@ -131,10 +131,12 @@ def test_clickhouse_client_breadcrumbs( assert actual_query_breadcrumbs == expected_breadcrumbs -def test_clickhouse_client_breadcrumbs_with_pii(sentry_init, capture_events) -> None: +def test_clickhouse_client_breadcrumbs_span_streaming( + sentry_init, capture_events +) -> None: sentry_init( integrations=[ClickhouseDriverIntegration()], - send_default_pii=True, + trace_lifecycle="stream", _experiments={"record_sql_params": True}, ) events = capture_events() @@ -157,11 +159,11 @@ def test_clickhouse_client_breadcrumbs_with_pii(sentry_init, capture_events) -> "category": "query", "data": { "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", "db.name": "", "db.user": "default", "server.address": "localhost", "server.port": 9000, - "db.result": [], }, "message": "DROP TABLE IF EXISTS test", "type": "default", @@ -170,11 +172,11 @@ def test_clickhouse_client_breadcrumbs_with_pii(sentry_init, capture_events) -> "category": "query", "data": { "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", "db.name": "", "db.user": "default", "server.address": "localhost", "server.port": 9000, - "db.result": [], }, "message": "CREATE TABLE test (x Int32) ENGINE = Memory", "type": "default", @@ -183,11 +185,11 @@ def test_clickhouse_client_breadcrumbs_with_pii(sentry_init, capture_events) -> "category": "query", "data": { "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", "db.name": "", "db.user": "default", "server.address": "localhost", "server.port": 9000, - "db.params": [{"x": 100}], }, "message": "INSERT INTO test (x) VALUES", "type": "default", @@ -196,11 +198,11 @@ def test_clickhouse_client_breadcrumbs_with_pii(sentry_init, capture_events) -> "category": "query", "data": { "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", "db.name": "", "db.user": "default", "server.address": "localhost", "server.port": 9000, - "db.params": [[170], [200]], }, "message": "INSERT INTO test (x) VALUES", "type": "default", @@ -209,36 +211,41 @@ def test_clickhouse_client_breadcrumbs_with_pii(sentry_init, capture_events) -> "category": "query", "data": { "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", "db.name": "", "db.user": "default", "server.address": "localhost", "server.port": 9000, - "db.result": [[370]], - "db.params": {"minv": 150}, }, "message": "SELECT sum(x) FROM test WHERE x > 150", "type": "default", }, ] - if not EXPECT_PARAMS_IN_SELECT: - expected_breadcrumbs[-1]["data"].pop("db.params", None) - for crumb in expected_breadcrumbs: crumb["data"] = ApproxDict(crumb["data"]) for crumb in event["breadcrumbs"]["values"]: crumb.pop("timestamp", None) - assert event["breadcrumbs"]["values"] == expected_breadcrumbs + actual_query_breadcrumbs = [ + breadcrumb + for breadcrumb in event["breadcrumbs"]["values"] + if breadcrumb["category"] == "query" + ] + assert actual_query_breadcrumbs == expected_breadcrumbs -def test_clickhouse_client_breadcrumbs_with_data_collection( - sentry_init, capture_events -) -> None: + # In span streaming mode, db.params is never present + for crumb in actual_query_breadcrumbs: + assert "db.params" not in crumb["data"] + + +def test_clickhouse_client_breadcrumbs_with_pii(sentry_init, capture_events) -> None: sentry_init( integrations=[ClickhouseDriverIntegration()], - _experiments={"data_collection": {"database_query_data": True}}, + send_default_pii=True, + _experiments={"record_sql_params": True}, ) events = capture_events() @@ -336,12 +343,14 @@ def test_clickhouse_client_breadcrumbs_with_data_collection( assert event["breadcrumbs"]["values"] == expected_breadcrumbs -def test_clickhouse_client_breadcrumbs_with_data_collection_disabled( +def test_clickhouse_client_breadcrumbs_span_streaming_with_pii( sentry_init, capture_events ) -> None: sentry_init( integrations=[ClickhouseDriverIntegration()], - _experiments={"data_collection": {"database_query_data": False}}, + send_default_pii=True, + trace_lifecycle="stream", + _experiments={"record_sql_params": True}, ) events = capture_events() @@ -363,6 +372,7 @@ def test_clickhouse_client_breadcrumbs_with_data_collection_disabled( "category": "query", "data": { "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", "db.name": "", "db.user": "default", "server.address": "localhost", @@ -375,6 +385,7 @@ def test_clickhouse_client_breadcrumbs_with_data_collection_disabled( "category": "query", "data": { "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", "db.name": "", "db.user": "default", "server.address": "localhost", @@ -387,6 +398,7 @@ def test_clickhouse_client_breadcrumbs_with_data_collection_disabled( "category": "query", "data": { "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", "db.name": "", "db.user": "default", "server.address": "localhost", @@ -399,6 +411,7 @@ def test_clickhouse_client_breadcrumbs_with_data_collection_disabled( "category": "query", "data": { "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", "db.name": "", "db.user": "default", "server.address": "localhost", @@ -411,6 +424,7 @@ def test_clickhouse_client_breadcrumbs_with_data_collection_disabled( "category": "query", "data": { "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", "db.name": "", "db.user": "default", "server.address": "localhost", @@ -429,19 +443,17 @@ def test_clickhouse_client_breadcrumbs_with_data_collection_disabled( assert event["breadcrumbs"]["values"] == expected_breadcrumbs - # ApproxDict is subset matching, so explicitly assert params/result absence + # In span streaming mode, db.params is never present for crumb in event["breadcrumbs"]["values"]: assert "db.params" not in crumb["data"] - assert "db.result" not in crumb["data"] -def test_clickhouse_client_breadcrumbs_data_collection_overrides_pii( +def test_clickhouse_client_breadcrumbs_with_data_collection( sentry_init, capture_events ) -> None: sentry_init( integrations=[ClickhouseDriverIntegration()], - send_default_pii=True, - _experiments={"data_collection": {"database_query_data": False}}, + _experiments={"data_collection": {"database_query_data": True}}, ) events = capture_events() @@ -467,6 +479,7 @@ def test_clickhouse_client_breadcrumbs_data_collection_overrides_pii( "db.user": "default", "server.address": "localhost", "server.port": 9000, + "db.result": [], }, "message": "DROP TABLE IF EXISTS test", "type": "default", @@ -479,6 +492,7 @@ def test_clickhouse_client_breadcrumbs_data_collection_overrides_pii( "db.user": "default", "server.address": "localhost", "server.port": 9000, + "db.result": [], }, "message": "CREATE TABLE test (x Int32) ENGINE = Memory", "type": "default", @@ -491,6 +505,7 @@ def test_clickhouse_client_breadcrumbs_data_collection_overrides_pii( "db.user": "default", "server.address": "localhost", "server.port": 9000, + "db.params": [{"x": 100}], }, "message": "INSERT INTO test (x) VALUES", "type": "default", @@ -503,6 +518,7 @@ def test_clickhouse_client_breadcrumbs_data_collection_overrides_pii( "db.user": "default", "server.address": "localhost", "server.port": 9000, + "db.params": [[170], [200]], }, "message": "INSERT INTO test (x) VALUES", "type": "default", @@ -515,12 +531,17 @@ def test_clickhouse_client_breadcrumbs_data_collection_overrides_pii( "db.user": "default", "server.address": "localhost", "server.port": 9000, + "db.result": [[370]], + "db.params": {"minv": 150}, }, "message": "SELECT sum(x) FROM test WHERE x > 150", "type": "default", }, ] + if not EXPECT_PARAMS_IN_SELECT: + expected_breadcrumbs[-1]["data"].pop("db.params", None) + for crumb in expected_breadcrumbs: crumb["data"] = ApproxDict(crumb["data"]) @@ -529,18 +550,14 @@ def test_clickhouse_client_breadcrumbs_data_collection_overrides_pii( assert event["breadcrumbs"]["values"] == expected_breadcrumbs - # ApproxDict is subset matching, so explicitly assert params/result absence - for crumb in event["breadcrumbs"]["values"]: - assert "db.params" not in crumb["data"] - assert "db.result" not in crumb["data"] - -def test_clickhouse_client_breadcrumbs_with_data_collection_default( +def test_clickhouse_client_breadcrumbs_span_streaming_with_data_collection( sentry_init, capture_events ) -> None: sentry_init( integrations=[ClickhouseDriverIntegration()], - _experiments={"data_collection": {}}, + trace_lifecycle="stream", + _experiments={"data_collection": {"database_query_data": True}}, ) events = capture_events() @@ -562,11 +579,11 @@ def test_clickhouse_client_breadcrumbs_with_data_collection_default( "category": "query", "data": { "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", "db.name": "", "db.user": "default", "server.address": "localhost", "server.port": 9000, - "db.result": [], }, "message": "DROP TABLE IF EXISTS test", "type": "default", @@ -575,11 +592,11 @@ def test_clickhouse_client_breadcrumbs_with_data_collection_default( "category": "query", "data": { "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", "db.name": "", "db.user": "default", "server.address": "localhost", "server.port": 9000, - "db.result": [], }, "message": "CREATE TABLE test (x Int32) ENGINE = Memory", "type": "default", @@ -588,11 +605,11 @@ def test_clickhouse_client_breadcrumbs_with_data_collection_default( "category": "query", "data": { "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", "db.name": "", "db.user": "default", "server.address": "localhost", "server.port": 9000, - "db.params": [{"x": 100}], }, "message": "INSERT INTO test (x) VALUES", "type": "default", @@ -601,11 +618,11 @@ def test_clickhouse_client_breadcrumbs_with_data_collection_default( "category": "query", "data": { "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", "db.name": "", "db.user": "default", "server.address": "localhost", "server.port": 9000, - "db.params": [[170], [200]], }, "message": "INSERT INTO test (x) VALUES", "type": "default", @@ -614,21 +631,17 @@ def test_clickhouse_client_breadcrumbs_with_data_collection_default( "category": "query", "data": { "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", "db.name": "", "db.user": "default", "server.address": "localhost", "server.port": 9000, - "db.result": [[370]], - "db.params": {"minv": 150}, }, "message": "SELECT sum(x) FROM test WHERE x > 150", "type": "default", }, ] - if not EXPECT_PARAMS_IN_SELECT: - expected_breadcrumbs[-1]["data"].pop("db.params", None) - for crumb in expected_breadcrumbs: crumb["data"] = ApproxDict(crumb["data"]) @@ -637,81 +650,116 @@ def test_clickhouse_client_breadcrumbs_with_data_collection_default( assert event["breadcrumbs"]["values"] == expected_breadcrumbs - -def test_clickhouse_client_span_streaming_with_data_collection( - sentry_init, capture_items -) -> None: - # Streamed spans never carry db.params/db.result, regardless of the - # data_collection configuration - sentry_init( - integrations=[ClickhouseDriverIntegration()], - traces_sample_rate=1.0, - trace_lifecycle="stream", - _experiments={ - "data_collection": {"database_query_data": True}, - }, - ) - items = capture_items("span") - - with sentry_sdk.traces.start_span(name="custom parent"): - client = Client("localhost") - client.execute("DROP TABLE IF EXISTS test") - client.execute("CREATE TABLE test (x Int32) ENGINE = Memory") - client.execute("INSERT INTO test (x) VALUES", [{"x": 100}]) - client.execute("INSERT INTO test (x) VALUES", ((i,) for i in range(3))) - - res = client.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 2}) - assert res[0][0] == 100 - - sentry_sdk.flush() - - spans = [item.payload for item in items] - assert len(spans) > 1 # sanity check that db spans were actually streamed - - for span in spans: - attribute_keys = { - attribute["name"] if isinstance(attribute, dict) else attribute - for attribute in span.get("attributes", {}) - } - assert "db.params" not in attribute_keys - assert "db.result" not in attribute_keys + # In span streaming mode, db.params is never present + for crumb in event["breadcrumbs"]["values"]: + assert "db.params" not in crumb["data"] -def test_clickhouse_client_send_data_generator_with_data_collection( +def test_clickhouse_client_breadcrumbs_with_data_collection_disabled( sentry_init, capture_events ) -> None: sentry_init( integrations=[ClickhouseDriverIntegration()], - _experiments={"data_collection": {"database_query_data": True}}, + _experiments={"data_collection": {"database_query_data": False}}, ) events = capture_events() client = Client("localhost") client.execute("DROP TABLE IF EXISTS test") client.execute("CREATE TABLE test (x Int32) ENGINE = Memory") - client.execute("INSERT INTO test (x) VALUES", ((i,) for i in range(3))) + client.execute("INSERT INTO test (x) VALUES", [{"x": 100}]) + client.execute("INSERT INTO test (x) VALUES", [[170], [200]]) - res = client.execute("SELECT sum(x) FROM test") - assert res[0][0] == 3 + res = client.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) + assert res[0][0] == 370 capture_message("hi") (event,) = events - (insert_breadcrumb,) = [ - crumb - for crumb in event["breadcrumbs"]["values"] - if crumb["message"] == "INSERT INTO test (x) VALUES" + expected_breadcrumbs = [ + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + }, + "message": "DROP TABLE IF EXISTS test", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + }, + "message": "CREATE TABLE test (x Int32) ENGINE = Memory", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + }, + "message": "INSERT INTO test (x) VALUES", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + }, + "message": "INSERT INTO test (x) VALUES", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + }, + "message": "SELECT sum(x) FROM test WHERE x > 150", + "type": "default", + }, ] - assert insert_breadcrumb["data"]["db.params"] == [[0], [1], [2]] + for crumb in expected_breadcrumbs: + crumb["data"] = ApproxDict(crumb["data"]) + + for crumb in event["breadcrumbs"]["values"]: + crumb.pop("timestamp", None) + assert event["breadcrumbs"]["values"] == expected_breadcrumbs -def test_clickhouse_client_send_data_generator_with_data_collection_disabled( + # ApproxDict is subset matching, so explicitly assert params/result absence + for crumb in event["breadcrumbs"]["values"]: + assert "db.params" not in crumb["data"] + assert "db.result" not in crumb["data"] + + +def test_clickhouse_client_breadcrumbs_span_streaming_with_data_collection_disabled( sentry_init, capture_events ) -> None: sentry_init( integrations=[ClickhouseDriverIntegration()], + trace_lifecycle="stream", _experiments={"data_collection": {"database_query_data": False}}, ) events = capture_events() @@ -719,320 +767,675 @@ def test_clickhouse_client_send_data_generator_with_data_collection_disabled( client = Client("localhost") client.execute("DROP TABLE IF EXISTS test") client.execute("CREATE TABLE test (x Int32) ENGINE = Memory") - client.execute("INSERT INTO test (x) VALUES", ((i,) for i in range(3))) + client.execute("INSERT INTO test (x) VALUES", [{"x": 100}]) + client.execute("INSERT INTO test (x) VALUES", [[170], [200]]) - res = client.execute("SELECT sum(x) FROM test") - assert res[0][0] == 3 + res = client.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) + assert res[0][0] == 370 capture_message("hi") (event,) = events - (insert_breadcrumb,) = [ - crumb - for crumb in event["breadcrumbs"]["values"] - if crumb["message"] == "INSERT INTO test (x) VALUES" - ] - - assert "db.params" not in insert_breadcrumb["data"] - - -@pytest.mark.parametrize("span_streaming", [True, False]) -def test_clickhouse_client_spans( - sentry_init, - capture_events, - capture_items, - span_streaming, -): - sentry_init( - integrations=[ClickhouseDriverIntegration()], - trace_lifecycle="stream" if span_streaming else "static", - _experiments={ - "record_sql_params": True, - }, - traces_sample_rate=1.0, - ) - if span_streaming: - items = capture_items("span") - - trace_id = None - span_id = None - - with sentry_sdk.traces.start_span(name="custom parent") as span: - trace_id = span.trace_id - span_id = span.span_id + expected_breadcrumbs = [ + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + }, + "message": "DROP TABLE IF EXISTS test", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + }, + "message": "CREATE TABLE test (x Int32) ENGINE = Memory", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + }, + "message": "INSERT INTO test (x) VALUES", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + }, + "message": "INSERT INTO test (x) VALUES", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + }, + "message": "SELECT sum(x) FROM test WHERE x > 150", + "type": "default", + }, + ] + + for crumb in expected_breadcrumbs: + crumb["data"] = ApproxDict(crumb["data"]) + + for crumb in event["breadcrumbs"]["values"]: + crumb.pop("timestamp", None) + + assert event["breadcrumbs"]["values"] == expected_breadcrumbs + + # In span streaming mode, db.params is never present + for crumb in event["breadcrumbs"]["values"]: + assert "db.params" not in crumb["data"] + + +def test_clickhouse_client_breadcrumbs_data_collection_overrides_pii( + sentry_init, capture_events +) -> None: + sentry_init( + integrations=[ClickhouseDriverIntegration()], + send_default_pii=True, + _experiments={"data_collection": {"database_query_data": False}}, + ) + events = capture_events() + + client = Client("localhost") + client.execute("DROP TABLE IF EXISTS test") + client.execute("CREATE TABLE test (x Int32) ENGINE = Memory") + client.execute("INSERT INTO test (x) VALUES", [{"x": 100}]) + client.execute("INSERT INTO test (x) VALUES", [[170], [200]]) + + res = client.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) + assert res[0][0] == 370 + + capture_message("hi") + + (event,) = events + + expected_breadcrumbs = [ + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + }, + "message": "DROP TABLE IF EXISTS test", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + }, + "message": "CREATE TABLE test (x Int32) ENGINE = Memory", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + }, + "message": "INSERT INTO test (x) VALUES", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + }, + "message": "INSERT INTO test (x) VALUES", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + }, + "message": "SELECT sum(x) FROM test WHERE x > 150", + "type": "default", + }, + ] + + for crumb in expected_breadcrumbs: + crumb["data"] = ApproxDict(crumb["data"]) + + for crumb in event["breadcrumbs"]["values"]: + crumb.pop("timestamp", None) + + assert event["breadcrumbs"]["values"] == expected_breadcrumbs + + # ApproxDict is subset matching, so explicitly assert params/result absence + for crumb in event["breadcrumbs"]["values"]: + assert "db.params" not in crumb["data"] + assert "db.result" not in crumb["data"] + + +def test_clickhouse_client_breadcrumbs_span_streaming_data_collection_overrides_pii( + sentry_init, capture_events +) -> None: + sentry_init( + integrations=[ClickhouseDriverIntegration()], + send_default_pii=True, + trace_lifecycle="stream", + _experiments={"data_collection": {"database_query_data": False}}, + ) + events = capture_events() + + client = Client("localhost") + client.execute("DROP TABLE IF EXISTS test") + client.execute("CREATE TABLE test (x Int32) ENGINE = Memory") + client.execute("INSERT INTO test (x) VALUES", [{"x": 100}]) + client.execute("INSERT INTO test (x) VALUES", [[170], [200]]) + + res = client.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) + assert res[0][0] == 370 + + capture_message("hi") + + (event,) = events + + expected_breadcrumbs = [ + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + }, + "message": "DROP TABLE IF EXISTS test", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + }, + "message": "CREATE TABLE test (x Int32) ENGINE = Memory", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + }, + "message": "INSERT INTO test (x) VALUES", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + }, + "message": "INSERT INTO test (x) VALUES", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + }, + "message": "SELECT sum(x) FROM test WHERE x > 150", + "type": "default", + }, + ] + + for crumb in expected_breadcrumbs: + crumb["data"] = ApproxDict(crumb["data"]) + + for crumb in event["breadcrumbs"]["values"]: + crumb.pop("timestamp", None) + + assert event["breadcrumbs"]["values"] == expected_breadcrumbs + + # In span streaming mode, db.params is never present + for crumb in event["breadcrumbs"]["values"]: + assert "db.params" not in crumb["data"] + + +def test_clickhouse_client_breadcrumbs_with_data_collection_default( + sentry_init, capture_events +) -> None: + sentry_init( + integrations=[ClickhouseDriverIntegration()], + _experiments={"data_collection": {}}, + ) + events = capture_events() + + client = Client("localhost") + client.execute("DROP TABLE IF EXISTS test") + client.execute("CREATE TABLE test (x Int32) ENGINE = Memory") + client.execute("INSERT INTO test (x) VALUES", [{"x": 100}]) + client.execute("INSERT INTO test (x) VALUES", [[170], [200]]) + + res = client.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) + assert res[0][0] == 370 + + capture_message("hi") + + (event,) = events + + expected_breadcrumbs = [ + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + "db.result": [], + }, + "message": "DROP TABLE IF EXISTS test", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + "db.result": [], + }, + "message": "CREATE TABLE test (x Int32) ENGINE = Memory", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + "db.params": [{"x": 100}], + }, + "message": "INSERT INTO test (x) VALUES", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + "db.params": [[170], [200]], + }, + "message": "INSERT INTO test (x) VALUES", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + "db.result": [[370]], + "db.params": {"minv": 150}, + }, + "message": "SELECT sum(x) FROM test WHERE x > 150", + "type": "default", + }, + ] - client = Client("localhost") - client.execute("DROP TABLE IF EXISTS test") - client.execute("CREATE TABLE test (x Int32) ENGINE = Memory") - client.execute("INSERT INTO test (x) VALUES", [{"x": 100}]) - client.execute("INSERT INTO test (x) VALUES", [[170], [200]]) + if not EXPECT_PARAMS_IN_SELECT: + expected_breadcrumbs[-1]["data"].pop("db.params", None) - res = client.execute( - "SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150} - ) - assert res[0][0] == 370 + for crumb in expected_breadcrumbs: + crumb["data"] = ApproxDict(crumb["data"]) - sentry_sdk.flush() - spans = [item.payload for item in items] + for crumb in event["breadcrumbs"]["values"]: + crumb.pop("timestamp", None) - expected_spans = [ - { - "name": "DROP TABLE IF EXISTS test", - "attributes": { - "db.system.name": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.namespace": "", - "db.user": "default", - "sentry.op": "db", - "db.query.text": "DROP TABLE IF EXISTS test", - "sentry.origin": "auto.db.clickhouse_driver", - "server.address": "localhost", - "server.port": 9000, - }, - "trace_id": trace_id, - "parent_span_id": span_id, - }, - { - "name": "CREATE TABLE test (x Int32) ENGINE = Memory", - "attributes": { - "db.system.name": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.namespace": "", - "db.user": "default", - "sentry.op": "db", - "db.query.text": "CREATE TABLE test (x Int32) ENGINE = Memory", - "sentry.origin": "auto.db.clickhouse_driver", - "server.address": "localhost", - "server.port": 9000, - }, - "trace_id": trace_id, - "parent_span_id": span_id, + assert event["breadcrumbs"]["values"] == expected_breadcrumbs + + +def test_clickhouse_client_breadcrumbs_span_streaming_with_data_collection_default( + sentry_init, capture_events +) -> None: + sentry_init( + integrations=[ClickhouseDriverIntegration()], + trace_lifecycle="stream", + _experiments={"data_collection": {}}, + ) + events = capture_events() + + client = Client("localhost") + client.execute("DROP TABLE IF EXISTS test") + client.execute("CREATE TABLE test (x Int32) ENGINE = Memory") + client.execute("INSERT INTO test (x) VALUES", [{"x": 100}]) + client.execute("INSERT INTO test (x) VALUES", [[170], [200]]) + + res = client.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) + assert res[0][0] == 370 + + capture_message("hi") + + (event,) = events + + expected_breadcrumbs = [ + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, }, - { - "name": "INSERT INTO test (x) VALUES", - "attributes": { - "db.system.name": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.namespace": "", - "db.user": "default", - "sentry.op": "db", - "db.query.text": "INSERT INTO test (x) VALUES", - "sentry.origin": "auto.db.clickhouse_driver", - "server.address": "localhost", - "server.port": 9000, - }, - "trace_id": trace_id, - "parent_span_id": span_id, + "message": "DROP TABLE IF EXISTS test", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, }, - { - "name": "INSERT INTO test (x) VALUES", - "attributes": { - "db.system.name": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.namespace": "", - "db.user": "default", - "sentry.op": "db", - "db.query.text": "INSERT INTO test (x) VALUES", - "sentry.origin": "auto.db.clickhouse_driver", - "server.address": "localhost", - "server.port": 9000, - }, - "trace_id": trace_id, - "parent_span_id": span_id, + "message": "CREATE TABLE test (x Int32) ENGINE = Memory", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, }, - { - "name": "SELECT sum(x) FROM test WHERE x > 150", - "attributes": { - "db.system.name": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.namespace": "", - "db.user": "default", - "sentry.op": "db", - "db.query.text": "SELECT sum(x) FROM test WHERE x > 150", - "sentry.origin": "auto.db.clickhouse_driver", - "server.address": "localhost", - "server.port": 9000, - }, - "trace_id": trace_id, - "parent_span_id": span_id, + "message": "INSERT INTO test (x) VALUES", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, }, - { - "name": "custom parent", - "attributes": [], - "trace_id": trace_id, + "message": "INSERT INTO test (x) VALUES", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, }, - ] + "message": "SELECT sum(x) FROM test WHERE x > 150", + "type": "default", + }, + ] + + for crumb in expected_breadcrumbs: + crumb["data"] = ApproxDict(crumb["data"]) + + for crumb in event["breadcrumbs"]["values"]: + crumb.pop("timestamp", None) + + assert event["breadcrumbs"]["values"] == expected_breadcrumbs + + # In span streaming mode, db.params is never present + for crumb in event["breadcrumbs"]["values"]: + assert "db.params" not in crumb["data"] + + +def test_clickhouse_client_span_streaming_with_data_collection( + sentry_init, capture_items +) -> None: + # Streamed spans never carry db.params/db.result, regardless of the + # data_collection configuration + sentry_init( + integrations=[ClickhouseDriverIntegration()], + traces_sample_rate=1.0, + trace_lifecycle="stream", + _experiments={ + "data_collection": {"database_query_data": True}, + }, + ) + items = capture_items("span") - for span in expected_spans: - span["attributes"] = ApproxDict(span["attributes"]) + with sentry_sdk.traces.start_span(name="custom parent"): + client = Client("localhost") + client.execute("DROP TABLE IF EXISTS test") + client.execute("CREATE TABLE test (x Int32) ENGINE = Memory") + client.execute("INSERT INTO test (x) VALUES", [{"x": 100}]) + client.execute("INSERT INTO test (x) VALUES", ((i,) for i in range(3))) - for span in spans: - span.pop("span_id", None) - span.pop("start_timestamp", None) - span.pop("end_timestamp", None) - span.pop("is_segment", None) - span.pop("status", None) + res = client.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 2}) + assert res[0][0] == 100 - assert spans == expected_spans - else: - events = capture_events() + sentry_sdk.flush() - transaction_trace_id = None - transaction_span_id = None + spans = [item.payload for item in items] + assert len(spans) > 1 # sanity check that db spans were actually streamed - with start_transaction(name="test_clickhouse_transaction") as transaction: - transaction_trace_id = transaction.trace_id - transaction_span_id = transaction.span_id + for span in spans: + attribute_keys = { + attribute["name"] if isinstance(attribute, dict) else attribute + for attribute in span.get("attributes", {}) + } + assert "db.params" not in attribute_keys + assert "db.result" not in attribute_keys - client = Client("localhost") - client.execute("DROP TABLE IF EXISTS test") - client.execute("CREATE TABLE test (x Int32) ENGINE = Memory") - client.execute("INSERT INTO test (x) VALUES", [{"x": 100}]) - client.execute("INSERT INTO test (x) VALUES", [[170], [200]]) - res = client.execute( - "SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150} - ) - assert res[0][0] == 370 +def test_clickhouse_client_send_data_generator_with_data_collection( + sentry_init, capture_events +) -> None: + sentry_init( + integrations=[ClickhouseDriverIntegration()], + _experiments={"data_collection": {"database_query_data": True}}, + ) + events = capture_events() - (event,) = events + client = Client("localhost") + client.execute("DROP TABLE IF EXISTS test") + client.execute("CREATE TABLE test (x Int32) ENGINE = Memory") + client.execute("INSERT INTO test (x) VALUES", ((i,) for i in range(3))) - expected_spans = [ - { - "op": "db", - "origin": "auto.db.clickhouse_driver", - "description": "DROP TABLE IF EXISTS test", - "data": { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - }, - "same_process_as_parent": True, - "trace_id": transaction_trace_id, - "parent_span_id": transaction_span_id, - }, - { - "op": "db", - "origin": "auto.db.clickhouse_driver", - "description": "CREATE TABLE test (x Int32) ENGINE = Memory", - "data": { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - }, - "same_process_as_parent": True, - "trace_id": transaction_trace_id, - "parent_span_id": transaction_span_id, - }, - { - "op": "db", - "origin": "auto.db.clickhouse_driver", - "description": "INSERT INTO test (x) VALUES", - "data": { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - }, - "same_process_as_parent": True, - "trace_id": transaction_trace_id, - "parent_span_id": transaction_span_id, - }, - { - "op": "db", - "origin": "auto.db.clickhouse_driver", - "description": "INSERT INTO test (x) VALUES", - "data": { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - }, - "same_process_as_parent": True, - "trace_id": transaction_trace_id, - "parent_span_id": transaction_span_id, - }, - { - "op": "db", - "origin": "auto.db.clickhouse_driver", - "description": "SELECT sum(x) FROM test WHERE x > 150", - "data": { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - }, - "same_process_as_parent": True, - "trace_id": transaction_trace_id, - "parent_span_id": transaction_span_id, - }, - ] + res = client.execute("SELECT sum(x) FROM test") + assert res[0][0] == 3 - if not EXPECT_PARAMS_IN_SELECT: - expected_spans[-1]["data"].pop("db.params", None) + capture_message("hi") - for span in expected_spans: - span["data"] = ApproxDict(span["data"]) + (event,) = events - for span in event["spans"]: - span.pop("span_id", None) - span.pop("start_timestamp", None) - span.pop("timestamp", None) + (insert_breadcrumb,) = [ + crumb + for crumb in event["breadcrumbs"]["values"] + if crumb["message"] == "INSERT INTO test (x) VALUES" + ] - assert event["spans"] == expected_spans + assert insert_breadcrumb["data"]["db.params"] == [[0], [1], [2]] -def test_clickhouse_spans_with_generator(sentry_init, capture_events): +def test_clickhouse_client_send_data_generator_with_data_collection_disabled( + sentry_init, capture_events +) -> None: sentry_init( integrations=[ClickhouseDriverIntegration()], - send_default_pii=True, - traces_sample_rate=1.0, + _experiments={"data_collection": {"database_query_data": False}}, ) events = capture_events() - # Use a generator to test that the integration obtains values from the generator, - # without consuming the generator. - values = ({"x": i} for i in range(3)) + client = Client("localhost") + client.execute("DROP TABLE IF EXISTS test") + client.execute("CREATE TABLE test (x Int32) ENGINE = Memory") + client.execute("INSERT INTO test (x) VALUES", ((i,) for i in range(3))) - with start_transaction(name="test_clickhouse_transaction"): - client = Client("localhost") - client.execute("DROP TABLE IF EXISTS test") - client.execute("CREATE TABLE test (x Int32) ENGINE = Memory") - client.execute("INSERT INTO test (x) VALUES", values) - res = client.execute("SELECT x FROM test") + res = client.execute("SELECT sum(x) FROM test") + assert res[0][0] == 3 - # Verify that the integration did not consume the generator - assert res == [(0,), (1,), (2,)] + capture_message("hi") (event,) = events - spans = event["spans"] - [span] = [ - span for span in spans if span["description"] == "INSERT INTO test (x) VALUES" + (insert_breadcrumb,) = [ + crumb + for crumb in event["breadcrumbs"]["values"] + if crumb["message"] == "INSERT INTO test (x) VALUES" ] - assert span["data"]["db.params"] == [{"x": 0}, {"x": 1}, {"x": 2}] + assert "db.params" not in insert_breadcrumb["data"] + + +def test_clickhouse_client_send_data_generator_span_streaming_with_data_collection( + sentry_init, capture_events +) -> None: + sentry_init( + integrations=[ClickhouseDriverIntegration()], + trace_lifecycle="stream", + _experiments={"data_collection": {"database_query_data": True}}, + ) + events = capture_events() + + client = Client("localhost") + client.execute("DROP TABLE IF EXISTS test") + client.execute("CREATE TABLE test (x Int32) ENGINE = Memory") + client.execute("INSERT INTO test (x) VALUES", ((i,) for i in range(3))) + + res = client.execute("SELECT sum(x) FROM test") + assert res[0][0] == 3 + + capture_message("hi") + + (event,) = events + + (insert_breadcrumb,) = [ + crumb + for crumb in event["breadcrumbs"]["values"] + if crumb["message"] == "INSERT INTO test (x) VALUES" + ] + + assert "db.params" not in insert_breadcrumb["data"] + + +def test_clickhouse_client_send_data_generator_span_streaming_with_data_collection_disabled( + sentry_init, capture_events +) -> None: + sentry_init( + integrations=[ClickhouseDriverIntegration()], + trace_lifecycle="stream", + _experiments={"data_collection": {"database_query_data": False}}, + ) + events = capture_events() + + client = Client("localhost") + client.execute("DROP TABLE IF EXISTS test") + client.execute("CREATE TABLE test (x Int32) ENGINE = Memory") + client.execute("INSERT INTO test (x) VALUES", ((i,) for i in range(3))) + + res = client.execute("SELECT sum(x) FROM test") + assert res[0][0] == 3 + + capture_message("hi") + + (event,) = events + + (insert_breadcrumb,) = [ + crumb + for crumb in event["breadcrumbs"]["values"] + if crumb["message"] == "INSERT INTO test (x) VALUES" + ] + + assert "db.params" not in insert_breadcrumb["data"] @pytest.mark.parametrize("span_streaming", [True, False]) -def test_clickhouse_client_spans_with_pii( +def test_clickhouse_client_spans( sentry_init, capture_events, capture_items, - capture_envelopes, span_streaming, ): sentry_init( @@ -1042,7 +1445,6 @@ def test_clickhouse_client_spans_with_pii( "record_sql_params": True, }, traces_sample_rate=1.0, - send_default_pii=True, ) if span_streaming: items = capture_items("span") @@ -1073,13 +1475,14 @@ def test_clickhouse_client_spans_with_pii( "name": "DROP TABLE IF EXISTS test", "attributes": { "db.system.name": "clickhouse", + "db.driver.name": "clickhouse-driver", "db.namespace": "", "db.user": "default", + "sentry.op": "db", "db.query.text": "DROP TABLE IF EXISTS test", + "sentry.origin": "auto.db.clickhouse_driver", "server.address": "localhost", "server.port": 9000, - "thread.id": mock.ANY, - "thread.name": mock.ANY, }, "trace_id": trace_id, "parent_span_id": span_id, @@ -1088,13 +1491,14 @@ def test_clickhouse_client_spans_with_pii( "name": "CREATE TABLE test (x Int32) ENGINE = Memory", "attributes": { "db.system.name": "clickhouse", + "db.driver.name": "clickhouse-driver", "db.namespace": "", "db.user": "default", + "sentry.op": "db", "db.query.text": "CREATE TABLE test (x Int32) ENGINE = Memory", + "sentry.origin": "auto.db.clickhouse_driver", "server.address": "localhost", "server.port": 9000, - "thread.id": mock.ANY, - "thread.name": mock.ANY, }, "trace_id": trace_id, "parent_span_id": span_id, @@ -1103,13 +1507,14 @@ def test_clickhouse_client_spans_with_pii( "name": "INSERT INTO test (x) VALUES", "attributes": { "db.system.name": "clickhouse", + "db.driver.name": "clickhouse-driver", "db.namespace": "", "db.user": "default", + "sentry.op": "db", "db.query.text": "INSERT INTO test (x) VALUES", + "sentry.origin": "auto.db.clickhouse_driver", "server.address": "localhost", "server.port": 9000, - "thread.id": mock.ANY, - "thread.name": mock.ANY, }, "trace_id": trace_id, "parent_span_id": span_id, @@ -1118,9 +1523,12 @@ def test_clickhouse_client_spans_with_pii( "name": "INSERT INTO test (x) VALUES", "attributes": { "db.system.name": "clickhouse", + "db.driver.name": "clickhouse-driver", "db.namespace": "", "db.user": "default", + "sentry.op": "db", "db.query.text": "INSERT INTO test (x) VALUES", + "sentry.origin": "auto.db.clickhouse_driver", "server.address": "localhost", "server.port": 9000, }, @@ -1131,9 +1539,12 @@ def test_clickhouse_client_spans_with_pii( "name": "SELECT sum(x) FROM test WHERE x > 150", "attributes": { "db.system.name": "clickhouse", + "db.driver.name": "clickhouse-driver", "db.namespace": "", "db.user": "default", + "sentry.op": "db", "db.query.text": "SELECT sum(x) FROM test WHERE x > 150", + "sentry.origin": "auto.db.clickhouse_driver", "server.address": "localhost", "server.port": 9000, }, @@ -1188,11 +1599,11 @@ def test_clickhouse_client_spans_with_pii( "description": "DROP TABLE IF EXISTS test", "data": { "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", "db.name": "", "db.user": "default", "server.address": "localhost", "server.port": 9000, - "db.result": [], }, "same_process_as_parent": True, "trace_id": transaction_trace_id, @@ -1204,11 +1615,11 @@ def test_clickhouse_client_spans_with_pii( "description": "CREATE TABLE test (x Int32) ENGINE = Memory", "data": { "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", "db.name": "", "db.user": "default", "server.address": "localhost", "server.port": 9000, - "db.result": [], }, "same_process_as_parent": True, "trace_id": transaction_trace_id, @@ -1220,11 +1631,11 @@ def test_clickhouse_client_spans_with_pii( "description": "INSERT INTO test (x) VALUES", "data": { "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", "db.name": "", "db.user": "default", "server.address": "localhost", "server.port": 9000, - "db.params": [{"x": 100}], }, "same_process_as_parent": True, "trace_id": transaction_trace_id, @@ -1236,11 +1647,11 @@ def test_clickhouse_client_spans_with_pii( "description": "INSERT INTO test (x) VALUES", "data": { "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", "db.name": "", "db.user": "default", "server.address": "localhost", "server.port": 9000, - "db.params": [[170], [200]], }, "same_process_as_parent": True, "trace_id": transaction_trace_id, @@ -1252,12 +1663,11 @@ def test_clickhouse_client_spans_with_pii( "description": "SELECT sum(x) FROM test WHERE x > 150", "data": { "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", "db.name": "", "db.user": "default", "server.address": "localhost", "server.port": 9000, - "db.params": {"minv": 150}, - "db.result": [[370]], }, "same_process_as_parent": True, "trace_id": transaction_trace_id, @@ -1279,212 +1689,40 @@ def test_clickhouse_client_spans_with_pii( assert event["spans"] == expected_spans -def test_clickhouse_dbapi_breadcrumbs(sentry_init, capture_events) -> None: - sentry_init( - integrations=[ClickhouseDriverIntegration()], - ) - events = capture_events() - - conn = connect("clickhouse://localhost") - cursor = conn.cursor() - cursor.execute("DROP TABLE IF EXISTS test") - cursor.execute("CREATE TABLE test (x Int32) ENGINE = Memory") - cursor.executemany("INSERT INTO test (x) VALUES", [{"x": 100}]) - cursor.executemany("INSERT INTO test (x) VALUES", [[170], [200]]) - cursor.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) - res = cursor.fetchall() - - assert res[0][0] == 370 - - capture_message("hi") - - (event,) = events - - expected_breadcrumbs = [ - { - "category": "query", - "data": { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - }, - "message": "DROP TABLE IF EXISTS test", - "type": "default", - }, - { - "category": "query", - "data": { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - }, - "message": "CREATE TABLE test (x Int32) ENGINE = Memory", - "type": "default", - }, - { - "category": "query", - "data": { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - }, - "message": "INSERT INTO test (x) VALUES", - "type": "default", - }, - { - "category": "query", - "data": { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - }, - "message": "INSERT INTO test (x) VALUES", - "type": "default", - }, - { - "category": "query", - "data": { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - }, - "message": "SELECT sum(x) FROM test WHERE x > 150", - "type": "default", - }, - ] - - if not EXPECT_PARAMS_IN_SELECT: - expected_breadcrumbs[-1]["data"].pop("db.params", None) - - for crumb in expected_breadcrumbs: - crumb["data"] = ApproxDict(crumb["data"]) - - for crumb in event["breadcrumbs"]["values"]: - crumb.pop("timestamp", None) - - assert event["breadcrumbs"]["values"] == expected_breadcrumbs - - -def test_clickhouse_dbapi_breadcrumbs_with_pii(sentry_init, capture_events) -> None: +def test_clickhouse_spans_with_generator(sentry_init, capture_events): sentry_init( integrations=[ClickhouseDriverIntegration()], send_default_pii=True, + traces_sample_rate=1.0, ) events = capture_events() - conn = connect("clickhouse://localhost") - cursor = conn.cursor() - cursor.execute("DROP TABLE IF EXISTS test") - cursor.execute("CREATE TABLE test (x Int32) ENGINE = Memory") - cursor.executemany("INSERT INTO test (x) VALUES", [{"x": 100}]) - cursor.executemany("INSERT INTO test (x) VALUES", [[170], [200]]) - cursor.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) - res = cursor.fetchall() + # Use a generator to test that the integration obtains values from the generator, + # without consuming the generator. + values = ({"x": i} for i in range(3)) - assert res[0][0] == 370 + with start_transaction(name="test_clickhouse_transaction"): + client = Client("localhost") + client.execute("DROP TABLE IF EXISTS test") + client.execute("CREATE TABLE test (x Int32) ENGINE = Memory") + client.execute("INSERT INTO test (x) VALUES", values) + res = client.execute("SELECT x FROM test") - capture_message("hi") + # Verify that the integration did not consume the generator + assert res == [(0,), (1,), (2,)] (event,) = events + spans = event["spans"] - expected_breadcrumbs = [ - { - "category": "query", - "data": { - "db.system": "clickhouse", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - "db.result": [[], []], - }, - "message": "DROP TABLE IF EXISTS test", - "type": "default", - }, - { - "category": "query", - "data": { - "db.system": "clickhouse", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - "db.result": [[], []], - }, - "message": "CREATE TABLE test (x Int32) ENGINE = Memory", - "type": "default", - }, - { - "category": "query", - "data": { - "db.system": "clickhouse", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - "db.params": [{"x": 100}], - }, - "message": "INSERT INTO test (x) VALUES", - "type": "default", - }, - { - "category": "query", - "data": { - "db.system": "clickhouse", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - "db.params": [[170], [200]], - }, - "message": "INSERT INTO test (x) VALUES", - "type": "default", - }, - { - "category": "query", - "data": { - "db.system": "clickhouse", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - "db.params": {"minv": 150}, - "db.result": [[["370"]], [["'sum(x)'", "'Int64'"]]], - }, - "message": "SELECT sum(x) FROM test WHERE x > 150", - "type": "default", - }, + [span] = [ + span for span in spans if span["description"] == "INSERT INTO test (x) VALUES" ] - if not EXPECT_PARAMS_IN_SELECT: - expected_breadcrumbs[-1]["data"].pop("db.params", None) - - for crumb in expected_breadcrumbs: - crumb["data"] = ApproxDict(crumb["data"]) - - for crumb in event["breadcrumbs"]["values"]: - crumb.pop("timestamp", None) - - assert event["breadcrumbs"]["values"] == expected_breadcrumbs + assert span["data"]["db.params"] == [{"x": 0}, {"x": 1}, {"x": 2}] @pytest.mark.parametrize("span_streaming", [True, False]) -def test_clickhouse_dbapi_spans( +def test_clickhouse_client_spans_with_pii( sentry_init, capture_events, capture_items, @@ -1498,6 +1736,7 @@ def test_clickhouse_dbapi_spans( "record_sql_params": True, }, traces_sample_rate=1.0, + send_default_pii=True, ) if span_streaming: items = capture_items("span") @@ -1509,14 +1748,16 @@ def test_clickhouse_dbapi_spans( trace_id = span.trace_id span_id = span.span_id - conn = connect("clickhouse://localhost") - cursor = conn.cursor() - cursor.execute("DROP TABLE IF EXISTS test") - cursor.execute("CREATE TABLE test (x Int32) ENGINE = Memory") - cursor.executemany("INSERT INTO test (x) VALUES", [{"x": 100}]) - cursor.executemany("INSERT INTO test (x) VALUES", [[170], [200]]) - cursor.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) - res = cursor.fetchall() + client = Client("localhost") + client.execute("DROP TABLE IF EXISTS test") + client.execute("CREATE TABLE test (x Int32) ENGINE = Memory") + client.execute("INSERT INTO test (x) VALUES", [{"x": 100}]) + client.execute("INSERT INTO test (x) VALUES", [[170], [200]]) + + res = client.execute( + "SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150} + ) + assert res[0][0] == 370 sentry_sdk.flush() spans = [item.payload for item in items] @@ -1526,14 +1767,13 @@ def test_clickhouse_dbapi_spans( "name": "DROP TABLE IF EXISTS test", "attributes": { "db.system.name": "clickhouse", - "db.driver.name": "clickhouse-driver", "db.namespace": "", "db.user": "default", "db.query.text": "DROP TABLE IF EXISTS test", - "sentry.op": "db", - "sentry.origin": "auto.db.clickhouse_driver", "server.address": "localhost", "server.port": 9000, + "thread.id": mock.ANY, + "thread.name": mock.ANY, }, "trace_id": trace_id, "parent_span_id": span_id, @@ -1542,14 +1782,13 @@ def test_clickhouse_dbapi_spans( "name": "CREATE TABLE test (x Int32) ENGINE = Memory", "attributes": { "db.system.name": "clickhouse", - "db.driver.name": "clickhouse-driver", "db.namespace": "", "db.user": "default", "db.query.text": "CREATE TABLE test (x Int32) ENGINE = Memory", - "sentry.op": "db", - "sentry.origin": "auto.db.clickhouse_driver", "server.address": "localhost", "server.port": 9000, + "thread.id": mock.ANY, + "thread.name": mock.ANY, }, "trace_id": trace_id, "parent_span_id": span_id, @@ -1558,14 +1797,13 @@ def test_clickhouse_dbapi_spans( "name": "INSERT INTO test (x) VALUES", "attributes": { "db.system.name": "clickhouse", - "db.driver.name": "clickhouse-driver", "db.namespace": "", "db.user": "default", "db.query.text": "INSERT INTO test (x) VALUES", - "sentry.op": "db", - "sentry.origin": "auto.db.clickhouse_driver", "server.address": "localhost", "server.port": 9000, + "thread.id": mock.ANY, + "thread.name": mock.ANY, }, "trace_id": trace_id, "parent_span_id": span_id, @@ -1574,12 +1812,9 @@ def test_clickhouse_dbapi_spans( "name": "INSERT INTO test (x) VALUES", "attributes": { "db.system.name": "clickhouse", - "db.driver.name": "clickhouse-driver", "db.namespace": "", "db.user": "default", "db.query.text": "INSERT INTO test (x) VALUES", - "sentry.op": "db", - "sentry.origin": "auto.db.clickhouse_driver", "server.address": "localhost", "server.port": 9000, }, @@ -1590,12 +1825,9 @@ def test_clickhouse_dbapi_spans( "name": "SELECT sum(x) FROM test WHERE x > 150", "attributes": { "db.system.name": "clickhouse", - "db.driver.name": "clickhouse-driver", "db.namespace": "", "db.user": "default", "db.query.text": "SELECT sum(x) FROM test WHERE x > 150", - "sentry.op": "db", - "sentry.origin": "auto.db.clickhouse_driver", "server.address": "localhost", "server.port": 9000, }, @@ -1630,16 +1862,16 @@ def test_clickhouse_dbapi_spans( transaction_trace_id = transaction.trace_id transaction_span_id = transaction.span_id - conn = connect("clickhouse://localhost") - cursor = conn.cursor() - cursor.execute("DROP TABLE IF EXISTS test") - cursor.execute("CREATE TABLE test (x Int32) ENGINE = Memory") - cursor.executemany("INSERT INTO test (x) VALUES", [{"x": 100}]) - cursor.executemany("INSERT INTO test (x) VALUES", [[170], [200]]) - cursor.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) - res = cursor.fetchall() + client = Client("localhost") + client.execute("DROP TABLE IF EXISTS test") + client.execute("CREATE TABLE test (x Int32) ENGINE = Memory") + client.execute("INSERT INTO test (x) VALUES", [{"x": 100}]) + client.execute("INSERT INTO test (x) VALUES", [[170], [200]]) - assert res[0][0] == 370 + res = client.execute( + "SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150} + ) + assert res[0][0] == 370 (event,) = events @@ -1650,11 +1882,11 @@ def test_clickhouse_dbapi_spans( "description": "DROP TABLE IF EXISTS test", "data": { "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", "db.name": "", "db.user": "default", "server.address": "localhost", "server.port": 9000, + "db.result": [], }, "same_process_as_parent": True, "trace_id": transaction_trace_id, @@ -1666,11 +1898,11 @@ def test_clickhouse_dbapi_spans( "description": "CREATE TABLE test (x Int32) ENGINE = Memory", "data": { "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", "db.name": "", "db.user": "default", "server.address": "localhost", "server.port": 9000, + "db.result": [], }, "same_process_as_parent": True, "trace_id": transaction_trace_id, @@ -1682,11 +1914,11 @@ def test_clickhouse_dbapi_spans( "description": "INSERT INTO test (x) VALUES", "data": { "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", "db.name": "", "db.user": "default", "server.address": "localhost", "server.port": 9000, + "db.params": [{"x": 100}], }, "same_process_as_parent": True, "trace_id": transaction_trace_id, @@ -1698,11 +1930,11 @@ def test_clickhouse_dbapi_spans( "description": "INSERT INTO test (x) VALUES", "data": { "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", "db.name": "", "db.user": "default", "server.address": "localhost", "server.port": 9000, + "db.params": [[170], [200]], }, "same_process_as_parent": True, "trace_id": transaction_trace_id, @@ -1714,11 +1946,12 @@ def test_clickhouse_dbapi_spans( "description": "SELECT sum(x) FROM test WHERE x > 150", "data": { "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", "db.name": "", "db.user": "default", "server.address": "localhost", "server.port": 9000, + "db.params": {"minv": 150}, + "db.result": [[370]], }, "same_process_as_parent": True, "trace_id": transaction_trace_id, @@ -1740,369 +1973,403 @@ def test_clickhouse_dbapi_spans( assert event["spans"] == expected_spans -@pytest.mark.parametrize("span_streaming", [True, False]) -def test_clickhouse_dbapi_spans_with_pii( - sentry_init, - capture_events, - capture_items, - capture_envelopes, - span_streaming, -): +def test_clickhouse_dbapi_breadcrumbs(sentry_init, capture_events) -> None: sentry_init( integrations=[ClickhouseDriverIntegration()], - trace_lifecycle="stream" if span_streaming else "static", - _experiments={ - "record_sql_params": True, - }, - traces_sample_rate=1.0, - send_default_pii=True, ) - if span_streaming: - items = capture_items("span") - - trace_id = None - span_id = None + events = capture_events() - with sentry_sdk.traces.start_span(name="custom parent") as span: - trace_id = span.trace_id - span_id = span.span_id + conn = connect("clickhouse://localhost") + cursor = conn.cursor() + cursor.execute("DROP TABLE IF EXISTS test") + cursor.execute("CREATE TABLE test (x Int32) ENGINE = Memory") + cursor.executemany("INSERT INTO test (x) VALUES", [{"x": 100}]) + cursor.executemany("INSERT INTO test (x) VALUES", [[170], [200]]) + cursor.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) + res = cursor.fetchall() - conn = connect("clickhouse://localhost") - cursor = conn.cursor() - cursor.execute("DROP TABLE IF EXISTS test") - cursor.execute("CREATE TABLE test (x Int32) ENGINE = Memory") - cursor.executemany("INSERT INTO test (x) VALUES", [{"x": 100}]) - cursor.executemany("INSERT INTO test (x) VALUES", [[170], [200]]) - cursor.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) - res = cursor.fetchall() + assert res[0][0] == 370 - assert res[0][0] == 370 + capture_message("hi") - sentry_sdk.flush() - spans = [item.payload for item in items] + (event,) = events - expected_spans = [ - { - "name": "DROP TABLE IF EXISTS test", - "attributes": { - "db.system.name": "clickhouse", - "db.namespace": "", - "db.user": "default", - "db.query.text": "DROP TABLE IF EXISTS test", - "sentry.op": "db", - "sentry.origin": "auto.db.clickhouse_driver", - "server.address": "localhost", - "server.port": 9000, - }, - "trace_id": trace_id, - "parent_span_id": span_id, - }, - { - "name": "CREATE TABLE test (x Int32) ENGINE = Memory", - "attributes": { - "db.system.name": "clickhouse", - "db.namespace": "", - "db.user": "default", - "db.query.text": "CREATE TABLE test (x Int32) ENGINE = Memory", - "sentry.op": "db", - "sentry.origin": "auto.db.clickhouse_driver", - "server.address": "localhost", - "server.port": 9000, - }, - "trace_id": trace_id, - "parent_span_id": span_id, + expected_breadcrumbs = [ + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, }, - { - "name": "INSERT INTO test (x) VALUES", - "attributes": { - "db.system.name": "clickhouse", - "db.namespace": "", - "db.user": "default", - "db.query.text": "INSERT INTO test (x) VALUES", - "sentry.op": "db", - "sentry.origin": "auto.db.clickhouse_driver", - "server.address": "localhost", - "server.port": 9000, - }, - "trace_id": trace_id, - "parent_span_id": span_id, + "message": "DROP TABLE IF EXISTS test", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, }, - { - "name": "INSERT INTO test (x) VALUES", - "attributes": { - "db.system.name": "clickhouse", - "db.namespace": "", - "db.user": "default", - "db.query.text": "INSERT INTO test (x) VALUES", - "sentry.op": "db", - "sentry.origin": "auto.db.clickhouse_driver", - "server.address": "localhost", - "server.port": 9000, - }, - "trace_id": trace_id, - "parent_span_id": span_id, + "message": "CREATE TABLE test (x Int32) ENGINE = Memory", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, }, - { - "name": "SELECT sum(x) FROM test WHERE x > 150", - "attributes": { - "db.system.name": "clickhouse", - "db.namespace": "", - "db.user": "default", - "db.query.text": "SELECT sum(x) FROM test WHERE x > 150", - "sentry.op": "db", - "sentry.origin": "auto.db.clickhouse_driver", - "server.address": "localhost", - "server.port": 9000, - }, - "trace_id": trace_id, - "parent_span_id": span_id, + "message": "INSERT INTO test (x) VALUES", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, }, - { - "name": "custom parent", - "attributes": [], - "trace_id": trace_id, + "message": "INSERT INTO test (x) VALUES", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, }, - ] + "message": "SELECT sum(x) FROM test WHERE x > 150", + "type": "default", + }, + ] - for span in expected_spans: - span["attributes"] = ApproxDict(span["attributes"]) + if not EXPECT_PARAMS_IN_SELECT: + expected_breadcrumbs[-1]["data"].pop("db.params", None) - for span in spans: - span.pop("span_id", None) - span.pop("start_timestamp", None) - span.pop("end_timestamp", None) - span.pop("is_segment", None) - span.pop("status", None) + for crumb in expected_breadcrumbs: + crumb["data"] = ApproxDict(crumb["data"]) + + for crumb in event["breadcrumbs"]["values"]: + crumb.pop("timestamp", None) - assert spans == expected_spans - else: - events = capture_events() + assert event["breadcrumbs"]["values"] == expected_breadcrumbs - transaction_trace_id = None - transaction_span_id = None - with start_transaction(name="test_clickhouse_transaction") as transaction: - transaction_trace_id = transaction.trace_id - transaction_span_id = transaction.span_id +def test_clickhouse_dbapi_breadcrumbs_span_streaming( + sentry_init, capture_events +) -> None: + sentry_init( + integrations=[ClickhouseDriverIntegration()], + trace_lifecycle="stream", + ) + events = capture_events() - conn = connect("clickhouse://localhost") - cursor = conn.cursor() - cursor.execute("DROP TABLE IF EXISTS test") - cursor.execute("CREATE TABLE test (x Int32) ENGINE = Memory") - cursor.executemany("INSERT INTO test (x) VALUES", [{"x": 100}]) - cursor.executemany("INSERT INTO test (x) VALUES", [[170], [200]]) - cursor.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) - res = cursor.fetchall() + conn = connect("clickhouse://localhost") + cursor = conn.cursor() + cursor.execute("DROP TABLE IF EXISTS test") + cursor.execute("CREATE TABLE test (x Int32) ENGINE = Memory") + cursor.executemany("INSERT INTO test (x) VALUES", [{"x": 100}]) + cursor.executemany("INSERT INTO test (x) VALUES", [[170], [200]]) + cursor.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) + res = cursor.fetchall() - assert res[0][0] == 370 + assert res[0][0] == 370 - (event,) = events + capture_message("hi") - expected_spans = [ - { - "op": "db", - "origin": "auto.db.clickhouse_driver", - "description": "DROP TABLE IF EXISTS test", - "data": { - "db.system": "clickhouse", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - "db.result": [[], []], - }, - "same_process_as_parent": True, - "trace_id": transaction_trace_id, - "parent_span_id": transaction_span_id, + (event,) = events + + expected_breadcrumbs = [ + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, }, - { - "op": "db", - "origin": "auto.db.clickhouse_driver", - "description": "CREATE TABLE test (x Int32) ENGINE = Memory", - "data": { - "db.system": "clickhouse", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - "db.result": [[], []], - }, - "same_process_as_parent": True, - "trace_id": transaction_trace_id, - "parent_span_id": transaction_span_id, + "message": "DROP TABLE IF EXISTS test", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, }, - { - "op": "db", - "origin": "auto.db.clickhouse_driver", - "description": "INSERT INTO test (x) VALUES", - "data": { - "db.system": "clickhouse", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - "db.params": [{"x": 100}], - }, - "same_process_as_parent": True, - "trace_id": transaction_trace_id, - "parent_span_id": transaction_span_id, + "message": "CREATE TABLE test (x Int32) ENGINE = Memory", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, }, - { - "op": "db", - "origin": "auto.db.clickhouse_driver", - "description": "INSERT INTO test (x) VALUES", - "data": { - "db.system": "clickhouse", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - "db.params": [[170], [200]], - }, - "same_process_as_parent": True, - "trace_id": transaction_trace_id, - "parent_span_id": transaction_span_id, + "message": "INSERT INTO test (x) VALUES", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, }, - { - "op": "db", - "origin": "auto.db.clickhouse_driver", - "description": "SELECT sum(x) FROM test WHERE x > 150", - "data": { - "db.system": "clickhouse", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - "db.params": {"minv": 150}, - "db.result": [[[370]], [["sum(x)", "Int64"]]], - }, - "same_process_as_parent": True, - "trace_id": transaction_trace_id, - "parent_span_id": transaction_span_id, + "message": "INSERT INTO test (x) VALUES", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, }, - ] + "message": "SELECT sum(x) FROM test WHERE x > 150", + "type": "default", + }, + ] - if not EXPECT_PARAMS_IN_SELECT: - expected_spans[-1]["data"].pop("db.params", None) + for crumb in expected_breadcrumbs: + crumb["data"] = ApproxDict(crumb["data"]) - for span in expected_spans: - span["data"] = ApproxDict(span["data"]) + for crumb in event["breadcrumbs"]["values"]: + crumb.pop("timestamp", None) - for span in event["spans"]: - span.pop("span_id", None) - span.pop("start_timestamp", None) - span.pop("timestamp", None) + assert event["breadcrumbs"]["values"] == expected_breadcrumbs - assert event["spans"] == expected_spans + # In span streaming mode, db.params is never present + for crumb in event["breadcrumbs"]["values"]: + assert "db.params" not in crumb["data"] -@pytest.mark.parametrize("span_streaming", [True, False]) -def test_span_origin( - sentry_init, - capture_events, - capture_items, - capture_envelopes, - span_streaming, -): +def test_clickhouse_dbapi_breadcrumbs_with_pii(sentry_init, capture_events) -> None: sentry_init( integrations=[ClickhouseDriverIntegration()], - traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", + send_default_pii=True, ) - if span_streaming: - items = capture_items("span") + events = capture_events() - with sentry_sdk.traces.start_span(name="custom parent"): - conn = connect("clickhouse://localhost") - cursor = conn.cursor() - cursor.execute("SELECT 1") + conn = connect("clickhouse://localhost") + cursor = conn.cursor() + cursor.execute("DROP TABLE IF EXISTS test") + cursor.execute("CREATE TABLE test (x Int32) ENGINE = Memory") + cursor.executemany("INSERT INTO test (x) VALUES", [{"x": 100}]) + cursor.executemany("INSERT INTO test (x) VALUES", [[170], [200]]) + cursor.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) + res = cursor.fetchall() - sentry_sdk.flush() - spans = [item.payload for item in items] + assert res[0][0] == 370 - assert spans[1]["attributes"]["sentry.origin"] == "manual" - assert spans[0]["attributes"]["sentry.origin"] == "auto.db.clickhouse_driver" - else: - events = capture_events() + capture_message("hi") - with start_transaction(name="test_clickhouse_transaction"): - conn = connect("clickhouse://localhost") - cursor = conn.cursor() - cursor.execute("SELECT 1") + (event,) = events + + expected_breadcrumbs = [ + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + "db.result": [[], []], + }, + "message": "DROP TABLE IF EXISTS test", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + "db.result": [[], []], + }, + "message": "CREATE TABLE test (x Int32) ENGINE = Memory", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + "db.params": [{"x": 100}], + }, + "message": "INSERT INTO test (x) VALUES", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + "db.params": [[170], [200]], + }, + "message": "INSERT INTO test (x) VALUES", + "type": "default", + }, + { + "category": "query", + "data": { + "db.system": "clickhouse", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + "db.params": {"minv": 150}, + "db.result": [[["370"]], [["'sum(x)'", "'Int64'"]]], + }, + "message": "SELECT sum(x) FROM test WHERE x > 150", + "type": "default", + }, + ] - (event,) = events + if not EXPECT_PARAMS_IN_SELECT: + expected_breadcrumbs[-1]["data"].pop("db.params", None) - assert event["contexts"]["trace"]["origin"] == "manual" - assert event["spans"][0]["origin"] == "auto.db.clickhouse_driver" + for crumb in expected_breadcrumbs: + crumb["data"] = ApproxDict(crumb["data"]) + for crumb in event["breadcrumbs"]["values"]: + crumb.pop("timestamp", None) -# ---- Span-first (streaming) breadcrumb tests ---- -# These mirror the breadcrumb tests above but specifically target the -# span-first code path (trace_lifecycle="stream"). In span-first mode -# breadcrumbs are emitted from the dedicated breadcrumb_data dict set on -# the connection, NOT derived from the span. db.params and db.result are -# never attached to breadcrumbs in this mode. + assert event["breadcrumbs"]["values"] == expected_breadcrumbs -def test_clickhouse_client_breadcrumbs_span_streaming( +def test_clickhouse_dbapi_breadcrumbs_span_streaming_with_pii( sentry_init, capture_events ) -> None: sentry_init( integrations=[ClickhouseDriverIntegration()], + send_default_pii=True, trace_lifecycle="stream", ) events = capture_events() - client = Client("localhost") - client.execute("DROP TABLE IF EXISTS test") - client.execute("CREATE TABLE test (x Int32) ENGINE = Memory") - client.execute("INSERT INTO test (x) VALUES", [{"x": 100}]) - client.execute("INSERT INTO test (x) VALUES", [[170], [200]]) + conn = connect("clickhouse://localhost") + cursor = conn.cursor() + cursor.execute("DROP TABLE IF EXISTS test") + cursor.execute("CREATE TABLE test (x Int32) ENGINE = Memory") + cursor.executemany("INSERT INTO test (x) VALUES", [{"x": 100}]) + cursor.executemany("INSERT INTO test (x) VALUES", [[170], [200]]) + cursor.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) + res = cursor.fetchall() - res = client.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) assert res[0][0] == 370 capture_message("hi") (event,) = events - expected_base = { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - } - expected_breadcrumbs = [ { "category": "query", - "data": {**expected_base}, + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + }, "message": "DROP TABLE IF EXISTS test", "type": "default", }, { "category": "query", - "data": {**expected_base}, + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + }, "message": "CREATE TABLE test (x Int32) ENGINE = Memory", "type": "default", }, { "category": "query", - "data": {**expected_base}, + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + }, "message": "INSERT INTO test (x) VALUES", "type": "default", }, { "category": "query", - "data": {**expected_base}, + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + }, "message": "INSERT INTO test (x) VALUES", "type": "default", }, { "category": "query", - "data": {**expected_base}, + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + }, "message": "SELECT sum(x) FROM test WHERE x > 150", "type": "default", }, @@ -2114,303 +2381,561 @@ def test_clickhouse_client_breadcrumbs_span_streaming( for crumb in event["breadcrumbs"]["values"]: crumb.pop("timestamp", None) - actual_query_breadcrumbs = [ - breadcrumb - for breadcrumb in event["breadcrumbs"]["values"] - if breadcrumb["category"] == "query" - ] - - assert actual_query_breadcrumbs == expected_breadcrumbs + assert event["breadcrumbs"]["values"] == expected_breadcrumbs - # Span-first breadcrumbs never carry db.params - for crumb in actual_query_breadcrumbs: + # In span streaming mode, db.params is never present + for crumb in event["breadcrumbs"]["values"]: assert "db.params" not in crumb["data"] -@pytest.mark.parametrize("send_default_pii", [True, False]) -def test_clickhouse_client_breadcrumbs_span_streaming_with_pii( - sentry_init, capture_events, send_default_pii -) -> None: +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_clickhouse_dbapi_spans( + sentry_init, + capture_events, + capture_items, + capture_envelopes, + span_streaming, +): sentry_init( integrations=[ClickhouseDriverIntegration()], - send_default_pii=send_default_pii, - trace_lifecycle="stream", + trace_lifecycle="stream" if span_streaming else "static", + _experiments={ + "record_sql_params": True, + }, + traces_sample_rate=1.0, ) - events = capture_events() - - client = Client("localhost") - client.execute("DROP TABLE IF EXISTS test") - client.execute("CREATE TABLE test (x Int32) ENGINE = Memory") - client.execute("INSERT INTO test (x) VALUES", [{"x": 100}]) - client.execute("INSERT INTO test (x) VALUES", [[170], [200]]) - - res = client.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) - assert res[0][0] == 370 + if span_streaming: + items = capture_items("span") - capture_message("hi") + trace_id = None + span_id = None - (event,) = events + with sentry_sdk.traces.start_span(name="custom parent") as span: + trace_id = span.trace_id + span_id = span.span_id - expected_base = { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - } + conn = connect("clickhouse://localhost") + cursor = conn.cursor() + cursor.execute("DROP TABLE IF EXISTS test") + cursor.execute("CREATE TABLE test (x Int32) ENGINE = Memory") + cursor.executemany("INSERT INTO test (x) VALUES", [{"x": 100}]) + cursor.executemany("INSERT INTO test (x) VALUES", [[170], [200]]) + cursor.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) + res = cursor.fetchall() - actual_query_breadcrumbs = [ - crumb - for crumb in event["breadcrumbs"]["values"] - if crumb["category"] == "query" - ] + sentry_sdk.flush() + spans = [item.payload for item in items] - assert len(actual_query_breadcrumbs) == 5 + expected_spans = [ + { + "name": "DROP TABLE IF EXISTS test", + "attributes": { + "db.system.name": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.namespace": "", + "db.user": "default", + "db.query.text": "DROP TABLE IF EXISTS test", + "sentry.op": "db", + "sentry.origin": "auto.db.clickhouse_driver", + "server.address": "localhost", + "server.port": 9000, + }, + "trace_id": trace_id, + "parent_span_id": span_id, + }, + { + "name": "CREATE TABLE test (x Int32) ENGINE = Memory", + "attributes": { + "db.system.name": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.namespace": "", + "db.user": "default", + "db.query.text": "CREATE TABLE test (x Int32) ENGINE = Memory", + "sentry.op": "db", + "sentry.origin": "auto.db.clickhouse_driver", + "server.address": "localhost", + "server.port": 9000, + }, + "trace_id": trace_id, + "parent_span_id": span_id, + }, + { + "name": "INSERT INTO test (x) VALUES", + "attributes": { + "db.system.name": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.namespace": "", + "db.user": "default", + "db.query.text": "INSERT INTO test (x) VALUES", + "sentry.op": "db", + "sentry.origin": "auto.db.clickhouse_driver", + "server.address": "localhost", + "server.port": 9000, + }, + "trace_id": trace_id, + "parent_span_id": span_id, + }, + { + "name": "INSERT INTO test (x) VALUES", + "attributes": { + "db.system.name": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.namespace": "", + "db.user": "default", + "db.query.text": "INSERT INTO test (x) VALUES", + "sentry.op": "db", + "sentry.origin": "auto.db.clickhouse_driver", + "server.address": "localhost", + "server.port": 9000, + }, + "trace_id": trace_id, + "parent_span_id": span_id, + }, + { + "name": "SELECT sum(x) FROM test WHERE x > 150", + "attributes": { + "db.system.name": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.namespace": "", + "db.user": "default", + "db.query.text": "SELECT sum(x) FROM test WHERE x > 150", + "sentry.op": "db", + "sentry.origin": "auto.db.clickhouse_driver", + "server.address": "localhost", + "server.port": 9000, + }, + "trace_id": trace_id, + "parent_span_id": span_id, + }, + { + "name": "custom parent", + "attributes": [], + "trace_id": trace_id, + }, + ] - for crumb in actual_query_breadcrumbs: - crumb.pop("timestamp", None) - assert crumb["type"] == "default" - assert crumb["category"] == "query" - assert crumb["data"] == ApproxDict(expected_base) - - assert actual_query_breadcrumbs[0]["message"] == "DROP TABLE IF EXISTS test" - assert ( - actual_query_breadcrumbs[1]["message"] - == "CREATE TABLE test (x Int32) ENGINE = Memory" - ) - assert actual_query_breadcrumbs[2]["message"] == "INSERT INTO test (x) VALUES" - assert actual_query_breadcrumbs[3]["message"] == "INSERT INTO test (x) VALUES" - assert ( - actual_query_breadcrumbs[4]["message"] - == "SELECT sum(x) FROM test WHERE x > 150" - ) + for span in expected_spans: + span["attributes"] = ApproxDict(span["attributes"]) - # Span-first breadcrumbs never carry db.params regardless of PII setting - for crumb in actual_query_breadcrumbs: - assert "db.params" not in crumb["data"] + for span in spans: + span.pop("span_id", None) + span.pop("start_timestamp", None) + span.pop("end_timestamp", None) + span.pop("is_segment", None) + span.pop("status", None) + assert spans == expected_spans + else: + events = capture_events() -@pytest.mark.parametrize( - "data_collection_setting", - [ - pytest.param({"database_query_data": True}, id="enabled"), - pytest.param({"database_query_data": False}, id="disabled"), - pytest.param({}, id="default"), - ], -) -def test_clickhouse_client_breadcrumbs_span_streaming_data_collection( - sentry_init, capture_events, data_collection_setting -) -> None: - sentry_init( - integrations=[ClickhouseDriverIntegration()], - trace_lifecycle="stream", - _experiments={"data_collection": data_collection_setting}, - ) - events = capture_events() + transaction_trace_id = None + transaction_span_id = None - client = Client("localhost") - client.execute("DROP TABLE IF EXISTS test") - client.execute("CREATE TABLE test (x Int32) ENGINE = Memory") - client.execute("INSERT INTO test (x) VALUES", [{"x": 100}]) - client.execute("INSERT INTO test (x) VALUES", [[170], [200]]) + with start_transaction(name="test_clickhouse_transaction") as transaction: + transaction_trace_id = transaction.trace_id + transaction_span_id = transaction.span_id - res = client.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) - assert res[0][0] == 370 + conn = connect("clickhouse://localhost") + cursor = conn.cursor() + cursor.execute("DROP TABLE IF EXISTS test") + cursor.execute("CREATE TABLE test (x Int32) ENGINE = Memory") + cursor.executemany("INSERT INTO test (x) VALUES", [{"x": 100}]) + cursor.executemany("INSERT INTO test (x) VALUES", [[170], [200]]) + cursor.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) + res = cursor.fetchall() - capture_message("hi") + assert res[0][0] == 370 - (event,) = events + (event,) = events - expected_base = { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - } + expected_spans = [ + { + "op": "db", + "origin": "auto.db.clickhouse_driver", + "description": "DROP TABLE IF EXISTS test", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + }, + "same_process_as_parent": True, + "trace_id": transaction_trace_id, + "parent_span_id": transaction_span_id, + }, + { + "op": "db", + "origin": "auto.db.clickhouse_driver", + "description": "CREATE TABLE test (x Int32) ENGINE = Memory", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + }, + "same_process_as_parent": True, + "trace_id": transaction_trace_id, + "parent_span_id": transaction_span_id, + }, + { + "op": "db", + "origin": "auto.db.clickhouse_driver", + "description": "INSERT INTO test (x) VALUES", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + }, + "same_process_as_parent": True, + "trace_id": transaction_trace_id, + "parent_span_id": transaction_span_id, + }, + { + "op": "db", + "origin": "auto.db.clickhouse_driver", + "description": "INSERT INTO test (x) VALUES", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + }, + "same_process_as_parent": True, + "trace_id": transaction_trace_id, + "parent_span_id": transaction_span_id, + }, + { + "op": "db", + "origin": "auto.db.clickhouse_driver", + "description": "SELECT sum(x) FROM test WHERE x > 150", + "data": { + "db.system": "clickhouse", + "db.driver.name": "clickhouse-driver", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + }, + "same_process_as_parent": True, + "trace_id": transaction_trace_id, + "parent_span_id": transaction_span_id, + }, + ] - actual_query_breadcrumbs = [ - crumb - for crumb in event["breadcrumbs"]["values"] - if crumb["category"] == "query" - ] + if not EXPECT_PARAMS_IN_SELECT: + expected_spans[-1]["data"].pop("db.params", None) - assert len(actual_query_breadcrumbs) == 5 + for span in expected_spans: + span["data"] = ApproxDict(span["data"]) - for crumb in actual_query_breadcrumbs: - crumb.pop("timestamp", None) - assert crumb["data"] == ApproxDict(expected_base) + for span in event["spans"]: + span.pop("span_id", None) + span.pop("start_timestamp", None) + span.pop("timestamp", None) - # Span-first breadcrumbs never carry db.params regardless of data_collection - for crumb in actual_query_breadcrumbs: - assert "db.params" not in crumb["data"] + assert event["spans"] == expected_spans -def test_clickhouse_client_breadcrumbs_span_streaming_data_collection_overrides_pii( - sentry_init, capture_events -) -> None: - """data_collection disabled takes precedence over send_default_pii=True.""" +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_clickhouse_dbapi_spans_with_pii( + sentry_init, + capture_events, + capture_items, + capture_envelopes, + span_streaming, +): sentry_init( integrations=[ClickhouseDriverIntegration()], + trace_lifecycle="stream" if span_streaming else "static", + _experiments={ + "record_sql_params": True, + }, + traces_sample_rate=1.0, send_default_pii=True, - trace_lifecycle="stream", - _experiments={"data_collection": {"database_query_data": False}}, ) - events = capture_events() + if span_streaming: + items = capture_items("span") - client = Client("localhost") - client.execute("DROP TABLE IF EXISTS test") - client.execute("CREATE TABLE test (x Int32) ENGINE = Memory") - client.execute("INSERT INTO test (x) VALUES", [{"x": 100}]) - client.execute("INSERT INTO test (x) VALUES", [[170], [200]]) + trace_id = None + span_id = None - res = client.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) - assert res[0][0] == 370 + with sentry_sdk.traces.start_span(name="custom parent") as span: + trace_id = span.trace_id + span_id = span.span_id - capture_message("hi") + conn = connect("clickhouse://localhost") + cursor = conn.cursor() + cursor.execute("DROP TABLE IF EXISTS test") + cursor.execute("CREATE TABLE test (x Int32) ENGINE = Memory") + cursor.executemany("INSERT INTO test (x) VALUES", [{"x": 100}]) + cursor.executemany("INSERT INTO test (x) VALUES", [[170], [200]]) + cursor.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) + res = cursor.fetchall() - (event,) = events + assert res[0][0] == 370 - actual_query_breadcrumbs = [ - crumb - for crumb in event["breadcrumbs"]["values"] - if crumb["category"] == "query" - ] + sentry_sdk.flush() + spans = [item.payload for item in items] - assert len(actual_query_breadcrumbs) == 5 + expected_spans = [ + { + "name": "DROP TABLE IF EXISTS test", + "attributes": { + "db.system.name": "clickhouse", + "db.namespace": "", + "db.user": "default", + "db.query.text": "DROP TABLE IF EXISTS test", + "sentry.op": "db", + "sentry.origin": "auto.db.clickhouse_driver", + "server.address": "localhost", + "server.port": 9000, + }, + "trace_id": trace_id, + "parent_span_id": span_id, + }, + { + "name": "CREATE TABLE test (x Int32) ENGINE = Memory", + "attributes": { + "db.system.name": "clickhouse", + "db.namespace": "", + "db.user": "default", + "db.query.text": "CREATE TABLE test (x Int32) ENGINE = Memory", + "sentry.op": "db", + "sentry.origin": "auto.db.clickhouse_driver", + "server.address": "localhost", + "server.port": 9000, + }, + "trace_id": trace_id, + "parent_span_id": span_id, + }, + { + "name": "INSERT INTO test (x) VALUES", + "attributes": { + "db.system.name": "clickhouse", + "db.namespace": "", + "db.user": "default", + "db.query.text": "INSERT INTO test (x) VALUES", + "sentry.op": "db", + "sentry.origin": "auto.db.clickhouse_driver", + "server.address": "localhost", + "server.port": 9000, + }, + "trace_id": trace_id, + "parent_span_id": span_id, + }, + { + "name": "INSERT INTO test (x) VALUES", + "attributes": { + "db.system.name": "clickhouse", + "db.namespace": "", + "db.user": "default", + "db.query.text": "INSERT INTO test (x) VALUES", + "sentry.op": "db", + "sentry.origin": "auto.db.clickhouse_driver", + "server.address": "localhost", + "server.port": 9000, + }, + "trace_id": trace_id, + "parent_span_id": span_id, + }, + { + "name": "SELECT sum(x) FROM test WHERE x > 150", + "attributes": { + "db.system.name": "clickhouse", + "db.namespace": "", + "db.user": "default", + "db.query.text": "SELECT sum(x) FROM test WHERE x > 150", + "sentry.op": "db", + "sentry.origin": "auto.db.clickhouse_driver", + "server.address": "localhost", + "server.port": 9000, + }, + "trace_id": trace_id, + "parent_span_id": span_id, + }, + { + "name": "custom parent", + "attributes": [], + "trace_id": trace_id, + }, + ] - for crumb in actual_query_breadcrumbs: - assert "db.params" not in crumb["data"] + for span in expected_spans: + span["attributes"] = ApproxDict(span["attributes"]) + for span in spans: + span.pop("span_id", None) + span.pop("start_timestamp", None) + span.pop("end_timestamp", None) + span.pop("is_segment", None) + span.pop("status", None) -def test_clickhouse_dbapi_breadcrumbs_span_streaming( - sentry_init, capture_events -) -> None: - sentry_init( - integrations=[ClickhouseDriverIntegration()], - trace_lifecycle="stream", - ) - events = capture_events() + assert spans == expected_spans + else: + events = capture_events() - conn = connect("clickhouse://localhost") - cursor = conn.cursor() - cursor.execute("DROP TABLE IF EXISTS test") - cursor.execute("CREATE TABLE test (x Int32) ENGINE = Memory") - cursor.executemany("INSERT INTO test (x) VALUES", [{"x": 100}]) - cursor.executemany("INSERT INTO test (x) VALUES", [[170], [200]]) - cursor.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) - res = cursor.fetchall() + transaction_trace_id = None + transaction_span_id = None - assert res[0][0] == 370 + with start_transaction(name="test_clickhouse_transaction") as transaction: + transaction_trace_id = transaction.trace_id + transaction_span_id = transaction.span_id - capture_message("hi") + conn = connect("clickhouse://localhost") + cursor = conn.cursor() + cursor.execute("DROP TABLE IF EXISTS test") + cursor.execute("CREATE TABLE test (x Int32) ENGINE = Memory") + cursor.executemany("INSERT INTO test (x) VALUES", [{"x": 100}]) + cursor.executemany("INSERT INTO test (x) VALUES", [[170], [200]]) + cursor.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) + res = cursor.fetchall() - (event,) = events + assert res[0][0] == 370 - expected_base = { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - } + (event,) = events - expected_breadcrumbs = [ - { - "category": "query", - "data": {**expected_base}, - "message": "DROP TABLE IF EXISTS test", - "type": "default", - }, - { - "category": "query", - "data": {**expected_base}, - "message": "CREATE TABLE test (x Int32) ENGINE = Memory", - "type": "default", - }, - { - "category": "query", - "data": {**expected_base}, - "message": "INSERT INTO test (x) VALUES", - "type": "default", - }, - { - "category": "query", - "data": {**expected_base}, - "message": "INSERT INTO test (x) VALUES", - "type": "default", - }, - { - "category": "query", - "data": {**expected_base}, - "message": "SELECT sum(x) FROM test WHERE x > 150", - "type": "default", - }, - ] + expected_spans = [ + { + "op": "db", + "origin": "auto.db.clickhouse_driver", + "description": "DROP TABLE IF EXISTS test", + "data": { + "db.system": "clickhouse", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + "db.result": [[], []], + }, + "same_process_as_parent": True, + "trace_id": transaction_trace_id, + "parent_span_id": transaction_span_id, + }, + { + "op": "db", + "origin": "auto.db.clickhouse_driver", + "description": "CREATE TABLE test (x Int32) ENGINE = Memory", + "data": { + "db.system": "clickhouse", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + "db.result": [[], []], + }, + "same_process_as_parent": True, + "trace_id": transaction_trace_id, + "parent_span_id": transaction_span_id, + }, + { + "op": "db", + "origin": "auto.db.clickhouse_driver", + "description": "INSERT INTO test (x) VALUES", + "data": { + "db.system": "clickhouse", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + "db.params": [{"x": 100}], + }, + "same_process_as_parent": True, + "trace_id": transaction_trace_id, + "parent_span_id": transaction_span_id, + }, + { + "op": "db", + "origin": "auto.db.clickhouse_driver", + "description": "INSERT INTO test (x) VALUES", + "data": { + "db.system": "clickhouse", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + "db.params": [[170], [200]], + }, + "same_process_as_parent": True, + "trace_id": transaction_trace_id, + "parent_span_id": transaction_span_id, + }, + { + "op": "db", + "origin": "auto.db.clickhouse_driver", + "description": "SELECT sum(x) FROM test WHERE x > 150", + "data": { + "db.system": "clickhouse", + "db.name": "", + "db.user": "default", + "server.address": "localhost", + "server.port": 9000, + "db.params": {"minv": 150}, + "db.result": [[[370]], [["sum(x)", "Int64"]]], + }, + "same_process_as_parent": True, + "trace_id": transaction_trace_id, + "parent_span_id": transaction_span_id, + }, + ] - for crumb in expected_breadcrumbs: - crumb["data"] = ApproxDict(crumb["data"]) + if not EXPECT_PARAMS_IN_SELECT: + expected_spans[-1]["data"].pop("db.params", None) - for crumb in event["breadcrumbs"]["values"]: - crumb.pop("timestamp", None) + for span in expected_spans: + span["data"] = ApproxDict(span["data"]) - assert event["breadcrumbs"]["values"] == expected_breadcrumbs + for span in event["spans"]: + span.pop("span_id", None) + span.pop("start_timestamp", None) + span.pop("timestamp", None) - # Span-first breadcrumbs never carry db.params - for crumb in event["breadcrumbs"]["values"]: - assert "db.params" not in crumb["data"] + assert event["spans"] == expected_spans -@pytest.mark.parametrize("send_default_pii", [True, False]) -def test_clickhouse_dbapi_breadcrumbs_span_streaming_with_pii( - sentry_init, capture_events, send_default_pii -) -> None: +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_span_origin( + sentry_init, + capture_events, + capture_items, + capture_envelopes, + span_streaming, +): sentry_init( integrations=[ClickhouseDriverIntegration()], - send_default_pii=send_default_pii, - trace_lifecycle="stream", + traces_sample_rate=1.0, + trace_lifecycle="stream" if span_streaming else "static", ) - events = capture_events() - - conn = connect("clickhouse://localhost") - cursor = conn.cursor() - cursor.execute("DROP TABLE IF EXISTS test") - cursor.execute("CREATE TABLE test (x Int32) ENGINE = Memory") - cursor.executemany("INSERT INTO test (x) VALUES", [{"x": 100}]) - cursor.executemany("INSERT INTO test (x) VALUES", [[170], [200]]) - cursor.execute("SELECT sum(x) FROM test WHERE x > %(minv)i", {"minv": 150}) - res = cursor.fetchall() - - assert res[0][0] == 370 - - capture_message("hi") + if span_streaming: + items = capture_items("span") - (event,) = events + with sentry_sdk.traces.start_span(name="custom parent"): + conn = connect("clickhouse://localhost") + cursor = conn.cursor() + cursor.execute("SELECT 1") - expected_base = { - "db.system": "clickhouse", - "db.driver.name": "clickhouse-driver", - "db.name": "", - "db.user": "default", - "server.address": "localhost", - "server.port": 9000, - } + sentry_sdk.flush() + spans = [item.payload for item in items] - actual_query_breadcrumbs = [ - crumb - for crumb in event["breadcrumbs"]["values"] - if crumb["category"] == "query" - ] + assert spans[1]["attributes"]["sentry.origin"] == "manual" + assert spans[0]["attributes"]["sentry.origin"] == "auto.db.clickhouse_driver" + else: + events = capture_events() - assert len(actual_query_breadcrumbs) == 5 + with start_transaction(name="test_clickhouse_transaction"): + conn = connect("clickhouse://localhost") + cursor = conn.cursor() + cursor.execute("SELECT 1") - for crumb in actual_query_breadcrumbs: - crumb.pop("timestamp", None) - assert crumb["data"] == ApproxDict(expected_base) + (event,) = events - # Span-first breadcrumbs never carry db.params regardless of PII setting - for crumb in actual_query_breadcrumbs: - assert "db.params" not in crumb["data"] + assert event["contexts"]["trace"]["origin"] == "manual" + assert event["spans"][0]["origin"] == "auto.db.clickhouse_driver" From 0430ec6ea84207447891ea52ff41c1695c830e1b Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 1 Sep 2026 14:13:26 +0200 Subject: [PATCH 13/14] . --- sentry_sdk/integrations/clickhouse_driver.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/clickhouse_driver.py b/sentry_sdk/integrations/clickhouse_driver.py index ebc456a55a..b2e440a50f 100644 --- a/sentry_sdk/integrations/clickhouse_driver.py +++ b/sentry_sdk/integrations/clickhouse_driver.py @@ -8,7 +8,7 @@ from sentry_sdk.traces import StreamedSpan from sentry_sdk.tracing import Span from sentry_sdk.tracing_utils import has_span_streaming_enabled -from sentry_sdk.utils import has_data_collection_enabled +from sentry_sdk.utils import capture_internal_exceptions, has_data_collection_enabled # Hack to get new Python features working in older versions # without introducing a hard dependency on `typing_extensions` @@ -180,6 +180,11 @@ def _inner_end(*args: "P.args", **kwargs: "P.kwargs") -> "T": elif should_send_default_pii(): span.set_data("db.result", res) + with capture_internal_exceptions(): + span.scope.add_breadcrumb( + message=span._data.pop("query"), category="query", data=span._data + ) + span.finish() return res From 324db128b9487a8c40d536ce6eb5077232666e99 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Tue, 1 Sep 2026 14:18:38 +0200 Subject: [PATCH 14/14] rename tests --- .../test_clickhouse_driver.py | 44 +++---------------- 1 file changed, 6 insertions(+), 38 deletions(-) diff --git a/tests/integrations/clickhouse_driver/test_clickhouse_driver.py b/tests/integrations/clickhouse_driver/test_clickhouse_driver.py index c521b8b79e..b6d45a2365 100644 --- a/tests/integrations/clickhouse_driver/test_clickhouse_driver.py +++ b/tests/integrations/clickhouse_driver/test_clickhouse_driver.py @@ -236,10 +236,6 @@ def test_clickhouse_client_breadcrumbs_span_streaming( assert actual_query_breadcrumbs == expected_breadcrumbs - # In span streaming mode, db.params is never present - for crumb in actual_query_breadcrumbs: - assert "db.params" not in crumb["data"] - def test_clickhouse_client_breadcrumbs_with_pii(sentry_init, capture_events) -> None: sentry_init( @@ -343,7 +339,7 @@ def test_clickhouse_client_breadcrumbs_with_pii(sentry_init, capture_events) -> assert event["breadcrumbs"]["values"] == expected_breadcrumbs -def test_clickhouse_client_breadcrumbs_span_streaming_with_pii( +def test_clickhouse_client_breadcrumbs_with_pii_span_streaming( sentry_init, capture_events ) -> None: sentry_init( @@ -443,10 +439,6 @@ def test_clickhouse_client_breadcrumbs_span_streaming_with_pii( assert event["breadcrumbs"]["values"] == expected_breadcrumbs - # In span streaming mode, db.params is never present - for crumb in event["breadcrumbs"]["values"]: - assert "db.params" not in crumb["data"] - def test_clickhouse_client_breadcrumbs_with_data_collection( sentry_init, capture_events @@ -551,7 +543,7 @@ def test_clickhouse_client_breadcrumbs_with_data_collection( assert event["breadcrumbs"]["values"] == expected_breadcrumbs -def test_clickhouse_client_breadcrumbs_span_streaming_with_data_collection( +def test_clickhouse_client_breadcrumbs_with_data_collection_span_streaming( sentry_init, capture_events ) -> None: sentry_init( @@ -650,10 +642,6 @@ def test_clickhouse_client_breadcrumbs_span_streaming_with_data_collection( assert event["breadcrumbs"]["values"] == expected_breadcrumbs - # In span streaming mode, db.params is never present - for crumb in event["breadcrumbs"]["values"]: - assert "db.params" not in crumb["data"] - def test_clickhouse_client_breadcrumbs_with_data_collection_disabled( sentry_init, capture_events @@ -754,7 +742,7 @@ def test_clickhouse_client_breadcrumbs_with_data_collection_disabled( assert "db.result" not in crumb["data"] -def test_clickhouse_client_breadcrumbs_span_streaming_with_data_collection_disabled( +def test_clickhouse_client_breadcrumbs_with_data_collection_disabled_span_streaming( sentry_init, capture_events ) -> None: sentry_init( @@ -853,10 +841,6 @@ def test_clickhouse_client_breadcrumbs_span_streaming_with_data_collection_disab assert event["breadcrumbs"]["values"] == expected_breadcrumbs - # In span streaming mode, db.params is never present - for crumb in event["breadcrumbs"]["values"]: - assert "db.params" not in crumb["data"] - def test_clickhouse_client_breadcrumbs_data_collection_overrides_pii( sentry_init, capture_events @@ -958,7 +942,7 @@ def test_clickhouse_client_breadcrumbs_data_collection_overrides_pii( assert "db.result" not in crumb["data"] -def test_clickhouse_client_breadcrumbs_span_streaming_data_collection_overrides_pii( +def test_clickhouse_client_breadcrumbs_data_collection_overrides_pii_span_streaming( sentry_init, capture_events ) -> None: sentry_init( @@ -1058,10 +1042,6 @@ def test_clickhouse_client_breadcrumbs_span_streaming_data_collection_overrides_ assert event["breadcrumbs"]["values"] == expected_breadcrumbs - # In span streaming mode, db.params is never present - for crumb in event["breadcrumbs"]["values"]: - assert "db.params" not in crumb["data"] - def test_clickhouse_client_breadcrumbs_with_data_collection_default( sentry_init, capture_events @@ -1166,7 +1146,7 @@ def test_clickhouse_client_breadcrumbs_with_data_collection_default( assert event["breadcrumbs"]["values"] == expected_breadcrumbs -def test_clickhouse_client_breadcrumbs_span_streaming_with_data_collection_default( +def test_clickhouse_client_breadcrumbs_with_data_collection_default_span_streaming( sentry_init, capture_events ) -> None: sentry_init( @@ -1265,10 +1245,6 @@ def test_clickhouse_client_breadcrumbs_span_streaming_with_data_collection_defau assert event["breadcrumbs"]["values"] == expected_breadcrumbs - # In span streaming mode, db.params is never present - for crumb in event["breadcrumbs"]["values"]: - assert "db.params" not in crumb["data"] - def test_clickhouse_client_span_streaming_with_data_collection( sentry_init, capture_items @@ -2174,10 +2150,6 @@ def test_clickhouse_dbapi_breadcrumbs_span_streaming( assert event["breadcrumbs"]["values"] == expected_breadcrumbs - # In span streaming mode, db.params is never present - for crumb in event["breadcrumbs"]["values"]: - assert "db.params" not in crumb["data"] - def test_clickhouse_dbapi_breadcrumbs_with_pii(sentry_init, capture_events) -> None: sentry_init( @@ -2282,7 +2254,7 @@ def test_clickhouse_dbapi_breadcrumbs_with_pii(sentry_init, capture_events) -> N assert event["breadcrumbs"]["values"] == expected_breadcrumbs -def test_clickhouse_dbapi_breadcrumbs_span_streaming_with_pii( +def test_clickhouse_dbapi_breadcrumbs_with_pii_span_streaming( sentry_init, capture_events ) -> None: sentry_init( @@ -2383,10 +2355,6 @@ def test_clickhouse_dbapi_breadcrumbs_span_streaming_with_pii( assert event["breadcrumbs"]["values"] == expected_breadcrumbs - # In span streaming mode, db.params is never present - for crumb in event["breadcrumbs"]["values"]: - assert "db.params" not in crumb["data"] - @pytest.mark.parametrize("span_streaming", [True, False]) def test_clickhouse_dbapi_spans(