feat(utils): Filter stack frame variables via data_collection option (#7274)

serialize_frame now respects the KeyValueCollectionBehaviour configured
for stack_frame_variables in the data_collection experiment, applying
allowlist/denylist filtering to frame locals before serialization. This
takes precedence over the include_local_variables option when
data_collection is enabled.

Refs PY-2578
Refs #6738

Author: ericapisani

sentry_sdk/data_collection.py

@@ -194,9 +194,6 @@ def _resolve_explicit(
     ``data_collection`` dict, filling in spec defaults for any omitted or
     partially-specified field.
     """
-    # frame_context_lines accepts an integer or a boolean fallback (spec: True
-    # -> platform default of 5, False -> 0). bool is a subclass of int, so
-    # coerce explicitly before treating it as a line count.
     frame_context_lines = d.get("frame_context_lines")
     if frame_context_lines is None:
         frame_context_lines = _DEFAULT_FRAME_CONTEXT_LINES
@@ -211,10 +208,11 @@ def _resolve_explicit(
     raw_stack_frame_variables = d.get("stack_frame_variables", True)
     stack_frame_variables: "Union[bool, KeyValueCollectionBehaviour]"
 
-    if isinstance(raw_stack_frame_variables, dict):
-        stack_frame_variables = _kvcb_from_value(raw_stack_frame_variables)
-    else:
-        stack_frame_variables = bool(raw_stack_frame_variables)
+    stack_frame_variables = (
+        _kvcb_from_value(raw_stack_frame_variables)
+        if isinstance(raw_stack_frame_variables, dict)
+        else bool(raw_stack_frame_variables)
+    )
 
     # http_bodies: omitted means "all valid types"; [] is the explicit opt-out.
     http_bodies = d.get("http_bodies")

sentry_sdk/utils.py

@@ -19,6 +19,8 @@
 from numbers import Real
 from urllib.parse import parse_qs, unquote, urlencode, urlsplit, urlunsplit
 
+from sentry_sdk.data_collection import _apply_key_value_collection_filtering
+
 try:
     # Python 3.11
     from builtins import BaseExceptionGroup
@@ -589,6 +591,8 @@ def serialize_frame(
     max_value_length: "Optional[int]" = None,
     custom_repr: "Optional[Callable[..., Optional[str]]]" = None,
 ) -> "Dict[str, Any]":
+    from sentry_sdk.serializer import serialize
+
     f_code = getattr(frame, "f_code", None)
     if not f_code:
         abs_path = None
@@ -628,9 +632,30 @@ def serialize_frame(
             frame, tb_lineno, max_value_length
         )
 
-    if include_local_variables:
-        from sentry_sdk.serializer import serialize
+    if has_data_collection_enabled(client_options):
+        dc_stack_frame_vars_config = client_options["data_collection"][
+            "stack_frame_variables"
+        ]
+
+        if isinstance(dc_stack_frame_vars_config, bool):
+            if dc_stack_frame_vars_config:
+                rv["vars"] = serialize(
+                    dict(frame.f_locals), is_vars=True, custom_repr=custom_repr
+                )
+        else:
+            local_variables_to_send = _apply_key_value_collection_filtering(
+                items=dict(frame.f_locals),
+                behaviour=dc_stack_frame_vars_config,
+            )
+
+            if local_variables_to_send:
+                serialized_variables = serialize(
+                    local_variables_to_send, is_vars=True, custom_repr=custom_repr
+                )
+
+                rv["vars"] = serialized_variables
 
+    elif include_local_variables:
         rv["vars"] = serialize(
             dict(frame.f_locals), is_vars=True, custom_repr=custom_repr
         )

tests/test_utils.py

@@ -529,6 +529,255 @@ def test_include_source_context_when_serializing_frame(
     assert ("post_context" in result) is expected_source_context
 
 
+def _frame_with_locals():
+    safe_value = "not sensitive"  # noqa: F841
+    password = "ada123"  # noqa: F841
+    api_key = "abc123"  # noqa: F841
+    nickname = "Beans"  # noqa: F841
+    return sys._getframe()
+
+
+@pytest.mark.parametrize(
+    "data_collection,include_local_variables,expected_vars",
+    [
+        pytest.param(
+            {"stack_frame_variables": True},
+            False,
+            True,
+            id="data_collection_stack_frame_variables_true_overrides_include_false",
+        ),
+        pytest.param(
+            {"stack_frame_variables": False},
+            True,
+            False,
+            id="data_collection_stack_frame_variables_false_overrides_include_true",
+        ),
+        pytest.param(
+            {},
+            False,
+            True,
+            id="data_collection_stack_frame_variables_spec_default_is_true",
+        ),
+    ],
+)
+def test_stack_frame_variables_bool_when_serializing_frame(
+    sentry_init, data_collection, include_local_variables, expected_vars
+):
+    sentry_init(_experiments={"data_collection": data_collection})
+
+    result = serialize_frame(
+        _frame_with_locals(), include_local_variables=include_local_variables
+    )
+
+    assert ("vars" in result) is expected_vars
+
+
+def test_stack_frame_variables_true_does_not_filter_sensitive_locals(sentry_init):
+    sentry_init(_experiments={"data_collection": {"stack_frame_variables": True}})
+
+    result = serialize_frame(_frame_with_locals())
+
+    assert result["vars"]["safe_value"] == "'not sensitive'"
+    assert result["vars"]["password"] == "'ada123'"
+
+
+@pytest.mark.parametrize(
+    "behaviour,expected_vars",
+    [
+        pytest.param(
+            {"mode": "denylist"},
+            {
+                "safe_value": "'not sensitive'",
+                "password": "'[Filtered]'",
+                "api_key": "'[Filtered]'",
+                "nickname": "'Beans'",
+            },
+            id="data_collection_stack_frame_variables_denylist_builtin_terms_only",
+        ),
+        pytest.param(
+            {"mode": "denylist", "terms": ["nickname"]},
+            {
+                "safe_value": "'not sensitive'",
+                "password": "'[Filtered]'",
+                "api_key": "'[Filtered]'",
+                "nickname": "'[Filtered]'",
+            },
+            id="data_collection_stack_frame_variables_denylist_user_terms",
+        ),
+        pytest.param(
+            {"mode": "allowlist", "terms": ["safe"]},
+            {
+                "safe_value": "'not sensitive'",
+                "password": "'[Filtered]'",
+                "api_key": "'[Filtered]'",
+                "nickname": "'[Filtered]'",
+            },
+            id="data_collection_stack_frame_variables_allowlist_user_terms",
+        ),
+        pytest.param(
+            {"mode": "allowlist", "terms": ["safe", "api_key"]},
+            {
+                "safe_value": "'not sensitive'",
+                "password": "'[Filtered]'",
+                "api_key": "'[Filtered]'",
+                "nickname": "'[Filtered]'",
+            },
+            id="data_collection_stack_frame_variables_allowlist_cannot_allow_sensitive_term",
+        ),
+    ],
+)
+def test_stack_frame_variables_filtering_when_serializing_frame(
+    sentry_init, behaviour, expected_vars
+):
+    sentry_init(_experiments={"data_collection": {"stack_frame_variables": behaviour}})
+
+    result = serialize_frame(_frame_with_locals())
+
+    assert result["vars"] == expected_vars
+
+
+def test_stack_frame_variables_off_omits_vars(sentry_init):
+    sentry_init(
+        _experiments={"data_collection": {"stack_frame_variables": {"mode": "off"}}}
+    )
+
+    result = serialize_frame(_frame_with_locals())
+
+    assert "vars" not in result
+
+
+def test_stack_frame_variables_omits_vars_when_frame_has_no_locals(sentry_init):
+    def _frame_without_locals():
+        return sys._getframe()
+
+    sentry_init(
+        _experiments={
+            "data_collection": {"stack_frame_variables": {"mode": "denylist"}}
+        }
+    )
+
+    result = serialize_frame(_frame_without_locals())
+
+    assert "vars" not in result
+
+
+def test_stack_frame_variables_filtering_uses_custom_repr(sentry_init):
+    sentry_init(
+        _experiments={
+            "data_collection": {"stack_frame_variables": {"mode": "denylist"}}
+        }
+    )
+
+    def custom_repr(value):
+        return "CUSTOM" if value == "not sensitive" else None
+
+    result = serialize_frame(_frame_with_locals(), custom_repr=custom_repr)
+
+    assert result["vars"]["safe_value"] == "CUSTOM"
+    assert result["vars"]["password"] == "'[Filtered]'"
+
+
+@pytest.mark.parametrize(
+    "options,include_local_variables,expected_vars",
+    [
+        pytest.param(
+            {},
+            True,
+            True,
+            id="no_data_collection-include_local_variables_true",
+        ),
+        pytest.param(
+            {},
+            False,
+            False,
+            id="no_data_collection-include_local_variables_false",
+        ),
+    ],
+)
+def test_include_local_variables_when_data_collection_is_unset(
+    sentry_init, options, include_local_variables, expected_vars
+):
+    sentry_init(**options)
+
+    result = serialize_frame(
+        _frame_with_locals(), include_local_variables=include_local_variables
+    )
+
+    assert ("vars" in result) is expected_vars
+
+
+def test_data_collection_stack_frame_variables_overrides_include_local_variables_option(
+    sentry_init, capture_events
+):
+    sentry_init(
+        include_local_variables=False,
+        _experiments={"data_collection": {"stack_frame_variables": True}},
+    )
+    events = capture_events()
+
+    def raise_with_locals():
+        safe_value = "not sensitive"  # noqa: F841
+        raise ValueError("boom")
+
+    try:
+        raise_with_locals()
+    except ValueError:
+        sentry_sdk.capture_exception()
+
+    (event,) = events
+    frame = event["exception"]["values"][0]["stacktrace"]["frames"][-1]
+    assert frame["vars"]["safe_value"] == "'not sensitive'"
+
+
+def test_data_collection_stack_frame_variables_filtering_applies_to_captured_exception(
+    sentry_init, capture_events
+):
+    sentry_init(
+        _experiments={
+            "data_collection": {
+                "stack_frame_variables": {"mode": "denylist", "terms": ["nickname"]}
+            }
+        }
+    )
+    events = capture_events()
+
+    def raise_with_locals():
+        safe_value = "not sensitive"  # noqa: F841
+        password = "hunter2"  # noqa: F841
+        nickname = "Bugsy"  # noqa: F841
+        raise ValueError("boom")
+
+    try:
+        raise_with_locals()
+    except ValueError:
+        sentry_sdk.capture_exception()
+
+    (event,) = events
+    frame = event["exception"]["values"][0]["stacktrace"]["frames"][-1]
+
+    assert frame["vars"]["safe_value"] == "'not sensitive'"
+    assert frame["vars"]["password"] == "'[Filtered]'"
+    assert frame["vars"]["nickname"] == "'[Filtered]'"
+
+
+def test_serialize_frame_variables_serializer_failure(sentry_init):
+    sentry_init(
+        _experiments={
+            "data_collection": {
+                "stack_frame_variables": {"mode": "denylist", "terms": ["password"]}
+            }
+        }
+    )
+
+    failure_message = "<failed to serialize, use init(debug=True) to see error logs>"
+
+    frame = sys._getframe()
+    with mock.patch("sentry_sdk.serializer.serialize", return_value=failure_message):
+        result = serialize_frame(frame)
+
+    assert result["vars"] == failure_message
+
+
 @pytest.mark.parametrize(
     "item,regex_list,expected_result",
     [