Skip to content

Commit 2bd07e4

Browse files
feat(boto3): Gate Bedrock prompts on data_collection and include_prompts
Record Bedrock prompts and responses like the other gen_ai integrations: `data_collection.gen_ai.inputs`/`outputs` decide when `data_collection` is set, otherwise `send_default_pii` and the new `include_prompts` option of Boto3Integration do. Refs #5470 Co-Authored-By: Sergiy Dybskiy <3760543+sergical@users.noreply.github.com>
1 parent 2212084 commit 2bd07e4

3 files changed

Lines changed: 84 additions & 30 deletions

File tree

‎sentry_sdk/integrations/boto3/__init__.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,11 @@ class Boto3Integration(Integration):
1313
identifier = IDENTIFIER
1414
origin = ORIGIN
1515

16+
def __init__(self: "Boto3Integration", include_prompts: bool = True) -> None:
17+
# whether to record gen_ai prompts and responses, e.g. of Bedrock Runtime
18+
# calls, when `send_default_pii` is on and `data_collection` isn't set.
19+
self.include_prompts = include_prompts
20+
1621
@staticmethod
1722
def setup_once() -> None:
1823
version = parse_version(BOTOCORE_VERSION)

‎sentry_sdk/integrations/boto3/_services/bedrock_runtime.py‎

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,16 @@
11
import json
22
from collections.abc import Mapping
3-
from typing import TYPE_CHECKING
3+
from typing import TYPE_CHECKING, cast
44

5+
import sentry_sdk
56
from sentry_sdk.consts import OP, SPANDATA
67
from sentry_sdk.integrations.boto3._services.base import _ServiceExtension
7-
from sentry_sdk.integrations.boto3.consts import AI_ORIGIN
8-
from sentry_sdk.tracing_utils import _should_collect_gen_ai
9-
from sentry_sdk.utils import capture_internal_exceptions
8+
from sentry_sdk.integrations.boto3.consts import AI_ORIGIN, IDENTIFIER
9+
from sentry_sdk.scope import should_send_default_pii
10+
from sentry_sdk.utils import capture_internal_exceptions, has_data_collection_enabled
1011

1112
if TYPE_CHECKING:
12-
from typing import Any, Optional, Union
13+
from typing import Any, Literal, Optional, Union
1314

1415
from sentry_sdk._types import (
1516
Attributes,
@@ -21,6 +22,7 @@
2122
ToolCallResponsePart,
2223
ToolDefinition,
2324
)
25+
from sentry_sdk.integrations.boto3 import Boto3Integration
2426
from sentry_sdk.integrations.boto3._context import AwsCallContext
2527

2628
ContentPart = Union[TextPart, ReasoningPart, ToolCallPart, ToolCallResponsePart]
@@ -94,6 +96,24 @@ def get_response_attributes(
9496
return _get_converse_response_attributes(response)
9597

9698

99+
def _should_record_gen_ai(kind: 'Literal["inputs", "outputs"]') -> bool:
100+
"""
101+
Record prompts (`inputs`) or responses (`outputs`) as `data_collection` allows,
102+
or, if it isn't set, with `send_default_pii` and `include_prompts`, like the
103+
other gen_ai integrations.
104+
"""
105+
client = sentry_sdk.get_client()
106+
if has_data_collection_enabled(client.options):
107+
return bool(client.options["data_collection"]["gen_ai"][kind])
108+
109+
integration = cast("Optional[Boto3Integration]", client.get_integration(IDENTIFIER))
110+
return (
111+
should_send_default_pii()
112+
and integration is not None
113+
and integration.include_prompts
114+
)
115+
116+
97117
def _get_converse_request_attributes(params: "Mapping[str, Any]") -> "Attributes":
98118
"""
99119
https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html#API_runtime_Converse_RequestSyntax
@@ -117,7 +137,7 @@ def _get_converse_request_attributes(params: "Mapping[str, Any]") -> "Attributes
117137
inference_config["stopSequences"]
118138
)
119139

120-
if not _should_collect_gen_ai("inputs"):
140+
if not _should_record_gen_ai("inputs"):
121141
return attributes
122142

123143
with capture_internal_exceptions():
@@ -160,7 +180,7 @@ def _get_converse_response_attributes(response: "Any") -> "Attributes":
160180
if isinstance(usage, Mapping):
161181
attributes.update(_get_usage_attributes(usage))
162182

163-
if not _should_collect_gen_ai("outputs"):
183+
if not _should_record_gen_ai("outputs"):
164184
return attributes
165185

166186
with capture_internal_exceptions():

‎tests/integrations/boto3/test_bedrock_runtime.py‎

Lines changed: 52 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -53,13 +53,59 @@ def _has_shape_member(shape_name, member_name):
5353
),
5454
]
5555

56+
# `sentry_init()` options, `include_prompts`, and whether prompts (inputs) and
57+
# responses (outputs) are recorded.
58+
GEN_AI_DATA_COLLECTION_CASES = [
59+
pytest.param({}, True, False, False, id="default"),
60+
pytest.param({"send_default_pii": True}, True, True, True, id="send_default_pii"),
61+
pytest.param(
62+
{"send_default_pii": True},
63+
False,
64+
False,
65+
False,
66+
id="send_default_pii_without_include_prompts",
67+
),
68+
pytest.param({"data_collection": {}}, True, True, True, id="data_collection"),
69+
pytest.param(
70+
{"data_collection": {"gen_ai": {"inputs": True, "outputs": False}}},
71+
True,
72+
True,
73+
False,
74+
id="data_collection_inputs_only",
75+
),
76+
pytest.param(
77+
{"data_collection": {"gen_ai": {"inputs": False, "outputs": True}}},
78+
True,
79+
False,
80+
True,
81+
id="data_collection_outputs_only",
82+
),
83+
pytest.param(
84+
{
85+
"send_default_pii": True,
86+
"data_collection": {"gen_ai": {"inputs": False, "outputs": False}},
87+
},
88+
True,
89+
False,
90+
False,
91+
id="data_collection_wins_over_send_default_pii",
92+
),
93+
pytest.param(
94+
{"data_collection": {"gen_ai": {"inputs": True, "outputs": True}}},
95+
False,
96+
True,
97+
True,
98+
id="data_collection_ignores_include_prompts",
99+
),
100+
]
101+
56102

57103
@pytest.fixture
58104
def init_bedrock(sentry_init):
59-
def inner(trace_mode, **kwargs):
105+
def inner(trace_mode, include_prompts=True, **kwargs):
60106
sentry_init(
61107
traces_sample_rate=1.0,
62-
integrations=[Boto3Integration()],
108+
integrations=[Boto3Integration(include_prompts=include_prompts)],
63109
server_name="",
64110
**trace_mode,
65111
**kwargs,
@@ -119,36 +165,19 @@ def _get_bedrock_span(spans, origin=AI_ORIGIN):
119165

120166
@pytest.mark.parametrize("trace_mode", TRACE_MODES)
121167
@pytest.mark.parametrize(
122-
"init_kwargs, record_inputs, record_outputs",
123-
[
124-
pytest.param({}, False, False, id="default"),
125-
pytest.param({"send_default_pii": True}, True, True, id="send_default_pii"),
126-
pytest.param(
127-
{"data_collection": {"gen_ai": {"inputs": True, "outputs": False}}},
128-
True,
129-
False,
130-
id="data_collection_inputs_only",
131-
),
132-
pytest.param(
133-
{
134-
"send_default_pii": True,
135-
"data_collection": {"gen_ai": {"inputs": False, "outputs": True}},
136-
},
137-
False,
138-
True,
139-
id="data_collection_wins_over_send_default_pii",
140-
),
141-
],
168+
"init_kwargs, include_prompts, record_inputs, record_outputs",
169+
GEN_AI_DATA_COLLECTION_CASES,
142170
)
143171
def test_converse(
144172
init_bedrock,
145173
capture_items,
146174
trace_mode,
147175
init_kwargs,
176+
include_prompts,
148177
record_inputs,
149178
record_outputs,
150179
):
151-
client = init_bedrock(trace_mode, **init_kwargs)
180+
client = init_bedrock(trace_mode, include_prompts=include_prompts, **init_kwargs)
152181
params = {
153182
"modelId": MODEL_ID,
154183
"system": [{"text": "You are a helpful assistant."}],

0 commit comments

Comments
 (0)