Skip to content

Commit db1a5a4

Browse files
feat(bedrock): Set gen_ai.output.messages for streamed responses
1 parent 444f537 commit db1a5a4

2 files changed

Lines changed: 280 additions & 20 deletions

File tree

‎sentry_sdk/integrations/boto3/_services/bedrock.py‎

Lines changed: 115 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -93,37 +93,132 @@ def sentry_patched_iter(
9393
span.set_attribute if isinstance(span, StreamedSpan) else span.set_data
9494
)
9595

96+
role: "Optional[str]" = None
97+
content_blocks: "dict[int, dict[str, Any]]" = {}
98+
9699
try:
97100
for event in original_iter(self):
98-
if "metadata" not in event:
99-
yield event
100-
continue
101+
if "metadata" in event:
102+
usage = event["metadata"]["usage"]
101103

102-
usage = event["metadata"]["usage"]
104+
input_tokens = usage["inputTokens"]
105+
set_on_span(
106+
SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS, usage["outputTokens"]
107+
)
108+
set_on_span(
109+
SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS, usage["totalTokens"]
110+
)
103111

104-
input_tokens = usage["inputTokens"]
105-
set_on_span(SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS, usage["outputTokens"])
106-
set_on_span(SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS, usage["totalTokens"])
112+
cache_read_tokens = usage.get("cacheReadInputTokens")
113+
if cache_read_tokens is not None:
114+
input_tokens += cache_read_tokens
115+
set_on_span(
116+
SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED, cache_read_tokens
117+
)
118+
119+
cache_write_tokens = usage.get("cacheWriteInputTokens")
120+
if cache_write_tokens is not None:
121+
input_tokens += cache_write_tokens
122+
set_on_span(
123+
SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE,
124+
cache_write_tokens,
125+
)
107126

108-
cache_read_tokens = usage.get("cacheReadInputTokens")
109-
if cache_read_tokens is not None:
110-
input_tokens += cache_read_tokens
111127
set_on_span(
112-
SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED, cache_read_tokens
128+
SPANDATA.GEN_AI_USAGE_INPUT_TOKENS,
129+
input_tokens,
113130
)
114131

115-
cache_write_tokens = usage.get("cacheWriteInputTokens")
116-
if cache_write_tokens is not None:
117-
input_tokens += cache_write_tokens
132+
yield event
133+
continue
134+
135+
client = sentry_sdk.get_client()
136+
if (
137+
not has_data_collection_enabled(client.options)
138+
or not client.options["data_collection"]["gen_ai"]["outputs"]
139+
) and (
140+
has_data_collection_enabled(client.options)
141+
or not should_send_default_pii()
142+
):
143+
yield event
144+
continue
145+
146+
if "messageStart" in event:
147+
role = event["messageStart"]["role"]
148+
149+
yield event
150+
continue
151+
152+
if "contentBlockStart" in event:
153+
block_start = event["contentBlockStart"]
154+
tool_use = block_start["start"].get("toolUse")
155+
if tool_use is not None:
156+
tool_use_block: "dict[str, Any]" = {"input": ""}
157+
if "name" in tool_use:
158+
tool_use_block["name"] = tool_use["name"]
159+
content_blocks[block_start["contentBlockIndex"]] = {
160+
"toolUse": tool_use_block
161+
}
162+
163+
yield event
164+
continue
165+
166+
if "contentBlockDelta" in event:
167+
index = event["contentBlockDelta"]["contentBlockIndex"]
168+
delta = event["contentBlockDelta"]["delta"]
169+
170+
if "text" in delta:
171+
block = content_blocks.setdefault(index, {"text": ""})
172+
block["text"] += delta["text"]
173+
elif "toolUse" in delta:
174+
block = content_blocks.setdefault(
175+
index, {"toolUse": {"input": ""}}
176+
)
177+
block["toolUse"]["input"] += delta["toolUse"]["input"]
178+
elif "reasoningContent" in delta:
179+
block = content_blocks.setdefault(
180+
index, {"reasoningContent": {}}
181+
)
182+
reasoning_text = delta["reasoningContent"].get("text")
183+
if reasoning_text is not None:
184+
block["reasoningContent"].setdefault(
185+
"reasoningText", {"text": ""}
186+
)["text"] += reasoning_text
187+
188+
yield event
189+
continue
190+
191+
if "messageStop" in event and role is not None:
192+
for block in content_blocks.values():
193+
if "toolUse" not in block:
194+
continue
195+
# Tool input is streamed as a JSON string; non-streaming responses contain the parsed object.
196+
try:
197+
block["toolUse"]["input"] = json.loads(
198+
block["toolUse"]["input"]
199+
)
200+
except ValueError:
201+
pass
202+
118203
set_on_span(
119-
SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE,
120-
cache_write_tokens,
204+
SPANDATA.GEN_AI_OUTPUT_MESSAGES,
205+
json.dumps(
206+
_transform_messages(
207+
[
208+
{
209+
"role": role,
210+
"content": [
211+
content_blocks[index]
212+
for index in sorted(content_blocks)
213+
],
214+
}
215+
]
216+
)
217+
),
121218
)
122219

123-
set_on_span(
124-
SPANDATA.GEN_AI_USAGE_INPUT_TOKENS,
125-
input_tokens,
126-
)
220+
yield event
221+
continue
127222

128223
yield event
129224
except GeneratorExit:

‎tests/integrations/boto3/test_bedrock.py‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1320,3 +1320,168 @@ def test_converse_stream_input_attributes(
13201320
json.loads(span["data"][SPANDATA.GEN_AI_INPUT_MESSAGES])
13211321
== expected_input_messages
13221322
)
1323+
1324+
1325+
@pytest.mark.parametrize("data_collection", [True, False])
1326+
@pytest.mark.parametrize("span_streaming", [True, False])
1327+
@pytest.mark.parametrize("stream_gen_ai_spans", [True, False])
1328+
def test_converse_stream_output_attributes(
1329+
sentry_init,
1330+
capture_items,
1331+
capture_events,
1332+
data_collection,
1333+
span_streaming,
1334+
stream_gen_ai_spans,
1335+
converse_stream_response,
1336+
):
1337+
if data_collection:
1338+
sentry_init(
1339+
traces_sample_rate=1.0,
1340+
integrations=[Boto3Integration()],
1341+
data_collection={"gen_ai": {"outputs": True}},
1342+
stream_gen_ai_spans=stream_gen_ai_spans,
1343+
trace_lifecycle="stream" if span_streaming else "static",
1344+
)
1345+
else:
1346+
sentry_init(
1347+
traces_sample_rate=1.0,
1348+
integrations=[Boto3Integration()],
1349+
send_default_pii=True,
1350+
stream_gen_ai_spans=stream_gen_ai_spans,
1351+
trace_lifecycle="stream" if span_streaming else "static",
1352+
)
1353+
1354+
events = [
1355+
("messageStart", {"role": "assistant"}),
1356+
(
1357+
"contentBlockDelta",
1358+
{
1359+
"contentBlockIndex": 0,
1360+
"delta": {"reasoningContent": {"text": "The user wants "}},
1361+
},
1362+
),
1363+
(
1364+
"contentBlockDelta",
1365+
{
1366+
"contentBlockIndex": 0,
1367+
"delta": {"reasoningContent": {"text": "the weather."}},
1368+
},
1369+
),
1370+
(
1371+
"contentBlockDelta",
1372+
{
1373+
"contentBlockIndex": 0,
1374+
"delta": {"reasoningContent": {"signature": "sig"}},
1375+
},
1376+
),
1377+
("contentBlockStop", {"contentBlockIndex": 0}),
1378+
(
1379+
"contentBlockDelta",
1380+
{"contentBlockIndex": 1, "delta": {"text": "Let me "}},
1381+
),
1382+
(
1383+
"contentBlockDelta",
1384+
{"contentBlockIndex": 1, "delta": {"text": "check."}},
1385+
),
1386+
("contentBlockStop", {"contentBlockIndex": 1}),
1387+
(
1388+
"contentBlockStart",
1389+
{
1390+
"contentBlockIndex": 2,
1391+
"start": {"toolUse": {"toolUseId": "tool-1", "name": "get_weather"}},
1392+
},
1393+
),
1394+
(
1395+
"contentBlockDelta",
1396+
{"contentBlockIndex": 2, "delta": {"toolUse": {"input": '{"city": '}}},
1397+
),
1398+
(
1399+
"contentBlockDelta",
1400+
{"contentBlockIndex": 2, "delta": {"toolUse": {"input": '"Paris"}'}}},
1401+
),
1402+
("contentBlockStop", {"contentBlockIndex": 2}),
1403+
("messageStop", {"stopReason": "tool_use"}),
1404+
(
1405+
"metadata",
1406+
{
1407+
"usage": {"inputTokens": 10, "outputTokens": 20, "totalTokens": 30},
1408+
"metrics": {"latencyMs": 500},
1409+
},
1410+
),
1411+
]
1412+
converse_stream_response["stream"]._event_generator = iter(
1413+
EventStreamMessage(
1414+
prelude=None,
1415+
headers={":message-type": "event", ":event-type": event_type},
1416+
payload=json.dumps(payload).encode(),
1417+
crc=None,
1418+
)
1419+
for event_type, payload in events
1420+
)
1421+
1422+
expected_output_messages = [
1423+
{
1424+
"role": "assistant",
1425+
"parts": [
1426+
{"type": "reasoning", "content": "The user wants the weather."},
1427+
{"type": "text", "content": "Let me check."},
1428+
{
1429+
"type": "tool_call",
1430+
"name": "get_weather",
1431+
"arguments": {"city": "Paris"},
1432+
},
1433+
],
1434+
}
1435+
]
1436+
1437+
session = boto3.Session(
1438+
aws_access_key_id="-", aws_secret_access_key="-", region_name="us-east-1"
1439+
)
1440+
client = session.client("bedrock-runtime")
1441+
mock_http = Mock(status_code=200)
1442+
client.meta.events.register(
1443+
"before-call.bedrock-runtime.ConverseStream",
1444+
lambda **kwargs: (mock_http, converse_stream_response),
1445+
)
1446+
1447+
if span_streaming or stream_gen_ai_spans:
1448+
items = capture_items("span")
1449+
1450+
with sentry_sdk.start_transaction(name="parent"):
1451+
response = client.converse_stream(
1452+
modelId="anthropic.claude-3-sonnet-20240229-v1:0",
1453+
messages=[{"role": "user", "content": [{"text": "Hi"}]}],
1454+
)
1455+
for _ in response["stream"]:
1456+
pass
1457+
1458+
sentry_sdk.flush()
1459+
spans = [item.payload for item in items]
1460+
(span,) = (
1461+
span
1462+
for span in spans
1463+
if span["attributes"].get("sentry.op") == OP.GEN_AI_CHAT
1464+
)
1465+
1466+
assert (
1467+
json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES])
1468+
== expected_output_messages
1469+
)
1470+
else:
1471+
events = capture_events()
1472+
1473+
with sentry_sdk.start_transaction(name="parent"):
1474+
response = client.converse_stream(
1475+
modelId="anthropic.claude-3-sonnet-20240229-v1:0",
1476+
messages=[{"role": "user", "content": [{"text": "Hi"}]}],
1477+
)
1478+
for _ in response["stream"]:
1479+
pass
1480+
1481+
(transaction,) = events
1482+
(span,) = transaction["spans"]
1483+
1484+
assert (
1485+
json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES])
1486+
== expected_output_messages
1487+
)

0 commit comments

Comments
 (0)