Skip to content

Commit d920520

Browse files
patch __iter__ and close()
1 parent d0233d7 commit d920520

3 files changed

Lines changed: 196 additions & 91 deletions

File tree

‎sentry_sdk/integrations/boto3/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
from sentry_sdk.integrations import DidNotEnable, Integration, _check_minimum_version
22
from sentry_sdk.integrations.boto3._client import _patch_botocore_client
3+
from sentry_sdk.integrations.boto3._services.bedrock import _patch_event_stream
34
from sentry_sdk.integrations.boto3.consts import IDENTIFIER, ORIGIN
45
from sentry_sdk.utils import parse_version
56

@@ -19,3 +20,4 @@ def setup_once() -> None:
1920
_check_minimum_version(Boto3Integration, version, "botocore")
2021

2122
_patch_botocore_client()
23+
_patch_event_stream()
Lines changed: 122 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,27 @@
11
import sys
2-
from contextlib import contextmanager
32
from typing import TYPE_CHECKING, cast
43

54
from sentry_sdk.consts import OP, SPANDATA
5+
from sentry_sdk.integrations import DidNotEnable
66
from sentry_sdk.integrations.boto3._services.base import _ServiceExtension
77
from sentry_sdk.traces import SpanStatus, StreamedSpan
88
from sentry_sdk.tracing import SPANSTATUS
9-
from sentry_sdk.utils import capture_internal_exceptions, reraise
9+
from sentry_sdk.utils import (
10+
capture_internal_exceptions,
11+
reraise,
12+
should_be_treated_as_error,
13+
)
14+
15+
try:
16+
from botocore.eventstream import EventStream
17+
except ImportError:
18+
raise DidNotEnable("botocore is not installed")
1019

1120
if TYPE_CHECKING:
1221
from typing import Any, Iterator, Optional, Union
1322

14-
from botocore.eventstream import EventStream
1523
from mypy_boto3_bedrock_runtime.type_defs import (
1624
ConverseResponseTypeDef,
17-
ConverseStreamOutputTypeDef,
1825
ConverseStreamResponseTypeDef,
1926
)
2027

@@ -23,31 +30,113 @@
2330
from sentry_sdk.tracing import Span
2431

2532

26-
@contextmanager
27-
def _stream_span_context(span: "Union[Span, StreamedSpan]"):
33+
def _finish_span(
34+
stream: "EventStream[Any]",
35+
exc_type: "Optional[type[BaseException]]",
36+
exc_val: "Optional[BaseException]",
37+
exc_tb: "Optional[Any]",
38+
) -> None:
2839
"""
29-
Closes the provided span and set the span's status on exit.
40+
Finishes the span attached to the stream.
41+
Is a no-op if the stream has no span set, i.e., when the span has already been finished.
3042
"""
31-
end_span = span.end if isinstance(span, StreamedSpan) else span.finish
32-
33-
try:
34-
yield
35-
except GeneratorExit: # Raised when close() is called on the stream.
36-
exc_info = sys.exc_info()
37-
with capture_internal_exceptions():
38-
end_span()
39-
reraise(*exc_info)
40-
except BaseException:
41-
exc_info = sys.exc_info()
42-
with capture_internal_exceptions():
43-
if isinstance(span, StreamedSpan):
44-
span.status = SpanStatus.ERROR
45-
else:
46-
span.status = SPANSTATUS.INTERNAL_ERROR
47-
end_span()
48-
reraise(*exc_info)
43+
if not hasattr(stream, "_span"):
44+
return
45+
46+
span = stream._span
47+
del stream._span
48+
49+
if should_be_treated_as_error(exc_type, exc_val):
50+
if isinstance(span, StreamedSpan):
51+
span.status = SpanStatus.ERROR
52+
else:
53+
span.status = SPANSTATUS.INTERNAL_ERROR
54+
55+
if isinstance(span, StreamedSpan):
56+
span.end()
4957
else:
50-
end_span()
58+
span.finish()
59+
60+
61+
def _patch_event_stream() -> None:
62+
original_iter = EventStream.__iter__
63+
original_close = EventStream.close
64+
65+
def sentry_patched_iter(
66+
self: "EventStream[Any]",
67+
) -> "Iterator[Any]":
68+
"""
69+
Sets information received while iterating the response stream on the AI Client Span.
70+
Responsible for closing the AI Client Span unless the span has already been closed in the close() patch.
71+
"""
72+
span = getattr(self, "_span", None)
73+
if span is None:
74+
yield from original_iter(self)
75+
return
76+
77+
set_on_span = (
78+
span.set_attribute if isinstance(span, StreamedSpan) else span.set_data
79+
)
80+
81+
try:
82+
for event in original_iter(self):
83+
if "metadata" not in event:
84+
yield event
85+
continue
86+
87+
usage = event["metadata"]["usage"]
88+
89+
input_tokens = usage["inputTokens"]
90+
set_on_span(SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS, usage["outputTokens"])
91+
set_on_span(SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS, usage["totalTokens"])
92+
93+
cache_read_tokens = usage.get("cacheReadInputTokens")
94+
if cache_read_tokens is not None:
95+
input_tokens += cache_read_tokens
96+
set_on_span(
97+
SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED, cache_read_tokens
98+
)
99+
100+
cache_write_tokens = usage.get("cacheWriteInputTokens")
101+
if cache_write_tokens is not None:
102+
input_tokens += cache_write_tokens
103+
set_on_span(
104+
SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE,
105+
cache_write_tokens,
106+
)
107+
108+
set_on_span(
109+
SPANDATA.GEN_AI_USAGE_INPUT_TOKENS,
110+
input_tokens,
111+
)
112+
113+
yield event
114+
except GeneratorExit:
115+
raise # Leave span open since underlying stream is not necessarily closed.
116+
except BaseException:
117+
exc_info = sys.exc_info()
118+
with capture_internal_exceptions():
119+
_finish_span(self, *exc_info)
120+
reraise(*exc_info)
121+
122+
_finish_span(self, None, None, None)
123+
124+
def sentry_patched_close(self: "EventStream[Any]") -> None:
125+
"""
126+
Closes the AI Client Span unless `sentry_patched_iter()` closes the span first.
127+
"""
128+
try:
129+
original_close(self)
130+
except BaseException:
131+
exc_info = sys.exc_info()
132+
with capture_internal_exceptions():
133+
_finish_span(self, *exc_info)
134+
reraise(*exc_info)
135+
136+
_finish_span(self, None, None, None)
137+
138+
EventStream.__iter__ = sentry_patched_iter # type: ignore[method-assign]
139+
EventStream.close = sentry_patched_close # type: ignore[method-assign]
51140

52141

53142
class _BedrockExtension(_ServiceExtension):
@@ -115,43 +204,13 @@ def on_success_response(
115204
span.finish()
116205
return
117206

118-
if ctx.operation_name != "ConverseStream":
207+
if ctx.operation_name == "ConverseStream":
208+
stream = cast("ConverseStreamResponseTypeDef", response)["stream"]
209+
stream._span = span # type: ignore[attr-defined]
119210
return
120211

121-
stream = cast("ConverseStreamResponseTypeDef", response)["stream"]
122-
response["stream"] = _wrap_converse_stream(stream, span)
123-
124-
125-
def _wrap_converse_stream(
126-
stream: "EventStream[ConverseStreamOutputTypeDef]",
127-
span: "Union[Span, StreamedSpan]",
128-
) -> "Iterator[ConverseStreamOutputTypeDef]":
129-
set_on_span = (
130-
span.set_attribute if isinstance(span, StreamedSpan) else span.set_data
131-
)
132-
133-
with _stream_span_context(span):
134-
for event in stream:
135-
if "metadata" not in event:
136-
yield event
137-
continue
138-
139-
usage = event["metadata"]["usage"]
140-
141-
set_on_span(SPANDATA.GEN_AI_USAGE_INPUT_TOKENS, usage["inputTokens"])
142-
set_on_span(SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS, usage["outputTokens"])
143-
set_on_span(SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS, usage["totalTokens"])
144-
145-
cache_read_tokens = usage.get("cacheReadInputTokens")
146-
if cache_read_tokens is not None:
147-
set_on_span(
148-
SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED, cache_read_tokens
149-
)
150-
151-
cache_write_tokens = usage.get("cacheWriteInputTokens")
152-
if cache_write_tokens is not None:
153-
set_on_span(
154-
SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE, cache_write_tokens
155-
)
212+
if isinstance(span, StreamedSpan):
213+
span.end()
214+
return
156215

157-
yield event
216+
span.finish()

‎tests/integrations/boto3/test_bedrock.py‎

Lines changed: 72 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,15 @@
1+
import io
12
from unittest.mock import Mock
23

34
import boto3
45
import botocore
6+
import botocore.session
57
import pytest
8+
import urllib3
9+
from botocore.eventstream import EventStream
610
from botocore.exceptions import ClientError, EventStreamError
711
from botocore.loaders import Loader
12+
from botocore.parsers import EventStreamJSONParser
813
from botocore.stub import Stubber
914
from packaging.version import Version
1015

@@ -16,6 +21,7 @@
1621

1722
BEFORE_CALL_EVENT = "before-call.bedrock-runtime.ConverseStream"
1823

24+
1925
pytestmark = pytest.mark.skipif(
2026
"bedrock-runtime" not in Loader().list_available_services("service-2"),
2127
reason="botocore is too old for bedrock-runtime.",
@@ -44,13 +50,64 @@ def converse_stream_response():
4450
usage["cacheReadInputTokens"] = 80
4551
usage["cacheWriteInputTokens"] = 20
4652

47-
return {
48-
"stream": [
53+
output_shape = (
54+
botocore.session.get_session()
55+
.get_service_model("bedrock-runtime")
56+
.operation_model("ConverseStream")
57+
.output_shape
58+
)
59+
raw_stream = urllib3.HTTPResponse(body=io.BytesIO(b""), preload_content=False)
60+
stream = EventStream(
61+
raw_stream, output_shape, EventStreamJSONParser(), "ConverseStream"
62+
)
63+
stream._event_generator = iter(
64+
[
4965
{"messageStart": {"role": "assistant"}},
5066
{"contentBlockDelta": {"contentBlockIndex": 0, "delta": {"text": "Hello"}}},
5167
{"messageStop": {"stopReason": "end_turn"}},
5268
{"metadata": {"usage": usage, "metrics": {"latencyMs": 500}}},
53-
],
69+
]
70+
)
71+
stream._parse_event = lambda event: event
72+
73+
return {
74+
"stream": stream,
75+
"ResponseMetadata": {
76+
"RequestId": "test-id",
77+
"HTTPStatusCode": 200,
78+
"HTTPHeaders": {},
79+
"RetryAttempts": 0,
80+
},
81+
}
82+
83+
84+
@pytest.fixture
85+
def converse_stream_error_response():
86+
error = EventStreamError(
87+
{"Error": {"Code": "throttlingException", "Message": "Rate exceeded"}},
88+
"ConverseStream",
89+
)
90+
91+
def error_events():
92+
yield {"messageStart": {"role": "assistant"}}
93+
yield {"contentBlockDelta": {"contentBlockIndex": 0, "delta": {"text": "Hi"}}}
94+
raise error
95+
96+
output_shape = (
97+
botocore.session.get_session()
98+
.get_service_model("bedrock-runtime")
99+
.operation_model("ConverseStream")
100+
.output_shape
101+
)
102+
raw_stream = urllib3.HTTPResponse(body=io.BytesIO(b""), preload_content=False)
103+
stream = EventStream(
104+
raw_stream, output_shape, EventStreamJSONParser(), "ConverseStream"
105+
)
106+
stream._event_generator = error_events()
107+
stream._parse_event = lambda event: event
108+
109+
return {
110+
"stream": stream,
54111
"ResponseMetadata": {
55112
"RequestId": "test-id",
56113
"HTTPStatusCode": 200,
@@ -267,11 +324,12 @@ def test_converse_stream(
267324
assert span["attributes"][SPANDATA.GEN_AI_PROVIDER_NAME] == "aws.bedrock"
268325
assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat"
269326
assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True
270-
assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10
271327
assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20
272328
assert span["attributes"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30
273329
if BOTOCORE_VERSION < Version("1.37.24"):
330+
assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10
274331
return
332+
assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 110
275333
assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 80
276334
assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 20
277335
else:
@@ -291,19 +349,25 @@ def test_converse_stream(
291349
assert span["data"][SPANDATA.GEN_AI_PROVIDER_NAME] == "aws.bedrock"
292350
assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat"
293351
assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True
294-
assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10
295352
assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20
296353
assert span["data"][SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS] == 30
297354
if BOTOCORE_VERSION < Version("1.37.24"):
355+
assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10
298356
return
357+
assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 110
299358
assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHED] == 80
300359
assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE] == 20
301360

302361

303362
@pytest.mark.parametrize("span_streaming", [True, False])
304363
@pytest.mark.parametrize("stream_gen_ai_spans", [True, False])
305364
def test_converse_stream_error_during_iteration(
306-
sentry_init, capture_items, capture_events, span_streaming, stream_gen_ai_spans
365+
sentry_init,
366+
capture_items,
367+
capture_events,
368+
span_streaming,
369+
stream_gen_ai_spans,
370+
converse_stream_error_response,
307371
):
308372
sentry_init(
309373
traces_sample_rate=1.0,
@@ -316,30 +380,10 @@ def test_converse_stream_error_during_iteration(
316380
aws_access_key_id="-", aws_secret_access_key="-", region_name="us-east-1"
317381
)
318382
client = session.client("bedrock-runtime")
319-
320-
error = EventStreamError(
321-
{"Error": {"Code": "throttlingException", "Message": "Rate exceeded"}},
322-
"ConverseStream",
323-
)
324-
325-
def error_stream():
326-
yield {"messageStart": {"role": "assistant"}}
327-
yield {"contentBlockDelta": {"contentBlockIndex": 0, "delta": {"text": "Hi"}}}
328-
raise error
329-
330-
response = {
331-
"stream": error_stream(),
332-
"ResponseMetadata": {
333-
"RequestId": "test-id",
334-
"HTTPStatusCode": 200,
335-
"HTTPHeaders": {},
336-
"RetryAttempts": 0,
337-
},
338-
}
339-
340383
mock_http = Mock(status_code=200)
341384
client.meta.events.register(
342-
BEFORE_CALL_EVENT, lambda **kwargs: (mock_http, response)
385+
BEFORE_CALL_EVENT,
386+
lambda **kwargs: (mock_http, converse_stream_error_response),
343387
)
344388

345389
if span_streaming or stream_gen_ai_spans:

0 commit comments

Comments
 (0)