Skip to content

Commit fbfe80f

Browse files
tests more realistic
1 parent 3053b22 commit fbfe80f

1 file changed

Lines changed: 68 additions & 144 deletions

File tree

‎tests/integrations/boto3/test_bedrock.py‎

Lines changed: 68 additions & 144 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@
77
import botocore.session
88
import pytest
99
import urllib3
10-
from botocore.eventstream import EventStream
11-
from botocore.exceptions import ClientError, EventStreamError
10+
from botocore.eventstream import EventStream, EventStreamMessage
11+
from botocore.exceptions import ClientError
1212
from botocore.loaders import Loader
1313
from botocore.parsers import EventStreamJSONParser
1414
from botocore.stub import Stubber
@@ -48,25 +48,48 @@ def converse_stream_response():
4848
usage["cacheReadInputTokens"] = 80
4949
usage["cacheWriteInputTokens"] = 20
5050

51-
output_shape = (
51+
stream_shape = (
5252
botocore.session.get_session()
5353
.get_service_model("bedrock-runtime")
5454
.operation_model("ConverseStream")
55-
.output_shape
55+
.output_shape.members["stream"]
5656
)
5757
raw_stream = urllib3.HTTPResponse(body=io.BytesIO(b""), preload_content=False)
5858
stream = EventStream(
59-
raw_stream, output_shape, EventStreamJSONParser(), "ConverseStream"
59+
raw_stream, stream_shape, EventStreamJSONParser(), "ConverseStream"
6060
)
6161
stream._event_generator = iter(
6262
[
63-
{"messageStart": {"role": "assistant"}},
64-
{"contentBlockDelta": {"contentBlockIndex": 0, "delta": {"text": "Hello"}}},
65-
{"messageStop": {"stopReason": "end_turn"}},
66-
{"metadata": {"usage": usage, "metrics": {"latencyMs": 500}}},
63+
EventStreamMessage(
64+
prelude=None,
65+
headers={":message-type": "event", ":event-type": "messageStart"},
66+
payload=json.dumps({"role": "assistant"}).encode(),
67+
crc=None,
68+
),
69+
EventStreamMessage(
70+
prelude=None,
71+
headers={":message-type": "event", ":event-type": "contentBlockDelta"},
72+
payload=json.dumps(
73+
{"contentBlockIndex": 0, "delta": {"text": "Hello"}}
74+
).encode(),
75+
crc=None,
76+
),
77+
EventStreamMessage(
78+
prelude=None,
79+
headers={":message-type": "event", ":event-type": "messageStop"},
80+
payload=json.dumps({"stopReason": "end_turn"}).encode(),
81+
crc=None,
82+
),
83+
EventStreamMessage(
84+
prelude=None,
85+
headers={":message-type": "event", ":event-type": "metadata"},
86+
payload=json.dumps(
87+
{"usage": usage, "metrics": {"latencyMs": 500}}
88+
).encode(),
89+
crc=None,
90+
),
6791
]
6892
)
69-
stream._parse_event = lambda event: event
7093

7194
return {
7295
"stream": stream,
@@ -81,28 +104,45 @@ def converse_stream_response():
81104

82105
@pytest.fixture
83106
def converse_stream_error_response():
84-
error = EventStreamError(
85-
{"Error": {"Code": "throttlingException", "Message": "Rate exceeded"}},
86-
"ConverseStream",
87-
)
88-
89-
def error_events():
90-
yield {"messageStart": {"role": "assistant"}}
91-
yield {"contentBlockDelta": {"contentBlockIndex": 0, "delta": {"text": "Hi"}}}
92-
raise error
107+
# Raw frames are left to botocore's own _parse_event, which raises
108+
# EventStreamError on the exception frame from inside EventStream.__iter__.
109+
frames = [
110+
EventStreamMessage(
111+
prelude=None,
112+
headers={":message-type": "event", ":event-type": "messageStart"},
113+
payload=json.dumps({"role": "assistant"}).encode(),
114+
crc=None,
115+
),
116+
EventStreamMessage(
117+
prelude=None,
118+
headers={":message-type": "event", ":event-type": "contentBlockDelta"},
119+
payload=json.dumps(
120+
{"contentBlockIndex": 0, "delta": {"text": "Hi"}}
121+
).encode(),
122+
crc=None,
123+
),
124+
EventStreamMessage(
125+
prelude=None,
126+
headers={
127+
":message-type": "exception",
128+
":exception-type": "modelStreamErrorException",
129+
},
130+
payload=json.dumps({"message": "Stream failed"}).encode(),
131+
crc=None,
132+
),
133+
]
93134

94-
output_shape = (
135+
stream_shape = (
95136
botocore.session.get_session()
96137
.get_service_model("bedrock-runtime")
97138
.operation_model("ConverseStream")
98-
.output_shape
139+
.output_shape.members["stream"]
99140
)
100141
raw_stream = urllib3.HTTPResponse(body=io.BytesIO(b""), preload_content=False)
101142
stream = EventStream(
102-
raw_stream, output_shape, EventStreamJSONParser(), "ConverseStream"
143+
raw_stream, stream_shape, EventStreamJSONParser(), "ConverseStream"
103144
)
104-
stream._event_generator = error_events()
105-
stream._parse_event = lambda event: event
145+
stream._event_generator = iter(frames)
106146

107147
return {
108148
"stream": stream,
@@ -579,12 +619,12 @@ def test_converse_stream_error_during_iteration(
579619
items = capture_items("span")
580620

581621
with sentry_sdk.start_transaction(name="parent"):
582-
result = client.converse_stream(
622+
response = client.converse_stream(
583623
modelId="anthropic.claude-3-sonnet-20240229-v1:0",
584624
messages=[{"role": "user", "content": [{"text": "Hi"}]}],
585625
)
586626
with pytest.raises(ClientError):
587-
for _ in result["stream"]:
627+
for _ in response["stream"]:
588628
pass
589629

590630
sentry_sdk.flush()
@@ -595,138 +635,22 @@ def test_converse_stream_error_during_iteration(
595635
if span["attributes"].get("sentry.op") == OP.GEN_AI_CHAT
596636
)
597637

598-
assert span["attributes"][SPANDATA.GEN_AI_PROVIDER_NAME] == "aws.bedrock"
599-
assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat"
600-
assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True
601638
assert span["status"] == "error"
602639
else:
603640
events = capture_events()
604641

605642
with sentry_sdk.start_transaction(name="parent"):
606-
result = client.converse_stream(
643+
response = client.converse_stream(
607644
modelId="anthropic.claude-3-sonnet-20240229-v1:0",
608645
messages=[{"role": "user", "content": [{"text": "Hi"}]}],
609646
)
610647
with pytest.raises(ClientError):
611-
for _ in result["stream"]:
648+
for _ in response["stream"]:
612649
pass
613650

614651
(transaction,) = events
615652
(span,) = transaction["spans"]
616653

617-
assert span["data"][SPANDATA.GEN_AI_PROVIDER_NAME] == "aws.bedrock"
618-
assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat"
619-
assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True
620-
assert span["status"] == "internal_error"
621-
622-
623-
@pytest.mark.parametrize("span_streaming", [True, False])
624-
@pytest.mark.parametrize("stream_gen_ai_spans", [True, False])
625-
def test_converse_stream_error(
626-
sentry_init, capture_items, capture_events, span_streaming, stream_gen_ai_spans
627-
):
628-
sentry_init(
629-
traces_sample_rate=1.0,
630-
integrations=[Boto3Integration()],
631-
stream_gen_ai_spans=stream_gen_ai_spans,
632-
trace_lifecycle="stream" if span_streaming else "static",
633-
)
634-
635-
session = boto3.Session(
636-
aws_access_key_id="-", aws_secret_access_key="-", region_name="us-east-1"
637-
)
638-
client = session.client("bedrock-runtime")
639-
original_exception = ClientError(
640-
error_response={
641-
"Error": {"Code": "AccessDeniedException", "Message": "Access denied"},
642-
"ResponseMetadata": {
643-
"RequestId": "test-id",
644-
"HostId": "",
645-
"HTTPStatusCode": 403,
646-
"HTTPHeaders": {},
647-
"RetryAttempts": 1,
648-
},
649-
},
650-
operation_name="ConverseStream",
651-
)
652-
653-
def raise_client_error(**kwargs):
654-
raise original_exception
655-
656-
client.meta.events.register("before-parameter-build", raise_client_error)
657-
658-
if span_streaming or stream_gen_ai_spans:
659-
items = capture_items("span")
660-
661-
with sentry_sdk.start_transaction(name="parent"), pytest.raises(ClientError):
662-
client.converse_stream(
663-
modelId="anthropic.claude-3-sonnet-20240229-v1:0",
664-
messages=[{"role": "user", "content": [{"text": "Hi"}]}],
665-
inferenceConfig={
666-
"maxTokens": 1024,
667-
"temperature": 0.7,
668-
"topP": 0.9,
669-
},
670-
)
671-
672-
sentry_sdk.flush()
673-
spans = [item.payload for item in items]
674-
(span,) = (
675-
span
676-
for span in spans
677-
if span["attributes"].get("sentry.op") == OP.GEN_AI_CHAT
678-
)
679-
680-
assert span["attributes"][SPANDATA.GEN_AI_PROVIDER_NAME] == "aws.bedrock"
681-
assert span["attributes"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat"
682-
assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True
683-
assert (
684-
span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL]
685-
== "anthropic.claude-3-sonnet-20240229-v1:0"
686-
)
687-
688-
assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 1024
689-
assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7
690-
assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9
691-
692-
assert span["attributes"][SPANDATA.ERROR_TYPE] == "AccessDeniedException"
693-
assert span["attributes"][SPANDATA.HTTP_STATUS_CODE] == 403
694-
assert span["attributes"][SPANDATA.AWS_REQUEST_ID] == "test-id"
695-
assert span["attributes"][SPANDATA.HTTP_REQUEST_RESEND_COUNT] == 1
696-
assert span["status"] == "error"
697-
else:
698-
events = capture_events()
699-
700-
with sentry_sdk.start_transaction(name="parent"), pytest.raises(ClientError):
701-
client.converse_stream(
702-
modelId="anthropic.claude-3-sonnet-20240229-v1:0",
703-
messages=[{"role": "user", "content": [{"text": "Hi"}]}],
704-
inferenceConfig={
705-
"maxTokens": 1024,
706-
"temperature": 0.7,
707-
"topP": 0.9,
708-
},
709-
)
710-
711-
(transaction,) = events
712-
(span,) = transaction["spans"]
713-
714-
assert span["data"][SPANDATA.GEN_AI_PROVIDER_NAME] == "aws.bedrock"
715-
assert span["data"][SPANDATA.GEN_AI_OPERATION_NAME] == "chat"
716-
assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True
717-
assert (
718-
span["data"][SPANDATA.GEN_AI_REQUEST_MODEL]
719-
== "anthropic.claude-3-sonnet-20240229-v1:0"
720-
)
721-
722-
assert span["data"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 1024
723-
assert span["data"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7
724-
assert span["data"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9
725-
726-
assert span["data"][SPANDATA.ERROR_TYPE] == "AccessDeniedException"
727-
assert span["data"][SPANDATA.HTTP_STATUS_CODE] == 403
728-
assert span["data"][SPANDATA.AWS_REQUEST_ID] == "test-id"
729-
assert span["data"][SPANDATA.HTTP_REQUEST_RESEND_COUNT] == 1
730654
assert span["status"] == "internal_error"
731655

732656

0 commit comments

Comments
 (0)