Skip to content

Commit 85232d7

Browse files
feat(bedrock): Record inferenceConfig parameters (#8232)
Set the max tokens, temperature, and top-p request parameters from the `inferenceConfig` parameter passed to the Bedrock API.
1 parent f73ce4e commit 85232d7

2 files changed

Lines changed: 89 additions & 0 deletions

File tree

‎sentry_sdk/integrations/boto3/_services/bedrock.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,23 @@ def get_request_attributes(self, ctx: "AwsCallContext") -> "Attributes":
3838
if model_id is not None:
3939
attributes[SPANDATA.GEN_AI_REQUEST_MODEL] = model_id
4040

41+
inference_config = ctx.params.get("inferenceConfig")
42+
43+
if not isinstance(inference_config, dict):
44+
return attributes
45+
46+
max_tokens = inference_config.get("maxTokens")
47+
if max_tokens is not None:
48+
attributes[SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] = max_tokens
49+
50+
temperature = inference_config.get("temperature")
51+
if temperature is not None:
52+
attributes[SPANDATA.GEN_AI_REQUEST_TEMPERATURE] = temperature
53+
54+
top_p = inference_config.get("topP")
55+
if top_p is not None:
56+
attributes[SPANDATA.GEN_AI_REQUEST_TOP_P] = top_p
57+
4158
return attributes
4259

4360
def get_response_attributes(

‎tests/integrations/boto3/test_bedrock.py‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,11 @@ def test_converse(
5555
client.converse(
5656
modelId="anthropic.claude-3-sonnet-20240229-v1:0",
5757
messages=[{"role": "user", "content": [{"text": "Hi"}]}],
58+
inferenceConfig={
59+
"maxTokens": 1024,
60+
"temperature": 0.7,
61+
"topP": 0.9,
62+
},
5863
)
5964

6065
sentry_sdk.flush()
@@ -72,13 +77,22 @@ def test_converse(
7277
span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL]
7378
== "anthropic.claude-3-sonnet-20240229-v1:0"
7479
)
80+
81+
assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 1024
82+
assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7
83+
assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9
7584
else:
7685
events = capture_events()
7786

7887
with stubber, sentry_sdk.start_transaction(name="parent"):
7988
client.converse(
8089
modelId="anthropic.claude-3-sonnet-20240229-v1:0",
8190
messages=[{"role": "user", "content": [{"text": "Hi"}]}],
91+
inferenceConfig={
92+
"maxTokens": 1024,
93+
"temperature": 0.7,
94+
"topP": 0.9,
95+
},
8296
)
8397

8498
(transaction,) = events
@@ -92,6 +106,10 @@ def test_converse(
92106
== "anthropic.claude-3-sonnet-20240229-v1:0"
93107
)
94108

109+
assert span["data"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 1024
110+
assert span["data"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7
111+
assert span["data"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9
112+
95113

96114
@pytest.mark.parametrize("span_streaming", [True, False])
97115
@pytest.mark.parametrize("stream_gen_ai_spans", [True, False])
@@ -135,6 +153,11 @@ def raise_client_error(**kwargs):
135153
client.converse(
136154
modelId="anthropic.claude-3-sonnet-20240229-v1:0",
137155
messages=[{"role": "user", "content": [{"text": "Hi"}]}],
156+
inferenceConfig={
157+
"maxTokens": 1024,
158+
"temperature": 0.7,
159+
"topP": 0.9,
160+
},
138161
)
139162

140163
sentry_sdk.flush()
@@ -152,6 +175,10 @@ def raise_client_error(**kwargs):
152175
== "anthropic.claude-3-sonnet-20240229-v1:0"
153176
)
154177

178+
assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 1024
179+
assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7
180+
assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9
181+
155182
assert span["attributes"][SPANDATA.ERROR_TYPE] == "AccessDeniedException"
156183
assert span["attributes"][SPANDATA.HTTP_STATUS_CODE] == 403
157184
assert span["attributes"][SPANDATA.AWS_REQUEST_ID] == "test-id"
@@ -164,6 +191,11 @@ def raise_client_error(**kwargs):
164191
client.converse(
165192
modelId="anthropic.claude-3-sonnet-20240229-v1:0",
166193
messages=[{"role": "user", "content": [{"text": "Hi"}]}],
194+
inferenceConfig={
195+
"maxTokens": 1024,
196+
"temperature": 0.7,
197+
"topP": 0.9,
198+
},
167199
)
168200

169201
(transaction,) = events
@@ -176,6 +208,10 @@ def raise_client_error(**kwargs):
176208
== "anthropic.claude-3-sonnet-20240229-v1:0"
177209
)
178210

211+
assert span["data"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 1024
212+
assert span["data"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7
213+
assert span["data"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9
214+
179215
assert span["data"][SPANDATA.ERROR_TYPE] == "AccessDeniedException"
180216
assert span["data"][SPANDATA.HTTP_STATUS_CODE] == 403
181217
assert span["data"][SPANDATA.AWS_REQUEST_ID] == "test-id"
@@ -209,6 +245,11 @@ def test_converse_stream(
209245
client.converse_stream(
210246
modelId="anthropic.claude-3-sonnet-20240229-v1:0",
211247
messages=[{"role": "user", "content": [{"text": "Hi"}]}],
248+
inferenceConfig={
249+
"maxTokens": 1024,
250+
"temperature": 0.7,
251+
"topP": 0.9,
252+
},
212253
)
213254

214255
sentry_sdk.flush()
@@ -226,13 +267,22 @@ def test_converse_stream(
226267
span["attributes"][SPANDATA.GEN_AI_REQUEST_MODEL]
227268
== "anthropic.claude-3-sonnet-20240229-v1:0"
228269
)
270+
271+
assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 1024
272+
assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7
273+
assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9
229274
else:
230275
events = capture_events()
231276

232277
with stubber, sentry_sdk.start_transaction(name="parent"):
233278
client.converse_stream(
234279
modelId="anthropic.claude-3-sonnet-20240229-v1:0",
235280
messages=[{"role": "user", "content": [{"text": "Hi"}]}],
281+
inferenceConfig={
282+
"maxTokens": 1024,
283+
"temperature": 0.7,
284+
"topP": 0.9,
285+
},
236286
)
237287

238288
(transaction,) = events
@@ -246,6 +296,10 @@ def test_converse_stream(
246296
== "anthropic.claude-3-sonnet-20240229-v1:0"
247297
)
248298

299+
assert span["data"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 1024
300+
assert span["data"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7
301+
assert span["data"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9
302+
249303

250304
@pytest.mark.parametrize("span_streaming", [True, False])
251305
@pytest.mark.parametrize("stream_gen_ai_spans", [True, False])
@@ -289,6 +343,11 @@ def raise_client_error(**kwargs):
289343
client.converse_stream(
290344
modelId="anthropic.claude-3-sonnet-20240229-v1:0",
291345
messages=[{"role": "user", "content": [{"text": "Hi"}]}],
346+
inferenceConfig={
347+
"maxTokens": 1024,
348+
"temperature": 0.7,
349+
"topP": 0.9,
350+
},
292351
)
293352

294353
sentry_sdk.flush()
@@ -307,6 +366,10 @@ def raise_client_error(**kwargs):
307366
== "anthropic.claude-3-sonnet-20240229-v1:0"
308367
)
309368

369+
assert span["attributes"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 1024
370+
assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7
371+
assert span["attributes"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9
372+
310373
assert span["attributes"][SPANDATA.ERROR_TYPE] == "AccessDeniedException"
311374
assert span["attributes"][SPANDATA.HTTP_STATUS_CODE] == 403
312375
assert span["attributes"][SPANDATA.AWS_REQUEST_ID] == "test-id"
@@ -319,6 +382,11 @@ def raise_client_error(**kwargs):
319382
client.converse_stream(
320383
modelId="anthropic.claude-3-sonnet-20240229-v1:0",
321384
messages=[{"role": "user", "content": [{"text": "Hi"}]}],
385+
inferenceConfig={
386+
"maxTokens": 1024,
387+
"temperature": 0.7,
388+
"topP": 0.9,
389+
},
322390
)
323391

324392
(transaction,) = events
@@ -332,6 +400,10 @@ def raise_client_error(**kwargs):
332400
== "anthropic.claude-3-sonnet-20240229-v1:0"
333401
)
334402

403+
assert span["data"][SPANDATA.GEN_AI_REQUEST_MAX_TOKENS] == 1024
404+
assert span["data"][SPANDATA.GEN_AI_REQUEST_TEMPERATURE] == 0.7
405+
assert span["data"][SPANDATA.GEN_AI_REQUEST_TOP_P] == 0.9
406+
335407
assert span["data"][SPANDATA.ERROR_TYPE] == "AccessDeniedException"
336408
assert span["data"][SPANDATA.HTTP_STATUS_CODE] == 403
337409
assert span["data"][SPANDATA.AWS_REQUEST_ID] == "test-id"

0 commit comments

Comments
 (0)