Skip to content

Commit 8abf619

Browse files
pabloDeputtersentry-junior[bot]dingsdaxsentrivana
authored
feat(boto3): Add SQS/SNS trace propagation (#8145)
### Description Propagates the trace through SQS and SNS messages, so the consumer can be linked back to the request that sent the message. **Send side** (`SendMessage`, `SendMessageBatch`, `Publish`, `PublishBatch`): - Adds `sentry-trace` and `baggage` as `String` message attributes, on the message or on each batch entry. - Only when `trace_propagation_targets` match the endpoint. - Skips a message if the caller already set these attributes, if it would go over the 10-attribute limit, or if it would go over 256 KiB. AWS would reject the message otherwise. - Works on a copy of the params. The caller's dict isn't changed. **Receive side** (`ReceiveMessage`): - Adds `sentry-trace` and `baggage` to `MessageAttributeNames`, so SQS returns them. Only when the span is recorded, and not when the caller already asks for `All` or `.*`. - Adds a span link to each distinct producer context in the received messages, with `sentry.link.type: message_origin`. - Reads the context from SQS message attributes, and from the JSON envelope of an SNS notification without raw message delivery. **Tests**: injection and extraction for single and batch calls, the limits above, and a round-trip test: send a message, receive what was injected, and check that the link points at the send span. Stacked on #8142 (SNS), which is stacked on #8137 (SQS). #### Issues * resolves: PY-3008 <!-- junior-request-attribution:start --> via @dingsdax. <!-- junior-request-attribution:end --> <!-- junior-session-footer:start --> <!-- junior-conversation-id:slack%3AC0C9X9V5NMT%3A1793904671.980379 --> -- [View Session](https://junior-prod.sentry.dev/conversations/slack%3AC0C9X9V5NMT%3A1793904671.980379) [[Sentry]](https://sentry.sentry.io/explore/conversations/slack%3AC0C9X9V5NMT%3A1793904671.980379/?project=4510944073809921) <!-- junior-session-footer:end --> --------- Co-authored-by: sentry-junior[bot] <264270552+sentry-junior[bot]@users.noreply.github.com> Co-authored-by: Johannes Daxböck <114897+dingsdax@users.noreply.github.com> Co-authored-by: Ivana Kellyer <ivana.kellyer@sentry.io> Co-authored-by: Ivana Kellyer <131587164+sentrivana@users.noreply.github.com>
1 parent 49473c4 commit 8abf619

6 files changed

Lines changed: 478 additions & 5 deletions

File tree

‎sentry_sdk/consts.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1424,6 +1424,12 @@ class SPANDATA:
14241424
Example: "client", "server", "producer", "consumer", "internal"
14251425
"""
14261426

1427+
SENTRY_LINK_TYPE = "sentry.link.type"
1428+
"""
1429+
Set on a span link. Describes the relationship between the span and the linked span.
1430+
Example: "message_origin"
1431+
"""
1432+
14271433
SENTRY_OP = "sentry.op"
14281434
"""
14291435
The operation of a span.

‎sentry_sdk/integrations/boto3/_services/sns.py‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,13 @@
66
from sentry_sdk.integrations.boto3._utils import (
77
_get_aws_arn_attributes,
88
_get_batch_error_type,
9+
_inject_message_attributes,
910
)
1011
from sentry_sdk.traces import SpanStatus
1112
from sentry_sdk.utils import capture_internal_exceptions
1213

1314
if TYPE_CHECKING:
14-
from typing import Any, Optional
15+
from typing import Any, Dict, Optional
1516

1617
from sentry_sdk._types import Attributes
1718
from sentry_sdk.integrations.boto3._context import AwsCallContext
@@ -129,3 +130,13 @@ def on_success_response(
129130

130131
span.set_attributes(attributes)
131132
span.end()
133+
134+
def get_request_params(
135+
self, ctx: "AwsCallContext", span: "Optional[Span]"
136+
) -> "Optional[Dict[str, Any]]":
137+
if ctx.operation_name not in _PUBLISH_OPERATIONS:
138+
return None
139+
140+
return _inject_message_attributes(
141+
ctx, span, "Message", "PublishBatchRequestEntries"
142+
)

‎sentry_sdk/integrations/boto3/_services/sqs.py‎

Lines changed: 89 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import json
12
import re
23
from typing import TYPE_CHECKING
34
from urllib.parse import unquote, urlsplit
@@ -7,14 +8,21 @@
78
from sentry_sdk.integrations.boto3._utils import (
89
_get_aws_arn_attributes,
910
_get_batch_error_type,
11+
_inject_message_attributes,
1012
)
11-
from sentry_sdk.traces import SpanStatus
13+
from sentry_sdk.traces import (
14+
BAGGAGE_HEADER_NAME,
15+
SENTRY_TRACE_HEADER_NAME,
16+
NoOpSpan,
17+
SpanStatus,
18+
)
19+
from sentry_sdk.tracing_utils import extract_sentrytrace_data
1220
from sentry_sdk.utils import capture_internal_exceptions
1321

1422
if TYPE_CHECKING:
15-
from typing import Any, Optional
23+
from typing import Any, Dict, Optional, Tuple
1624

17-
from sentry_sdk._types import Attributes
25+
from sentry_sdk._types import Attributes, SpanLink
1826
from sentry_sdk.integrations.boto3._context import AwsCallContext
1927
from sentry_sdk.traces import Span
2028

@@ -28,6 +36,44 @@
2836
}
2937

3038

39+
def _message_link(message: "Dict[str, Any]") -> "Optional[SpanLink]":
40+
# message was sent straight to SQS, or through SNS with raw message delivery on.
41+
sentry_trace = (
42+
message.get("MessageAttributes", {})
43+
.get(SENTRY_TRACE_HEADER_NAME, {})
44+
.get("StringValue")
45+
)
46+
47+
# SNS wraps the message and its attributes in a JSON envelope when raw message delivery is off.
48+
# https://docs.aws.amazon.com/sns/latest/dg/sns-large-payload-raw-message-delivery.html
49+
body = message.get("Body", "")
50+
if sentry_trace is None and f'"{SENTRY_TRACE_HEADER_NAME}"' in body:
51+
try:
52+
envelope = json.loads(body)
53+
except ValueError:
54+
envelope = None
55+
if isinstance(envelope, dict) and envelope.get("Type") == "Notification":
56+
sentry_trace = (
57+
envelope.get("MessageAttributes", {})
58+
.get(SENTRY_TRACE_HEADER_NAME, {})
59+
.get("Value")
60+
)
61+
62+
data = extract_sentrytrace_data(sentry_trace)
63+
if data is None or not data["trace_id"] or not data["parent_span_id"]:
64+
return None
65+
66+
link: "SpanLink" = {
67+
"trace_id": str(data["trace_id"]),
68+
"span_id": str(data["parent_span_id"]),
69+
# Sentry drops other link attributes for now; add `messaging.message.id` in the future.
70+
"attributes": {SPANDATA.SENTRY_LINK_TYPE: "message_origin"},
71+
}
72+
if data["parent_sampled"] is not None:
73+
link["sampled"] = bool(data["parent_sampled"])
74+
return link
75+
76+
3177
class _SQSExtension(_ServiceExtension):
3278
def get_span_op(self, ctx: "AwsCallContext") -> "Optional[str]":
3379
if ctx.operation_name in _MESSAGING_OPERATIONS:
@@ -124,5 +170,45 @@ def on_success_response(
124170
span.status = SpanStatus.ERROR
125171
attributes[SPANDATA.ERROR_TYPE] = error_type
126172

173+
if ctx.operation_name == "ReceiveMessage":
174+
with capture_internal_exceptions():
175+
# one link per producer context; batch entries of one send share it.
176+
links: "Dict[Tuple[str, str], SpanLink]" = {}
177+
for message in response.get("Messages", ()):
178+
# returns a link to the producer context a received message carries, if valid.
179+
link = _message_link(message)
180+
if link is not None:
181+
links.setdefault((link["trace_id"], link["span_id"]), link)
182+
span.add_links(list(links.values()))
183+
127184
span.set_attributes(attributes)
128185
span.end()
186+
187+
def get_request_params(
188+
self, ctx: "AwsCallContext", span: "Optional[Span]"
189+
) -> "Optional[Dict[str, Any]]":
190+
if ctx.operation_name in ("SendMessage", "SendMessageBatch"):
191+
return _inject_message_attributes(ctx, span, "MessageBody", "Entries")
192+
193+
# only request the carrier when the receive span can record links.
194+
if (
195+
ctx.operation_name != "ReceiveMessage"
196+
or span is None
197+
or isinstance(span, NoOpSpan)
198+
):
199+
return None
200+
201+
# SQS only returns the message attributes named in `MessageAttributeNames`.
202+
# https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_ReceiveMessage.html
203+
names = ctx.params.get("MessageAttributeNames", [])
204+
missing = [
205+
name
206+
for name in (SENTRY_TRACE_HEADER_NAME, BAGGAGE_HEADER_NAME)
207+
if name not in names
208+
]
209+
# `All` and `.*` already return every attribute, including ours.
210+
if not missing or "All" in names or ".*" in names:
211+
return None
212+
213+
# append ours to the caller's names, in a copy of their params.
214+
return {**ctx.params, "MessageAttributeNames": [*names, *missing]}

‎sentry_sdk/integrations/boto3/_utils.py‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,15 @@
1515
# tuple of (request param name, span attribute name).
1616
_AttributeSpec = Tuple[str, str]
1717

18+
# SQS and SNS reject a message with more than 10 message attributes.
19+
# https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-message-metadata.html
20+
_MAX_MESSAGE_ATTRIBUTES = 10
21+
22+
# body and attributes must fit the queue's or topic's configurable `MaximumMessageSize`;
23+
# assume the 256 KiB default of SNS topics and older SQS queues; batch uses the same limit.
24+
# https://docs.aws.amazon.com/sns/latest/api/API_PublishBatch.html
25+
_MAX_MESSAGE_SIZE = 256 * 1024
26+
1827

1928
def _extract_attributes(
2029
source: "Dict[str, Any]", specs: "Sequence[_AttributeSpec]"
@@ -76,3 +85,56 @@ def _get_propagation_headers(
7685
return dict(
7786
sentry_sdk.get_current_scope().iter_trace_propagation_headers(span=span)
7887
)
88+
89+
90+
def _inject_message_attributes(
91+
ctx: "AwsCallContext", span: "Optional[Span]", body_key: str, entries_key: str
92+
) -> "Optional[Dict[str, Any]]":
93+
"""
94+
Return a copy of `ctx.params` with `sentry-trace` and `baggage` added in
95+
string message attributes, or to each batch entry with room for them.
96+
"""
97+
carrier = {
98+
name: {"DataType": "String", "StringValue": value}
99+
for name, value in _get_propagation_headers(ctx, span).items()
100+
}
101+
if not carrier:
102+
return None
103+
104+
def _get_message_size(body: str, attributes: "Dict[str, Any]") -> int:
105+
size = len(body.encode())
106+
107+
# names, types and values of message attributes count towards the message size.
108+
# https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-message-metadata.html
109+
for name, value in attributes.items():
110+
size += len(name.encode()) + len(value["DataType"].encode())
111+
if "StringValue" in value:
112+
size += len(value["StringValue"].encode())
113+
if "BinaryValue" in value:
114+
binary = value["BinaryValue"]
115+
size += len(binary.encode() if isinstance(binary, str) else binary)
116+
return size
117+
118+
is_batch = entries_key in ctx.params
119+
messages = []
120+
for message in ctx.params[entries_key] if is_batch else (ctx.params,):
121+
attributes = message.get("MessageAttributes", {})
122+
injected = {**attributes, **carrier}
123+
if (
124+
# never replace the caller's carrier.
125+
not carrier.keys() & attributes.keys()
126+
# don't make AWS reject the message.
127+
and len(injected) <= _MAX_MESSAGE_ATTRIBUTES
128+
and _get_message_size(message[body_key], injected) <= _MAX_MESSAGE_SIZE
129+
):
130+
message = {**message, "MessageAttributes": injected}
131+
messages.append(message)
132+
133+
total_size = sum(
134+
_get_message_size(message[body_key], message.get("MessageAttributes", {}))
135+
for message in messages
136+
)
137+
if total_size > _MAX_MESSAGE_SIZE:
138+
return None
139+
140+
return {**ctx.params, entries_key: messages} if is_batch else messages[0]

‎tests/integrations/boto3/test_sqs.py‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
import copy
2+
import json
3+
14
import pytest
25
from botocore.exceptions import ClientError
36
from botocore.stub import Stubber
@@ -321,3 +324,116 @@ def test_delete_message(
321324
assert attributes[SPANDATA.MESSAGING_OPERATION_NAME] == "settle"
322325
assert attributes.get(SPANDATA.MESSAGING_BATCH_MESSAGE_COUNT) == message_count
323326
assert "receipt-handle" not in str(span)
327+
328+
329+
@pytest.mark.parametrize(
330+
"names,sent_names",
331+
[
332+
pytest.param(None, ["sentry-trace", "baggage"], id="default"),
333+
pytest.param(
334+
["orderId", "baggage"],
335+
["orderId", "baggage", "sentry-trace"],
336+
id="caller-names",
337+
),
338+
pytest.param(
339+
["baggage", "sentry-trace"],
340+
["baggage", "sentry-trace"],
341+
id="already-requested",
342+
),
343+
pytest.param(["All"], ["All"], id="all"),
344+
pytest.param([".*"], [".*"], id="all-pattern"),
345+
],
346+
)
347+
def test_receive_message_requests_carrier(
348+
client_factory, capture_items, names, sent_names
349+
):
350+
client = client_factory("sqs")
351+
queue_url = "https://sqs.eu-north-1.amazonaws.com/123456789012/orders"
352+
params = {} if names is None else {"MessageAttributeNames": names}
353+
caller_params = copy.deepcopy(params)
354+
355+
with Stubber(client) as stubber:
356+
stubber.add_response(
357+
"receive_message",
358+
{},
359+
{"QueueUrl": queue_url, "MessageAttributeNames": sent_names},
360+
)
361+
capture_spans_by_op(
362+
lambda: client.receive_message(QueueUrl=queue_url, **params),
363+
capture_items,
364+
)
365+
366+
assert params == caller_params
367+
368+
369+
def test_receive_message_links(client_factory, capture_items):
370+
client = client_factory("sqs")
371+
sentry_trace = {
372+
"sentry-trace": {
373+
"DataType": "String",
374+
"StringValue": "771a43a4192642f0b136d5159a501700-a3e9e5ba6f99f4d6-1",
375+
}
376+
}
377+
sns_envelope = {
378+
"Type": "Notification",
379+
"MessageId": "sns-message-3",
380+
"TopicArn": "arn:aws:sns:eu-north-1:123456789012:alerts",
381+
"Message": "hi",
382+
"MessageAttributes": {
383+
"sentry-trace": {
384+
"Type": "String",
385+
"Value": "b2c1b1d6cb5f4a0d8a6a2f1c3e9d7b40-9c4f3a2b1e0d8f7a",
386+
}
387+
},
388+
}
389+
response = {
390+
"Messages": [
391+
# batch entries of one send share a context.
392+
{"MessageId": "1", "Body": "hi", "MessageAttributes": sentry_trace},
393+
{"MessageId": "2", "Body": "hi", "MessageAttributes": sentry_trace},
394+
# SNS-to-SQS delivery without raw message delivery.
395+
{"MessageId": "3", "Body": json.dumps(sns_envelope)},
396+
# trace id with malformed sentry-trace.
397+
{
398+
"MessageId": "4",
399+
"Body": "hi",
400+
"MessageAttributes": {
401+
"sentry-trace": {
402+
"DataType": "String",
403+
"StringValue": "771a43a4192642f0b136d5159a501700",
404+
}
405+
},
406+
},
407+
# JSON body that is not an SNS envelope.
408+
{"MessageId": "5", "Body": '{"sentry-trace": "not an SNS envelope"}'},
409+
{"MessageId": "6", "Body": "hi"},
410+
]
411+
}
412+
413+
with Stubber(client) as stubber:
414+
stubber.add_response("receive_message", response)
415+
spans = capture_spans_by_op(
416+
lambda: client.receive_message(
417+
QueueUrl="https://sqs.eu-north-1.amazonaws.com/123456789012/orders"
418+
),
419+
capture_items,
420+
)
421+
422+
(span,) = spans[OP.QUEUE_RECEIVE]
423+
link_type = {
424+
SPANDATA.SENTRY_LINK_TYPE: {"value": "message_origin", "type": "string"}
425+
}
426+
assert span["links"] == [
427+
# messages 1 and 2 share a context, so one link.
428+
{
429+
"trace_id": "771a43a4192642f0b136d5159a501700",
430+
"span_id": "a3e9e5ba6f99f4d6",
431+
"sampled": True,
432+
"attributes": link_type,
433+
},
434+
{
435+
"trace_id": "b2c1b1d6cb5f4a0d8a6a2f1c3e9d7b40",
436+
"span_id": "9c4f3a2b1e0d8f7a",
437+
"attributes": link_type,
438+
},
439+
]

0 commit comments

Comments
 (0)