Skip to content

Commit bbb516b

Browse files
fix(transport): Keep the longest active rate limit
A shorter rate limit replaced a longer one that was still active, both for repeated categories in one X-Sentry-Rate-Limits header and across responses, including the all-categories limit set by a 429. The SDK then sent that category before the longer quota expired. The deadlines are now stored in a private _RateLimits class. Its only write operation, update, keeps the later of the stored and the new deadline, so no update path can shorten an active limit. The backing dict is name-mangled and the class uses __slots__. Fixes #8296 Co-Authored-By: Daniel Szoke <7881302+szokeasaurusrex@users.noreply.github.com>
1 parent e540bd6 commit bbb516b

2 files changed

Lines changed: 66 additions & 17 deletions

File tree

‎sentry_sdk/transport.py‎

Lines changed: 37 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,35 @@ def _parse_rate_limits(
211211
continue
212212

213213

214+
class _RateLimits:
215+
"""Deadlines until which each data category is rate limited.
216+
217+
The ``None`` category holds the limit for all categories. ``update`` is the only
218+
write operation, and it keeps the later of the stored and the new deadline, so a
219+
shorter rate limit never replaces a longer one, as the rate limiting spec requires:
220+
https://develop.sentry.dev/sdk/foundations/transport/rate-limiting/#stage-2-determine-rate-limits
221+
"""
222+
223+
__slots__ = ("__deadlines",)
224+
225+
def __init__(self) -> None:
226+
self.__deadlines: "Dict[Optional[str], datetime]" = {}
227+
228+
def update(self, category: "Optional[str]", until: datetime) -> None:
229+
"""Extend the limit for ``category`` to ``until``, unless it already ends later."""
230+
current = self.__deadlines.get(category)
231+
if current is None or until > current:
232+
self.__deadlines[category] = until
233+
234+
def is_active(self, category: "Optional[str]") -> bool:
235+
until = self.__deadlines.get(category)
236+
return until is not None and until > datetime.now(timezone.utc)
237+
238+
def any_active(self) -> bool:
239+
now = datetime.now(timezone.utc)
240+
return any(until > now for until in self.__deadlines.values())
241+
242+
214243
class HttpTransportCore(Transport):
215244
"""Shared base class for sync and async transports."""
216245

@@ -224,7 +253,7 @@ def __init__(self: "Self", options: "Dict[str, Any]") -> None:
224253
self.options: "Dict[str, Any]" = options
225254
self._worker = self._create_worker(options)
226255
self._auth = self.parsed_dsn.to_auth("sentry.python/%s" % VERSION)
227-
self._disabled_until: "Dict[Optional[EventDataCategory], datetime]" = {}
256+
self._disabled_until = _RateLimits()
228257
# We only use this Retry() class for the `get_retry_after` method it exposes
229258
self._retry = urllib3.util.Retry()
230259
self._discarded_events: "DefaultDict[Tuple[EventDataCategory, str], int]" = (
@@ -343,7 +372,8 @@ def _update_rate_limits(
343372
header = self._get_header_value(response, "x-sentry-rate-limits")
344373
if header:
345374
logger.warning("Rate-limited via x-sentry-rate-limits")
346-
self._disabled_until.update(_parse_rate_limits(header))
375+
for category, until in _parse_rate_limits(header):
376+
self._disabled_until.update(category, until)
347377

348378
# old sentries only communicate global rate limit hits via the
349379
# retry-after header on 429. This header can also be emitted on new
@@ -356,8 +386,8 @@ def _update_rate_limits(
356386
if retry_after_value is not None
357387
else None
358388
) or 60
359-
self._disabled_until[None] = datetime.now(timezone.utc) + timedelta(
360-
seconds=retry_after
389+
self._disabled_until.update(
390+
None, datetime.now(timezone.utc) + timedelta(seconds=retry_after)
361391
)
362392

363393
def _handle_request_error(
@@ -463,16 +493,11 @@ def _fetch_pending_client_report(
463493
)
464494

465495
def _check_disabled(self, category: str) -> bool:
466-
def _disabled(bucket: "Any") -> bool:
467-
ts = self._disabled_until.get(bucket)
468-
return ts is not None and ts > datetime.now(timezone.utc)
469-
470-
return _disabled(category) or _disabled(None)
496+
rate_limits = self._disabled_until
497+
return rate_limits.is_active(category) or rate_limits.is_active(None)
471498

472499
def _is_rate_limited(self: "Self") -> bool:
473-
return any(
474-
ts > datetime.now(timezone.utc) for ts in self._disabled_until.values()
475-
)
500+
return self._disabled_until.any_active()
476501

477502
def _is_worker_full(self: "Self") -> bool:
478503
return self._worker.full()

‎tests/test_transport.py‎

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
AsyncHttpTransport,
4545
HttpTransport,
4646
_parse_rate_limits,
47+
_RateLimits,
4748
)
4849

4950
server = None
@@ -473,6 +474,22 @@ def test_parse_rate_limits(input, expected):
473474
assert dict(_parse_rate_limits(input, now=NOW)) == expected
474475

475476

477+
def test_rate_limits_keep_longest_deadline():
478+
rate_limits = _RateLimits()
479+
now = datetime.now(timezone.utc)
480+
481+
rate_limits.update("error", now - timedelta(seconds=1))
482+
assert not rate_limits.is_active("error")
483+
484+
# A longer deadline replaces a shorter one.
485+
rate_limits.update("error", now + timedelta(hours=1))
486+
assert rate_limits.is_active("error")
487+
488+
# A shorter deadline does not replace a longer one.
489+
rate_limits.update("error", now - timedelta(seconds=1))
490+
assert rate_limits.is_active("error")
491+
492+
476493
def test_envelope_too_large_response(capturing_server, make_client):
477494
client = make_client()
478495

@@ -511,7 +528,7 @@ def test_simple_rate_limits(capturing_server, make_client):
511528
assert capturing_server.captured[0].path == "/api/132/envelope/"
512529
capturing_server.clear_captured()
513530

514-
assert set(client.transport._disabled_until) == set([None])
531+
assert client.transport._disabled_until.is_active(None)
515532

516533
client.capture_event({"type": "transaction"})
517534
client.capture_event({"type": "event"})
@@ -547,7 +564,9 @@ def record_lost_event(reason, data_category=None, item=None):
547564
assert capturing_server.captured[0].path == "/api/132/envelope/"
548565
capturing_server.clear_captured()
549566

550-
assert set(client.transport._disabled_until) == set(["transaction"])
567+
rate_limits = client.transport._disabled_until
568+
assert rate_limits.is_active("transaction")
569+
assert not rate_limits.is_active(None)
551570

552571
client.capture_event({"type": "transaction"})
553572
client.capture_event({"type": "transaction"})
@@ -600,7 +619,10 @@ def intercepting_fetch(*args, **kwargs):
600619
assert capturing_server.captured[0].path == "/api/132/envelope/"
601620
capturing_server.clear_captured()
602621

603-
assert set(client.transport._disabled_until) == set(["attachment", "transaction"])
622+
rate_limits = client.transport._disabled_until
623+
assert rate_limits.is_active("attachment")
624+
assert rate_limits.is_active("transaction")
625+
assert not rate_limits.is_active(None)
604626

605627
client.capture_event({"type": "transaction"})
606628
client.capture_event({"type": "transaction"})
@@ -694,7 +716,7 @@ def test_complex_limits_without_data_category(
694716
assert capturing_server.captured[0].path == "/api/132/envelope/"
695717
capturing_server.clear_captured()
696718

697-
assert set(client.transport._disabled_until) == set([None])
719+
assert client.transport._disabled_until.is_active(None)
698720

699721
client.capture_event({"type": "transaction"})
700722
client.capture_event({"type": "transaction"})
@@ -826,7 +848,9 @@ def test_log_item_limits(capturing_server, response_code, item, make_client):
826848
assert capturing_server.captured[0].path == "/api/132/envelope/"
827849
capturing_server.clear_captured()
828850

829-
assert set(client.transport._disabled_until) == {"log_item"}
851+
rate_limits = client.transport._disabled_until
852+
assert rate_limits.is_active("log_item")
853+
assert not rate_limits.is_active(None)
830854

831855
client.transport.capture_envelope(envelope)
832856
client.capture_event({"type": "transaction"})

0 commit comments

Comments
 (0)