From c4afc6871522d83f173726bbad0cb0b208275fcc Mon Sep 17 00:00:00 2001 From: linzch3 <20380442+linzch3@users.noreply.github.com> Date: Thu, 19 Nov 2026 13:19:04 +0000 Subject: [PATCH] Bound Idempotency-Key retries by the advertised idempotency-key-lifetime --- pyiceberg/catalog/rest/__init__.py | 98 ++++++++++++++++++++-- tests/catalog/test_rest.py | 125 ++++++++++++++++++++++++++++- 2 files changed, 215 insertions(+), 8 deletions(-) diff --git a/pyiceberg/catalog/rest/__init__.py b/pyiceberg/catalog/rest/__init__.py index 2ad3684c9a..cecb3ee32a 100644 --- a/pyiceberg/catalog/rest/__init__.py +++ b/pyiceberg/catalog/rest/__init__.py @@ -24,10 +24,14 @@ import uuid from collections import deque from collections.abc import Mapping +from contextvars import ContextVar +from datetime import timedelta from enum import Enum, StrEnum +from types import TracebackType from typing import ( TYPE_CHECKING, Any, + Self, ) from urllib.parse import quote, unquote @@ -36,6 +40,9 @@ from requests.adapters import DEFAULT_RETRIES, HTTPAdapter from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt from typing_extensions import override +from urllib3.connectionpool import ConnectionPool +from urllib3.exceptions import MaxRetryError, ResponseError +from urllib3.response import BaseHTTPResponse from urllib3.util.retry import Retry from pyiceberg import __version__ @@ -502,6 +509,65 @@ class ListViewsResponse(IcebergBaseModel): _PLANNING_RESPONSE_ADAPTER = TypeAdapter(PlanningResponse) +_TIMEDELTA_ADAPTER = TypeAdapter(timedelta) + +# Monotonic deadline after which the Idempotency-Key of the request in flight must not be reused. +# Set by `_RetryTimeoutHTTPAdapter.send` for keyed requests and read by `_IdempotencyKeyLifetimeRetry`. +_IDEMPOTENCY_KEY_DEADLINE: ContextVar[float | None] = ContextVar("_IDEMPOTENCY_KEY_DEADLINE", default=None) + + +def _parse_idempotency_key_lifetime(lifetime: str | None) -> timedelta | None: + """Parse the ISO-8601 `idempotency-key-lifetime` advertised by the server. + + Returns None (no bound) when the value is missing or malformed, following the Java client. + A zero or negative lifetime is returned as zero, which disables retries of keyed requests. + """ + if lifetime is None: + return None + try: + parsed = _TIMEDELTA_ADAPTER.validate_python(lifetime) + except ValueError: + logger.warning("Ignoring malformed idempotency-key lifetime: %s", lifetime) + return None + return max(parsed, timedelta(0)) + + +class _IdempotencyKeyLifetimeRetry(Retry): + """Retry that declines to retry a keyed request once the next attempt would outlive the key. + + The REST spec says clients SHOULD NOT reuse an Idempotency-Key after the advertised lifetime, since + the server may have forgotten the key and apply the mutation again. Mirrors the Java client's + `ExponentialHttpRequestRetryStrategy`: a retry is declined when the time since the first attempt + plus the next sleep reaches the lifetime. + """ + + def increment( + self, + method: str | None = None, + url: str | None = None, + response: BaseHTTPResponse | None = None, + error: Exception | None = None, + _pool: ConnectionPool | None = None, + _stacktrace: TracebackType | None = None, + ) -> Self: + new_retry = super().increment(method, url, response=response, error=error, _pool=_pool, _stacktrace=_stacktrace) + deadline = _IDEMPOTENCY_KEY_DEADLINE.get() + if deadline is not None and time.monotonic() + new_retry._max_sleep(response) >= deadline: + # Same as running out of retries: urllib3 returns the last response (raise_on_status=False) + # so `_handle_non_200_response` raises the typed error, or re-raises the connection error. + reason = error or ResponseError(f"next retry would exceed the {IDEMPOTENCY_KEY_HEADER} lifetime") + raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] + return new_retry + + def _max_sleep(self, response: BaseHTTPResponse | None) -> float: + """Upper bound of what `Retry.sleep` will wait before the next attempt.""" + if self.respect_retry_after_header and response is not None: + if retry_after := self.get_retry_after(response): + return retry_after + # `get_backoff_time` adds a random jitter in [0, backoff_jitter), so take the worst case. + return min(self.backoff_max, self.new(backoff_jitter=0.0).get_backoff_time() + self.backoff_jitter) + + class _RetryTimeoutHTTPAdapter(HTTPAdapter): """HTTPAdapter that applies a default per-request timeout. @@ -510,8 +576,14 @@ class _RetryTimeoutHTTPAdapter(HTTPAdapter): The adapter applies `self._timeout` whenever a per-call timeout is not set. """ - def __init__(self, timeout: float | None = None, max_retries: Retry | int = DEFAULT_RETRIES) -> None: + def __init__( + self, + timeout: float | None = None, + max_retries: Retry | int = DEFAULT_RETRIES, + idempotency_key_lifetime: timedelta | None = None, + ) -> None: self._timeout = timeout + self._idempotency_key_lifetime = idempotency_key_lifetime super().__init__(max_retries=max_retries) def send( @@ -525,14 +597,23 @@ def send( ) -> Response: if timeout is None: timeout = self._timeout - return super().send(request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies) + if self._idempotency_key_lifetime is None or IDEMPOTENCY_KEY_HEADER not in request.headers: + return super().send(request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies) + token = _IDEMPOTENCY_KEY_DEADLINE.set(time.monotonic() + self._idempotency_key_lifetime.total_seconds()) + try: + return super().send(request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies) + finally: + _IDEMPOTENCY_KEY_DEADLINE.reset(token) -def _create_connection_adapter(properties: Properties, retry_post: bool = False) -> _RetryTimeoutHTTPAdapter | None: +def _create_connection_adapter( + properties: Properties, retry_post: bool = False, idempotency_key_lifetime: timedelta | None = None +) -> _RetryTimeoutHTTPAdapter | None: """Build a connection adapter from the optional `rest.client.*` properties. Returns None when no connection properties are supplied, leaving the default Session behavior unchanged. POST requests are retried only when `retry_post` is set. + Retries of requests carrying an Idempotency-Key stop before `idempotency_key_lifetime` elapses. Raises ValueError on invalid input. """ connection_timeout_ms = property_as_int(properties, REST_CLIENT_CONNECTION_TIMEOUT_MS) @@ -562,7 +643,8 @@ def _create_connection_adapter(properties: Properties, retry_post: bool = False) return _RetryTimeoutHTTPAdapter( timeout=timeout, - max_retries=Retry( + idempotency_key_lifetime=idempotency_key_lifetime, + max_retries=_IdempotencyKeyLifetimeRetry( # `retries` and `backoff_factor` fall back to a no-op Retry when unset, so a user can # configure only one without having to specify the rest of the policy. total=retries if retries is not None else DEFAULT_RETRIES, @@ -585,6 +667,7 @@ class RestCatalog(Catalog): _supported_endpoints: set[Endpoint] _namespace_separator: str _idempotency_enabled: bool = False + _idempotency_key_lifetime: timedelta | None = None def __init__(self, name: str, **properties: str): """Rest Catalog. @@ -607,7 +690,11 @@ def _create_session(self) -> Session: # Mount the retry/timeout adapter when `connection.*` properties are set. # SigV4's adapter mounted below at `self.uri` is a longer prefix and still wins for that host. - if (connection_adapter := _create_connection_adapter(self.properties, self._idempotency_enabled)) is not None: + if ( + connection_adapter := _create_connection_adapter( + self.properties, self._idempotency_enabled, self._idempotency_key_lifetime + ) + ) is not None: session.mount("http://", connection_adapter) session.mount("https://", connection_adapter) @@ -1083,6 +1170,7 @@ def _fetch_config(self) -> None: self.uri = config[URI] self._idempotency_enabled = config_response.idempotency_key_lifetime is not None + self._idempotency_key_lifetime = _parse_idempotency_key_lifetime(config_response.idempotency_key_lifetime) # Determine supported endpoints endpoints = config_response.endpoints diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index 3d2b64da92..10f32174bb 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -24,6 +24,7 @@ import uuid from collections.abc import Callable, Iterator from contextlib import contextmanager +from datetime import timedelta from typing import Any, cast from unittest import mock from urllib.parse import parse_qs, quote, urlsplit @@ -57,6 +58,7 @@ ScanPlanningMode, TableResponse, ViewResponse, + _parse_idempotency_key_lifetime, _RetryTimeoutHTTPAdapter, ) from pyiceberg.exceptions import ( @@ -2558,7 +2560,9 @@ def test_session_with_socket_timeout_only(rest_mock: Mocker) -> None: @contextmanager -def _local_rest_server_503_then_200(num_failures: int, idempotency_key_lifetime: str | None = None) -> Iterator[dict[str, Any]]: +def _local_rest_server_503_then_200( + num_failures: int, idempotency_key_lifetime: str | None = None, retry_after: str | None = None +) -> Iterator[dict[str, Any]]: """Stand up a loopback HTTP server that returns `num_failures` 503s for `/v1/namespaces` then a 200. Used in place of `requests_mock`, which replaces the HTTPAdapter and would bypass the retry logic. @@ -2591,16 +2595,18 @@ def do_GET(self) -> None: state["namespace_calls"] += 1 state["idempotency_keys"].append(self.headers.get(IDEMPOTENCY_KEY_HEADER)) if state["namespace_calls"] <= num_failures: - self._respond(503, b"") + self._respond(503, b"", {"Retry-After": retry_after} if retry_after is not None else {}) else: self._respond(200, json.dumps({"namespaces": [["foo"]]}).encode()) else: self._respond(404, b"") - def _respond(self, status: int, body: bytes) -> None: + def _respond(self, status: int, body: bytes, headers: dict[str, str] | None = None) -> None: self.send_response(status) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(body))) + for name, value in (headers or {}).items(): + self.send_header(name, value) self.end_headers() if body: self.wfile.write(body) @@ -2674,6 +2680,119 @@ def test_session_retries_keyed_post_with_the_same_key() -> None: assert server["idempotency_keys"] == [key, key] +def test_session_does_not_retry_keyed_post_past_the_key_lifetime() -> None: + # Retry-After (60s) is longer than the advertised lifetime (PT30S), so the retry would reuse an expired key. + with _local_rest_server_503_then_200(num_failures=1, idempotency_key_lifetime="PT30S", retry_after="60") as server: + catalog = RestCatalog( + "rest", + **{ + "uri": f"http://127.0.0.1:{server['port']}/", + "token": TEST_TOKEN, + REST_CLIENT_MAX_RETRIES: "3", + REST_CLIENT_RETRY_BACKOFF_FACTOR: "0", + }, + ) + with pytest.raises(ServiceUnavailableError): + catalog.create_namespace("foo") + assert server["namespace_calls"] == 1 + + +def test_session_does_not_retry_keyed_post_past_the_key_lifetime_due_to_backoff() -> None: + # urllib3 retries the first failure immediately; the backoff before the second retry (20s) alone + # exceeds the advertised lifetime (PT5S). + with _local_rest_server_503_then_200(num_failures=2, idempotency_key_lifetime="PT5S") as server: + catalog = RestCatalog( + "rest", + **{ + "uri": f"http://127.0.0.1:{server['port']}/", + "token": TEST_TOKEN, + REST_CLIENT_MAX_RETRIES: "3", + REST_CLIENT_RETRY_BACKOFF_FACTOR: "10", + }, + ) + with pytest.raises(ServiceUnavailableError): + catalog.create_namespace("foo") + assert server["namespace_calls"] == 2 + + +def test_session_retries_keyed_post_within_the_key_lifetime_with_retry_after() -> None: + with _local_rest_server_503_then_200(num_failures=1, idempotency_key_lifetime="PT30M", retry_after="0") as server: + catalog = RestCatalog( + "rest", + **{ + "uri": f"http://127.0.0.1:{server['port']}/", + "token": TEST_TOKEN, + REST_CLIENT_MAX_RETRIES: "1", + REST_CLIENT_RETRY_BACKOFF_FACTOR: "0", + }, + ) + catalog.create_namespace("foo") + assert server["namespace_calls"] == 2 + + +def test_session_never_retries_keyed_post_with_non_positive_key_lifetime() -> None: + for lifetime in ("PT0S", "-PT5M"): + with _local_rest_server_503_then_200(num_failures=1, idempotency_key_lifetime=lifetime) as server: + catalog = RestCatalog( + "rest", + **{ + "uri": f"http://127.0.0.1:{server['port']}/", + "token": TEST_TOKEN, + REST_CLIENT_MAX_RETRIES: "3", + REST_CLIENT_RETRY_BACKOFF_FACTOR: "0", + }, + ) + with pytest.raises(ServiceUnavailableError): + catalog.create_namespace("foo") + assert server["namespace_calls"] == 1, lifetime + # The key is still sent, only the retry is skipped. + assert server["idempotency_keys"][0] is not None + + +def test_session_key_lifetime_does_not_bound_unkeyed_requests() -> None: + # GETs carry no Idempotency-Key, so they are retried as before even though the lifetime is zero. + with _local_rest_server_503_then_200(num_failures=1, idempotency_key_lifetime="PT0S") as server: + catalog = RestCatalog( + "rest", + **{ + "uri": f"http://127.0.0.1:{server['port']}/", + "token": TEST_TOKEN, + REST_CLIENT_MAX_RETRIES: "1", + REST_CLIENT_RETRY_BACKOFF_FACTOR: "0", + }, + ) + assert catalog.list_namespaces() == [("foo",)] + assert server["namespace_calls"] == 2 + + +def test_session_retries_keyed_post_with_malformed_key_lifetime(caplog: pytest.LogCaptureFixture) -> None: + with _local_rest_server_503_then_200(num_failures=1, idempotency_key_lifetime="not-a-duration") as server: + catalog = RestCatalog( + "rest", + **{ + "uri": f"http://127.0.0.1:{server['port']}/", + "token": TEST_TOKEN, + REST_CLIENT_MAX_RETRIES: "1", + REST_CLIENT_RETRY_BACKOFF_FACTOR: "0", + }, + ) + catalog.create_namespace("foo") + + key = server["idempotency_keys"][0] + assert key is not None + assert server["idempotency_keys"] == [key, key] + assert "Ignoring malformed idempotency-key lifetime: not-a-duration" in caplog.text + + +def test_parse_idempotency_key_lifetime() -> None: + assert _parse_idempotency_key_lifetime(None) is None + assert _parse_idempotency_key_lifetime("PT30M") == timedelta(minutes=30) + assert _parse_idempotency_key_lifetime("P1DT2H") == timedelta(days=1, hours=2) + assert _parse_idempotency_key_lifetime("PT0S") == timedelta(0) + assert _parse_idempotency_key_lifetime("-PT5M") == timedelta(0) + assert _parse_idempotency_key_lifetime("garbage") is None + + def test_mutations_send_idempotency_key_when_advertised(requests_mock: Mocker) -> None: requests_mock.get(f"{TEST_URI}v1/config", json={"defaults": {}, "overrides": {}, "idempotency-key-lifetime": "PT30M"}) requests_mock.post(f"{TEST_URI}v1/namespaces", json={"namespace": ["foo"], "properties": {}})