Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 93 additions & 5 deletions pyiceberg/catalog/rest/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,14 @@
import uuid
from collections import deque
from collections.abc import Mapping
from contextvars import ContextVar
from datetime import timedelta
from enum import Enum, StrEnum
from types import TracebackType
from typing import (
TYPE_CHECKING,
Any,
Self,
)
from urllib.parse import quote, unquote

Expand All @@ -36,6 +40,9 @@
from requests.adapters import DEFAULT_RETRIES, HTTPAdapter
from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt
from typing_extensions import override
from urllib3.connectionpool import ConnectionPool
from urllib3.exceptions import MaxRetryError, ResponseError
from urllib3.response import BaseHTTPResponse
from urllib3.util.retry import Retry

from pyiceberg import __version__
Expand Down Expand Up @@ -502,6 +509,65 @@ class ListViewsResponse(IcebergBaseModel):
_PLANNING_RESPONSE_ADAPTER = TypeAdapter(PlanningResponse)


_TIMEDELTA_ADAPTER = TypeAdapter(timedelta)

# Monotonic deadline after which the Idempotency-Key of the request in flight must not be reused.
# Set by `_RetryTimeoutHTTPAdapter.send` for keyed requests and read by `_IdempotencyKeyLifetimeRetry`.
_IDEMPOTENCY_KEY_DEADLINE: ContextVar[float | None] = ContextVar("_IDEMPOTENCY_KEY_DEADLINE", default=None)


def _parse_idempotency_key_lifetime(lifetime: str | None) -> timedelta | None:
"""Parse the ISO-8601 `idempotency-key-lifetime` advertised by the server.

Returns None (no bound) when the value is missing or malformed, following the Java client.
A zero or negative lifetime is returned as zero, which disables retries of keyed requests.
"""
if lifetime is None:
return None
try:
parsed = _TIMEDELTA_ADAPTER.validate_python(lifetime)
except ValueError:
logger.warning("Ignoring malformed idempotency-key lifetime: %s", lifetime)
return None
return max(parsed, timedelta(0))


class _IdempotencyKeyLifetimeRetry(Retry):
"""Retry that declines to retry a keyed request once the next attempt would outlive the key.

The REST spec says clients SHOULD NOT reuse an Idempotency-Key after the advertised lifetime, since
the server may have forgotten the key and apply the mutation again. Mirrors the Java client's
`ExponentialHttpRequestRetryStrategy`: a retry is declined when the time since the first attempt
plus the next sleep reaches the lifetime.
"""

def increment(
self,
method: str | None = None,
url: str | None = None,
response: BaseHTTPResponse | None = None,
error: Exception | None = None,
_pool: ConnectionPool | None = None,
_stacktrace: TracebackType | None = None,
) -> Self:
new_retry = super().increment(method, url, response=response, error=error, _pool=_pool, _stacktrace=_stacktrace)
deadline = _IDEMPOTENCY_KEY_DEADLINE.get()
if deadline is not None and time.monotonic() + new_retry._max_sleep(response) >= deadline:
# Same as running out of retries: urllib3 returns the last response (raise_on_status=False)
# so `_handle_non_200_response` raises the typed error, or re-raises the connection error.
reason = error or ResponseError(f"next retry would exceed the {IDEMPOTENCY_KEY_HEADER} lifetime")
raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]
return new_retry

def _max_sleep(self, response: BaseHTTPResponse | None) -> float:
"""Upper bound of what `Retry.sleep` will wait before the next attempt."""
if self.respect_retry_after_header and response is not None:
if retry_after := self.get_retry_after(response):
return retry_after
# `get_backoff_time` adds a random jitter in [0, backoff_jitter), so take the worst case.
return min(self.backoff_max, self.new(backoff_jitter=0.0).get_backoff_time() + self.backoff_jitter)


class _RetryTimeoutHTTPAdapter(HTTPAdapter):
"""HTTPAdapter that applies a default per-request timeout.

Expand All @@ -510,8 +576,14 @@ class _RetryTimeoutHTTPAdapter(HTTPAdapter):
The adapter applies `self._timeout` whenever a per-call timeout is not set.
"""

def __init__(self, timeout: float | None = None, max_retries: Retry | int = DEFAULT_RETRIES) -> None:
def __init__(
self,
timeout: float | None = None,
max_retries: Retry | int = DEFAULT_RETRIES,
idempotency_key_lifetime: timedelta | None = None,
) -> None:
self._timeout = timeout
self._idempotency_key_lifetime = idempotency_key_lifetime
super().__init__(max_retries=max_retries)

def send(
Expand All @@ -525,14 +597,23 @@ def send(
) -> Response:
if timeout is None:
timeout = self._timeout
return super().send(request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies)
if self._idempotency_key_lifetime is None or IDEMPOTENCY_KEY_HEADER not in request.headers:
return super().send(request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies)
token = _IDEMPOTENCY_KEY_DEADLINE.set(time.monotonic() + self._idempotency_key_lifetime.total_seconds())
try:
return super().send(request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies)
finally:
_IDEMPOTENCY_KEY_DEADLINE.reset(token)


def _create_connection_adapter(properties: Properties, retry_post: bool = False) -> _RetryTimeoutHTTPAdapter | None:
def _create_connection_adapter(
properties: Properties, retry_post: bool = False, idempotency_key_lifetime: timedelta | None = None
) -> _RetryTimeoutHTTPAdapter | None:
"""Build a connection adapter from the optional `rest.client.*` properties.

Returns None when no connection properties are supplied, leaving the default
Session behavior unchanged. POST requests are retried only when `retry_post` is set.
Retries of requests carrying an Idempotency-Key stop before `idempotency_key_lifetime` elapses.
Raises ValueError on invalid input.
"""
connection_timeout_ms = property_as_int(properties, REST_CLIENT_CONNECTION_TIMEOUT_MS)
Expand Down Expand Up @@ -562,7 +643,8 @@ def _create_connection_adapter(properties: Properties, retry_post: bool = False)

return _RetryTimeoutHTTPAdapter(
timeout=timeout,
max_retries=Retry(
idempotency_key_lifetime=idempotency_key_lifetime,
max_retries=_IdempotencyKeyLifetimeRetry(
# `retries` and `backoff_factor` fall back to a no-op Retry when unset, so a user can
# configure only one without having to specify the rest of the policy.
total=retries if retries is not None else DEFAULT_RETRIES,
Expand All @@ -585,6 +667,7 @@ class RestCatalog(Catalog):
_supported_endpoints: set[Endpoint]
_namespace_separator: str
_idempotency_enabled: bool = False
_idempotency_key_lifetime: timedelta | None = None

def __init__(self, name: str, **properties: str):
"""Rest Catalog.
Expand All @@ -607,7 +690,11 @@ def _create_session(self) -> Session:

# Mount the retry/timeout adapter when `connection.*` properties are set.
# SigV4's adapter mounted below at `self.uri` is a longer prefix and still wins for that host.
if (connection_adapter := _create_connection_adapter(self.properties, self._idempotency_enabled)) is not None:
if (
connection_adapter := _create_connection_adapter(
self.properties, self._idempotency_enabled, self._idempotency_key_lifetime
)
) is not None:
session.mount("http://", connection_adapter)
session.mount("https://", connection_adapter)

Expand Down Expand Up @@ -1083,6 +1170,7 @@ def _fetch_config(self) -> None:
self.uri = config[URI]

self._idempotency_enabled = config_response.idempotency_key_lifetime is not None
self._idempotency_key_lifetime = _parse_idempotency_key_lifetime(config_response.idempotency_key_lifetime)

# Determine supported endpoints
endpoints = config_response.endpoints
Expand Down
125 changes: 122 additions & 3 deletions tests/catalog/test_rest.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import uuid
from collections.abc import Callable, Iterator
from contextlib import contextmanager
from datetime import timedelta
from typing import Any, cast
from unittest import mock
from urllib.parse import parse_qs, quote, urlsplit
Expand Down Expand Up @@ -57,6 +58,7 @@
ScanPlanningMode,
TableResponse,
ViewResponse,
_parse_idempotency_key_lifetime,
_RetryTimeoutHTTPAdapter,
)
from pyiceberg.exceptions import (
Expand Down Expand Up @@ -2558,7 +2560,9 @@ def test_session_with_socket_timeout_only(rest_mock: Mocker) -> None:


@contextmanager
def _local_rest_server_503_then_200(num_failures: int, idempotency_key_lifetime: str | None = None) -> Iterator[dict[str, Any]]:
def _local_rest_server_503_then_200(
num_failures: int, idempotency_key_lifetime: str | None = None, retry_after: str | None = None
) -> Iterator[dict[str, Any]]:
"""Stand up a loopback HTTP server that returns `num_failures` 503s for `/v1/namespaces` then a 200.

Used in place of `requests_mock`, which replaces the HTTPAdapter and would bypass the retry logic.
Expand Down Expand Up @@ -2591,16 +2595,18 @@ def do_GET(self) -> None:
state["namespace_calls"] += 1
state["idempotency_keys"].append(self.headers.get(IDEMPOTENCY_KEY_HEADER))
if state["namespace_calls"] <= num_failures:
self._respond(503, b"")
self._respond(503, b"", {"Retry-After": retry_after} if retry_after is not None else {})
else:
self._respond(200, json.dumps({"namespaces": [["foo"]]}).encode())
else:
self._respond(404, b"")

def _respond(self, status: int, body: bytes) -> None:
def _respond(self, status: int, body: bytes, headers: dict[str, str] | None = None) -> None:
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
for name, value in (headers or {}).items():
self.send_header(name, value)
self.end_headers()
if body:
self.wfile.write(body)
Expand Down Expand Up @@ -2674,6 +2680,119 @@ def test_session_retries_keyed_post_with_the_same_key() -> None:
assert server["idempotency_keys"] == [key, key]


def test_session_does_not_retry_keyed_post_past_the_key_lifetime() -> None:
# Retry-After (60s) is longer than the advertised lifetime (PT30S), so the retry would reuse an expired key.
with _local_rest_server_503_then_200(num_failures=1, idempotency_key_lifetime="PT30S", retry_after="60") as server:
catalog = RestCatalog(
"rest",
**{
"uri": f"http://127.0.0.1:{server['port']}/",
"token": TEST_TOKEN,
REST_CLIENT_MAX_RETRIES: "3",
REST_CLIENT_RETRY_BACKOFF_FACTOR: "0",
},
)
with pytest.raises(ServiceUnavailableError):
catalog.create_namespace("foo")
assert server["namespace_calls"] == 1


def test_session_does_not_retry_keyed_post_past_the_key_lifetime_due_to_backoff() -> None:
# urllib3 retries the first failure immediately; the backoff before the second retry (20s) alone
# exceeds the advertised lifetime (PT5S).
with _local_rest_server_503_then_200(num_failures=2, idempotency_key_lifetime="PT5S") as server:
catalog = RestCatalog(
"rest",
**{
"uri": f"http://127.0.0.1:{server['port']}/",
"token": TEST_TOKEN,
REST_CLIENT_MAX_RETRIES: "3",
REST_CLIENT_RETRY_BACKOFF_FACTOR: "10",
},
)
with pytest.raises(ServiceUnavailableError):
catalog.create_namespace("foo")
assert server["namespace_calls"] == 2


def test_session_retries_keyed_post_within_the_key_lifetime_with_retry_after() -> None:
with _local_rest_server_503_then_200(num_failures=1, idempotency_key_lifetime="PT30M", retry_after="0") as server:
catalog = RestCatalog(
"rest",
**{
"uri": f"http://127.0.0.1:{server['port']}/",
"token": TEST_TOKEN,
REST_CLIENT_MAX_RETRIES: "1",
REST_CLIENT_RETRY_BACKOFF_FACTOR: "0",
},
)
catalog.create_namespace("foo")
assert server["namespace_calls"] == 2


def test_session_never_retries_keyed_post_with_non_positive_key_lifetime() -> None:
for lifetime in ("PT0S", "-PT5M"):
with _local_rest_server_503_then_200(num_failures=1, idempotency_key_lifetime=lifetime) as server:
catalog = RestCatalog(
"rest",
**{
"uri": f"http://127.0.0.1:{server['port']}/",
"token": TEST_TOKEN,
REST_CLIENT_MAX_RETRIES: "3",
REST_CLIENT_RETRY_BACKOFF_FACTOR: "0",
},
)
with pytest.raises(ServiceUnavailableError):
catalog.create_namespace("foo")
assert server["namespace_calls"] == 1, lifetime
# The key is still sent, only the retry is skipped.
assert server["idempotency_keys"][0] is not None


def test_session_key_lifetime_does_not_bound_unkeyed_requests() -> None:
# GETs carry no Idempotency-Key, so they are retried as before even though the lifetime is zero.
with _local_rest_server_503_then_200(num_failures=1, idempotency_key_lifetime="PT0S") as server:
catalog = RestCatalog(
"rest",
**{
"uri": f"http://127.0.0.1:{server['port']}/",
"token": TEST_TOKEN,
REST_CLIENT_MAX_RETRIES: "1",
REST_CLIENT_RETRY_BACKOFF_FACTOR: "0",
},
)
assert catalog.list_namespaces() == [("foo",)]
assert server["namespace_calls"] == 2


def test_session_retries_keyed_post_with_malformed_key_lifetime(caplog: pytest.LogCaptureFixture) -> None:
with _local_rest_server_503_then_200(num_failures=1, idempotency_key_lifetime="not-a-duration") as server:
catalog = RestCatalog(
"rest",
**{
"uri": f"http://127.0.0.1:{server['port']}/",
"token": TEST_TOKEN,
REST_CLIENT_MAX_RETRIES: "1",
REST_CLIENT_RETRY_BACKOFF_FACTOR: "0",
},
)
catalog.create_namespace("foo")

key = server["idempotency_keys"][0]
assert key is not None
assert server["idempotency_keys"] == [key, key]
assert "Ignoring malformed idempotency-key lifetime: not-a-duration" in caplog.text


def test_parse_idempotency_key_lifetime() -> None:
assert _parse_idempotency_key_lifetime(None) is None
assert _parse_idempotency_key_lifetime("PT30M") == timedelta(minutes=30)
assert _parse_idempotency_key_lifetime("P1DT2H") == timedelta(days=1, hours=2)
assert _parse_idempotency_key_lifetime("PT0S") == timedelta(0)
assert _parse_idempotency_key_lifetime("-PT5M") == timedelta(0)
assert _parse_idempotency_key_lifetime("garbage") is None


def test_mutations_send_idempotency_key_when_advertised(requests_mock: Mocker) -> None:
requests_mock.get(f"{TEST_URI}v1/config", json={"defaults": {}, "overrides": {}, "idempotency-key-lifetime": "PT30M"})
requests_mock.post(f"{TEST_URI}v1/namespaces", json={"namespace": ["foo"], "properties": {}})
Expand Down
Loading