From b044ea96e3969a2f5b1db7d96b14e74918336090 Mon Sep 17 00:00:00 2001 From: Yuang Gao Date: Thu, 8 Oct 2026 22:18:50 -0700 Subject: [PATCH 1/4] feat: support freshness-aware table loading in RestCatalog --- mkdocs/docs/configuration.md | 2 ++ pyiceberg/catalog/rest/__init__.py | 29 ++++++++++++++-- tests/catalog/test_rest.py | 47 ++++++++++++++++++++++++++ tests/integration/test_rest_catalog.py | 23 +++++++++++++ 4 files changed, 99 insertions(+), 2 deletions(-) diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index 14cb9c7467..9f5362a6e2 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -413,6 +413,8 @@ catalog: | uri | | URI identifying the REST Server | | warehouse | myWarehouse | Warehouse location or identifier to request from the catalog service. May be used to determine server-side overrides, such as the warehouse location. | | snapshot-loading-mode | refs | The snapshots to return in the body of the metadata. Setting the value to `all` would return the full set of snapshots currently valid for the table. Setting the value to `refs` would load all snapshots referenced by branches or tags. | +| rest-table-cache.max-entries | 100 | Maximum number of loaded tables kept to answer `304 Not Modified` on reload, keyed by the table's `ETag`. Set to `0` to disable (default: 100). | +| rest-table-cache.expire-after-write-ms | 300000 | How long a loaded table stays cached for reload (default: 5 minutes). | | `header.X-Iceberg-Access-Delegation` | `vended-credentials` | Signal to the server that the client supports delegated access via a comma-separated list of access mechanisms. The server may choose to supply access via any or none of the requested mechanisms. When using `vended-credentials`, the server provides temporary credentials to the client. When using `remote-signing`, the server signs requests on behalf of the client. (default: `vended-credentials`) | | view-endpoints-supported | false | For backwards compatibility with older REST servers. Set to `true` if the server supports view endpoints but doesn't send the `endpoints` field in the ConfigResponse. | | scan-planning-mode | client | When set to `server`, and the catalog advertises the plan-table-scan endpoint, `table.scan()` uses REST server-side scan planning. May be set by the client, returned by the catalog via `GET /v1/config` defaults/overrides, or returned per table in the `config` of the `loadTable` response. The `loadTable` value takes precedence over the catalog-level value, which lets a server request server-side planning only for specific tables. Async plans (`status=submitted`) are polled via `GET .../plan/{plan-id}` until terminal state (completed / failed / cancelled). | diff --git a/pyiceberg/catalog/rest/__init__.py b/pyiceberg/catalog/rest/__init__.py index 785940ea54..da53cb6fa9 100644 --- a/pyiceberg/catalog/rest/__init__.py +++ b/pyiceberg/catalog/rest/__init__.py @@ -19,6 +19,7 @@ import logging import os import sys +import threading import time import uuid from collections import deque @@ -30,6 +31,7 @@ ) from urllib.parse import quote, unquote +from cachetools import TTLCache from pydantic import ConfigDict, Field, TypeAdapter, field_validator from requests import HTTPError, PreparedRequest, Response, Session from requests.adapters import DEFAULT_RETRIES, HTTPAdapter @@ -66,6 +68,7 @@ NoSuchTableError, NoSuchViewError, RemotePlanTimeoutError, + RESTError, TableAlreadyExistsError, UnauthorizedError, ViewAlreadyExistsError, @@ -306,6 +309,10 @@ class ScanPlanningMode(Enum): EMPTY_BODY_SHA256: str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" OAUTH2_SERVER_URI = "oauth2-server-uri" SNAPSHOT_LOADING_MODE = "snapshot-loading-mode" +REST_TABLE_CACHE_MAX_ENTRIES = "rest-table-cache.max-entries" +REST_TABLE_CACHE_MAX_ENTRIES_DEFAULT = 100 +REST_TABLE_CACHE_EXPIRE_AFTER_WRITE_MS = "rest-table-cache.expire-after-write-ms" +REST_TABLE_CACHE_EXPIRE_AFTER_WRITE_MS_DEFAULT = 5 * 60 * 1000 # 5 minutes, matches Java AUTH = "auth" CUSTOM = "custom" SCAN_PLANNING_MODE = "scan-planning-mode" @@ -578,6 +585,8 @@ class RestCatalog(Catalog): _supported_endpoints: set[Endpoint] _namespace_separator: str _idempotency_enabled: bool = False + # Stale entries are harmless: the server answers 304 only while the ETag matches. + _table_cache: TTLCache[Identifier, tuple[str, TableResponse]] | None def __init__(self, name: str, **properties: str): """Rest Catalog. @@ -593,6 +602,10 @@ def __init__(self, name: str, **properties: str): self.uri = properties[URI] self._fetch_config() self._session = self._create_session() + max_entries = int(self.properties.get(REST_TABLE_CACHE_MAX_ENTRIES, REST_TABLE_CACHE_MAX_ENTRIES_DEFAULT)) + ttl_ms = int(self.properties.get(REST_TABLE_CACHE_EXPIRE_AFTER_WRITE_MS, REST_TABLE_CACHE_EXPIRE_AFTER_WRITE_MS_DEFAULT)) + self._table_cache = TTLCache(maxsize=max_entries, ttl=ttl_ms / 1000) if max_entries > 0 else None + self._table_cache_lock = threading.Lock() def _create_session(self) -> Session: """Create a request session with provided catalog configuration.""" @@ -1475,16 +1488,28 @@ def load_table(self, identifier: str | Identifier) -> Table: else: raise ValueError("Invalid snapshot-loading-mode: {}") + identifier_tuple = self.identifier_to_tuple(identifier) + with self._table_cache_lock: + cached = self._table_cache.get(identifier_tuple) if self._table_cache is not None else None response = self._session.get( - self.url(Endpoints.load_table, prefixed=True, **self._split_identifier_for_path(identifier)), params=params + self.url(Endpoints.load_table, prefixed=True, **self._split_identifier_for_path(identifier)), + params=params, + headers={"If-None-Match": cached[0]} if cached else None, ) + if response.status_code == 304: + if cached is None: + raise RESTError("Received 304 Not Modified for a table load without If-None-Match") + return self._response_to_table(identifier_tuple, cached[1]) try: response.raise_for_status() except HTTPError as exc: _handle_non_200_response(exc, {404: NoSuchTableError}) table_response = TableResponse.model_validate_json(response.text) - return self._response_to_table(self.identifier_to_tuple(identifier), table_response) + if self._table_cache is not None and (etag := response.headers.get("ETag")): + with self._table_cache_lock: + self._table_cache[identifier_tuple] = (etag, table_response) + return self._response_to_table(identifier_tuple, table_response) @retry(**_RETRY_ARGS) def _load_credentials( diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index a11adc55ee..53b01e020a 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -45,6 +45,7 @@ REST_CLIENT_MAX_RETRIES, REST_CLIENT_RETRY_BACKOFF_FACTOR, REST_CLIENT_SOCKET_TIMEOUT_MS, + REST_TABLE_CACHE_MAX_ENTRIES, SIGV4_MAX_RETRIES, SIGV4_MAX_RETRIES_DEFAULT, SNAPSHOT_LOADING_MODE, @@ -66,6 +67,7 @@ NoSuchTableError, NoSuchViewError, OAuthError, + RESTError, ServerError, ServiceUnavailableError, TableAlreadyExistsError, @@ -1493,6 +1495,51 @@ def test_load_table_200_loading_mode( assert actual == expected +def test_load_table_reuses_cached_table_on_not_modified( + rest_mock: Mocker, example_table_metadata_with_snapshot_v1_rest_json: dict[str, Any] +) -> None: + rest_mock.get( + f"{TEST_URI}v1/namespaces/fokko/tables/table", + [ + {"json": example_table_metadata_with_snapshot_v1_rest_json, "status_code": 200, "headers": {"ETag": '"v1"'}}, + {"status_code": 304}, + ], + ) + catalog = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN) + + loaded = catalog.load_table(("fokko", "table")) + reloaded = catalog.load_table(("fokko", "table")) + + assert rest_mock.last_request.headers["If-None-Match"] == '"v1"' + assert reloaded == loaded + assert reloaded is not loaded + + +def test_load_table_not_modified_without_cached_table(rest_mock: Mocker) -> None: + rest_mock.get(f"{TEST_URI}v1/namespaces/fokko/tables/table", status_code=304) + catalog = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN) + + with pytest.raises(RESTError, match="304 Not Modified"): + catalog.load_table(("fokko", "table")) + + +def test_load_table_cache_disabled_with_zero_max_entries( + rest_mock: Mocker, example_table_metadata_with_snapshot_v1_rest_json: dict[str, Any] +) -> None: + rest_mock.get( + f"{TEST_URI}v1/namespaces/fokko/tables/table", + json=example_table_metadata_with_snapshot_v1_rest_json, + status_code=200, + headers={"ETag": '"v1"'}, + ) + catalog = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN, **{REST_TABLE_CACHE_MAX_ENTRIES: "0"}) + + catalog.load_table(("fokko", "table")) + catalog.load_table(("fokko", "table")) + + assert "If-None-Match" not in rest_mock.last_request.headers + + def test_load_table_honor_access_delegation( rest_mock: Mocker, example_table_metadata_with_snapshot_v1_rest_json: dict[str, Any] ) -> None: diff --git a/tests/integration/test_rest_catalog.py b/tests/integration/test_rest_catalog.py index 1705dcfb27..dc2fadb702 100644 --- a/tests/integration/test_rest_catalog.py +++ b/tests/integration/test_rest_catalog.py @@ -17,6 +17,7 @@ # pylint:disable=redefined-outer-name import time +from unittest import mock import pytest from pytest_lazy_fixtures import lf @@ -69,6 +70,28 @@ def test_create_namespace_if_already_existing(catalog: RestCatalog) -> None: assert catalog.namespace_exists(TEST_NAMESPACE_IDENTIFIER) +@pytest.mark.integration +@pytest.mark.parametrize("catalog", [lf("session_catalog")]) +def test_load_table_reuses_cached_table_on_not_modified( + catalog: RestCatalog, table_schema_nested: Schema, database_name: str, table_name: str +) -> None: + identifier = (database_name, table_name) + catalog.create_namespace_if_not_exists(database_name) + catalog.create_table(identifier, table_schema_nested) + statuses: list[int] = [] + + # The fixture predates apache/iceberg#17598, so its unquoted ETags get a `--gzip` suffix that Jetty never strips. + with ( + mock.patch.dict(catalog._session.headers, {"Accept-Encoding": "identity"}), + mock.patch.object(catalog._session, "hooks", {"response": [lambda r, **_: statuses.append(r.status_code)]}), + ): + loaded = catalog.load_table(identifier) + reloaded = catalog.load_table(identifier) + + assert statuses == [200, 304] + assert reloaded.metadata_location == loaded.metadata_location + + @pytest.mark.integration @pytest.mark.parametrize("catalog", [lf("session_catalog")]) def test_load_view(catalog: RestCatalog, table_schema_nested: Schema, database_name: str, view_name: str) -> None: From 056c4b15ad64b11c02e0c972a3063afc10099aa9 Mon Sep 17 00:00:00 2001 From: Yuang Gao Date: Fri, 9 Oct 2026 23:01:50 -0700 Subject: [PATCH 2/4] fix: reject non-positive table cache expiry --- mkdocs/docs/configuration.md | 2 +- pyiceberg/catalog/rest/__init__.py | 2 ++ tests/catalog/test_rest.py | 7 +++++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index 9f5362a6e2..0dc3381a51 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -414,7 +414,7 @@ catalog: | warehouse | myWarehouse | Warehouse location or identifier to request from the catalog service. May be used to determine server-side overrides, such as the warehouse location. | | snapshot-loading-mode | refs | The snapshots to return in the body of the metadata. Setting the value to `all` would return the full set of snapshots currently valid for the table. Setting the value to `refs` would load all snapshots referenced by branches or tags. | | rest-table-cache.max-entries | 100 | Maximum number of loaded tables kept to answer `304 Not Modified` on reload, keyed by the table's `ETag`. Set to `0` to disable (default: 100). | -| rest-table-cache.expire-after-write-ms | 300000 | How long a loaded table stays cached for reload (default: 5 minutes). | +| rest-table-cache.expire-after-write-ms | 300000 | How long a loaded table stays cached for reload. Must be positive (default: 5 minutes). | | `header.X-Iceberg-Access-Delegation` | `vended-credentials` | Signal to the server that the client supports delegated access via a comma-separated list of access mechanisms. The server may choose to supply access via any or none of the requested mechanisms. When using `vended-credentials`, the server provides temporary credentials to the client. When using `remote-signing`, the server signs requests on behalf of the client. (default: `vended-credentials`) | | view-endpoints-supported | false | For backwards compatibility with older REST servers. Set to `true` if the server supports view endpoints but doesn't send the `endpoints` field in the ConfigResponse. | | scan-planning-mode | client | When set to `server`, and the catalog advertises the plan-table-scan endpoint, `table.scan()` uses REST server-side scan planning. May be set by the client, returned by the catalog via `GET /v1/config` defaults/overrides, or returned per table in the `config` of the `loadTable` response. The `loadTable` value takes precedence over the catalog-level value, which lets a server request server-side planning only for specific tables. Async plans (`status=submitted`) are polled via `GET .../plan/{plan-id}` until terminal state (completed / failed / cancelled). | diff --git a/pyiceberg/catalog/rest/__init__.py b/pyiceberg/catalog/rest/__init__.py index da53cb6fa9..16ddb53f06 100644 --- a/pyiceberg/catalog/rest/__init__.py +++ b/pyiceberg/catalog/rest/__init__.py @@ -604,6 +604,8 @@ def __init__(self, name: str, **properties: str): self._session = self._create_session() max_entries = int(self.properties.get(REST_TABLE_CACHE_MAX_ENTRIES, REST_TABLE_CACHE_MAX_ENTRIES_DEFAULT)) ttl_ms = int(self.properties.get(REST_TABLE_CACHE_EXPIRE_AFTER_WRITE_MS, REST_TABLE_CACHE_EXPIRE_AFTER_WRITE_MS_DEFAULT)) + if ttl_ms <= 0: + raise ValueError(f"Invalid value for {REST_TABLE_CACHE_EXPIRE_AFTER_WRITE_MS}: {ttl_ms} (must be positive)") self._table_cache = TTLCache(maxsize=max_entries, ttl=ttl_ms / 1000) if max_entries > 0 else None self._table_cache_lock = threading.Lock() diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index 53b01e020a..37e1a413c6 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -45,6 +45,7 @@ REST_CLIENT_MAX_RETRIES, REST_CLIENT_RETRY_BACKOFF_FACTOR, REST_CLIENT_SOCKET_TIMEOUT_MS, + REST_TABLE_CACHE_EXPIRE_AFTER_WRITE_MS, REST_TABLE_CACHE_MAX_ENTRIES, SIGV4_MAX_RETRIES, SIGV4_MAX_RETRIES_DEFAULT, @@ -1540,6 +1541,12 @@ def test_load_table_cache_disabled_with_zero_max_entries( assert "If-None-Match" not in rest_mock.last_request.headers +@pytest.mark.parametrize("ttl_ms", ["0", "-1"]) +def test_table_cache_rejects_non_positive_expiry(rest_mock: Mocker, ttl_ms: str) -> None: + with pytest.raises(ValueError, match=REST_TABLE_CACHE_EXPIRE_AFTER_WRITE_MS): + RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN, **{REST_TABLE_CACHE_EXPIRE_AFTER_WRITE_MS: ttl_ms}) + + def test_load_table_honor_access_delegation( rest_mock: Mocker, example_table_metadata_with_snapshot_v1_rest_json: dict[str, Any] ) -> None: From 09cf09ac190ad540d37bd59c882b7075c6ab0c3e Mon Sep 17 00:00:00 2001 From: Yuang Gao Date: Mon, 19 Oct 2026 19:48:44 -0700 Subject: [PATCH 3/4] fix: invalidate table cache on changes and skip vended credentials --- mkdocs/docs/configuration.md | 2 +- pyiceberg/catalog/rest/__init__.py | 33 +++++++++- tests/catalog/test_rest.py | 98 +++++++++++++++++++++++++++++- 3 files changed, 128 insertions(+), 5 deletions(-) diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index 0dc3381a51..fcc1bfe732 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -413,7 +413,7 @@ catalog: | uri | | URI identifying the REST Server | | warehouse | myWarehouse | Warehouse location or identifier to request from the catalog service. May be used to determine server-side overrides, such as the warehouse location. | | snapshot-loading-mode | refs | The snapshots to return in the body of the metadata. Setting the value to `all` would return the full set of snapshots currently valid for the table. Setting the value to `refs` would load all snapshots referenced by branches or tags. | -| rest-table-cache.max-entries | 100 | Maximum number of loaded tables kept to answer `304 Not Modified` on reload, keyed by the table's `ETag`. Set to `0` to disable (default: 100). | +| rest-table-cache.max-entries | 100 | Maximum number of loaded tables kept to answer `304 Not Modified` on reload, keyed by the table's `ETag`. Loads that vend short-lived storage credentials are never cached. Set to `0` to disable (default: 100). | | rest-table-cache.expire-after-write-ms | 300000 | How long a loaded table stays cached for reload. Must be positive (default: 5 minutes). | | `header.X-Iceberg-Access-Delegation` | `vended-credentials` | Signal to the server that the client supports delegated access via a comma-separated list of access mechanisms. The server may choose to supply access via any or none of the requested mechanisms. When using `vended-credentials`, the server provides temporary credentials to the client. When using `remote-signing`, the server signs requests on behalf of the client. (default: `vended-credentials`) | | view-endpoints-supported | false | For backwards compatibility with older REST servers. Set to `true` if the server supports view endpoints but doesn't send the `endpoints` field in the ConfigResponse. | diff --git a/pyiceberg/catalog/rest/__init__.py b/pyiceberg/catalog/rest/__init__.py index 16ddb53f06..7789461c2a 100644 --- a/pyiceberg/catalog/rest/__init__.py +++ b/pyiceberg/catalog/rest/__init__.py @@ -74,11 +74,14 @@ ViewAlreadyExistsError, ) from pyiceberg.io import ( + ADLS_SAS_TOKEN, AWS_ACCESS_KEY_ID, AWS_PROFILE_NAME, AWS_REGION, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, + GCS_TOKEN, + S3_SESSION_TOKEN, FileIO, load_file_io, ) @@ -362,6 +365,12 @@ def _uuid7() -> uuid.UUID: return uuid.UUID(int=value) +def _is_short_lived_credential(key: str) -> bool: + """Whether a loadTable config key carries a short-lived storage credential.""" + key = key.lower() + return key in (S3_SESSION_TOKEN, GCS_TOKEN) or key.startswith(ADLS_SAS_TOKEN) + + def _retry_hook(retry_state: RetryCallState) -> None: rest_catalog: RestCatalog = retry_state.args[0] rest_catalog._refresh_token() # pylint: disable=protected-access @@ -585,7 +594,7 @@ class RestCatalog(Catalog): _supported_endpoints: set[Endpoint] _namespace_separator: str _idempotency_enabled: bool = False - # Stale entries are harmless: the server answers 304 only while the ETag matches. + # Loads that vend short-lived storage credentials are not cached, since a 304 cannot refresh them. _table_cache: TTLCache[Identifier, tuple[str, TableResponse]] | None def __init__(self, name: str, **properties: str): @@ -609,6 +618,11 @@ def __init__(self, name: str, **properties: str): self._table_cache = TTLCache(maxsize=max_entries, ttl=ttl_ms / 1000) if max_entries > 0 else None self._table_cache_lock = threading.Lock() + def _invalidate_cached_table(self, identifier: str | Identifier) -> None: + if self._table_cache is not None: + with self._table_cache_lock: + self._table_cache.pop(self.identifier_to_tuple(identifier), None) + def _create_session(self) -> Session: """Create a request session with provided catalog configuration.""" session = Session() @@ -1403,6 +1417,7 @@ def register_table(self, identifier: str | Identifier, metadata_location: str, o overwrite=overwrite, ) serialized_json = request.model_dump_json().encode(UTF8) + self._invalidate_cached_table(identifier) response = self._session.post( self.url(Endpoints.register_table, namespace=namespace_and_table["namespace"]), data=serialized_json, @@ -1505,12 +1520,21 @@ def load_table(self, identifier: str | Identifier) -> Table: try: response.raise_for_status() except HTTPError as exc: + if response.status_code == 404: + self._invalidate_cached_table(identifier) _handle_non_200_response(exc, {404: NoSuchTableError}) table_response = TableResponse.model_validate_json(response.text) - if self._table_cache is not None and (etag := response.headers.get("ETag")): + if self._table_cache is not None: + etag = response.headers.get("ETag") + vends_credentials = bool(table_response.storage_credentials) or any( + map(_is_short_lived_credential, table_response.config) + ) with self._table_cache_lock: - self._table_cache[identifier_tuple] = (etag, table_response) + if etag and not vends_credentials: + self._table_cache[identifier_tuple] = (etag, table_response) + else: + self._table_cache.pop(identifier_tuple, None) return self._response_to_table(identifier_tuple, table_response) @retry(**_RETRY_ARGS) @@ -1543,6 +1567,7 @@ def load_credentials( @override def drop_table(self, identifier: str | Identifier, purge_requested: bool = False) -> None: self._check_endpoint(Capability.V1_DELETE_TABLE) + self._invalidate_cached_table(identifier) response = self._session.delete( self.url(Endpoints.drop_table, prefixed=True, **self._split_identifier_for_path(identifier)), params={"purgeRequested": purge_requested}, @@ -1576,6 +1601,7 @@ def rename_table(self, from_identifier: str | Identifier, to_identifier: str | I if not self.namespace_exists(destination_namespace): raise NoSuchNamespaceError(f"Destination namespace does not exist: {destination_namespace}") + self._invalidate_cached_table(from_identifier) response = self._session.post(self.url(Endpoints.rename_table), json=payload, headers=self._mutation_headers()) try: response.raise_for_status() @@ -1677,6 +1703,7 @@ def commit_table( if table_token := table.config.get(TOKEN): headers[AUTHORIZATION_HEADER] = f"{BEARER_PREFIX} {table_token}" + self._invalidate_cached_table(identifier) response = self._session.post( self.url(Endpoints.update_table, prefixed=True, **self._split_identifier_for_path(table_request.identifier)), data=table_request.model_dump_json().encode(UTF8), diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index 37e1a413c6..2760a66f9c 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -30,7 +30,7 @@ from pydantic import ValidationError from requests import Request from requests.adapters import HTTPAdapter -from requests.exceptions import HTTPError +from requests.exceptions import ConnectionError, HTTPError from requests_mock import Mocker import pyiceberg @@ -57,6 +57,7 @@ ScanPlanningMode, TableResponse, ViewResponse, + _is_short_lived_credential, _RetryTimeoutHTTPAdapter, ) from pyiceberg.exceptions import ( @@ -1547,6 +1548,101 @@ def test_table_cache_rejects_non_positive_expiry(rest_mock: Mocker, ttl_ms: str) RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN, **{REST_TABLE_CACHE_EXPIRE_AFTER_WRITE_MS: ttl_ms}) +@pytest.mark.parametrize("change", ["drop", "rename", "register", "commit"]) +def test_table_changes_invalidate_cached_table( + rest_mock: Mocker, example_table_metadata_with_snapshot_v1_rest_json: dict[str, Any], change: str +) -> None: + table_url = f"{TEST_URI}v1/namespaces/fokko/tables/table" + body = example_table_metadata_with_snapshot_v1_rest_json + rest_mock.get(table_url, json=body, headers={"ETag": '"v1"'}) + rest_mock.get(f"{TEST_URI}v1/namespaces/fokko/tables/renamed", json=body) + rest_mock.head(f"{TEST_URI}v1/namespaces/fokko", status_code=204) + rest_mock.delete(table_url, status_code=204) + rest_mock.post(f"{TEST_URI}v1/tables/rename", status_code=204) + rest_mock.post(f"{TEST_URI}v1/namespaces/fokko/register", json=body) + rest_mock.post(table_url, json=body) + catalog = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN) + identifier = ("fokko", "table") + table = catalog.load_table(identifier) + + { + "drop": lambda: catalog.drop_table(identifier), + "rename": lambda: catalog.rename_table(identifier, ("fokko", "renamed")), + "register": lambda: catalog.register_table(identifier, body["metadata-location"]), + "commit": lambda: catalog.commit_table(table, (), ()), + }[change]() + catalog.load_table(identifier) + + assert "If-None-Match" not in rest_mock.last_request.headers + + +def test_failed_drop_still_invalidates_cached_table( + rest_mock: Mocker, example_table_metadata_with_snapshot_v1_rest_json: dict[str, Any] +) -> None: + table_url = f"{TEST_URI}v1/namespaces/fokko/tables/table" + rest_mock.get(table_url, json=example_table_metadata_with_snapshot_v1_rest_json, headers={"ETag": '"v1"'}) + rest_mock.delete(table_url, exc=ConnectionError) + catalog = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN) + + catalog.load_table(("fokko", "table")) + with pytest.raises(ConnectionError): + catalog.drop_table(("fokko", "table")) + catalog.load_table(("fokko", "table")) + + assert "If-None-Match" not in rest_mock.last_request.headers + + +def test_load_table_404_invalidates_cached_table( + rest_mock: Mocker, example_table_metadata_with_snapshot_v1_rest_json: dict[str, Any] +) -> None: + found = {"json": example_table_metadata_with_snapshot_v1_rest_json, "headers": {"ETag": '"v1"'}} + rest_mock.get(f"{TEST_URI}v1/namespaces/fokko/tables/table", [found, {"status_code": 404, "json": {}}, found]) + catalog = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN) + + catalog.load_table(("fokko", "table")) + with pytest.raises(NoSuchTableError): + catalog.load_table(("fokko", "table")) + catalog.load_table(("fokko", "table")) + + assert "If-None-Match" not in rest_mock.last_request.headers + + +@pytest.mark.parametrize( + "vended", + [ + {"config": {"s3.session-token": "token"}}, + {"storage-credentials": [{"prefix": "s3://", "config": {"s3.session-token": "token"}}]}, + ], +) +def test_load_table_vending_credentials_evicts_cached_table( + rest_mock: Mocker, example_table_metadata_with_snapshot_v1_rest_json: dict[str, Any], vended: dict[str, Any] +) -> None: + plain = {"json": example_table_metadata_with_snapshot_v1_rest_json, "headers": {"ETag": '"v1"'}} + vending = {"json": {**example_table_metadata_with_snapshot_v1_rest_json, **vended}, "headers": {"ETag": '"v2"'}} + rest_mock.get(f"{TEST_URI}v1/namespaces/fokko/tables/table", [plain, vending, plain]) + catalog = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN) + + for _ in range(3): + catalog.load_table(("fokko", "table")) + + assert "If-None-Match" not in rest_mock.last_request.headers + + +@pytest.mark.parametrize( + "key, expected", + [ + ("S3.Session-Token", True), + ("gcs.oauth2.token", True), + ("adls.sas-token.account", True), + ("s3.sse.key", False), + ("s3.access-key-id", False), + ("encryption.key-id", False), + ], +) +def test_is_short_lived_credential(key: str, expected: bool) -> None: + assert _is_short_lived_credential(key) is expected + + def test_load_table_honor_access_delegation( rest_mock: Mocker, example_table_metadata_with_snapshot_v1_rest_json: dict[str, Any] ) -> None: From a45e934cbfb5a299c22ee5cff8dc175a02cfef0b Mon Sep 17 00:00:00 2001 From: Yuang Gao Date: Wed, 11 Nov 2026 19:16:50 -0800 Subject: [PATCH 4/4] fix: keep RestCatalog picklable and treat more vended tokens as short-lived --- pyiceberg/catalog/rest/__init__.py | 16 +++++++++++++++- tests/catalog/test_rest.py | 3 +++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/pyiceberg/catalog/rest/__init__.py b/pyiceberg/catalog/rest/__init__.py index 7789461c2a..d48916c1c7 100644 --- a/pyiceberg/catalog/rest/__init__.py +++ b/pyiceberg/catalog/rest/__init__.py @@ -75,6 +75,7 @@ ) from pyiceberg.io import ( ADLS_SAS_TOKEN, + ADLS_TOKEN, AWS_ACCESS_KEY_ID, AWS_PROFILE_NAME, AWS_REGION, @@ -368,7 +369,7 @@ def _uuid7() -> uuid.UUID: def _is_short_lived_credential(key: str) -> bool: """Whether a loadTable config key carries a short-lived storage credential.""" key = key.lower() - return key in (S3_SESSION_TOKEN, GCS_TOKEN) or key.startswith(ADLS_SAS_TOKEN) + return key in (S3_SESSION_TOKEN, AWS_SESSION_TOKEN, GCS_TOKEN, ADLS_TOKEN, TOKEN) or key.startswith(ADLS_SAS_TOKEN) def _retry_hook(retry_state: RetryCallState) -> None: @@ -618,6 +619,19 @@ def __init__(self, name: str, **properties: str): self._table_cache = TTLCache(maxsize=max_entries, ttl=ttl_ms / 1000) if max_entries > 0 else None self._table_cache_lock = threading.Lock() + def __getstate__(self) -> dict[str, Any]: + """Create a dictionary of the RestCatalog fields used when pickling, with an empty table cache.""" + state = self.__dict__.copy() + if (cache := state["_table_cache"]) is not None: + state["_table_cache"] = TTLCache(maxsize=cache.maxsize, ttl=cache.ttl) + del state["_table_cache_lock"] + return state + + def __setstate__(self, state: dict[str, Any]) -> None: + """Deserialize the state into a RestCatalog instance.""" + self.__dict__ = state + self._table_cache_lock = threading.Lock() + def _invalidate_cached_table(self, identifier: str | Identifier) -> None: if self._table_cache is not None: with self._table_cache_lock: diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index 2760a66f9c..c02b3711b7 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -1632,8 +1632,11 @@ def test_load_table_vending_credentials_evicts_cached_table( "key, expected", [ ("S3.Session-Token", True), + ("client.session-token", True), ("gcs.oauth2.token", True), + ("adls.token", True), ("adls.sas-token.account", True), + ("token", True), ("s3.sse.key", False), ("s3.access-key-id", False), ("encryption.key-id", False),