Skip to content

Commit a45e934

Browse files
committed
fix: keep RestCatalog picklable and treat more vended tokens as short-lived
1 parent 09cf09a commit a45e934

2 files changed

Lines changed: 18 additions & 1 deletion

File tree

‎pyiceberg/catalog/rest/__init__.py‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@
7575
)
7676
from pyiceberg.io import (
7777
ADLS_SAS_TOKEN,
78+
ADLS_TOKEN,
7879
AWS_ACCESS_KEY_ID,
7980
AWS_PROFILE_NAME,
8081
AWS_REGION,
@@ -368,7 +369,7 @@ def _uuid7() -> uuid.UUID:
368369
def _is_short_lived_credential(key: str) -> bool:
369370
"""Whether a loadTable config key carries a short-lived storage credential."""
370371
key = key.lower()
371-
return key in (S3_SESSION_TOKEN, GCS_TOKEN) or key.startswith(ADLS_SAS_TOKEN)
372+
return key in (S3_SESSION_TOKEN, AWS_SESSION_TOKEN, GCS_TOKEN, ADLS_TOKEN, TOKEN) or key.startswith(ADLS_SAS_TOKEN)
372373

373374

374375
def _retry_hook(retry_state: RetryCallState) -> None:
@@ -618,6 +619,19 @@ def __init__(self, name: str, **properties: str):
618619
self._table_cache = TTLCache(maxsize=max_entries, ttl=ttl_ms / 1000) if max_entries > 0 else None
619620
self._table_cache_lock = threading.Lock()
620621

622+
def __getstate__(self) -> dict[str, Any]:
623+
"""Create a dictionary of the RestCatalog fields used when pickling, with an empty table cache."""
624+
state = self.__dict__.copy()
625+
if (cache := state["_table_cache"]) is not None:
626+
state["_table_cache"] = TTLCache(maxsize=cache.maxsize, ttl=cache.ttl)
627+
del state["_table_cache_lock"]
628+
return state
629+
630+
def __setstate__(self, state: dict[str, Any]) -> None:
631+
"""Deserialize the state into a RestCatalog instance."""
632+
self.__dict__ = state
633+
self._table_cache_lock = threading.Lock()
634+
621635
def _invalidate_cached_table(self, identifier: str | Identifier) -> None:
622636
if self._table_cache is not None:
623637
with self._table_cache_lock:

‎tests/catalog/test_rest.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1632,8 +1632,11 @@ def test_load_table_vending_credentials_evicts_cached_table(
16321632
"key, expected",
16331633
[
16341634
("S3.Session-Token", True),
1635+
("client.session-token", True),
16351636
("gcs.oauth2.token", True),
1637+
("adls.token", True),
16361638
("adls.sas-token.account", True),
1639+
("token", True),
16371640
("s3.sse.key", False),
16381641
("s3.access-key-id", False),
16391642
("encryption.key-id", False),

0 commit comments

Comments
 (0)