Skip to content

Commit 488c82c

Browse files
committed
Add a shared mixin for state that cannot be pickled
Locks, thread-locals and caches have to be dropped when pickling and rebuilt afterwards. Each class has handled this with its own __getstate__ and __setstate__, and new locks keep landing without it. Add _TransientStateMixin and port FsspecFileIO and PyArrowFileIO to it.
1 parent 0008d65 commit 488c82c

6 files changed

Lines changed: 99 additions & 32 deletions

File tree

‎pyiceberg/catalog/rest/auth.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030

3131
from pyiceberg.catalog.rest.response import TokenResponse, _handle_non_200_response
3232
from pyiceberg.exceptions import OAuthError
33+
from pyiceberg.utils.transient import _TransientStateMixin
3334

3435
AUTH_MANAGER = "auth.manager"
3536

@@ -127,7 +128,7 @@ def auth_header(self) -> str:
127128
return f"Bearer {self._token}"
128129

129130

130-
class OAuth2TokenProvider:
131+
class OAuth2TokenProvider(_TransientStateMixin):
131132
"""Thread-safe OAuth2 token provider with token refresh support."""
132133

133134
client_id: str
@@ -141,6 +142,9 @@ class OAuth2TokenProvider:
141142
_expires_at: int
142143
_lock: threading.Lock
143144

145+
# The token expiry is based on time.monotonic(), which is only meaningful within one process.
146+
_transient_attrs = ("_token", "_expires_at", "_lock")
147+
144148
def __init__(
145149
self,
146150
client_id: str,
@@ -156,7 +160,9 @@ def __init__(
156160
self.scope = scope
157161
self.refresh_margin = refresh_margin
158162
self.expires_in = expires_in
163+
self._init_transient()
159164

165+
def _init_transient(self) -> None:
160166
self._token = None
161167
self._expires_at = 0
162168
self._lock = threading.Lock()

‎pyiceberg/io/fsspec.py‎

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
import os
2424
import threading
2525
from collections.abc import Callable
26-
from copy import copy
2726
from functools import lru_cache
2827
from typing import (
2928
TYPE_CHECKING,
@@ -96,6 +95,7 @@
9695
from pyiceberg.typedef import Properties
9796
from pyiceberg.types import strtobool
9897
from pyiceberg.utils.properties import get_first_property_value, get_header_properties, property_as_bool
98+
from pyiceberg.utils.transient import _TransientStateMixin
9999

100100
logger = logging.getLogger(__name__)
101101

@@ -443,14 +443,19 @@ def to_input_file(self) -> FsspecInputFile:
443443
return FsspecInputFile(location=self.location, fs=self._fs)
444444

445445

446-
class FsspecFileIO(FileIO):
446+
class FsspecFileIO(_TransientStateMixin, FileIO):
447447
"""A FileIO implementation that uses fsspec."""
448448

449+
_transient_attrs = ("_thread_locals",)
450+
449451
def __init__(self, properties: Properties):
450452
self._scheme_to_fs: dict[str, Callable[..., AbstractFileSystem]] = dict(SCHEME_TO_FS)
451-
self._thread_locals = threading.local()
453+
self._init_transient()
452454
super().__init__(properties=properties)
453455

456+
def _init_transient(self) -> None:
457+
self._thread_locals = threading.local()
458+
454459
@override
455460
def new_input(self, location: str) -> FsspecInputFile:
456461
"""Get an FsspecInputFile instance to read bytes from the file at the given location.
@@ -521,14 +526,3 @@ def _get_fs(self, scheme: str, hostname: str | None = None) -> AbstractFileSyste
521526
return _adls(self.properties, hostname)
522527

523528
return self._scheme_to_fs[scheme](self.properties)
524-
525-
def __getstate__(self) -> dict[str, Any]:
526-
"""Create a dictionary of the FsSpecFileIO fields used when pickling."""
527-
fileio_copy = copy(self.__dict__)
528-
del fileio_copy["_thread_locals"]
529-
return fileio_copy
530-
531-
def __setstate__(self, state: dict[str, Any]) -> None:
532-
"""Deserialize the state into a FsSpecFileIO instance."""
533-
self.__dict__ = state
534-
self._thread_locals = threading.local()

‎pyiceberg/io/pyarrow.py‎

Lines changed: 7 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@
3939
import warnings
4040
from abc import ABC, abstractmethod
4141
from collections.abc import Callable, Iterable, Iterator
42-
from copy import copy
4342
from dataclasses import dataclass
4443
from decimal import Decimal
4544
from enum import Enum
@@ -193,6 +192,7 @@
193192
from pyiceberg.utils.decimal import unscaled_to_decimal
194193
from pyiceberg.utils.properties import get_first_property_value, property_as_bool, property_as_int
195194
from pyiceberg.utils.singleton import Singleton
195+
from pyiceberg.utils.transient import _TransientStateMixin
196196
from pyiceberg.utils.truncate import truncate_upper_bound_binary_string, truncate_upper_bound_text_string
197197

198198
if TYPE_CHECKING:
@@ -399,13 +399,17 @@ def to_input_file(self) -> PyArrowFile:
399399
return self
400400

401401

402-
class PyArrowFileIO(FileIO):
402+
class PyArrowFileIO(_TransientStateMixin, FileIO):
403403
fs_by_scheme: Callable[[str, str | None], FileSystem]
404+
_transient_attrs = ("fs_by_scheme",)
404405

405406
def __init__(self, properties: Properties = EMPTY_DICT):
406-
self.fs_by_scheme: Callable[[str, str | None], FileSystem] = lru_cache(self._initialize_fs)
407+
self._init_transient()
407408
super().__init__(properties=properties)
408409

410+
def _init_transient(self) -> None:
411+
self.fs_by_scheme = lru_cache(self._initialize_fs)
412+
409413
@staticmethod
410414
def parse_location(location: str, properties: Properties = EMPTY_DICT) -> tuple[str, str, str]:
411415
r"""Return (scheme, netloc, path) for the given location.
@@ -727,17 +731,6 @@ def delete(self, location: str | InputFile | OutputFile) -> None:
727731
raise PermissionError(f"Cannot delete file, access denied: {location}") from e
728732
raise # pragma: no cover - If some other kind of OSError, raise the raw error
729733

730-
def __getstate__(self) -> dict[str, Any]:
731-
"""Create a dictionary of the PyArrowFileIO fields used when pickling."""
732-
fileio_copy = copy(self.__dict__)
733-
fileio_copy["fs_by_scheme"] = None
734-
return fileio_copy
735-
736-
def __setstate__(self, state: dict[str, Any]) -> None:
737-
"""Deserialize the state into a PyArrowFileIO instance."""
738-
self.__dict__ = state
739-
self.fs_by_scheme = lru_cache(self._initialize_fs)
740-
741734

742735
def schema_to_pyarrow(
743736
schema: Schema | IcebergType,

‎pyiceberg/utils/transient.py‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
"""Support for pickling objects that hold process-local state such as locks and caches."""
18+
19+
from typing import Any, ClassVar
20+
21+
22+
class _TransientStateMixin:
23+
"""Excludes process-local attributes from pickling and rebuilds them on unpickle.
24+
25+
Subclasses list those attributes in `_transient_attrs`, create them in `_init_transient`,
26+
and call `_init_transient` from `__init__`. Place the mixin before other bases so its
27+
`__getstate__` and `__setstate__` take precedence.
28+
"""
29+
30+
_transient_attrs: ClassVar[tuple[str, ...]] = ()
31+
32+
def _init_transient(self) -> None:
33+
"""Create the attributes listed in `_transient_attrs`."""
34+
35+
def __getstate__(self) -> dict[str, Any]:
36+
"""Return the instance state without the transient attributes."""
37+
state = self.__dict__.copy()
38+
for attr in self._transient_attrs:
39+
state.pop(attr, None)
40+
return state
41+
42+
def __setstate__(self, state: dict[str, Any]) -> None:
43+
"""Restore the instance state and rebuild the transient attributes."""
44+
self.__dict__.update(state)
45+
self._init_transient()

‎tests/catalog/test_rest_auth.py‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,21 @@
1616
# under the License.
1717

1818
import base64
19+
import pickle
1920
from unittest.mock import MagicMock, patch
2021

2122
import pytest
2223
import requests
2324
from requests_mock import Mocker
2425

25-
from pyiceberg.catalog.rest.auth import AuthManagerAdapter, BasicAuthManager, EntraAuthManager, GoogleAuthManager, NoopAuthManager
26+
from pyiceberg.catalog.rest.auth import (
27+
AuthManagerAdapter,
28+
BasicAuthManager,
29+
EntraAuthManager,
30+
GoogleAuthManager,
31+
NoopAuthManager,
32+
OAuth2TokenProvider,
33+
)
2634

2735
TEST_URI = "https://iceberg-test-catalog/"
2836
GOOGLE_CREDS_URI = "https://oauth2.googleapis.com/token"
@@ -249,3 +257,19 @@ def test_entra_auth_manager_token_failure(mock_default_cred: MagicMock, rest_moc
249257
# Verify no requests were made with a blank/missing auth header
250258
history = rest_mock.request_history
251259
assert len(history) == 0
260+
261+
262+
def test_oauth2_token_provider_pickle_round_trip(requests_mock: Mocker) -> None:
263+
token_url = f"{TEST_URI}v1/oauth/tokens"
264+
requests_mock.post(
265+
token_url,
266+
[{"json": {"access_token": "first", "expires_in": 3600}}, {"json": {"access_token": "second", "expires_in": 3600}}],
267+
)
268+
provider = OAuth2TokenProvider(client_id="client", client_secret="secret", token_url=token_url)
269+
assert provider.get_token() == "first"
270+
271+
unpickled = pickle.loads(pickle.dumps(provider))
272+
273+
# The cached token is not carried over, since its expiry is relative to the original process' clock
274+
assert unpickled.get_token() == "second"
275+
assert unpickled.client_id == "client"

‎tests/io/test_io.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,13 @@
2626
from pyiceberg.io import (
2727
ARROW_FILE_IO,
2828
PY_IO_IMPL,
29+
FileIO,
2930
_import_file_io,
3031
_infer_file_io_from_scheme,
3132
_is_local_path,
3233
load_file_io,
3334
)
35+
from pyiceberg.io.fsspec import FsspecFileIO
3436
from pyiceberg.io.pyarrow import PyArrowFileIO
3537

3638

@@ -75,10 +77,13 @@ def test_custom_local_output_file() -> None:
7577
assert len(output_file) == 3
7678

7779

78-
def test_pickled_pyarrow_round_trip() -> None:
80+
@pytest.mark.parametrize("file_io_class", [PyArrowFileIO, FsspecFileIO])
81+
def test_pickled_file_io_round_trip(file_io_class: type[FileIO]) -> None:
7982
with tempfile.TemporaryDirectory() as tmpdirname:
8083
file_location = os.path.join(tmpdirname, "foo.txt")
81-
file_io = PyArrowFileIO()
84+
file_io = file_io_class(properties={})
85+
# Exercise the cached filesystem so the transient state is populated before pickling
86+
file_io.new_input(location=os.path.abspath(file_location)).exists()
8287
serialized_file_io = pickle.dumps(file_io)
8388
deserialized_file_io = pickle.loads(serialized_file_io)
8489
absolute_file_location = os.path.abspath(file_location)

0 commit comments

Comments
 (0)