Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion pyiceberg/catalog/rest/auth.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@

from pyiceberg.catalog.rest.response import TokenResponse, _handle_non_200_response
from pyiceberg.exceptions import OAuthError
from pyiceberg.utils.transient import _TransientStateMixin

AUTH_MANAGER = "auth.manager"

Expand Down Expand Up @@ -127,7 +128,7 @@ def auth_header(self) -> str:
return f"Bearer {self._token}"


class OAuth2TokenProvider:
class OAuth2TokenProvider(_TransientStateMixin):
"""Thread-safe OAuth2 token provider with token refresh support."""

client_id: str
Expand All @@ -141,6 +142,9 @@ class OAuth2TokenProvider:
_expires_at: int
_lock: threading.Lock

# The token expiry is based on time.monotonic(), which is only meaningful within one process.
_transient_attrs = ("_token", "_expires_at", "_lock")

def __init__(
self,
client_id: str,
Expand All @@ -156,7 +160,9 @@ def __init__(
self.scope = scope
self.refresh_margin = refresh_margin
self.expires_in = expires_in
self._init_transient()

def _init_transient(self) -> None:
self._token = None
self._expires_at = 0
self._lock = threading.Lock()
Expand Down
22 changes: 8 additions & 14 deletions pyiceberg/io/fsspec.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import os
import threading
from collections.abc import Callable
from copy import copy
from functools import lru_cache
from typing import (
TYPE_CHECKING,
Expand Down Expand Up @@ -96,6 +95,7 @@
from pyiceberg.typedef import Properties
from pyiceberg.types import strtobool
from pyiceberg.utils.properties import get_first_property_value, get_header_properties, property_as_bool
from pyiceberg.utils.transient import _TransientStateMixin

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -443,14 +443,19 @@ def to_input_file(self) -> FsspecInputFile:
return FsspecInputFile(location=self.location, fs=self._fs)


class FsspecFileIO(FileIO):
class FsspecFileIO(_TransientStateMixin, FileIO):
"""A FileIO implementation that uses fsspec."""

_transient_attrs = ("_thread_locals",)

def __init__(self, properties: Properties):
self._scheme_to_fs: dict[str, Callable[..., AbstractFileSystem]] = dict(SCHEME_TO_FS)
self._thread_locals = threading.local()
self._init_transient()
super().__init__(properties=properties)

def _init_transient(self) -> None:
self._thread_locals = threading.local()

@override
def new_input(self, location: str) -> FsspecInputFile:
"""Get an FsspecInputFile instance to read bytes from the file at the given location.
Expand Down Expand Up @@ -521,14 +526,3 @@ def _get_fs(self, scheme: str, hostname: str | None = None) -> AbstractFileSyste
return _adls(self.properties, hostname)

return self._scheme_to_fs[scheme](self.properties)

def __getstate__(self) -> dict[str, Any]:
"""Create a dictionary of the FsSpecFileIO fields used when pickling."""
fileio_copy = copy(self.__dict__)
del fileio_copy["_thread_locals"]
return fileio_copy

def __setstate__(self, state: dict[str, Any]) -> None:
"""Deserialize the state into a FsSpecFileIO instance."""
self.__dict__ = state
self._thread_locals = threading.local()
21 changes: 7 additions & 14 deletions pyiceberg/io/pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@
import warnings
from abc import ABC, abstractmethod
from collections.abc import Callable, Iterable, Iterator
from copy import copy
from dataclasses import dataclass
from decimal import Decimal
from enum import Enum
Expand Down Expand Up @@ -193,6 +192,7 @@
from pyiceberg.utils.decimal import unscaled_to_decimal
from pyiceberg.utils.properties import get_first_property_value, property_as_bool, property_as_int
from pyiceberg.utils.singleton import Singleton
from pyiceberg.utils.transient import _TransientStateMixin
from pyiceberg.utils.truncate import truncate_upper_bound_binary_string, truncate_upper_bound_text_string

if TYPE_CHECKING:
Expand Down Expand Up @@ -399,13 +399,17 @@ def to_input_file(self) -> PyArrowFile:
return self


class PyArrowFileIO(FileIO):
class PyArrowFileIO(_TransientStateMixin, FileIO):
fs_by_scheme: Callable[[str, str | None], FileSystem]
_transient_attrs = ("fs_by_scheme",)

def __init__(self, properties: Properties = EMPTY_DICT):
self.fs_by_scheme: Callable[[str, str | None], FileSystem] = lru_cache(self._initialize_fs)
self._init_transient()
super().__init__(properties=properties)

def _init_transient(self) -> None:
self.fs_by_scheme = lru_cache(self._initialize_fs)

@staticmethod
def parse_location(location: str, properties: Properties = EMPTY_DICT) -> tuple[str, str, str]:
r"""Return (scheme, netloc, path) for the given location.
Expand Down Expand Up @@ -727,17 +731,6 @@ def delete(self, location: str | InputFile | OutputFile) -> None:
raise PermissionError(f"Cannot delete file, access denied: {location}") from e
raise # pragma: no cover - If some other kind of OSError, raise the raw error

def __getstate__(self) -> dict[str, Any]:
"""Create a dictionary of the PyArrowFileIO fields used when pickling."""
fileio_copy = copy(self.__dict__)
fileio_copy["fs_by_scheme"] = None
return fileio_copy

def __setstate__(self, state: dict[str, Any]) -> None:
"""Deserialize the state into a PyArrowFileIO instance."""
self.__dict__ = state
self.fs_by_scheme = lru_cache(self._initialize_fs)


def schema_to_pyarrow(
schema: Schema | IcebergType,
Expand Down
45 changes: 45 additions & 0 deletions pyiceberg/utils/transient.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Support for pickling objects that hold process-local state such as locks and caches."""

from typing import Any, ClassVar


class _TransientStateMixin:
"""Excludes process-local attributes from pickling and rebuilds them on unpickle.

Subclasses list those attributes in `_transient_attrs`, create them in `_init_transient`,
and call `_init_transient` from `__init__`. Place the mixin before other bases so its
`__getstate__` and `__setstate__` take precedence.
"""

_transient_attrs: ClassVar[tuple[str, ...]] = ()

def _init_transient(self) -> None:
"""Create the attributes listed in `_transient_attrs`."""

def __getstate__(self) -> dict[str, Any]:

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: __getstate__/__setstate__ here only round-trip self.__dict__, so a subclass that uses __slots__ (and doesn't mirror its slots into __dict__) would silently lose its slotted state on pickle. The hand-written versions on main had the same behavior, so this is not a regression, but since this is now the central mechanism future adopters might hit it. Maybe worth a line in the docstring that slotted subclasses need their own handling.

"""Return the instance state without the transient attributes."""
state = self.__dict__.copy()
for attr in self._transient_attrs:
state.pop(attr, None)
return state

def __setstate__(self, state: dict[str, Any]) -> None:
"""Restore the instance state and rebuild the transient attributes."""
self.__dict__.update(state)
self._init_transient()
26 changes: 25 additions & 1 deletion tests/catalog/test_rest_auth.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,21 @@
# under the License.

import base64
import pickle
from unittest.mock import MagicMock, patch

import pytest
import requests
from requests_mock import Mocker

from pyiceberg.catalog.rest.auth import AuthManagerAdapter, BasicAuthManager, EntraAuthManager, GoogleAuthManager, NoopAuthManager
from pyiceberg.catalog.rest.auth import (
AuthManagerAdapter,
BasicAuthManager,
EntraAuthManager,
GoogleAuthManager,
NoopAuthManager,
OAuth2TokenProvider,
)

TEST_URI = "https://iceberg-test-catalog/"
GOOGLE_CREDS_URI = "https://oauth2.googleapis.com/token"
Expand Down Expand Up @@ -249,3 +257,19 @@ def test_entra_auth_manager_token_failure(mock_default_cred: MagicMock, rest_moc
# Verify no requests were made with a blank/missing auth header
history = rest_mock.request_history
assert len(history) == 0


def test_oauth2_token_provider_pickle_round_trip(requests_mock: Mocker) -> None:
token_url = f"{TEST_URI}v1/oauth/tokens"
requests_mock.post(
token_url,
[{"json": {"access_token": "first", "expires_in": 3600}}, {"json": {"access_token": "second", "expires_in": 3600}}],
)
provider = OAuth2TokenProvider(client_id="client", client_secret="secret", token_url=token_url)
assert provider.get_token() == "first"

unpickled = pickle.loads(pickle.dumps(provider))

# The cached token is not carried over, since its expiry is relative to the original process' clock
assert unpickled.get_token() == "second"
assert unpickled.client_id == "client"
9 changes: 7 additions & 2 deletions tests/io/test_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,13 @@
from pyiceberg.io import (
ARROW_FILE_IO,
PY_IO_IMPL,
FileIO,
_import_file_io,
_infer_file_io_from_scheme,
_is_local_path,
load_file_io,
)
from pyiceberg.io.fsspec import FsspecFileIO
from pyiceberg.io.pyarrow import PyArrowFileIO


Expand Down Expand Up @@ -75,10 +77,13 @@ def test_custom_local_output_file() -> None:
assert len(output_file) == 3


def test_pickled_pyarrow_round_trip() -> None:
@pytest.mark.parametrize("file_io_class", [PyArrowFileIO, FsspecFileIO])
def test_pickled_file_io_round_trip(file_io_class: type[FileIO]) -> None:
with tempfile.TemporaryDirectory() as tmpdirname:
file_location = os.path.join(tmpdirname, "foo.txt")
file_io = PyArrowFileIO()
file_io = file_io_class(properties={})
# Exercise the cached filesystem so the transient state is populated before pickling
file_io.new_input(location=os.path.abspath(file_location)).exists()
serialized_file_io = pickle.dumps(file_io)
deserialized_file_io = pickle.loads(serialized_file_io)
absolute_file_location = os.path.abspath(file_location)
Expand Down
Loading