|
39 | 39 | import warnings |
40 | 40 | from abc import ABC, abstractmethod |
41 | 41 | from collections.abc import Callable, Iterable, Iterator |
42 | | -from copy import copy |
43 | 42 | from dataclasses import dataclass |
44 | 43 | from decimal import Decimal |
45 | 44 | from enum import Enum |
|
193 | 192 | from pyiceberg.utils.decimal import unscaled_to_decimal |
194 | 193 | from pyiceberg.utils.properties import get_first_property_value, property_as_bool, property_as_int |
195 | 194 | from pyiceberg.utils.singleton import Singleton |
| 195 | +from pyiceberg.utils.transient import _TransientStateMixin |
196 | 196 | from pyiceberg.utils.truncate import truncate_upper_bound_binary_string, truncate_upper_bound_text_string |
197 | 197 |
|
198 | 198 | if TYPE_CHECKING: |
@@ -399,13 +399,17 @@ def to_input_file(self) -> PyArrowFile: |
399 | 399 | return self |
400 | 400 |
|
401 | 401 |
|
402 | | -class PyArrowFileIO(FileIO): |
| 402 | +class PyArrowFileIO(_TransientStateMixin, FileIO): |
403 | 403 | fs_by_scheme: Callable[[str, str | None], FileSystem] |
| 404 | + _transient_attrs = ("fs_by_scheme",) |
404 | 405 |
|
405 | 406 | def __init__(self, properties: Properties = EMPTY_DICT): |
406 | | - self.fs_by_scheme: Callable[[str, str | None], FileSystem] = lru_cache(self._initialize_fs) |
| 407 | + self._init_transient() |
407 | 408 | super().__init__(properties=properties) |
408 | 409 |
|
| 410 | + def _init_transient(self) -> None: |
| 411 | + self.fs_by_scheme = lru_cache(self._initialize_fs) |
| 412 | + |
409 | 413 | @staticmethod |
410 | 414 | def parse_location(location: str, properties: Properties = EMPTY_DICT) -> tuple[str, str, str]: |
411 | 415 | r"""Return (scheme, netloc, path) for the given location. |
@@ -727,17 +731,6 @@ def delete(self, location: str | InputFile | OutputFile) -> None: |
727 | 731 | raise PermissionError(f"Cannot delete file, access denied: {location}") from e |
728 | 732 | raise # pragma: no cover - If some other kind of OSError, raise the raw error |
729 | 733 |
|
730 | | - def __getstate__(self) -> dict[str, Any]: |
731 | | - """Create a dictionary of the PyArrowFileIO fields used when pickling.""" |
732 | | - fileio_copy = copy(self.__dict__) |
733 | | - fileio_copy["fs_by_scheme"] = None |
734 | | - return fileio_copy |
735 | | - |
736 | | - def __setstate__(self, state: dict[str, Any]) -> None: |
737 | | - """Deserialize the state into a PyArrowFileIO instance.""" |
738 | | - self.__dict__ = state |
739 | | - self.fs_by_scheme = lru_cache(self._initialize_fs) |
740 | | - |
741 | 734 |
|
742 | 735 | def schema_to_pyarrow( |
743 | 736 | schema: Schema | IcebergType, |
|
0 commit comments