diff --git a/pyiceberg/io/__init__.py b/pyiceberg/io/__init__.py index ec232a02eb..be7bbfeff8 100644 --- a/pyiceberg/io/__init__.py +++ b/pyiceberg/io/__init__.py @@ -38,7 +38,9 @@ ) from urllib.parse import urlparse +from pyiceberg.io.content_cache import ContentCache from pyiceberg.typedef import EMPTY_DICT, Properties +from pyiceberg.utils.properties import property_as_bool, property_as_int logger = logging.getLogger(__name__) @@ -118,6 +120,15 @@ def _is_local_path(path: str) -> bool: HF_ENDPOINT = "hf.endpoint" HF_TOKEN = "hf.token" +IO_MANIFEST_CACHE_ENABLED = "io.manifest.cache-enabled" +IO_MANIFEST_CACHE_MAX_TOTAL_BYTES = "io.manifest.cache.max-total-bytes" +IO_MANIFEST_CACHE_MAX_CONTENT_LENGTH = "io.manifest.cache.max-content-length" +IO_MANIFEST_CACHE_EXPIRATION_INTERVAL_MS = "io.manifest.cache.expiration-interval-ms" + +IO_MANIFEST_CACHE_MAX_TOTAL_BYTES_DEFAULT = 100 * 1024 * 1024 +IO_MANIFEST_CACHE_MAX_CONTENT_LENGTH_DEFAULT = 8 * 1024 * 1024 +IO_MANIFEST_CACHE_EXPIRATION_INTERVAL_MS_DEFAULT = 60_000 + @runtime_checkable class InputStream(Protocol): @@ -272,6 +283,26 @@ def create(self, overwrite: bool = False) -> OutputStream: """ +def _content_cache_from_properties(properties: Properties) -> ContentCache | None: + if not property_as_bool(properties, IO_MANIFEST_CACHE_ENABLED, False): + return None + + max_total_bytes: int = property_as_int( # type: ignore[assignment] + properties, IO_MANIFEST_CACHE_MAX_TOTAL_BYTES, IO_MANIFEST_CACHE_MAX_TOTAL_BYTES_DEFAULT + ) + max_content_length: int = property_as_int( # type: ignore[assignment] + properties, IO_MANIFEST_CACHE_MAX_CONTENT_LENGTH, IO_MANIFEST_CACHE_MAX_CONTENT_LENGTH_DEFAULT + ) + expiration_interval_ms: int = property_as_int( # type: ignore[assignment] + properties, IO_MANIFEST_CACHE_EXPIRATION_INTERVAL_MS, IO_MANIFEST_CACHE_EXPIRATION_INTERVAL_MS_DEFAULT + ) + return ContentCache( + max_total_bytes=max_total_bytes, + max_content_length=max_content_length, + expiration_interval_ms=expiration_interval_ms, + ) + + class FileIO(ABC): """A base class for FileIO implementations.""" @@ -279,6 +310,12 @@ class FileIO(ABC): def __init__(self, properties: Properties = EMPTY_DICT): self.properties = properties + self._content_cache = _content_cache_from_properties(properties) + + @property + def content_cache(self) -> ContentCache | None: + """Return the cache for immutable metadata file contents, when enabled.""" + return self._content_cache @abstractmethod def new_input(self, location: str) -> InputFile: diff --git a/pyiceberg/io/content_cache.py b/pyiceberg/io/content_cache.py new file mode 100644 index 0000000000..776b055497 --- /dev/null +++ b/pyiceberg/io/content_cache.py @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""A bounded in-memory cache for the bytes of immutable metadata files.""" + +from __future__ import annotations + +import time +from collections import OrderedDict +from collections.abc import Callable + + +class ContentCache: + """Cache file contents by location, bounded by total size and by entry age. + + Args: + max_total_bytes: The maximum number of bytes to keep across all entries. + max_content_length: Entries larger than this are not cached. + expiration_interval_ms: Entries older than this are treated as absent. + clock: Returns the current time in seconds, replaced in tests. + """ + + def __init__( + self, + max_total_bytes: int, + max_content_length: int, + expiration_interval_ms: int, + clock: Callable[[], float] = time.monotonic, + ) -> None: + self.max_total_bytes = max_total_bytes + self.max_content_length = max_content_length + self.expiration_interval_seconds = expiration_interval_ms / 1000 + self._clock = clock + self._entries: OrderedDict[str, tuple[bytes, float]] = OrderedDict() + self._total_bytes = 0 + + def get(self, location: str) -> bytes | None: + """Return the cached bytes for a location, or None when absent or expired.""" + entry = self._entries.get(location) + if entry is None: + return None + + data, inserted_at = entry + if self._clock() - inserted_at >= self.expiration_interval_seconds: + self._remove(location) + return None + + self._entries.move_to_end(location) + return data + + def put(self, location: str, data: bytes) -> None: + """Cache the bytes for a location, evicting the least recently used entries when needed.""" + if len(data) > self.max_content_length or len(data) > self.max_total_bytes or self.max_content_length <= 0: + return + + self._remove(location) + self._purge_expired() + while self._entries and self._total_bytes + len(data) > self.max_total_bytes: + _, (evicted, _) = self._entries.popitem(last=False) + self._total_bytes -= len(evicted) + + self._entries[location] = (data, self._clock()) + self._total_bytes += len(data) + + def clear(self) -> None: + """Remove every entry.""" + self._entries.clear() + self._total_bytes = 0 + + def _remove(self, location: str) -> None: + entry = self._entries.pop(location, None) + if entry is not None: + self._total_bytes -= len(entry[0]) + + def _purge_expired(self) -> None: + now = self._clock() + for location in [ + location + for location, (_, inserted_at) in self._entries.items() + if now - inserted_at >= self.expiration_interval_seconds + ]: + self._remove(location) diff --git a/pyiceberg/manifest.py b/pyiceberg/manifest.py index 52e5890651..df54aadf6b 100644 --- a/pyiceberg/manifest.py +++ b/pyiceberg/manifest.py @@ -22,6 +22,7 @@ from collections.abc import Callable, Iterator from copy import copy from enum import Enum, StrEnum +from io import BytesIO from types import TracebackType from typing import ( Any, @@ -37,7 +38,7 @@ from pyiceberg.encryption.io import EncryptedInputFile from pyiceberg.encryption.key_metadata import StandardKeyMetadata from pyiceberg.exceptions import ValidationError -from pyiceberg.io import FileIO, InputFile, OutputFile +from pyiceberg.io import FileIO, InputFile, InputStream, OutputFile from pyiceberg.partitioning import PartitionSpec from pyiceberg.schema import Schema from pyiceberg.typedef import Record, TableVersion @@ -62,6 +63,38 @@ INITIAL_SEQUENCE_NUMBER = 0 +class _MemoryInputFile(InputFile): + """An InputFile that serves bytes already read from the file at the location.""" + + def __init__(self, location: str, data: bytes) -> None: + super().__init__(location) + self._data = data + + def __len__(self) -> int: + return len(self._data) + + def exists(self) -> bool: + return True + + def open(self, seekable: bool = True) -> InputStream: + return BytesIO(self._data) + + +def _open_metadata_file(io: FileIO, location: str) -> InputFile: + """Open a metadata file, reading its bytes through the content cache when the FileIO has one.""" + cache = io.content_cache + if cache is None: + return io.new_input(location) + + if (data := cache.get(location)) is None: + input_file = io.new_input(location) + with input_file.open() as stream: + data = stream.read() + cache.put(location, data) + + return _MemoryInputFile(location, data) + + class DataFileContent(int, Enum): DATA = 0 POSITION_DELETES = 1 @@ -916,7 +949,7 @@ def fetch_manifest_entry( Returns: An Iterator of manifest entries. """ - input_file = io.new_input(self.manifest_path) + input_file = _open_metadata_file(io, self.manifest_path) if self.key_metadata is not None: input_file = EncryptedInputFile(input_file, StandardKeyMetadata.from_bytes(self.key_metadata)) with AvroFile[ManifestEntry]( @@ -1033,7 +1066,7 @@ def _manifests(io: FileIO, manifest_list: str, key_metadata: StandardKeyMetadata Returns: A tuple of ManifestFile objects. """ - file = io.new_input(manifest_list) + file = _open_metadata_file(io, manifest_list) if key_metadata is not None: file = EncryptedInputFile(file, key_metadata) manifest_files = list(read_manifest_list(file)) diff --git a/tests/io/test_content_cache.py b/tests/io/test_content_cache.py new file mode 100644 index 0000000000..ce52d37465 --- /dev/null +++ b/tests/io/test_content_cache.py @@ -0,0 +1,173 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from unittest.mock import patch + +import pyarrow as pa +import pytest + +from pyiceberg.catalog import Catalog +from pyiceberg.io import ( + IO_MANIFEST_CACHE_ENABLED, + IO_MANIFEST_CACHE_EXPIRATION_INTERVAL_MS, + IO_MANIFEST_CACHE_MAX_CONTENT_LENGTH, + IO_MANIFEST_CACHE_MAX_TOTAL_BYTES, + IO_MANIFEST_CACHE_MAX_TOTAL_BYTES_DEFAULT, +) +from pyiceberg.io.content_cache import ContentCache +from pyiceberg.io.pyarrow import PyArrowFileIO + + +class _FakeClock: + def __init__(self) -> None: + self.now = 0.0 + + def __call__(self) -> float: + return self.now + + +def test_cache_disabled_by_default() -> None: + assert PyArrowFileIO().content_cache is None + + +def test_cache_enabled_by_properties() -> None: + cache = PyArrowFileIO({IO_MANIFEST_CACHE_ENABLED: "true"}).content_cache + + assert cache is not None + assert cache.max_total_bytes == IO_MANIFEST_CACHE_MAX_TOTAL_BYTES_DEFAULT + + +def test_cache_returns_put_bytes() -> None: + cache = ContentCache(max_total_bytes=10, max_content_length=10, expiration_interval_ms=60_000) + + assert cache.get("a") is None + cache.put("a", b"123") + assert cache.get("a") == b"123" + + +def test_cache_skips_entries_over_max_content_length() -> None: + cache = ContentCache(max_total_bytes=100, max_content_length=3, expiration_interval_ms=60_000) + + cache.put("a", b"1234") + + assert cache.get("a") is None + + +def test_cache_skips_entries_over_max_total_bytes() -> None: + cache = ContentCache(max_total_bytes=3, max_content_length=10, expiration_interval_ms=60_000) + + cache.put("a", b"1234") + + assert cache.get("a") is None + + +def test_cache_evicts_least_recently_used() -> None: + cache = ContentCache(max_total_bytes=4, max_content_length=4, expiration_interval_ms=60_000) + + cache.put("a", b"11") + cache.put("b", b"22") + assert cache.get("a") == b"11" + cache.put("c", b"33") + + assert cache.get("b") is None + assert cache.get("a") == b"11" + assert cache.get("c") == b"33" + + +def test_cache_expires_entries() -> None: + clock = _FakeClock() + cache = ContentCache(max_total_bytes=10, max_content_length=10, expiration_interval_ms=1_000, clock=clock) + + cache.put("a", b"11") + clock.now = 1.0 + + assert cache.get("a") is None + + +def test_manifest_list_is_read_once(catalog: Catalog) -> None: + catalog.create_namespace("default") + table = catalog.create_table("default.cached_manifest_list", schema=pa.schema([pa.field("x", pa.int64())])) + table.append(pa.table({"x": [1, 2, 3]})) + + snapshot = table.current_snapshot() + assert snapshot is not None + + io = PyArrowFileIO({IO_MANIFEST_CACHE_ENABLED: "true"}) + with patch.object(io, "new_input", side_effect=io.new_input) as new_input: + snapshot.manifests(io) + snapshot.manifests(io) + + assert new_input.call_count == 1 + + +def test_manifest_entries_are_read_once(catalog: Catalog) -> None: + catalog.create_namespace("default") + table = catalog.create_table("default.cached_manifest", schema=pa.schema([pa.field("x", pa.int64())])) + table.append(pa.table({"x": [1, 2, 3]})) + + snapshot = table.current_snapshot() + assert snapshot is not None + + io = PyArrowFileIO({IO_MANIFEST_CACHE_ENABLED: "true"}) + manifest = snapshot.manifests(io)[0] + + with patch.object(io, "new_input", side_effect=io.new_input) as new_input: + manifest.fetch_manifest_entry(io) + manifest.fetch_manifest_entry(io) + + assert new_input.call_count == 1 + + +def test_reads_are_not_cached_when_disabled(catalog: Catalog) -> None: + catalog.create_namespace("default") + table = catalog.create_table("default.uncached_manifest", schema=pa.schema([pa.field("x", pa.int64())])) + table.append(pa.table({"x": [1, 2, 3]})) + + snapshot = table.current_snapshot() + assert snapshot is not None + + io = PyArrowFileIO() + with patch.object(io, "new_input", side_effect=io.new_input) as new_input: + snapshot.manifests(io) + snapshot.manifests(io) + + assert new_input.call_count == 2 + + +def test_cache_properties_are_read_from_table_properties(catalog: Catalog) -> None: + catalog.create_namespace("default") + catalog.create_table( + "default.cache_properties", + schema=pa.schema([pa.field("x", pa.int64())]), + properties={ + IO_MANIFEST_CACHE_ENABLED: "true", + IO_MANIFEST_CACHE_MAX_TOTAL_BYTES: "1024", + IO_MANIFEST_CACHE_MAX_CONTENT_LENGTH: "512", + IO_MANIFEST_CACHE_EXPIRATION_INTERVAL_MS: "1000", + }, + ) + + cache = catalog.load_table("default.cache_properties").io.content_cache + + assert cache is not None + assert cache.max_total_bytes == 1024 + assert cache.max_content_length == 512 + assert cache.expiration_interval_seconds == 1.0 + + +@pytest.mark.parametrize("enabled", ["false", "0", ""]) +def test_cache_stays_disabled_when_property_is_not_truthy(enabled: str) -> None: + assert PyArrowFileIO({IO_MANIFEST_CACHE_ENABLED: enabled}).content_cache is None