Skip to content

Commit 89c5f91

Browse files
committed
Tighten manifest cache configuration
1 parent 6144557 commit 89c5f91

4 files changed

Lines changed: 84 additions & 54 deletions

File tree

‎mkdocs/docs/configuration.md‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,15 +51,15 @@ For example, `PYICEBERG_CATALOG__DEFAULT__S3__ACCESS_KEY_ID`, sets `s3.access-ke
5151
## Manifest Caching
5252

5353
PyIceberg caches `ManifestFile` objects locally and uses an LRU policy to bound the cache size. By default, up to `128`
54-
manifest entries are retained.
54+
distinct manifest files are retained.
5555

5656
You can tune the `manifest-cache-size` configuration in `.pyiceberg.yaml`:
5757

5858
```yaml
5959
manifest-cache-size: 256
6060
```
6161
62-
Permitted values: Any `int` between `0` and `math.inf`.
62+
Permitted values: any non-negative integer. Set the value to `0` to disable manifest caching entirely.
6363

6464
You can also set it with the `PYICEBERG_MANIFEST_CACHE_SIZE` environment variable:
6565

@@ -69,7 +69,7 @@ export PYICEBERG_MANIFEST_CACHE_SIZE=256
6969

7070
The memory used by this cache depends on the size and number of distinct manifests your workload touches. Lower the value
7171
if you want a tighter memory bound, or call `clear_manifest_cache()` to proactively release cached manifest metadata in
72-
long-lived processes. Setting `manifest-cache-size` or `PYICEBERG_MANIFEST_CACHE_SIZE` to `0` disables manifest caching entirely.
72+
long-lived processes.
7373

7474
## Tables
7575

‎pyiceberg/manifest.py‎

Lines changed: 54 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
import math
2020
import threading
2121
from abc import ABC, abstractmethod
22-
from collections.abc import Iterator, MutableMapping
22+
from collections.abc import Iterator
2323
from copy import copy
2424
from enum import Enum
2525
from types import TracebackType
@@ -892,18 +892,57 @@ def __hash__(self) -> int:
892892
return hash(self.manifest_path)
893893

894894

895-
# Global cache for ManifestFile objects, keyed by manifest_path.
896-
# This deduplicates ManifestFile objects across manifest lists, which commonly
897-
# share manifests after append operations.
898-
_DEFAULT_MANIFEST_CACHE_SIZE = 128
899-
_configured_manifest_cache_size = Config().get_int("manifest-cache-size")
900-
_manifest_cache_size = (
901-
max(_configured_manifest_cache_size, 0) if _configured_manifest_cache_size is not None else _DEFAULT_MANIFEST_CACHE_SIZE
902-
)
895+
class _ManifestCache:
896+
"""Process-wide ManifestFile cache keyed by manifest_path.
897+
898+
Consecutive snapshots often reference the same manifests after append
899+
operations, so reusing ManifestFile instances avoids retaining duplicate
900+
objects.
901+
"""
902+
903+
DEFAULT_SIZE = 128
904+
905+
_cache: LRUCache[str, ManifestFile] | None
906+
907+
def __init__(self) -> None:
908+
self.maxsize = self._load_configured_size()
909+
self._cache = LRUCache(maxsize=self.maxsize) if self.maxsize > 0 else None
910+
self._lock = threading.RLock()
911+
912+
@classmethod
913+
def _load_configured_size(cls) -> int:
914+
configured_size = Config().get_int("manifest-cache-size")
915+
if configured_size is None:
916+
return cls.DEFAULT_SIZE
917+
if configured_size < 0:
918+
raise ValueError(
919+
f"manifest-cache-size should be a non-negative integer or left unset. Current value: {configured_size}"
920+
)
921+
return configured_size
922+
923+
def clear(self) -> None:
924+
with self._lock:
925+
if self._cache is not None:
926+
self._cache.clear()
903927

904-
# Lock for thread-safe cache access
905-
_manifest_cache_lock = threading.RLock()
906-
_manifest_cache: MutableMapping[str, ManifestFile] = LRUCache(maxsize=_manifest_cache_size) if _manifest_cache_size > 0 else {}
928+
def get_or_cache(self, manifest_file: ManifestFile) -> ManifestFile:
929+
if self._cache is None:
930+
return manifest_file
931+
932+
with self._lock:
933+
manifest_path = manifest_file.manifest_path
934+
if manifest_path in self._cache:
935+
return self._cache[manifest_path]
936+
937+
self._cache[manifest_path] = manifest_file
938+
return manifest_file
939+
940+
def __len__(self) -> int:
941+
with self._lock:
942+
return len(self._cache) if self._cache is not None else 0
943+
944+
945+
_manifest_cache = _ManifestCache()
907946

908947

909948
def clear_manifest_cache() -> None:
@@ -912,12 +951,11 @@ def clear_manifest_cache() -> None:
912951
This is primarily useful in long-lived or memory-sensitive processes that
913952
want to release cached manifest metadata between bursts of table reads.
914953
"""
915-
with _manifest_cache_lock:
916-
_manifest_cache.clear()
954+
_manifest_cache.clear()
917955

918956

919957
def _manifests(io: FileIO, manifest_list: str) -> tuple[ManifestFile, ...]:
920-
"""Read manifests from a manifest list, deduplicating ManifestFile objects via cache.
958+
"""Read manifests from a manifest list, reusing cached ManifestFile objects.
921959
922960
Caches individual ManifestFile objects by manifest_path. This is memory-efficient
923961
because consecutive manifest lists typically share most of their manifests:
@@ -943,20 +981,7 @@ def _manifests(io: FileIO, manifest_list: str) -> tuple[ManifestFile, ...]:
943981
file = io.new_input(manifest_list)
944982
manifest_files = list(read_manifest_list(file))
945983

946-
if _manifest_cache_size == 0:
947-
return tuple(manifest_files)
948-
949-
result = []
950-
with _manifest_cache_lock:
951-
for manifest_file in manifest_files:
952-
manifest_path = manifest_file.manifest_path
953-
if manifest_path in _manifest_cache:
954-
result.append(_manifest_cache[manifest_path])
955-
else:
956-
_manifest_cache[manifest_path] = manifest_file
957-
result.append(manifest_file)
958-
959-
return tuple(result)
984+
return tuple(_manifest_cache.get_or_cache(manifest_file) for manifest_file in manifest_files)
960985

961986

962987
def read_manifest_list(input_file: InputFile) -> Iterator[ManifestFile]:

‎tests/benchmark/test_memory_benchmark.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,8 +73,8 @@ def clear_caches() -> None:
7373
def test_manifest_cache_memory_growth(memory_catalog: InMemoryCatalog) -> None:
7474
"""Benchmark memory growth of manifest cache during repeated appends.
7575
76-
This test reproduces the issue from GitHub #2325 where each append creates
77-
a new manifest list entry in the cache, causing memory to grow.
76+
This test reproduces the issue from GitHub #2325 where the old cache stored
77+
each manifest list result, causing memory to grow.
7878
7979
With the old caching strategy (tuple per manifest list), memory grew as O(N²).
8080
With the new strategy (individual ManifestFile objects), memory grows as O(N).

‎tests/utils/test_manifest.py‎

Lines changed: 25 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -808,9 +808,9 @@ def test_manifest_cache_deduplicates_manifest_files() -> None:
808808

809809
# Verify cache size - should only have 3 unique ManifestFile objects
810810
# instead of 1 + 2 + 3 = 6 objects as with the old approach
811-
cache = manifest_module._manifest_cache
812-
assert cache is not None, "Manifest cache should be enabled for this test"
813-
assert len(cache) == 3, f"Cache should contain exactly 3 unique ManifestFile objects, but has {len(cache)}"
811+
assert len(manifest_module._manifest_cache) == 3, (
812+
f"Cache should contain exactly 3 unique ManifestFile objects, but has {len(manifest_module._manifest_cache)}"
813+
)
814814

815815

816816
def test_manifest_cache_efficiency_with_many_overlapping_lists() -> None:
@@ -883,11 +883,9 @@ def test_manifest_cache_efficiency_with_many_overlapping_lists() -> None:
883883
# With the new approach, we should have exactly N objects
884884

885885
# Verify cache has exactly N unique entries
886-
cache = manifest_module._manifest_cache
887-
assert cache is not None, "Manifest cache should be enabled for this test"
888-
assert len(cache) == num_manifests, (
886+
assert len(manifest_module._manifest_cache) == num_manifests, (
889887
f"Cache should contain exactly {num_manifests} ManifestFile objects, "
890-
f"but has {len(cache)}. "
888+
f"but has {len(manifest_module._manifest_cache)}. "
891889
f"Old approach would have {num_manifests * (num_manifests + 1) // 2} objects."
892890
)
893891

@@ -1035,27 +1033,22 @@ def test_clear_manifest_cache() -> None:
10351033
_manifests(io, list_path)
10361034

10371035
# Verify cache has entries
1038-
cache = manifest_module._manifest_cache
1039-
assert cache is not None, "Cache should be enabled"
1040-
assert len(cache) > 0, "Cache should have entries after reading manifests"
1036+
assert len(manifest_module._manifest_cache) > 0, "Cache should have entries after reading manifests"
10411037

10421038
# Clear the cache
10431039
clear_manifest_cache()
10441040

10451041
# Verify cache is empty but still enabled
1046-
cache_after = manifest_module._manifest_cache
1047-
assert cache_after is not None, "Cache should still be enabled after clear"
1048-
assert len(cache_after) == 0, "Cache should be empty after clear"
1042+
assert len(manifest_module._manifest_cache) == 0, "Cache should be empty after clear"
10491043

10501044

1051-
@pytest.mark.parametrize("cache_size", ["0", "-1"])
1052-
def test_manifest_cache_can_be_disabled_with_non_positive_size(monkeypatch: pytest.MonkeyPatch, cache_size: str) -> None:
1053-
"""Test that non-positive manifest-cache-size values disable caching."""
1054-
monkeypatch.setenv("PYICEBERG_MANIFEST_CACHE_SIZE", cache_size)
1045+
def test_manifest_cache_can_be_disabled_with_size_zero(monkeypatch: pytest.MonkeyPatch) -> None:
1046+
"""Test that manifest-cache-size=0 disables caching."""
1047+
monkeypatch.setenv("PYICEBERG_MANIFEST_CACHE_SIZE", "0")
10551048
importlib.reload(manifest_module)
10561049

10571050
try:
1058-
assert manifest_module._manifest_cache_size == 0
1051+
assert manifest_module._manifest_cache.maxsize == 0
10591052
assert len(manifest_module._manifest_cache) == 0
10601053

10611054
io = PyArrowFileIO()
@@ -1079,7 +1072,7 @@ def test_manifest_cache_respects_positive_env_size(monkeypatch: pytest.MonkeyPat
10791072
importlib.reload(manifest_module)
10801073

10811074
try:
1082-
assert manifest_module._manifest_cache_size == 1
1075+
assert manifest_module._manifest_cache.maxsize == 1
10831076

10841077
io = PyArrowFileIO()
10851078

@@ -1112,7 +1105,7 @@ def test_manifest_cache_reads_size_from_configuration_file(monkeypatch: pytest.M
11121105
importlib.reload(manifest_module)
11131106

11141107
try:
1115-
assert manifest_module._manifest_cache_size == 2
1108+
assert manifest_module._manifest_cache.maxsize == 2
11161109

11171110
io = PyArrowFileIO()
11181111

@@ -1141,3 +1134,15 @@ def test_invalid_manifest_cache_size_raises_value_error(monkeypatch: pytest.Monk
11411134
finally:
11421135
monkeypatch.delenv("PYICEBERG_MANIFEST_CACHE_SIZE", raising=False)
11431136
importlib.reload(manifest_module)
1137+
1138+
1139+
def test_negative_manifest_cache_size_raises_value_error(monkeypatch: pytest.MonkeyPatch) -> None:
1140+
"""Test that negative manifest-cache-size values raise a helpful error."""
1141+
monkeypatch.setenv("PYICEBERG_MANIFEST_CACHE_SIZE", "-1")
1142+
1143+
try:
1144+
with pytest.raises(ValueError, match="manifest-cache-size should be a non-negative integer or left unset"):
1145+
importlib.reload(manifest_module)
1146+
finally:
1147+
monkeypatch.delenv("PYICEBERG_MANIFEST_CACHE_SIZE", raising=False)
1148+
importlib.reload(manifest_module)

0 commit comments

Comments
 (0)