Skip to content

Commit 4bbc007

Browse files
committed
Fix snapshot history after expiring snapshots
Discard snapshot-log entries through the last expired occurrence so timestamp lookups cannot fall back across an expired interval. Add metadata and catalog-backed regressions for #4185.
1 parent 06a529d commit 4bbc007

3 files changed

Lines changed: 100 additions & 6 deletions

File tree

‎pyiceberg/table/update/__init__.py‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -544,11 +544,14 @@ def _(update: RemoveSnapshotsUpdate, base_metadata: TableMetadata, context: _Tab
544544
for snapshot in base_metadata.snapshots
545545
if snapshot.snapshot_id not in update.snapshot_ids
546546
]
547-
snapshot_log = [
548-
snapshot_log_entry
549-
for snapshot_log_entry in base_metadata.snapshot_log
550-
if snapshot_log_entry.snapshot_id not in update.snapshot_ids
551-
]
547+
expired_ids = set(update.snapshot_ids)
548+
snapshot_log: list[SnapshotLogEntry] = []
549+
for entry in base_metadata.snapshot_log:
550+
if entry.snapshot_id in expired_ids:
551+
# Earlier entries would make time travel resolve across an expired gap.
552+
snapshot_log.clear()
553+
else:
554+
snapshot_log.append(entry)
552555

553556
remove_ref_updates = (
554557
RemoveSnapshotRefUpdate(ref_name=ref_name)

‎tests/table/test_expire_snapshots.py‎

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,18 +14,69 @@
1414
# KIND, either express or implied. See the License for the
1515
# specific language governing permissions and limitations
1616
# under the License.
17+
import json
1718
import threading
18-
from datetime import datetime, timedelta
19+
from datetime import UTC, datetime, timedelta
20+
from types import SimpleNamespace
1921
from unittest.mock import MagicMock, Mock
2022
from uuid import uuid4
2123

2224
import pytest
2325

26+
from pyiceberg.catalog import Catalog
2427
from pyiceberg.table import CommitTableResponse, Table
2528
from pyiceberg.table.update import RemoveSnapshotsUpdate, update_table_metadata
2629
from pyiceberg.table.update.snapshot import ExpireSnapshots
2730

2831

32+
@pytest.mark.parametrize("expiration", ["by_id", "older_than"])
33+
def test_expiration_does_not_substitute_older_history(catalog: Catalog, monkeypatch: pytest.MonkeyPatch, expiration: str) -> None:
34+
import pyarrow as pa
35+
36+
catalog.create_namespace("history")
37+
table = catalog.create_table("history.events", schema=pa.schema([("x", pa.int64())]))
38+
# Keep snapshot and metadata timestamps consistent; commits and file IO remain real.
39+
clock_ms = table.metadata.last_updated_ms
40+
monkeypatch.setattr("pyiceberg.table.snapshots.time", SimpleNamespace(time=lambda: clock_ms / 1000))
41+
monkeypatch.setattr(
42+
"pyiceberg.table.update.datetime", SimpleNamespace(now=lambda: datetime.fromtimestamp(clock_ms / 1000, UTC))
43+
)
44+
for value in (1, 2, 3):
45+
clock_ms += 1000
46+
table.append(pa.table({"x": [value]}))
47+
first, middle, latest = table.history()
48+
assert first.timestamp_ms < middle.timestamp_ms < latest.timestamp_ms
49+
timestamp_ms = latest.timestamp_ms - 1
50+
before = table.snapshot_as_of_timestamp(timestamp_ms)
51+
assert before is not None and before.snapshot_id == middle.snapshot_id
52+
53+
clock_ms += 1000
54+
table.manage_snapshots().create_tag(first.snapshot_id, "keep_first").commit()
55+
clock_ms += 1000
56+
if expiration == "by_id":
57+
table.maintenance.expire_snapshots().by_id(middle.snapshot_id).commit()
58+
else:
59+
cutoff = datetime.fromtimestamp(latest.timestamp_ms / 1000, UTC)
60+
table.maintenance.expire_snapshots().older_than(cutoff).commit()
61+
62+
reloaded = catalog.load_table("history.events")
63+
for actual in (table, reloaded):
64+
assert actual.metadata.last_updated_ms == clock_ms
65+
assert actual.history() == [latest]
66+
assert actual.snapshot_as_of_timestamp(timestamp_ms) is None
67+
assert actual.snapshot_as_of_timestamp(latest.timestamp_ms, inclusive=False) is None
68+
at_latest = actual.snapshot_as_of_timestamp(latest.timestamp_ms)
69+
assert at_latest is not None and at_latest.snapshot_id == latest.snapshot_id
70+
assert actual.snapshot_by_id(middle.snapshot_id) is None
71+
assert actual.refs()["keep_first"].snapshot_id == first.snapshot_id
72+
assert actual.scan(snapshot_id=first.snapshot_id).to_arrow()["x"].to_pylist() == [1]
73+
assert sorted(actual.scan().to_arrow()["x"].to_pylist()) == [1, 2, 3]
74+
75+
with reloaded.io.new_input(reloaded.metadata_location).open() as stream:
76+
persisted_metadata = json.load(stream)
77+
assert persisted_metadata["snapshot-log"] == [{"timestamp-ms": latest.timestamp_ms, "snapshot-id": latest.snapshot_id}]
78+
79+
2980
def test_cannot_expire_protected_head_snapshot(table_v2: Table) -> None:
3081
"""Test that a HEAD (branch) snapshot cannot be expired."""
3182
HEAD_SNAPSHOT = 3051729675574597004

‎tests/table/test_init.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1186,6 +1186,46 @@ def test_update_remove_snapshots_doesnt_exist(table_v2: Table) -> None:
11861186
update_table_metadata(table_v2.metadata, (update,))
11871187

11881188

1189+
@pytest.mark.parametrize("table", [lf("table_v1"), lf("table_v2"), lf("table_v3")])
1190+
def test_remove_snapshots_retains_contiguous_history(table: Table) -> None:
1191+
snapshots = [
1192+
Snapshot(snapshot_id=snapshot_id, timestamp_ms=snapshot_id, manifest_list=f"file:///manifest-{snapshot_id}.avro")
1193+
for snapshot_id in (10, 20, 30, 40)
1194+
]
1195+
cases: list[tuple[str, list[int], list[int], list[int]]] = [
1196+
("middle", [10, 20, 30, 40], [20], [30, 40]),
1197+
("prefix", [10, 20, 30, 40], [10], [20, 30, 40]),
1198+
("separated-gaps", [10, 20, 30, 40], [10, 30], [40]),
1199+
("adjacent-gaps", [10, 20, 30, 40], [20, 30], [40]),
1200+
("repeated-expired-snapshot", [10, 20, 30, 20, 40], [20], [40]),
1201+
("rollback-to-retained-snapshot", [10, 20, 10, 30, 40], [20], [10, 30, 40]),
1202+
("no-expiration", [10, 20, 30, 40], [], [10, 20, 30, 40]),
1203+
("snapshot-outside-main-history", [10, 30, 40], [20], [10, 30, 40]),
1204+
("empty-history", [], [20], []),
1205+
("last-entry", [10, 20, 30, 40], [40], []),
1206+
]
1207+
for case, history_ids, expired_ids, expected_ids in cases:
1208+
history = [
1209+
SnapshotLogEntry(snapshot_id=snapshot_id, timestamp_ms=100 + index) for index, snapshot_id in enumerate(history_ids)
1210+
]
1211+
original = table.metadata.model_copy(
1212+
update={
1213+
"snapshots": snapshots,
1214+
"snapshot_log": history,
1215+
"current_snapshot_id": 40,
1216+
"refs": {"main": SnapshotRef(snapshot_id=40, snapshot_ref_type="branch")},
1217+
}
1218+
)
1219+
updated = update_table_metadata(original, (RemoveSnapshotsUpdate(snapshot_ids=expired_ids),), enforce_validation=True)
1220+
1221+
assert [entry.snapshot_id for entry in updated.snapshot_log] == expected_ids, case
1222+
expected_history = history[-len(expected_ids) :] if expected_ids else []
1223+
assert updated.snapshot_log == expected_history, case
1224+
assert {snapshot.snapshot_id for snapshot in updated.snapshots} == {10, 20, 30, 40} - set(expired_ids), case
1225+
assert original.snapshot_log == history, case
1226+
assert [entry.snapshot_id for entry in original.snapshot_log] == history_ids, case
1227+
1228+
11891229
def test_update_remove_snapshots_remove_current_snapshot_id(table_v2: Table) -> None:
11901230
update = RemoveSnapshotsUpdate(snapshot_ids=[3055729675574597004])
11911231
new_metadata = update_table_metadata(table_v2.metadata, (update,))

0 commit comments

Comments
 (0)