Skip to content

Commit 6ba3f58

Browse files
committed
Handle missing snapshots and simplify history regression tests
Check history against retained snapshot IDs, including snapshots already absent before expiration. Split history scenarios into named tests and remove clock patches from catalog tests in response to review on #4212.
1 parent 4bbc007 commit 6ba3f58

3 files changed

Lines changed: 146 additions & 57 deletions

File tree

‎pyiceberg/table/update/__init__.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -544,11 +544,11 @@ def _(update: RemoveSnapshotsUpdate, base_metadata: TableMetadata, context: _Tab
544544
for snapshot in base_metadata.snapshots
545545
if snapshot.snapshot_id not in update.snapshot_ids
546546
]
547-
expired_ids = set(update.snapshot_ids)
547+
retained_snapshot_ids = {snapshot.snapshot_id for snapshot in snapshots}
548548
snapshot_log: list[SnapshotLogEntry] = []
549549
for entry in base_metadata.snapshot_log:
550-
if entry.snapshot_id in expired_ids:
551-
# Earlier entries would make time travel resolve across an expired gap.
550+
if entry.snapshot_id not in retained_snapshot_ids:
551+
# Earlier entries would make time travel resolve across a missing snapshot.
552552
snapshot_log.clear()
553553
else:
554554
snapshot_log.append(entry)

‎tests/table/test_expire_snapshots.py‎

Lines changed: 4 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
import json
1818
import threading
1919
from datetime import UTC, datetime, timedelta
20-
from types import SimpleNamespace
2120
from unittest.mock import MagicMock, Mock
2221
from uuid import uuid4
2322

@@ -30,40 +29,27 @@
3029

3130

3231
@pytest.mark.parametrize("expiration", ["by_id", "older_than"])
33-
def test_expiration_does_not_substitute_older_history(catalog: Catalog, monkeypatch: pytest.MonkeyPatch, expiration: str) -> None:
32+
def test_expiration_retains_history_after_gap(catalog: Catalog, expiration: str) -> None:
3433
import pyarrow as pa
3534

3635
catalog.create_namespace("history")
3736
table = catalog.create_table("history.events", schema=pa.schema([("x", pa.int64())]))
38-
# Keep snapshot and metadata timestamps consistent; commits and file IO remain real.
39-
clock_ms = table.metadata.last_updated_ms
40-
monkeypatch.setattr("pyiceberg.table.snapshots.time", SimpleNamespace(time=lambda: clock_ms / 1000))
41-
monkeypatch.setattr(
42-
"pyiceberg.table.update.datetime", SimpleNamespace(now=lambda: datetime.fromtimestamp(clock_ms / 1000, UTC))
43-
)
4437
for value in (1, 2, 3):
45-
clock_ms += 1000
4638
table.append(pa.table({"x": [value]}))
4739
first, middle, latest = table.history()
48-
assert first.timestamp_ms < middle.timestamp_ms < latest.timestamp_ms
49-
timestamp_ms = latest.timestamp_ms - 1
50-
before = table.snapshot_as_of_timestamp(timestamp_ms)
51-
assert before is not None and before.snapshot_id == middle.snapshot_id
5240

53-
clock_ms += 1000
5441
table.manage_snapshots().create_tag(first.snapshot_id, "keep_first").commit()
55-
clock_ms += 1000
5642
if expiration == "by_id":
5743
table.maintenance.expire_snapshots().by_id(middle.snapshot_id).commit()
5844
else:
59-
cutoff = datetime.fromtimestamp(latest.timestamp_ms / 1000, UTC)
45+
# Include all snapshots; the tag and main branch protect the first and latest.
46+
cutoff = datetime.fromtimestamp(max(snapshot.timestamp_ms for snapshot in table.snapshots()) / 1000, UTC)
47+
cutoff += timedelta(milliseconds=1)
6048
table.maintenance.expire_snapshots().older_than(cutoff).commit()
6149

6250
reloaded = catalog.load_table("history.events")
6351
for actual in (table, reloaded):
64-
assert actual.metadata.last_updated_ms == clock_ms
6552
assert actual.history() == [latest]
66-
assert actual.snapshot_as_of_timestamp(timestamp_ms) is None
6753
assert actual.snapshot_as_of_timestamp(latest.timestamp_ms, inclusive=False) is None
6854
at_latest = actual.snapshot_as_of_timestamp(latest.timestamp_ms)
6955
assert at_latest is not None and at_latest.snapshot_id == latest.snapshot_id

‎tests/table/test_init.py‎

Lines changed: 139 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1186,44 +1186,147 @@ def test_update_remove_snapshots_doesnt_exist(table_v2: Table) -> None:
11861186
update_table_metadata(table_v2.metadata, (update,))
11871187

11881188

1189-
@pytest.mark.parametrize("table", [lf("table_v1"), lf("table_v2"), lf("table_v3")])
1190-
def test_remove_snapshots_retains_contiguous_history(table: Table) -> None:
1189+
@pytest.fixture(params=[lf("table_v1"), lf("table_v2"), lf("table_v3")])
1190+
def table_with_snapshot_history(request: pytest.FixtureRequest) -> Table:
1191+
table: Table = copy(request.param)
11911192
snapshots = [
1192-
Snapshot(snapshot_id=snapshot_id, timestamp_ms=snapshot_id, manifest_list=f"file:///manifest-{snapshot_id}.avro")
1193-
for snapshot_id in (10, 20, 30, 40)
1193+
Snapshot(snapshot_id=snapshot_id, timestamp_ms=snapshot_id * 1000, manifest_list=f"file:///manifest-{snapshot_id}.avro")
1194+
for snapshot_id in range(1, 5)
11941195
]
1195-
cases: list[tuple[str, list[int], list[int], list[int]]] = [
1196-
("middle", [10, 20, 30, 40], [20], [30, 40]),
1197-
("prefix", [10, 20, 30, 40], [10], [20, 30, 40]),
1198-
("separated-gaps", [10, 20, 30, 40], [10, 30], [40]),
1199-
("adjacent-gaps", [10, 20, 30, 40], [20, 30], [40]),
1200-
("repeated-expired-snapshot", [10, 20, 30, 20, 40], [20], [40]),
1201-
("rollback-to-retained-snapshot", [10, 20, 10, 30, 40], [20], [10, 30, 40]),
1202-
("no-expiration", [10, 20, 30, 40], [], [10, 20, 30, 40]),
1203-
("snapshot-outside-main-history", [10, 30, 40], [20], [10, 30, 40]),
1204-
("empty-history", [], [20], []),
1205-
("last-entry", [10, 20, 30, 40], [40], []),
1206-
]
1207-
for case, history_ids, expired_ids, expected_ids in cases:
1208-
history = [
1209-
SnapshotLogEntry(snapshot_id=snapshot_id, timestamp_ms=100 + index) for index, snapshot_id in enumerate(history_ids)
1210-
]
1211-
original = table.metadata.model_copy(
1212-
update={
1213-
"snapshots": snapshots,
1214-
"snapshot_log": history,
1215-
"current_snapshot_id": 40,
1216-
"refs": {"main": SnapshotRef(snapshot_id=40, snapshot_ref_type="branch")},
1217-
}
1218-
)
1219-
updated = update_table_metadata(original, (RemoveSnapshotsUpdate(snapshot_ids=expired_ids),), enforce_validation=True)
1220-
1221-
assert [entry.snapshot_id for entry in updated.snapshot_log] == expected_ids, case
1222-
expected_history = history[-len(expected_ids) :] if expected_ids else []
1223-
assert updated.snapshot_log == expected_history, case
1224-
assert {snapshot.snapshot_id for snapshot in updated.snapshots} == {10, 20, 30, 40} - set(expired_ids), case
1225-
assert original.snapshot_log == history, case
1226-
assert [entry.snapshot_id for entry in original.snapshot_log] == history_ids, case
1196+
table.metadata = table.metadata.model_copy(
1197+
update={
1198+
"snapshots": snapshots,
1199+
"snapshot_log": [
1200+
SnapshotLogEntry(snapshot_id=snapshot.snapshot_id, timestamp_ms=snapshot.timestamp_ms) for snapshot in snapshots
1201+
],
1202+
"current_snapshot_id": snapshots[-1].snapshot_id,
1203+
"refs": {"main": SnapshotRef(snapshot_id=snapshots[-1].snapshot_id, snapshot_ref_type="branch")},
1204+
}
1205+
)
1206+
return table
1207+
1208+
1209+
def _remove_snapshots_from_history(table: Table, *expired: SnapshotLogEntry) -> Table:
1210+
original = table.metadata.model_copy(deep=True)
1211+
expired_ids = [entry.snapshot_id for entry in expired]
1212+
updated = copy(table)
1213+
updated.metadata = update_table_metadata(
1214+
table.metadata, (RemoveSnapshotsUpdate(snapshot_ids=expired_ids),), enforce_validation=True
1215+
)
1216+
assert {snapshot.snapshot_id for snapshot in updated.snapshots()} == {
1217+
snapshot.snapshot_id for snapshot in original.snapshots
1218+
} - set(expired_ids)
1219+
assert table.metadata == original
1220+
return updated
1221+
1222+
1223+
def test_remove_snapshots_drops_history_before_middle_gap(table_with_snapshot_history: Table) -> None:
1224+
table = table_with_snapshot_history
1225+
first, middle, later, latest = table.history()
1226+
assert table.snapshot_as_of_timestamp(middle.timestamp_ms) == table.snapshot_by_id(middle.snapshot_id)
1227+
1228+
updated = _remove_snapshots_from_history(table, middle)
1229+
1230+
assert updated.history() == [later, latest]
1231+
assert updated.snapshot_as_of_timestamp(middle.timestamp_ms) is None
1232+
assert updated.snapshot_as_of_timestamp(later.timestamp_ms, inclusive=False) is None
1233+
assert updated.snapshot_as_of_timestamp(later.timestamp_ms) == updated.snapshot_by_id(later.snapshot_id)
1234+
assert updated.snapshot_by_id(first.snapshot_id) is not None
1235+
1236+
1237+
def test_remove_snapshots_drops_history_prefix(table_with_snapshot_history: Table) -> None:
1238+
first, second, third, latest = table_with_snapshot_history.history()
1239+
updated = _remove_snapshots_from_history(table_with_snapshot_history, first)
1240+
assert updated.history() == [second, third, latest]
1241+
1242+
1243+
def test_remove_snapshots_drops_history_before_last_separated_gap(table_with_snapshot_history: Table) -> None:
1244+
first, _, third, latest = table_with_snapshot_history.history()
1245+
updated = _remove_snapshots_from_history(table_with_snapshot_history, first, third)
1246+
assert updated.history() == [latest]
1247+
1248+
1249+
def test_remove_snapshots_drops_history_before_adjacent_gaps(table_with_snapshot_history: Table) -> None:
1250+
_, second, third, latest = table_with_snapshot_history.history()
1251+
updated = _remove_snapshots_from_history(table_with_snapshot_history, second, third)
1252+
assert updated.history() == [latest]
1253+
1254+
1255+
def test_remove_snapshots_drops_history_before_repeated_expired_snapshot(table_with_snapshot_history: Table) -> None:
1256+
table = table_with_snapshot_history
1257+
first, expired, later, latest = table.history()
1258+
rollback = expired.model_copy(update={"timestamp_ms": later.timestamp_ms + 1})
1259+
table.metadata = table.metadata.model_copy(update={"snapshot_log": [first, expired, later, rollback, latest]})
1260+
1261+
updated = _remove_snapshots_from_history(table, expired)
1262+
1263+
assert updated.history() == [latest]
1264+
1265+
1266+
def test_remove_snapshots_retains_rollback_after_gap(table_with_snapshot_history: Table) -> None:
1267+
table = table_with_snapshot_history
1268+
first, expired, later, latest = table.history()
1269+
rollback = first.model_copy(update={"timestamp_ms": expired.timestamp_ms + 1})
1270+
table.metadata = table.metadata.model_copy(update={"snapshot_log": [first, expired, rollback, later, latest]})
1271+
1272+
updated = _remove_snapshots_from_history(table, expired)
1273+
1274+
assert updated.history() == [rollback, later, latest]
1275+
1276+
1277+
def test_remove_snapshots_without_expiration_preserves_history(table_with_snapshot_history: Table) -> None:
1278+
updated = _remove_snapshots_from_history(table_with_snapshot_history)
1279+
assert updated.history() == table_with_snapshot_history.history()
1280+
1281+
1282+
def test_remove_snapshots_outside_main_history_preserves_history(table_with_snapshot_history: Table) -> None:
1283+
table = table_with_snapshot_history
1284+
first, outside_history, later, latest = table.history()
1285+
table.metadata = table.metadata.model_copy(update={"snapshot_log": [first, later, latest]})
1286+
1287+
updated = _remove_snapshots_from_history(table, outside_history)
1288+
1289+
assert updated.history() == table.history()
1290+
1291+
1292+
def test_remove_snapshots_preserves_empty_history(table_with_snapshot_history: Table) -> None:
1293+
table = table_with_snapshot_history
1294+
_, expired, _, _ = table.history()
1295+
table.metadata = table.metadata.model_copy(update={"snapshot_log": []})
1296+
1297+
updated = _remove_snapshots_from_history(table, expired)
1298+
1299+
assert updated.history() == []
1300+
1301+
1302+
def test_remove_snapshots_clears_history_when_last_entry_expires(table_with_snapshot_history: Table) -> None:
1303+
latest = table_with_snapshot_history.history()[-1]
1304+
updated = _remove_snapshots_from_history(table_with_snapshot_history, latest)
1305+
assert updated.history() == []
1306+
1307+
1308+
def test_remove_snapshots_drops_history_before_already_missing_snapshot(table_with_snapshot_history: Table) -> None:
1309+
table = table_with_snapshot_history
1310+
_, expired, missing, latest = table.history()
1311+
table.metadata = table.metadata.model_copy(
1312+
update={"snapshots": [snapshot for snapshot in table.snapshots() if snapshot.snapshot_id != missing.snapshot_id]}
1313+
)
1314+
1315+
updated = _remove_snapshots_from_history(table, expired)
1316+
1317+
assert updated.history() == [latest]
1318+
1319+
1320+
def test_remove_snapshots_cleans_missing_history_without_expiration(table_with_snapshot_history: Table) -> None:
1321+
table = table_with_snapshot_history
1322+
_, missing, later, latest = table.history()
1323+
table.metadata = table.metadata.model_copy(
1324+
update={"snapshots": [snapshot for snapshot in table.snapshots() if snapshot.snapshot_id != missing.snapshot_id]}
1325+
)
1326+
1327+
updated = _remove_snapshots_from_history(table)
1328+
1329+
assert updated.history() == [later, latest]
12271330

12281331

12291332
def test_update_remove_snapshots_remove_current_snapshot_id(table_v2: Table) -> None:

0 commit comments

Comments
 (0)