Skip to content

Commit 2e94e2b

Browse files
committed
Pick the latest timestamp in snapshot_as_of_timestamp instead of the last log entry
The snapshot log is not guaranteed to be ordered by timestamp, so iterating it in reverse could return an older snapshot when entries were appended out of order.
1 parent 3228b17 commit 2e94e2b

2 files changed

Lines changed: 37 additions & 4 deletions

File tree

‎pyiceberg/table/__init__.py‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1628,10 +1628,15 @@ def snapshot_as_of_timestamp(self, timestamp_ms: int, inclusive: bool = True) ->
16281628
timestamp_ms: Find snapshot that was current at/before this timestamp
16291629
inclusive: Includes timestamp_ms in search when True. Excludes timestamp_ms when False
16301630
"""
1631-
for log_entry in reversed(self.history()):
1632-
if (inclusive and log_entry.timestamp_ms <= timestamp_ms) or log_entry.timestamp_ms < timestamp_ms:
1633-
return self.snapshot_by_id(log_entry.snapshot_id)
1634-
return None
1631+
# The snapshot log is not guaranteed to be ordered by timestamp, so pick the latest timestamp rather than the last entry
1632+
candidate: SnapshotLogEntry | None = None
1633+
for log_entry in self.history():
1634+
if log_entry.timestamp_ms > timestamp_ms or (not inclusive and log_entry.timestamp_ms == timestamp_ms):
1635+
continue
1636+
# On equal timestamps the later entry wins, it is the state that was current last
1637+
if candidate is None or log_entry.timestamp_ms >= candidate.timestamp_ms:
1638+
candidate = log_entry
1639+
return self.snapshot_by_id(candidate.snapshot_id) if candidate is not None else None
16351640

16361641
def history(self) -> list[SnapshotLogEntry]:
16371642
"""Get the snapshot history of this table."""

‎tests/table/test_init.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -236,6 +236,34 @@ def test_snapshot_by_timestamp(table_v2: Table) -> None:
236236
assert table_v2.snapshot_as_of_timestamp(1515100955770, inclusive=False) is None
237237

238238

239+
def test_snapshot_by_timestamp_with_unordered_snapshot_log(table_v2: Table) -> None:
240+
# Writers with slightly skewed clocks can append snapshot log entries out of timestamp order
241+
first, second = table_v2.metadata.snapshot_log
242+
now = second.timestamp_ms
243+
log = [
244+
SnapshotLogEntry(snapshot_id=second.snapshot_id, timestamp_ms=now),
245+
SnapshotLogEntry(snapshot_id=first.snapshot_id, timestamp_ms=now - 10_000),
246+
]
247+
table_v2.metadata = table_v2.metadata.model_copy(update={"snapshot_log": log})
248+
249+
latest = table_v2.snapshot_as_of_timestamp(now)
250+
assert latest is not None
251+
assert latest.snapshot_id == second.snapshot_id
252+
253+
earlier = table_v2.snapshot_as_of_timestamp(now - 5_000)
254+
assert earlier is not None
255+
assert earlier.snapshot_id == first.snapshot_id
256+
257+
assert table_v2.snapshot_as_of_timestamp(now - 10_000, inclusive=False) is None
258+
259+
# On equal timestamps the later entry wins
260+
tied = [SnapshotLogEntry(snapshot_id=first.snapshot_id, timestamp_ms=now), log[0]]
261+
table_v2.metadata = table_v2.metadata.model_copy(update={"snapshot_log": tied})
262+
result = table_v2.snapshot_as_of_timestamp(now)
263+
assert result is not None
264+
assert result.snapshot_id == second.snapshot_id
265+
266+
239267
def test_snapshot_by_id_does_not_exist(table_v2: Table) -> None:
240268
assert table_v2.snapshot_by_id(-1) is None
241269

0 commit comments

Comments
 (0)