Skip to content

Commit 2376629

Browse files
committed
Read a naive datetime in older_than as local time
expire_snapshots().older_than() converted its cutoff with datetime_to_millis, which reads a naive datetime as UTC. The docs call it with datetime.now() - timedelta(...), a naive local time, so east of UTC the cutoff landed later and expired snapshots that should be kept, and west of UTC it landed earlier. A naive datetime is now read as local time, as datetime.timestamp() does; an aware one is unchanged. Closes #4182
1 parent 06a529d commit 2376629

2 files changed

Lines changed: 32 additions & 0 deletions

File tree

‎pyiceberg/table/update/snapshot.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1418,11 +1418,14 @@ def older_than(self, dt: datetime) -> ExpireSnapshots:
14181418
14191419
Args:
14201420
dt (datetime): Only snapshots with datetime < this value will be expired.
1421+
A naive datetime is read as local time, like datetime.now() returns it.
14211422
14221423
Returns:
14231424
This for method chaining.
14241425
"""
14251426
protected_ids = self._get_protected_snapshot_ids()
1427+
if dt.tzinfo is None:
1428+
dt = dt.astimezone()
14261429
expire_from = datetime_to_millis(dt)
14271430
for snapshot in self._transaction.table_metadata.snapshots:
14281431
if snapshot.timestamp_ms < expire_from and snapshot.snapshot_id not in protected_ids:

‎tests/table/test_expire_snapshots.py‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
# specific language governing permissions and limitations
1616
# under the License.
1717
import threading
18+
import time
1819
from datetime import datetime, timedelta
1920
from unittest.mock import MagicMock, Mock
2021
from uuid import uuid4
@@ -174,6 +175,34 @@ def test_expire_snapshots_by_timestamp_skips_protected(table_v2: Table) -> None:
174175
assert remove_update.snapshot_ids == []
175176

176177

178+
def test_expire_snapshots_older_than_reads_naive_datetime_as_local_time(table_v2: Table, monkeypatch: pytest.MonkeyPatch) -> None:
179+
"""Test that older_than reads a naive datetime, as datetime.now() returns it, in the local time zone."""
180+
if not hasattr(time, "tzset"):
181+
pytest.skip("time.tzset is not available on Windows")
182+
EXPIRE_SNAPSHOT = 3051729675574597004
183+
KEEP_SNAPSHOT = 3055729675574597004
184+
expire_snapshot_ms = 1515100955770
185+
186+
table_v2.metadata = table_v2.metadata.model_copy(
187+
update={"refs": {"main": MagicMock(snapshot_id=KEEP_SNAPSHOT, snapshot_ref_type="branch")}}
188+
)
189+
190+
# A zone east of UTC, so reading local time as UTC moves the cutoff nine hours earlier
191+
monkeypatch.setenv("TZ", "Asia/Tokyo")
192+
time.tzset()
193+
try:
194+
local_snapshot_time = datetime.fromtimestamp(expire_snapshot_ms / 1000)
195+
for cutoff, expected in (
196+
(local_snapshot_time - timedelta(minutes=1), set()),
197+
(local_snapshot_time + timedelta(minutes=1), {EXPIRE_SNAPSHOT}),
198+
):
199+
expire = table_v2.maintenance.expire_snapshots().older_than(cutoff)
200+
assert expire._snapshot_ids_to_expire == expected
201+
finally:
202+
monkeypatch.undo()
203+
time.tzset()
204+
205+
177206
def test_expire_snapshots_by_ids(table_v2: Table) -> None:
178207
"""Test that multiple unprotected snapshots can be expired by IDs."""
179208
EXPIRE_SNAPSHOT_1 = 3051729675574597004

0 commit comments

Comments
 (0)