diff --git a/mkdocs/docs/api.md b/mkdocs/docs/api.md index aa7c111329..f762273741 100644 --- a/mkdocs/docs/api.md +++ b/mkdocs/docs/api.md @@ -1607,6 +1607,11 @@ table.maintenance.expire_snapshots().older_than( datetime.now() - timedelta(days=3) ).commit() +# Expire snapshots older than three days, but keep the 10 most recent ancestors of each branch +table.maintenance.expire_snapshots().older_than( + datetime.now() - timedelta(days=3) +).retain_last(10).commit() + # Expire a specific snapshot by ID table.maintenance.expire_snapshots().by_id(12345).commit() diff --git a/pyiceberg/table/update/snapshot.py b/pyiceberg/table/update/snapshot.py index 0c9831450e..b36dbde922 100644 --- a/pyiceberg/table/update/snapshot.py +++ b/pyiceberg/table/update/snapshot.py @@ -1369,12 +1369,16 @@ class ExpireSnapshots(UpdateTableMetadata["ExpireSnapshots"]): _updates: tuple[TableUpdate, ...] _requirements: tuple[TableRequirement, ...] _snapshot_ids_to_expire: set[int] + _snapshot_ids_older_than: set[int] + _retain_last: int | None def __init__(self, transaction: Transaction) -> None: super().__init__(transaction) self._updates = () self._requirements = () self._snapshot_ids_to_expire = set() + self._snapshot_ids_older_than = set() + self._retain_last = None def _commit(self) -> UpdatesAndRequirements: """ @@ -1386,6 +1390,7 @@ def _commit(self) -> UpdatesAndRequirements: Tuple of updates and requirements to be committed, as required by the calling parent apply functions. """ + self._snapshot_ids_to_expire |= self._snapshot_ids_older_than - self._get_retained_ancestor_ids() # Remove any protected snapshot IDs from the set to expire, just in case protected_ids = self._get_protected_snapshot_ids() self._snapshot_ids_to_expire -= protected_ids @@ -1408,6 +1413,19 @@ def _get_protected_snapshot_ids(self) -> set[int]: if ref.snapshot_ref_type in [SnapshotRefType.TAG, SnapshotRefType.BRANCH] } + def _get_retained_ancestor_ids(self) -> set[int]: + """Get the IDs of the most recent ancestors of each branch that retain_last keeps.""" + if self._retain_last is None: + return set() + + metadata = self._transaction.table_metadata + retained_ids: set[int] = set() + for ref in metadata.refs.values(): + if ref.snapshot_ref_type == SnapshotRefType.BRANCH: + ancestors = ancestors_of(metadata.snapshot_by_id(ref.snapshot_id), metadata) + retained_ids.update(snapshot.snapshot_id for snapshot in itertools.islice(ancestors, self._retain_last)) + return retained_ids + def by_id(self, snapshot_id: int) -> ExpireSnapshots: """ Expire a snapshot by its ID. @@ -1460,5 +1478,24 @@ def older_than(self, dt: datetime) -> ExpireSnapshots: expire_from = datetime_to_millis(dt) for snapshot in self._transaction.table_metadata.snapshots: if snapshot.timestamp_ms < expire_from and snapshot.snapshot_id not in protected_ids: - self._snapshot_ids_to_expire.add(snapshot.snapshot_id) + self._snapshot_ids_older_than.add(snapshot.snapshot_id) + return self + + def retain_last(self, num_snapshots: int) -> ExpireSnapshots: + """ + Retain the most recent ancestors of each branch. + + A snapshot that older_than would expire is kept if it is one of the num_snapshots most recent + ancestors of a branch. Snapshots expired by ID are still expired. + + Args: + num_snapshots (int): The number of ancestors to retain per branch, must be at least 1. + + Returns: + This for method chaining. + """ + if num_snapshots < 1: + raise ValueError(f"Number of snapshots to retain must be at least 1, cannot be: {num_snapshots}") + + self._retain_last = num_snapshots return self diff --git a/tests/table/test_expire_snapshots.py b/tests/table/test_expire_snapshots.py index dd6b6cc2d2..50d109c9fb 100644 --- a/tests/table/test_expire_snapshots.py +++ b/tests/table/test_expire_snapshots.py @@ -16,12 +16,16 @@ # under the License. import threading from datetime import datetime, timedelta +from typing import Any from unittest.mock import MagicMock, Mock from uuid import uuid4 import pytest +from pyiceberg.catalog.noop import NoopCatalog +from pyiceberg.io import load_file_io from pyiceberg.table import CommitTableResponse, Table +from pyiceberg.table.metadata import TableMetadataV2 from pyiceberg.table.statistics import PartitionStatisticsFile from pyiceberg.table.update import RemoveSnapshotsUpdate, SetPartitionStatisticsUpdate, update_table_metadata from pyiceberg.table.update.snapshot import ExpireSnapshots @@ -343,3 +347,77 @@ def test_update_remove_snapshots_with_partition_statistics(table_v2: Table) -> N assert [snapshot.snapshot_id for snapshot in new_metadata.snapshots] == [KEEP_SNAPSHOT] assert [stat.snapshot_id for stat in new_metadata.partition_statistics] == [KEEP_SNAPSHOT] + + +def _table_with_branches(example_table_metadata_v2: dict[str, Any]) -> Table: + """Create a table where main is 1 <- 2 <- 3 <- 4 and branch "b" is 1 <- 2 <- 5.""" + parents = {1: None, 2: 1, 3: 2, 4: 3, 5: 2} + snapshots = [ + { + "snapshot-id": snapshot_id, + **({"parent-snapshot-id": parent_id} if parent_id is not None else {}), + "timestamp-ms": snapshot_id * 1000, + "sequence-number": snapshot_id, + "summary": {"operation": "append"}, + "manifest-list": f"s3://a/b/{snapshot_id}.avro", + } + for snapshot_id, parent_id in parents.items() + ] + metadata = TableMetadataV2( + **{ + **example_table_metadata_v2, + "current-snapshot-id": 4, + "last-sequence-number": 5, + "snapshots": snapshots, + "snapshot-log": [], + "refs": {"main": {"snapshot-id": 4, "type": "branch"}, "b": {"snapshot-id": 5, "type": "branch"}}, + } + ) + return Table( + identifier=("database", "table"), + metadata=metadata, + metadata_location=f"{metadata.location}/uuid.metadata.json", + io=load_file_io(), + catalog=NoopCatalog("NoopCatalog"), + ) + + +@pytest.mark.parametrize( + "num_snapshots, expected_remaining", + [(1, {4, 5}), (2, {2, 3, 4, 5}), (10, {1, 2, 3, 4, 5})], +) +def test_expire_snapshots_retain_last( + example_table_metadata_v2: dict[str, Any], num_snapshots: int, expected_remaining: set[int] +) -> None: + table = _table_with_branches(example_table_metadata_v2) + future_datetime = datetime.now() + timedelta(days=1) + + txn = table.transaction() + ExpireSnapshots(txn).older_than(future_datetime).retain_last(num_snapshots).commit() + + assert {snapshot.snapshot_id for snapshot in txn.table_metadata.snapshots} == expected_remaining + + +def test_expire_snapshots_retain_last_before_older_than(example_table_metadata_v2: dict[str, Any]) -> None: + table = _table_with_branches(example_table_metadata_v2) + future_datetime = datetime.now() + timedelta(days=1) + + txn = table.transaction() + ExpireSnapshots(txn).retain_last(2).older_than(future_datetime).commit() + + assert {snapshot.snapshot_id for snapshot in txn.table_metadata.snapshots} == {2, 3, 4, 5} + + +def test_expire_snapshots_retain_last_does_not_retain_by_id(example_table_metadata_v2: dict[str, Any]) -> None: + table = _table_with_branches(example_table_metadata_v2) + + txn = table.transaction() + ExpireSnapshots(txn).retain_last(10).by_id(3).commit() + + assert {snapshot.snapshot_id for snapshot in txn.table_metadata.snapshots} == {1, 2, 4, 5} + + +@pytest.mark.parametrize("num_snapshots", [0, -1]) +def test_expire_snapshots_retain_last_invalid(table_v2: Table, num_snapshots: int) -> None: + with pytest.raises(ValueError, match=f"Number of snapshots to retain must be at least 1, cannot be: {num_snapshots}"): + table_v2.maintenance.expire_snapshots().retain_last(num_snapshots)