Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions mkdocs/docs/api.md
Original file line number Diff line number Diff line change
Expand Up @@ -1607,6 +1607,11 @@ table.maintenance.expire_snapshots().older_than(
datetime.now() - timedelta(days=3)
).commit()

# Expire snapshots older than three days, but keep the 10 most recent ancestors of each branch
table.maintenance.expire_snapshots().older_than(
datetime.now() - timedelta(days=3)
).retain_last(10).commit()

# Expire a specific snapshot by ID
table.maintenance.expire_snapshots().by_id(12345).commit()

Expand Down
39 changes: 38 additions & 1 deletion pyiceberg/table/update/snapshot.py
Original file line number Diff line number Diff line change
Expand Up @@ -1369,12 +1369,16 @@ class ExpireSnapshots(UpdateTableMetadata["ExpireSnapshots"]):
_updates: tuple[TableUpdate, ...]
_requirements: tuple[TableRequirement, ...]
_snapshot_ids_to_expire: set[int]
_snapshot_ids_older_than: set[int]
_retain_last: int | None

def __init__(self, transaction: Transaction) -> None:
super().__init__(transaction)
self._updates = ()
self._requirements = ()
self._snapshot_ids_to_expire = set()
self._snapshot_ids_older_than = set()
self._retain_last = None

def _commit(self) -> UpdatesAndRequirements:
"""
Expand All @@ -1386,6 +1390,7 @@ def _commit(self) -> UpdatesAndRequirements:
Tuple of updates and requirements to be committed,
as required by the calling parent apply functions.
"""
self._snapshot_ids_to_expire |= self._snapshot_ids_older_than - self._get_retained_ancestor_ids()
# Remove any protected snapshot IDs from the set to expire, just in case
protected_ids = self._get_protected_snapshot_ids()
self._snapshot_ids_to_expire -= protected_ids
Expand All @@ -1408,6 +1413,19 @@ def _get_protected_snapshot_ids(self) -> set[int]:
if ref.snapshot_ref_type in [SnapshotRefType.TAG, SnapshotRefType.BRANCH]
}

def _get_retained_ancestor_ids(self) -> set[int]:
"""Get the IDs of the most recent ancestors of each branch that retain_last keeps."""
if self._retain_last is None:
return set()

metadata = self._transaction.table_metadata
retained_ids: set[int] = set()
for ref in metadata.refs.values():
if ref.snapshot_ref_type == SnapshotRefType.BRANCH:
ancestors = ancestors_of(metadata.snapshot_by_id(ref.snapshot_id), metadata)
retained_ids.update(snapshot.snapshot_id for snapshot in itertools.islice(ancestors, self._retain_last))
return retained_ids

def by_id(self, snapshot_id: int) -> ExpireSnapshots:
"""
Expire a snapshot by its ID.
Expand Down Expand Up @@ -1460,5 +1478,24 @@ def older_than(self, dt: datetime) -> ExpireSnapshots:
expire_from = datetime_to_millis(dt)
for snapshot in self._transaction.table_metadata.snapshots:
if snapshot.timestamp_ms < expire_from and snapshot.snapshot_id not in protected_ids:
self._snapshot_ids_to_expire.add(snapshot.snapshot_id)
self._snapshot_ids_older_than.add(snapshot.snapshot_id)
return self

def retain_last(self, num_snapshots: int) -> ExpireSnapshots:

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In Java, retainLast(n) alone still expires snapshots older than history.expire.max-snapshot-age-ms, keeping n per branch (RemoveSnapshots, test: apache/iceberg#19049). Here it does nothing without older_than.

Could we note that in the docstring? Honoring history.expire.max-snapshot-age-ms and history.expire.min-snapshots-to-keep (and the ref-level overrides) looks like a separate follow-up.

"""
Retain the most recent ancestors of each branch.

A snapshot that older_than would expire is kept if it is one of the num_snapshots most recent
ancestors of a branch. Snapshots expired by ID are still expired.

Args:
num_snapshots (int): The number of ancestors to retain per branch, must be at least 1.

Returns:
This for method chaining.
"""
if num_snapshots < 1:
raise ValueError(f"Number of snapshots to retain must be at least 1, cannot be: {num_snapshots}")

self._retain_last = num_snapshots
return self
78 changes: 78 additions & 0 deletions tests/table/test_expire_snapshots.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,16 @@
# under the License.
import threading
from datetime import datetime, timedelta
from typing import Any
from unittest.mock import MagicMock, Mock
from uuid import uuid4

import pytest

from pyiceberg.catalog.noop import NoopCatalog
from pyiceberg.io import load_file_io
from pyiceberg.table import CommitTableResponse, Table
from pyiceberg.table.metadata import TableMetadataV2
from pyiceberg.table.statistics import PartitionStatisticsFile
from pyiceberg.table.update import RemoveSnapshotsUpdate, SetPartitionStatisticsUpdate, update_table_metadata
from pyiceberg.table.update.snapshot import ExpireSnapshots
Expand Down Expand Up @@ -343,3 +347,77 @@ def test_update_remove_snapshots_with_partition_statistics(table_v2: Table) -> N

assert [snapshot.snapshot_id for snapshot in new_metadata.snapshots] == [KEEP_SNAPSHOT]
assert [stat.snapshot_id for stat in new_metadata.partition_statistics] == [KEEP_SNAPSHOT]


def _table_with_branches(example_table_metadata_v2: dict[str, Any]) -> Table:
"""Create a table where main is 1 <- 2 <- 3 <- 4 and branch "b" is 1 <- 2 <- 5."""
parents = {1: None, 2: 1, 3: 2, 4: 3, 5: 2}
snapshots = [
{
"snapshot-id": snapshot_id,
**({"parent-snapshot-id": parent_id} if parent_id is not None else {}),
"timestamp-ms": snapshot_id * 1000,
"sequence-number": snapshot_id,
"summary": {"operation": "append"},
"manifest-list": f"s3://a/b/{snapshot_id}.avro",
}
for snapshot_id, parent_id in parents.items()
]
metadata = TableMetadataV2(
**{
**example_table_metadata_v2,
"current-snapshot-id": 4,
"last-sequence-number": 5,
"snapshots": snapshots,
"snapshot-log": [],
"refs": {"main": {"snapshot-id": 4, "type": "branch"}, "b": {"snapshot-id": 5, "type": "branch"}},
}
)
return Table(
identifier=("database", "table"),
metadata=metadata,
metadata_location=f"{metadata.location}/uuid.metadata.json",
io=load_file_io(),
catalog=NoopCatalog("NoopCatalog"),
)


@pytest.mark.parametrize(
"num_snapshots, expected_remaining",
[(1, {4, 5}), (2, {2, 3, 4, 5}), (10, {1, 2, 3, 4, 5})],
)
def test_expire_snapshots_retain_last(
example_table_metadata_v2: dict[str, Any], num_snapshots: int, expected_remaining: set[int]
) -> None:
table = _table_with_branches(example_table_metadata_v2)
future_datetime = datetime.now() + timedelta(days=1)

txn = table.transaction()
ExpireSnapshots(txn).older_than(future_datetime).retain_last(num_snapshots).commit()

assert {snapshot.snapshot_id for snapshot in txn.table_metadata.snapshots} == expected_remaining


def test_expire_snapshots_retain_last_before_older_than(example_table_metadata_v2: dict[str, Any]) -> None:
table = _table_with_branches(example_table_metadata_v2)
future_datetime = datetime.now() + timedelta(days=1)

txn = table.transaction()
ExpireSnapshots(txn).retain_last(2).older_than(future_datetime).commit()

assert {snapshot.snapshot_id for snapshot in txn.table_metadata.snapshots} == {2, 3, 4, 5}


def test_expire_snapshots_retain_last_does_not_retain_by_id(example_table_metadata_v2: dict[str, Any]) -> None:

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we add a test that retain_last alone expires nothing? It pins the current behavior, so it's a visible change if the history.expire.* defaults get wired in later.

def test_expire_snapshots_retain_last_without_older_than(example_table_metadata_v2: dict[str, Any]) -> None:
    table = _table_with_branches(example_table_metadata_v2)

    txn = table.transaction()
    ExpireSnapshots(txn).retain_last(1).commit()

    assert {snapshot.snapshot_id for snapshot in txn.table_metadata.snapshots} == {1, 2, 3, 4, 5}

table = _table_with_branches(example_table_metadata_v2)

txn = table.transaction()
ExpireSnapshots(txn).retain_last(10).by_id(3).commit()

assert {snapshot.snapshot_id for snapshot in txn.table_metadata.snapshots} == {1, 2, 4, 5}


@pytest.mark.parametrize("num_snapshots", [0, -1])
def test_expire_snapshots_retain_last_invalid(table_v2: Table, num_snapshots: int) -> None:
with pytest.raises(ValueError, match=f"Number of snapshots to retain must be at least 1, cannot be: {num_snapshots}"):
table_v2.maintenance.expire_snapshots().retain_last(num_snapshots)
Loading