Skip to content

Commit 85dbedc

Browse files
Remove partition statistics of removed snapshots (#4364)
RemoveSnapshotsUpdate dropped the statistics files of removed snapshots but kept their partition statistics files, so expire_snapshots left partition-statistics entries that point at snapshots that no longer exist. Iceberg Java removes both in TableMetadata.Builder. Claude-Session: https://claude.ai/code/session_012XPuJeHi7A5CeHSKHjHmwx Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 7c425ed commit 85dbedc

2 files changed

Lines changed: 34 additions & 2 deletions

File tree

‎pyiceberg/table/update/__init__.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -563,7 +563,12 @@ def _(update: RemoveSnapshotsUpdate, base_metadata: TableMetadata, context: _Tab
563563
for statistics_file in base_metadata.statistics
564564
if statistics_file.snapshot_id in update.snapshot_ids
565565
)
566-
updates = itertools.chain(remove_ref_updates, remove_statistics_updates)
566+
remove_partition_statistics_updates = (
567+
RemovePartitionStatisticsUpdate(snapshot_id=partition_statistics_file.snapshot_id)
568+
for partition_statistics_file in base_metadata.partition_statistics
569+
if partition_statistics_file.snapshot_id in update.snapshot_ids
570+
)
571+
updates = itertools.chain(remove_ref_updates, remove_statistics_updates, remove_partition_statistics_updates)
567572
new_metadata = base_metadata
568573
for upd in updates:
569574
new_metadata = _apply_table_update(upd, new_metadata, context)

‎tests/table/test_expire_snapshots.py‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@
2222
import pytest
2323

2424
from pyiceberg.table import CommitTableResponse, Table
25-
from pyiceberg.table.update import RemoveSnapshotsUpdate, update_table_metadata
25+
from pyiceberg.table.statistics import PartitionStatisticsFile
26+
from pyiceberg.table.update import RemoveSnapshotsUpdate, SetPartitionStatisticsUpdate, update_table_metadata
2627
from pyiceberg.table.update.snapshot import ExpireSnapshots
2728

2829

@@ -316,3 +317,29 @@ def test_update_remove_snapshots_with_statistics(table_v2_with_statistics: Table
316317
assert not any(stat.snapshot_id == REMOVE_SNAPSHOT for stat in new_metadata.statistics), (
317318
"Statistics for removed snapshot should be gone"
318319
)
320+
321+
322+
def test_update_remove_snapshots_with_partition_statistics(table_v2: Table) -> None:
323+
"""Removing a snapshot also removes its partition statistics, as Iceberg Java does."""
324+
REMOVE_SNAPSHOT = 3051729675574597004
325+
KEEP_SNAPSHOT = 3055729675574597004
326+
327+
metadata = update_table_metadata(
328+
table_v2.metadata,
329+
tuple(
330+
SetPartitionStatisticsUpdate(
331+
partition_statistics=PartitionStatisticsFile(
332+
snapshot_id=snapshot_id,
333+
statistics_path=f"s3://bucket/warehouse/partition-stats-{snapshot_id}.parquet",
334+
file_size_in_bytes=124,
335+
)
336+
)
337+
for snapshot_id in (REMOVE_SNAPSHOT, KEEP_SNAPSHOT)
338+
),
339+
)
340+
assert len(metadata.partition_statistics) == 2
341+
342+
new_metadata = update_table_metadata(metadata, (RemoveSnapshotsUpdate(snapshot_ids=[REMOVE_SNAPSHOT]),))
343+
344+
assert [snapshot.snapshot_id for snapshot in new_metadata.snapshots] == [KEEP_SNAPSHOT]
345+
assert [stat.snapshot_id for stat in new_metadata.partition_statistics] == [KEEP_SNAPSHOT]

0 commit comments

Comments
 (0)