Skip to content

Commit e1fc90e

Browse files
rahulsmahadevIsaac
andauthored
Add update_partition_statistics() to Table and Transaction (#4397)
* Add update_partition_statistics() to Table and Transaction Add an UpdatePartitionStatistics builder with set_partition_statistics() and remove_partition_statistics(), so callers no longer build update objects by hand. Sets apply before removals and absent removals are no-ops, as in Java. The RemovePartitionStatisticsUpdate handler therefore no longer raises. Co-authored-by: Isaac <no-reply@databricks.com> * Take partition statistics fields instead of the file object Review feedback: set_partition_statistics takes snapshot_id, statistics_path and file_size_in_bytes and builds the PartitionStatisticsFile, so the model can evolve separately from the API. Add docstrings to both public methods. Co-authored-by: Isaac <no-reply@databricks.com> --------- Co-authored-by: Isaac <no-reply@databricks.com>
1 parent 710d15e commit e1fc90e

6 files changed

Lines changed: 296 additions & 11 deletions

File tree

‎mkdocs/docs/api.md‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1788,6 +1788,30 @@ with table.update_statistics() as update:
17881788
update.remove_statistics(snapshot_id2)
17891789
```
17901790

1791+
## Table Partition Statistics Management
1792+
1793+
Manage partition statistics files with operations through the `Table` API:
1794+
1795+
```python
1796+
# Setting a file for a snapshot replaces the file already set for that snapshot
1797+
table.update_partition_statistics().set_partition_statistics(
1798+
snapshot_id=snapshot_id, statistics_path="s3://bucket/partition-stats.parquet", file_size_in_bytes=1024
1799+
).commit()
1800+
table.update_partition_statistics().remove_partition_statistics(snapshot_id).commit()
1801+
```
1802+
1803+
You can also use context managers to make more changes:
1804+
1805+
```python
1806+
with table.update_partition_statistics() as update:
1807+
update.set_partition_statistics(
1808+
snapshot_id=snapshot_id, statistics_path="s3://bucket/partition-stats.parquet", file_size_in_bytes=1024
1809+
)
1810+
update.remove_partition_statistics(snapshot_id)
1811+
```
1812+
1813+
Within one update, sets are applied before removals, as in Java, so a removal wins over a set for the same snapshot regardless of the order of the calls. Removing partition statistics for a snapshot that has no file is a no-op.
1814+
17911815
## Query the data
17921816

17931817
To query a table, a table scan is needed. A table scan accepts a filter, columns, optionally a limit and a snapshot ID:

‎pyiceberg/table/__init__.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@
9797
UpgradeFormatVersionUpdate,
9898
update_table_metadata,
9999
)
100+
from pyiceberg.table.update.partition_statistics import UpdatePartitionStatistics
100101
from pyiceberg.table.update.schema import UpdateSchema
101102
from pyiceberg.table.update.snapshot import ManageSnapshots, UpdateSnapshot, _FastAppendFiles
102103
from pyiceberg.table.update.sorting import UpdateSortOrder
@@ -510,6 +511,15 @@ def update_statistics(self) -> UpdateStatistics:
510511
"""
511512
return UpdateStatistics(transaction=self)
512513

514+
def update_partition_statistics(self) -> UpdatePartitionStatistics:
515+
"""
516+
Create a new UpdatePartitionStatistics to update the partition statistics of the table.
517+
518+
Returns:
519+
A new UpdatePartitionStatistics
520+
"""
521+
return UpdatePartitionStatistics(transaction=self)
522+
513523
def append(
514524
self,
515525
df: pa.Table | pa.RecordBatchReader,
@@ -1704,6 +1714,23 @@ def update_statistics(self) -> UpdateStatistics:
17041714
"""
17051715
return UpdateStatistics(transaction=Transaction(self, autocommit=True))
17061716

1717+
def update_partition_statistics(self) -> UpdatePartitionStatistics:
1718+
"""
1719+
Shorthand to run partition statistics management operations like set and remove partition statistics.
1720+
1721+
Use table.update_partition_statistics().<operation>().commit() to run a specific operation.
1722+
Use table.update_partition_statistics().<operation-one>().<operation-two>().commit() to run multiple operations.
1723+
1724+
Pending changes are applied on commit.
1725+
1726+
We can also use context managers to make more changes. For example:
1727+
1728+
with table.update_partition_statistics() as update:
1729+
update.set_partition_statistics(partition_statistics_file=partition_statistics_file)
1730+
update.remove_partition_statistics(snapshot_id=2)
1731+
"""
1732+
return UpdatePartitionStatistics(transaction=Transaction(self, autocommit=True))
1733+
17071734
def update_schema(self, allow_incompatible_changes: bool = False, case_sensitive: bool = True) -> UpdateSchema:
17081735
"""Create a new UpdateSchema to alter the columns of this table.
17091736

‎pyiceberg/table/update/__init__.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -692,8 +692,9 @@ def _(update: SetPartitionStatisticsUpdate, base_metadata: TableMetadata, contex
692692
def _(
693693
update: RemovePartitionStatisticsUpdate, base_metadata: TableMetadata, context: _TableMetadataUpdateContext
694694
) -> TableMetadata:
695+
# Removing an absent entry is a no-op, so the metadata is not changed
695696
if not any(part_stat.snapshot_id == update.snapshot_id for part_stat in base_metadata.partition_statistics):
696-
raise ValueError(f"Partition Statistics with snapshot id {update.snapshot_id} does not exist")
697+
return base_metadata
697698

698699
statistics = filter_statistics_by_snapshot_id(base_metadata.partition_statistics, update.snapshot_id)
699700
context.add_update(update)
Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
from typing import TYPE_CHECKING
18+
19+
from pyiceberg.table.statistics import PartitionStatisticsFile
20+
from pyiceberg.table.update import (
21+
RemovePartitionStatisticsUpdate,
22+
SetPartitionStatisticsUpdate,
23+
TableUpdate,
24+
UpdatesAndRequirements,
25+
UpdateTableMetadata,
26+
)
27+
28+
if TYPE_CHECKING:
29+
from pyiceberg.table import Transaction
30+
31+
32+
class UpdatePartitionStatistics(UpdateTableMetadata["UpdatePartitionStatistics"]):
33+
"""
34+
Run partition statistics management operations using APIs.
35+
36+
APIs include set_partition_statistics and remove_partition_statistics operations.
37+
38+
Use table.update_partition_statistics().<operation>().commit() to run a specific operation.
39+
Use table.update_partition_statistics().<operation-one>().<operation-two>().commit() to run multiple operations.
40+
41+
Pending changes are applied on commit. Sets are applied before removals, so a removal wins over a set
42+
for the same snapshot, regardless of the order of the calls.
43+
44+
We can also use context managers to make more changes. For example:
45+
46+
with table.update_partition_statistics() as update:
47+
update.set_partition_statistics(snapshot_id=1, statistics_path="s3://bucket/stats.parquet", file_size_in_bytes=124)
48+
update.remove_partition_statistics(snapshot_id=2)
49+
"""
50+
51+
_sets: dict[int, SetPartitionStatisticsUpdate]
52+
_removals: dict[int, RemovePartitionStatisticsUpdate]
53+
54+
def __init__(self, transaction: "Transaction") -> None:
55+
super().__init__(transaction)
56+
self._sets = {}
57+
self._removals = {}
58+
59+
def set_partition_statistics(
60+
self, snapshot_id: int, statistics_path: str, file_size_in_bytes: int
61+
) -> "UpdatePartitionStatistics":
62+
"""Set the partition statistics file of a snapshot, replacing the file already set for that snapshot.
63+
64+
Args:
65+
snapshot_id: The ID of the snapshot that the statistics file describes.
66+
statistics_path: The location of the partition statistics file.
67+
file_size_in_bytes: The size of the partition statistics file in bytes.
68+
69+
Returns:
70+
This for method chaining.
71+
"""
72+
partition_statistics_file = PartitionStatisticsFile(
73+
snapshot_id=snapshot_id, statistics_path=statistics_path, file_size_in_bytes=file_size_in_bytes
74+
)
75+
self._sets[snapshot_id] = SetPartitionStatisticsUpdate(partition_statistics=partition_statistics_file)
76+
77+
return self
78+
79+
def remove_partition_statistics(self, snapshot_id: int) -> "UpdatePartitionStatistics":
80+
"""Remove the partition statistics file of a snapshot. Removing a snapshot that has no file is a no-op.
81+
82+
Args:
83+
snapshot_id: The ID of the snapshot whose partition statistics file is removed.
84+
85+
Returns:
86+
This for method chaining.
87+
"""
88+
self._removals[snapshot_id] = RemovePartitionStatisticsUpdate(snapshot_id=snapshot_id)
89+
90+
return self
91+
92+
def _commit(self) -> UpdatesAndRequirements:
93+
# Sets come before removals, as in Java's SetPartitionStatistics, so a removal wins over a set
94+
updates: tuple[TableUpdate, ...] = (*self._sets.values(), *self._removals.values())
95+
return updates, ()

‎tests/table/test_init.py‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2215,16 +2215,17 @@ def test_remove_partition_statistics_update(table_v2_with_statistics: Table) ->
22152215
assert len(remove_metadata.partition_statistics) == 0
22162216

22172217

2218-
def test_remove_partition_statistics_update_with_invalid_snapshot_id(table_v2_with_statistics: Table) -> None:
2219-
# Remove the same partition statistics file.
2220-
with pytest.raises(
2221-
ValueError,
2222-
match="Partition Statistics with snapshot id 123456789 does not exist",
2223-
):
2224-
update_table_metadata(
2225-
table_v2_with_statistics.metadata,
2226-
(RemovePartitionStatisticsUpdate(snapshot_id=123456789),),
2227-
)
2218+
def test_remove_partition_statistics_update_without_file_is_noop(table_v2_with_statistics: Table) -> None:
2219+
# Removing a partition statistics file that does not exist leaves the metadata unchanged, also when repeated
2220+
new_metadata = update_table_metadata(
2221+
table_v2_with_statistics.metadata,
2222+
(
2223+
RemovePartitionStatisticsUpdate(snapshot_id=123456789),
2224+
RemovePartitionStatisticsUpdate(snapshot_id=123456789),
2225+
),
2226+
)
2227+
2228+
assert new_metadata == table_v2_with_statistics.metadata
22282229

22292230

22302231
def test_add_encryption_key_update(table_v3: Table) -> None:

‎tests/table/test_statistics.py‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,11 @@
1414
# KIND, either express or implied. See the License for the
1515
# specific language governing permissions and limitations
1616
# under the License.
17+
import pyarrow as pa
18+
import pytest
19+
20+
from pyiceberg.catalog import Catalog
21+
from pyiceberg.table import Table
1722
from pyiceberg.table.statistics import BlobMetadata, PartitionStatisticsFile, StatisticsFile
1823

1924

@@ -52,3 +57,135 @@ def test_statistics_file() -> None:
5257
)
5358

5459
assert statistics_file.model_dump_json() == statistics_file_json
60+
61+
62+
def _create_table_with_two_snapshots(catalog: Catalog) -> tuple[Table, int, int]:
63+
arrow_table = pa.table({"id": [1, 2, 3]})
64+
catalog.create_namespace("default")
65+
table = catalog.create_table("default.test_partition_statistics", schema=arrow_table.schema)
66+
table.append(arrow_table)
67+
table.append(arrow_table)
68+
snapshot_id_1, snapshot_id_2 = [snapshot.snapshot_id for snapshot in table.snapshots()]
69+
return table, snapshot_id_1, snapshot_id_2
70+
71+
72+
def _partition_statistics_file(snapshot_id: int, statistics_path: str) -> PartitionStatisticsFile:
73+
return PartitionStatisticsFile(snapshot_id=snapshot_id, statistics_path=statistics_path, file_size_in_bytes=124)
74+
75+
76+
def test_update_partition_statistics(catalog: Catalog) -> None:
77+
table, snapshot_id_1, snapshot_id_2 = _create_table_with_two_snapshots(catalog)
78+
stats_1 = _partition_statistics_file(snapshot_id_1, "s3://bucket/partition-stats-1.parquet")
79+
stats_1_replaced = _partition_statistics_file(snapshot_id_1, "s3://bucket/partition-stats-1-v2.parquet")
80+
stats_2 = _partition_statistics_file(snapshot_id_2, "s3://bucket/partition-stats-2.parquet")
81+
82+
table.update_partition_statistics().set_partition_statistics(**stats_1.model_dump(by_alias=False)).commit()
83+
assert table.metadata.partition_statistics == [stats_1]
84+
85+
# A file set for a snapshot that already has one replaces it
86+
table.update_partition_statistics().set_partition_statistics(**stats_1_replaced.model_dump(by_alias=False)).commit()
87+
assert table.metadata.partition_statistics == [stats_1_replaced]
88+
89+
with table.update_partition_statistics() as update:
90+
update.set_partition_statistics(**stats_2.model_dump(by_alias=False))
91+
assert table.metadata.partition_statistics == [stats_1_replaced, stats_2]
92+
93+
table.update_partition_statistics().remove_partition_statistics(snapshot_id_1).commit()
94+
assert table.metadata.partition_statistics == [stats_2]
95+
assert catalog.load_table(table.name()).metadata.partition_statistics == [stats_2]
96+
97+
98+
def test_update_partition_statistics_set_and_remove_in_one_update(catalog: Catalog) -> None:
99+
table, snapshot_id_1, snapshot_id_2 = _create_table_with_two_snapshots(catalog)
100+
stats_1 = _partition_statistics_file(snapshot_id_1, "s3://bucket/partition-stats-1.parquet")
101+
stats_1_replaced = _partition_statistics_file(snapshot_id_1, "s3://bucket/partition-stats-1-v2.parquet")
102+
stats_2 = _partition_statistics_file(snapshot_id_2, "s3://bucket/partition-stats-2.parquet")
103+
104+
# Operations on different snapshots are both applied, in either call order
105+
table.update_partition_statistics().set_partition_statistics(**stats_1.model_dump(by_alias=False)).commit()
106+
table.update_partition_statistics().set_partition_statistics(
107+
**stats_2.model_dump(by_alias=False)
108+
).remove_partition_statistics(snapshot_id_1).commit()
109+
assert table.metadata.partition_statistics == [stats_2]
110+
111+
with table.update_partition_statistics() as update:
112+
update.set_partition_statistics(**stats_1.model_dump(by_alias=False))
113+
update.remove_partition_statistics(snapshot_id_2)
114+
assert table.metadata.partition_statistics == [stats_1]
115+
116+
# On the same snapshot, the removal wins in either call order, with or without an existing file
117+
for has_existing_file in (False, True):
118+
for remove_first in (False, True):
119+
if has_existing_file:
120+
table.update_partition_statistics().set_partition_statistics(**stats_1.model_dump(by_alias=False)).commit()
121+
else:
122+
table.update_partition_statistics().remove_partition_statistics(snapshot_id_1).commit()
123+
with table.update_partition_statistics() as update:
124+
if remove_first:
125+
update.remove_partition_statistics(snapshot_id_1).set_partition_statistics(
126+
**stats_1_replaced.model_dump(by_alias=False)
127+
)
128+
else:
129+
update.set_partition_statistics(**stats_1_replaced.model_dump(by_alias=False)).remove_partition_statistics(
130+
snapshot_id_1
131+
)
132+
assert table.metadata.partition_statistics == []
133+
134+
# Repeated operations: the last file set wins, and repeated removals are no-ops
135+
with table.update_partition_statistics() as update:
136+
update.set_partition_statistics(**stats_1.model_dump(by_alias=False)).set_partition_statistics(
137+
**stats_1_replaced.model_dump(by_alias=False)
138+
)
139+
assert table.metadata.partition_statistics == [stats_1_replaced]
140+
141+
with table.update_partition_statistics() as update:
142+
update.remove_partition_statistics(snapshot_id_1)
143+
update.remove_partition_statistics(snapshot_id_1)
144+
assert table.metadata.partition_statistics == []
145+
146+
147+
def test_update_partition_statistics_in_transaction(catalog: Catalog) -> None:
148+
table, snapshot_id_1, snapshot_id_2 = _create_table_with_two_snapshots(catalog)
149+
stats_1 = _partition_statistics_file(snapshot_id_1, "s3://bucket/partition-stats-1.parquet")
150+
stats_2 = _partition_statistics_file(snapshot_id_2, "s3://bucket/partition-stats-2.parquet")
151+
152+
with table.transaction() as transaction:
153+
transaction.update_partition_statistics().set_partition_statistics(**stats_1.model_dump(by_alias=False)).commit()
154+
with transaction.update_partition_statistics() as update:
155+
update.set_partition_statistics(**stats_2.model_dump(by_alias=False))
156+
157+
# The pending changes are visible in the transaction only, until it commits
158+
assert transaction.table_metadata.partition_statistics == [stats_1, stats_2]
159+
assert table.metadata.partition_statistics == []
160+
assert catalog.load_table(table.name()).metadata.partition_statistics == []
161+
162+
assert table.metadata.partition_statistics == [stats_1, stats_2]
163+
assert catalog.load_table(table.name()).metadata.partition_statistics == [stats_1, stats_2]
164+
165+
# An exception raised inside the transaction rolls back its pending changes
166+
def abort_transaction() -> None:
167+
with table.transaction() as transaction:
168+
transaction.update_partition_statistics().remove_partition_statistics(snapshot_id_1).commit()
169+
raise RuntimeError("abort")
170+
171+
with pytest.raises(RuntimeError, match="abort"):
172+
abort_transaction()
173+
174+
assert table.metadata.partition_statistics == [stats_1, stats_2]
175+
assert catalog.load_table(table.name()).metadata.partition_statistics == [stats_1, stats_2]
176+
177+
178+
def test_remove_partition_statistics_without_file_is_noop(catalog: Catalog) -> None:
179+
table, snapshot_id_1, snapshot_id_2 = _create_table_with_two_snapshots(catalog)
180+
stats_2 = _partition_statistics_file(snapshot_id_2, "s3://bucket/partition-stats-2.parquet")
181+
table.update_partition_statistics().set_partition_statistics(**stats_2.model_dump(by_alias=False)).commit()
182+
metadata_before = table.metadata
183+
184+
# Removing a snapshot without a file commits without error, once or repeated, and leaves the metadata unchanged
185+
table.update_partition_statistics().remove_partition_statistics(snapshot_id_1).commit()
186+
with table.update_partition_statistics() as update:
187+
update.remove_partition_statistics(snapshot_id_1)
188+
update.remove_partition_statistics(snapshot_id_1)
189+
190+
assert table.metadata == metadata_before
191+
assert catalog.load_table(table.name()).metadata.partition_statistics == [stats_2]

0 commit comments

Comments
 (0)