|
14 | 14 | # KIND, either express or implied. See the License for the |
15 | 15 | # specific language governing permissions and limitations |
16 | 16 | # under the License. |
| 17 | +import pyarrow as pa |
| 18 | +import pytest |
| 19 | + |
| 20 | +from pyiceberg.catalog import Catalog |
| 21 | +from pyiceberg.table import Table |
17 | 22 | from pyiceberg.table.statistics import BlobMetadata, PartitionStatisticsFile, StatisticsFile |
18 | 23 |
|
19 | 24 |
|
@@ -52,3 +57,135 @@ def test_statistics_file() -> None: |
52 | 57 | ) |
53 | 58 |
|
54 | 59 | assert statistics_file.model_dump_json() == statistics_file_json |
| 60 | + |
| 61 | + |
| 62 | +def _create_table_with_two_snapshots(catalog: Catalog) -> tuple[Table, int, int]: |
| 63 | + arrow_table = pa.table({"id": [1, 2, 3]}) |
| 64 | + catalog.create_namespace("default") |
| 65 | + table = catalog.create_table("default.test_partition_statistics", schema=arrow_table.schema) |
| 66 | + table.append(arrow_table) |
| 67 | + table.append(arrow_table) |
| 68 | + snapshot_id_1, snapshot_id_2 = [snapshot.snapshot_id for snapshot in table.snapshots()] |
| 69 | + return table, snapshot_id_1, snapshot_id_2 |
| 70 | + |
| 71 | + |
| 72 | +def _partition_statistics_file(snapshot_id: int, statistics_path: str) -> PartitionStatisticsFile: |
| 73 | + return PartitionStatisticsFile(snapshot_id=snapshot_id, statistics_path=statistics_path, file_size_in_bytes=124) |
| 74 | + |
| 75 | + |
| 76 | +def test_update_partition_statistics(catalog: Catalog) -> None: |
| 77 | + table, snapshot_id_1, snapshot_id_2 = _create_table_with_two_snapshots(catalog) |
| 78 | + stats_1 = _partition_statistics_file(snapshot_id_1, "s3://bucket/partition-stats-1.parquet") |
| 79 | + stats_1_replaced = _partition_statistics_file(snapshot_id_1, "s3://bucket/partition-stats-1-v2.parquet") |
| 80 | + stats_2 = _partition_statistics_file(snapshot_id_2, "s3://bucket/partition-stats-2.parquet") |
| 81 | + |
| 82 | + table.update_partition_statistics().set_partition_statistics(**stats_1.model_dump(by_alias=False)).commit() |
| 83 | + assert table.metadata.partition_statistics == [stats_1] |
| 84 | + |
| 85 | + # A file set for a snapshot that already has one replaces it |
| 86 | + table.update_partition_statistics().set_partition_statistics(**stats_1_replaced.model_dump(by_alias=False)).commit() |
| 87 | + assert table.metadata.partition_statistics == [stats_1_replaced] |
| 88 | + |
| 89 | + with table.update_partition_statistics() as update: |
| 90 | + update.set_partition_statistics(**stats_2.model_dump(by_alias=False)) |
| 91 | + assert table.metadata.partition_statistics == [stats_1_replaced, stats_2] |
| 92 | + |
| 93 | + table.update_partition_statistics().remove_partition_statistics(snapshot_id_1).commit() |
| 94 | + assert table.metadata.partition_statistics == [stats_2] |
| 95 | + assert catalog.load_table(table.name()).metadata.partition_statistics == [stats_2] |
| 96 | + |
| 97 | + |
| 98 | +def test_update_partition_statistics_set_and_remove_in_one_update(catalog: Catalog) -> None: |
| 99 | + table, snapshot_id_1, snapshot_id_2 = _create_table_with_two_snapshots(catalog) |
| 100 | + stats_1 = _partition_statistics_file(snapshot_id_1, "s3://bucket/partition-stats-1.parquet") |
| 101 | + stats_1_replaced = _partition_statistics_file(snapshot_id_1, "s3://bucket/partition-stats-1-v2.parquet") |
| 102 | + stats_2 = _partition_statistics_file(snapshot_id_2, "s3://bucket/partition-stats-2.parquet") |
| 103 | + |
| 104 | + # Operations on different snapshots are both applied, in either call order |
| 105 | + table.update_partition_statistics().set_partition_statistics(**stats_1.model_dump(by_alias=False)).commit() |
| 106 | + table.update_partition_statistics().set_partition_statistics( |
| 107 | + **stats_2.model_dump(by_alias=False) |
| 108 | + ).remove_partition_statistics(snapshot_id_1).commit() |
| 109 | + assert table.metadata.partition_statistics == [stats_2] |
| 110 | + |
| 111 | + with table.update_partition_statistics() as update: |
| 112 | + update.set_partition_statistics(**stats_1.model_dump(by_alias=False)) |
| 113 | + update.remove_partition_statistics(snapshot_id_2) |
| 114 | + assert table.metadata.partition_statistics == [stats_1] |
| 115 | + |
| 116 | + # On the same snapshot, the removal wins in either call order, with or without an existing file |
| 117 | + for has_existing_file in (False, True): |
| 118 | + for remove_first in (False, True): |
| 119 | + if has_existing_file: |
| 120 | + table.update_partition_statistics().set_partition_statistics(**stats_1.model_dump(by_alias=False)).commit() |
| 121 | + else: |
| 122 | + table.update_partition_statistics().remove_partition_statistics(snapshot_id_1).commit() |
| 123 | + with table.update_partition_statistics() as update: |
| 124 | + if remove_first: |
| 125 | + update.remove_partition_statistics(snapshot_id_1).set_partition_statistics( |
| 126 | + **stats_1_replaced.model_dump(by_alias=False) |
| 127 | + ) |
| 128 | + else: |
| 129 | + update.set_partition_statistics(**stats_1_replaced.model_dump(by_alias=False)).remove_partition_statistics( |
| 130 | + snapshot_id_1 |
| 131 | + ) |
| 132 | + assert table.metadata.partition_statistics == [] |
| 133 | + |
| 134 | + # Repeated operations: the last file set wins, and repeated removals are no-ops |
| 135 | + with table.update_partition_statistics() as update: |
| 136 | + update.set_partition_statistics(**stats_1.model_dump(by_alias=False)).set_partition_statistics( |
| 137 | + **stats_1_replaced.model_dump(by_alias=False) |
| 138 | + ) |
| 139 | + assert table.metadata.partition_statistics == [stats_1_replaced] |
| 140 | + |
| 141 | + with table.update_partition_statistics() as update: |
| 142 | + update.remove_partition_statistics(snapshot_id_1) |
| 143 | + update.remove_partition_statistics(snapshot_id_1) |
| 144 | + assert table.metadata.partition_statistics == [] |
| 145 | + |
| 146 | + |
| 147 | +def test_update_partition_statistics_in_transaction(catalog: Catalog) -> None: |
| 148 | + table, snapshot_id_1, snapshot_id_2 = _create_table_with_two_snapshots(catalog) |
| 149 | + stats_1 = _partition_statistics_file(snapshot_id_1, "s3://bucket/partition-stats-1.parquet") |
| 150 | + stats_2 = _partition_statistics_file(snapshot_id_2, "s3://bucket/partition-stats-2.parquet") |
| 151 | + |
| 152 | + with table.transaction() as transaction: |
| 153 | + transaction.update_partition_statistics().set_partition_statistics(**stats_1.model_dump(by_alias=False)).commit() |
| 154 | + with transaction.update_partition_statistics() as update: |
| 155 | + update.set_partition_statistics(**stats_2.model_dump(by_alias=False)) |
| 156 | + |
| 157 | + # The pending changes are visible in the transaction only, until it commits |
| 158 | + assert transaction.table_metadata.partition_statistics == [stats_1, stats_2] |
| 159 | + assert table.metadata.partition_statistics == [] |
| 160 | + assert catalog.load_table(table.name()).metadata.partition_statistics == [] |
| 161 | + |
| 162 | + assert table.metadata.partition_statistics == [stats_1, stats_2] |
| 163 | + assert catalog.load_table(table.name()).metadata.partition_statistics == [stats_1, stats_2] |
| 164 | + |
| 165 | + # An exception raised inside the transaction rolls back its pending changes |
| 166 | + def abort_transaction() -> None: |
| 167 | + with table.transaction() as transaction: |
| 168 | + transaction.update_partition_statistics().remove_partition_statistics(snapshot_id_1).commit() |
| 169 | + raise RuntimeError("abort") |
| 170 | + |
| 171 | + with pytest.raises(RuntimeError, match="abort"): |
| 172 | + abort_transaction() |
| 173 | + |
| 174 | + assert table.metadata.partition_statistics == [stats_1, stats_2] |
| 175 | + assert catalog.load_table(table.name()).metadata.partition_statistics == [stats_1, stats_2] |
| 176 | + |
| 177 | + |
| 178 | +def test_remove_partition_statistics_without_file_is_noop(catalog: Catalog) -> None: |
| 179 | + table, snapshot_id_1, snapshot_id_2 = _create_table_with_two_snapshots(catalog) |
| 180 | + stats_2 = _partition_statistics_file(snapshot_id_2, "s3://bucket/partition-stats-2.parquet") |
| 181 | + table.update_partition_statistics().set_partition_statistics(**stats_2.model_dump(by_alias=False)).commit() |
| 182 | + metadata_before = table.metadata |
| 183 | + |
| 184 | + # Removing a snapshot without a file commits without error, once or repeated, and leaves the metadata unchanged |
| 185 | + table.update_partition_statistics().remove_partition_statistics(snapshot_id_1).commit() |
| 186 | + with table.update_partition_statistics() as update: |
| 187 | + update.remove_partition_statistics(snapshot_id_1) |
| 188 | + update.remove_partition_statistics(snapshot_id_1) |
| 189 | + |
| 190 | + assert table.metadata == metadata_before |
| 191 | + assert catalog.load_table(table.name()).metadata.partition_statistics == [stats_2] |
0 commit comments