From 3308ff30b13b4644ce4d04e341978dcb1ef381ed Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Wed, 11 Nov 2026 14:36:39 +0530 Subject: [PATCH] Write added files under the partition spec they were planned with --- pyiceberg/table/update/snapshot.py | 35 +++++++++--------- tests/table/test_commit_retry.py | 57 ++++++++++++++++++++++++++++++ tests/table/test_snapshots.py | 4 ++- 3 files changed, 79 insertions(+), 17 deletions(-) diff --git a/pyiceberg/table/update/snapshot.py b/pyiceberg/table/update/snapshot.py index 5ca3eb7a73..6048d42c21 100644 --- a/pyiceberg/table/update/snapshot.py +++ b/pyiceberg/table/update/snapshot.py @@ -235,20 +235,24 @@ def _process_manifests(self, manifests: list[ManifestFile]) -> list[ManifestFile def _manifests(self) -> list[ManifestFile]: def _write_added_manifest() -> list[ManifestFile]: if self._added_data_files: - with self.new_manifest_writer( - spec=self._transaction.table_metadata.spec(), - ) as writer: - for data_file in self._added_data_files: - writer.add( - ManifestEntry.from_args( - status=ManifestEntryStatus.ADDED, - snapshot_id=self._snapshot_id, - sequence_number=None, - file_sequence_number=None, - data_file=data_file, + added_manifests = [] + partition_groups: dict[int, list[DataFile]] = defaultdict(list) + for data_file in self._added_data_files: + partition_groups[data_file.spec_id].append(data_file) + for spec_id, data_files in partition_groups.items(): + with self.new_manifest_writer(self.spec(spec_id)) as writer: + for data_file in data_files: + writer.add( + ManifestEntry.from_args( + status=ManifestEntryStatus.ADDED, + snapshot_id=self._snapshot_id, + sequence_number=None, + file_sequence_number=None, + data_file=data_file, + ) ) - ) - return [writer.to_manifest_file()] + added_manifests.append(writer.to_manifest_file()) + return added_manifests else: return [] @@ -286,7 +290,7 @@ def _summary(self, snapshot_properties: dict[str, str] = EMPTY_DICT) -> Summary: # avoid copying metadata for each data file table_metadata = self._transaction.table_metadata schema = table_metadata.schema() - default_spec = table_metadata.spec() + specs = table_metadata.specs() partition_summary_limit = int( table_metadata.properties.get( @@ -298,12 +302,11 @@ def _summary(self, snapshot_properties: dict[str, str] = EMPTY_DICT) -> Summary: for data_file in self._added_data_files: ssc.add_file( data_file=data_file, - partition_spec=default_spec, + partition_spec=specs[data_file.spec_id], schema=schema, ) if len(self._deleted_data_files) > 0: - specs = table_metadata.specs() for data_file in self._deleted_data_files: ssc.remove_file( data_file=data_file, diff --git a/tests/table/test_commit_retry.py b/tests/table/test_commit_retry.py index ab95457e23..4ae4901465 100644 --- a/tests/table/test_commit_retry.py +++ b/tests/table/test_commit_retry.py @@ -247,6 +247,63 @@ def counting_rebuild(self_tx: Any) -> None: assert len(result) == 5 +def test_retried_append_records_the_spec_it_was_planned_with(catalog: Catalog) -> None: + """A retried append keeps the partition spec of the files it planned, not the refreshed default.""" + catalog.create_namespace("default") + + import pyarrow as pa + + def row(x: int, p: int, q: int) -> pa.Table: + return pa.table({"x": [x], "p": [p], "q": [q]}) + + table = catalog.create_table("default.spec_retry_test", schema=row(1, 5, 50).schema) + with table.update_spec() as update: + update.add_identity("p") + + other = catalog.load_table("default.spec_retry_test") + with other.update_spec() as update: + update.remove_field("p") + update.add_identity("q") + other.append(row(2, 6, 60)) + + table.append(row(3, 7, 70)) + + refreshed = catalog.load_table("default.spec_retry_test") + assert refreshed.scan(row_filter="q == 70").to_arrow()["x"].to_pylist() == [3] + + snapshot = refreshed.current_snapshot() + assert snapshot is not None + assert {manifest.partition_spec_id for manifest in snapshot.manifests(io=refreshed.io)} == {1, 2} + + +def test_retried_append_with_a_spec_that_has_more_fields(catalog: Catalog) -> None: + """A retried append does not index the planned partitions with the refreshed spec's fields.""" + catalog.create_namespace("default") + + import pyarrow as pa + + def row(x: int, p: int, q: int) -> pa.Table: + return pa.table({"x": [x], "p": [p], "q": [q]}) + + table = catalog.create_table("default.spec_retry_more_fields_test", schema=row(1, 5, 50).schema) + with table.update_spec() as update: + update.add_identity("p") + + other = catalog.load_table("default.spec_retry_more_fields_test") + with other.update_spec() as update: + update.add_identity("q") + other.append(row(2, 6, 60)) + + table.append(row(3, 7, 70)) + + refreshed = catalog.load_table("default.spec_retry_more_fields_test") + assert sorted(refreshed.scan().to_arrow().to_pylist(), key=str) == [ + {"x": 2, "p": 6, "q": 60}, + {"x": 3, "p": 7, "q": 70}, + ] + assert refreshed.scan(row_filter="p == 7").to_arrow()["x"].to_pylist() == [3] + + def test_retry_exhaustion_raises_commit_failed(catalog: Catalog) -> None: """When retries are exhausted, CommitFailedException should be raised.""" catalog.create_namespace("default") diff --git a/tests/table/test_snapshots.py b/tests/table/test_snapshots.py index 39aeb7b349..479656e85b 100644 --- a/tests/table/test_snapshots.py +++ b/tests/table/test_snapshots.py @@ -671,7 +671,9 @@ def test_snapshot_producer_bounded_metadata_access(table_v2: Table) -> None: from pyiceberg.table.update.snapshot import _FastAppendFiles, _MergeAppendFiles def make_file() -> DataFile: - return DataFile.from_args(content=DataFileContent.DATA, record_count=1, file_size_in_bytes=1, partition=Record()) + return DataFile.from_args( + content=DataFileContent.DATA, record_count=1, file_size_in_bytes=1, partition=Record(), spec_id=0 + ) txn = table_v2.transaction()