Skip to content

Commit 3308ff3

Browse files
committed
Write added files under the partition spec they were planned with
1 parent aafad6e commit 3308ff3

3 files changed

Lines changed: 79 additions & 17 deletions

File tree

‎pyiceberg/table/update/snapshot.py‎

Lines changed: 19 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -235,20 +235,24 @@ def _process_manifests(self, manifests: list[ManifestFile]) -> list[ManifestFile
235235
def _manifests(self) -> list[ManifestFile]:
236236
def _write_added_manifest() -> list[ManifestFile]:
237237
if self._added_data_files:
238-
with self.new_manifest_writer(
239-
spec=self._transaction.table_metadata.spec(),
240-
) as writer:
241-
for data_file in self._added_data_files:
242-
writer.add(
243-
ManifestEntry.from_args(
244-
status=ManifestEntryStatus.ADDED,
245-
snapshot_id=self._snapshot_id,
246-
sequence_number=None,
247-
file_sequence_number=None,
248-
data_file=data_file,
238+
added_manifests = []
239+
partition_groups: dict[int, list[DataFile]] = defaultdict(list)
240+
for data_file in self._added_data_files:
241+
partition_groups[data_file.spec_id].append(data_file)
242+
for spec_id, data_files in partition_groups.items():
243+
with self.new_manifest_writer(self.spec(spec_id)) as writer:
244+
for data_file in data_files:
245+
writer.add(
246+
ManifestEntry.from_args(
247+
status=ManifestEntryStatus.ADDED,
248+
snapshot_id=self._snapshot_id,
249+
sequence_number=None,
250+
file_sequence_number=None,
251+
data_file=data_file,
252+
)
249253
)
250-
)
251-
return [writer.to_manifest_file()]
254+
added_manifests.append(writer.to_manifest_file())
255+
return added_manifests
252256
else:
253257
return []
254258

@@ -286,7 +290,7 @@ def _summary(self, snapshot_properties: dict[str, str] = EMPTY_DICT) -> Summary:
286290
# avoid copying metadata for each data file
287291
table_metadata = self._transaction.table_metadata
288292
schema = table_metadata.schema()
289-
default_spec = table_metadata.spec()
293+
specs = table_metadata.specs()
290294

291295
partition_summary_limit = int(
292296
table_metadata.properties.get(
@@ -298,12 +302,11 @@ def _summary(self, snapshot_properties: dict[str, str] = EMPTY_DICT) -> Summary:
298302
for data_file in self._added_data_files:
299303
ssc.add_file(
300304
data_file=data_file,
301-
partition_spec=default_spec,
305+
partition_spec=specs[data_file.spec_id],
302306
schema=schema,
303307
)
304308

305309
if len(self._deleted_data_files) > 0:
306-
specs = table_metadata.specs()
307310
for data_file in self._deleted_data_files:
308311
ssc.remove_file(
309312
data_file=data_file,

‎tests/table/test_commit_retry.py‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,63 @@ def counting_rebuild(self_tx: Any) -> None:
247247
assert len(result) == 5
248248

249249

250+
def test_retried_append_records_the_spec_it_was_planned_with(catalog: Catalog) -> None:
251+
"""A retried append keeps the partition spec of the files it planned, not the refreshed default."""
252+
catalog.create_namespace("default")
253+
254+
import pyarrow as pa
255+
256+
def row(x: int, p: int, q: int) -> pa.Table:
257+
return pa.table({"x": [x], "p": [p], "q": [q]})
258+
259+
table = catalog.create_table("default.spec_retry_test", schema=row(1, 5, 50).schema)
260+
with table.update_spec() as update:
261+
update.add_identity("p")
262+
263+
other = catalog.load_table("default.spec_retry_test")
264+
with other.update_spec() as update:
265+
update.remove_field("p")
266+
update.add_identity("q")
267+
other.append(row(2, 6, 60))
268+
269+
table.append(row(3, 7, 70))
270+
271+
refreshed = catalog.load_table("default.spec_retry_test")
272+
assert refreshed.scan(row_filter="q == 70").to_arrow()["x"].to_pylist() == [3]
273+
274+
snapshot = refreshed.current_snapshot()
275+
assert snapshot is not None
276+
assert {manifest.partition_spec_id for manifest in snapshot.manifests(io=refreshed.io)} == {1, 2}
277+
278+
279+
def test_retried_append_with_a_spec_that_has_more_fields(catalog: Catalog) -> None:
280+
"""A retried append does not index the planned partitions with the refreshed spec's fields."""
281+
catalog.create_namespace("default")
282+
283+
import pyarrow as pa
284+
285+
def row(x: int, p: int, q: int) -> pa.Table:
286+
return pa.table({"x": [x], "p": [p], "q": [q]})
287+
288+
table = catalog.create_table("default.spec_retry_more_fields_test", schema=row(1, 5, 50).schema)
289+
with table.update_spec() as update:
290+
update.add_identity("p")
291+
292+
other = catalog.load_table("default.spec_retry_more_fields_test")
293+
with other.update_spec() as update:
294+
update.add_identity("q")
295+
other.append(row(2, 6, 60))
296+
297+
table.append(row(3, 7, 70))
298+
299+
refreshed = catalog.load_table("default.spec_retry_more_fields_test")
300+
assert sorted(refreshed.scan().to_arrow().to_pylist(), key=str) == [
301+
{"x": 2, "p": 6, "q": 60},
302+
{"x": 3, "p": 7, "q": 70},
303+
]
304+
assert refreshed.scan(row_filter="p == 7").to_arrow()["x"].to_pylist() == [3]
305+
306+
250307
def test_retry_exhaustion_raises_commit_failed(catalog: Catalog) -> None:
251308
"""When retries are exhausted, CommitFailedException should be raised."""
252309
catalog.create_namespace("default")

‎tests/table/test_snapshots.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -671,7 +671,9 @@ def test_snapshot_producer_bounded_metadata_access(table_v2: Table) -> None:
671671
from pyiceberg.table.update.snapshot import _FastAppendFiles, _MergeAppendFiles
672672

673673
def make_file() -> DataFile:
674-
return DataFile.from_args(content=DataFileContent.DATA, record_count=1, file_size_in_bytes=1, partition=Record())
674+
return DataFile.from_args(
675+
content=DataFileContent.DATA, record_count=1, file_size_in_bytes=1, partition=Record(), spec_id=0
676+
)
675677

676678
txn = table_v2.transaction()
677679

0 commit comments

Comments
 (0)