diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 53db9cd6b7..ebf296792a 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -1884,18 +1884,23 @@ def _read_all_delete_files(io: FileIO, tasks: Iterable[FileScanTask]) -> dict[st deletes_per_file: dict[str, list[ChunkedArray]] = {} # DataFile equality uses only the path; DVs from different tasks may share a file. unique_deletes: dict[_DeleteFileKey, DataFile] = {} + # Data files whose task has the delete file attached; a delete file can also list rows of other data files. + attached_paths: dict[_DeleteFileKey, set[str]] = {} for task in tasks: for delete_file in task.delete_files: key = _DeleteFileKey(delete_file.file_path, delete_file.content_offset, delete_file.content_size_in_bytes) unique_deletes.setdefault(key, delete_file) + attached_paths.setdefault(key, set()).add(task.file.file_path) if len(unique_deletes) > 0: executor = ExecutorFactory.get_or_create() deletes_per_files: Iterator[dict[str, ChunkedArray]] = executor.map( lambda args: _read_deletes(*args), [(io, delete_file) for delete_file in unique_deletes.values()], ) - for delete in deletes_per_files: + for key, delete in zip(unique_deletes, deletes_per_files, strict=True): for file, arr in delete.items(): + if file not in attached_paths[key]: + continue if file in deletes_per_file: deletes_per_file[file].append(arr) else: diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index b0f1cdde37..e1b223e093 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -2253,6 +2253,39 @@ def test_read_all_delete_files_keeps_multiple_dv_content_ranges_for_same_path(tm } +def test_read_all_delete_files_only_loads_positions_for_tasks_it_is_attached_to(tmp_path: Path) -> None: + """A delete file that names rows of two data files only applies to the data files whose task has it attached.""" + first_path, second_path = f"{tmp_path}/data-0.parquet", f"{tmp_path}/data-1.parquet" + old_path, new_path = f"{tmp_path}/old-deletes.parquet", f"{tmp_path}/new-deletes.parquet" + pq.write_table(pa.table({"file_path": [first_path, second_path], "pos": [0, 0]}), old_path) + pq.write_table(pa.table({"file_path": [first_path], "pos": [1]}), new_path) + old_deletes = DataFile.from_args( + content=DataFileContent.POSITION_DELETES, file_path=old_path, file_format=FileFormat.PARQUET, record_count=2 + ) + new_deletes = DataFile.from_args( + content=DataFileContent.POSITION_DELETES, file_path=new_path, file_format=FileFormat.PARQUET, record_count=1 + ) + first_data = DataFile.from_args( + content=DataFileContent.DATA, file_path=first_path, file_format=FileFormat.PARQUET, record_count=10 + ) + second_data = DataFile.from_args( + content=DataFileContent.DATA, file_path=second_path, file_format=FileFormat.PARQUET, record_count=10 + ) + # (delete files attached to the first task, delete files attached to the second task) -> expected positions + shapes = [ + (set(), {old_deletes}, {second_path: [pa.chunked_array([[0]])]}), + ({old_deletes}, {old_deletes}, {first_path: [pa.chunked_array([[0]])], second_path: [pa.chunked_array([[0]])]}), + # The first data file has a newer delete file too; the old one must not add its row 0 to it. + ({new_deletes}, {old_deletes}, {first_path: [pa.chunked_array([[1]])], second_path: [pa.chunked_array([[0]])]}), + ] + for first_task_deletes, second_task_deletes, expected in shapes: + tasks = [ + FileScanTask(data_file=first_data, delete_files=first_task_deletes), + FileScanTask(data_file=second_data, delete_files=second_task_deletes), + ] + assert _read_all_delete_files(PyArrowFileIO(), tasks) == expected + + @pytest.mark.parametrize("selected_indices", [(0, 1, 2, 3, 4, 5), (0, 3, 5)]) @pytest.mark.parametrize("planning_mode", ["local", "rest"]) def test_read_all_delete_files_from_external_packed_deletion_vectors( diff --git a/tests/table/test_commit_retry.py b/tests/table/test_commit_retry.py index 67653be957..deed091d60 100644 --- a/tests/table/test_commit_retry.py +++ b/tests/table/test_commit_retry.py @@ -544,11 +544,13 @@ def test_file_overwrite_validates_concurrent_file_delete( def commit_partition_position_delete( - table: Table, partition: Record, file_path: str, pos: int, row: dict[str, Any] | None = None + table: Table, partition: Record, rows: list[tuple[str, int]], row: dict[str, Any] | None = None ) -> None: """PyIceberg cannot write delete files, so commit one without path bounds, as other engines write them. - If ``row`` is given, it is stored as the deleted row, which gives the delete file bounds on the data columns. + Each ``(file_path, pos)`` in ``rows`` goes into the same delete file, which may name several data files. + If ``row`` is given, it is stored as the deleted row of a single-row delete file, which gives the delete file + bounds on the data columns. """ import pyarrow as pa import pyarrow.parquet as pq @@ -571,7 +573,8 @@ def new_delete_manifest_writer(producer: _OverwriteFiles, spec: PartitionSpec) - spec, producer.schema(), producer.new_manifest_output(), producer._snapshot_id, producer._compression ) - columns: dict[str, list[Any]] = {"file_path": [file_path], "pos": [pos]} + rows = sorted(rows) # the spec requires delete rows sorted by file_path, then pos + columns: dict[str, list[Any]] = {"file_path": [file_path for file_path, _ in rows], "pos": [pos for _, pos in rows]} bounds = None if row is not None: columns["row"] = [row] @@ -585,7 +588,7 @@ def new_delete_manifest_writer(producer: _OverwriteFiles, spec: PartitionSpec) - file_path=delete_path, file_format=FileFormat.PARQUET, partition=partition, - record_count=1, + record_count=len(rows), file_size_in_bytes=len(table.io.new_input(delete_path)), lower_bounds=bounds, upper_bounds=bounds, @@ -628,7 +631,7 @@ def test_file_overwrite_validates_concurrent_position_delete_in_nan_partition(ca overwrite.delete_data_file(file_to_replace) overwrite.append_data_file(replacement_file) - commit_partition_position_delete(deleting_table, Record(float("nan")), file_to_replace.file_path, pos=0) + commit_partition_position_delete(deleting_table, Record(float("nan")), [(file_to_replace.file_path, 0)]) with pytest.raises(ValidationException, match="found new delete for replace data file"): replacing_transaction.commit_transaction() @@ -650,7 +653,7 @@ def test_partial_delete_validates_concurrent_delete_outside_the_filter(catalog: deleting_table = catalog.load_table(identifier) # The delete file stores the deleted row (2, 'b'), so its bounds can't match k = 1. - commit_partition_position_delete(table, Record(), data_file.file_path, pos=1, row={"k": 2, "v": "b"}) + commit_partition_position_delete(table, Record(), [(data_file.file_path, 1)], row={"k": 2, "v": "b"}) with pytest.raises(ValidationException, match="found new delete for replace data file"): deleting_table.delete("k = 1") diff --git a/tests/table/test_init.py b/tests/table/test_init.py index af9a11a452..a2b9a34576 100644 --- a/tests/table/test_init.py +++ b/tests/table/test_init.py @@ -2470,6 +2470,40 @@ def test_overwrite_delete_data_file_on_bucket_partition(catalog: Catalog) -> Non assert table.scan().to_arrow().num_rows == 0 +def test_position_delete_respects_sequence_number_of_readded_file(catalog: Catalog) -> None: + """A position delete that names two data files applies to a file only if the file is not newer than the delete.""" + import pyarrow as pa + + catalog.create_namespace("default") + schema = Schema(NestedField(1, "id", LongType(), required=False)) + shapes = [ + # The delete predates the re-registered file P, so it keeps P's row 0 but still removes Q's row 0. + (True, [*range(10), *range(101, 110)]), + # The delete is committed after P is re-registered, so it removes row 0 of both files. + (False, [*range(1, 10), *range(101, 110)]), + ] + for index, (delete_before_readd, expected) in enumerate(shapes): + table = catalog.create_table( + f"default.readded_position_delete_{index}", schema=schema, properties={"format-version": "2"} + ) + table.append(pa.table({"id": pa.array(range(10), pa.int64())})) + table.append(pa.table({"id": pa.array(range(100, 110), pa.int64())})) + p_path = next(task.file.file_path for task in table.scan(row_filter="id = 0").plan_files()) + q_path = next(task.file.file_path for task in table.scan(row_filter="id = 100").plan_files()) + rows = [(p_path, 0), (q_path, 0)] + + if delete_before_readd: + commit_partition_position_delete(table, Record(), rows) + table.delete("id < 10") # removes P, the whole file + table.add_files([p_path]) # registers P again with a newer data sequence number + if not delete_before_readd: + commit_partition_position_delete(table, Record(), rows) + + assert sorted(table.scan().to_arrow()["id"].to_pylist()) == expected + assert table.scan().count() == len(expected) + assert sorted(table.scan(row_filter="id < 10").to_arrow()["id"].to_pylist()) == [i for i in expected if i < 10] + + def test_partial_delete_keeps_earlier_deletes_outside_the_filter(catalog: Catalog) -> None: """Rewriting a file applies all of its delete files, not only those whose bounds can match the delete filter.""" import pyarrow as pa @@ -2481,7 +2515,7 @@ def test_partial_delete_keeps_earlier_deletes_outside_the_filter(catalog: Catalo data_file = next(iter(table.scan().plan_files())).file # The delete file stores the deleted row (2, 'b'), so a scan for k = 1 skips it. - commit_partition_position_delete(table, Record(), data_file.file_path, pos=1, row={"k": 2, "v": "b"}) + commit_partition_position_delete(table, Record(), [(data_file.file_path, 1)], row={"k": 2, "v": "b"}) assert [task.delete_files for task in table.scan(row_filter="k = 1").plan_files()] == [set()] table.delete("k = 1")