From 7b109f396c2e9a4828ecb31009e370847b96745a Mon Sep 17 00:00:00 2001 From: ghoshp83 Date: Thu, 5 Nov 2026 21:01:04 +0000 Subject: [PATCH 1/3] Apply every delete file when rewriting files for a partial delete A partial delete reads each candidate file in full and writes back the surviving rows, but planned the rewrite with the delete filter, which also prunes delete files by column bounds. Rows removed by a pruned delete file came back in the rewritten file. Plan the rewrite without pruning delete files; scans are unchanged. --- pyiceberg/table/__init__.py | 25 ++++- tests/catalog/test_catalog_behaviors.py | 131 +++++++++++++++++++++++- 2 files changed, 151 insertions(+), 5 deletions(-) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 7020cb482b..08a05f5dee 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -810,7 +810,20 @@ def delete( file_scan = self._scan(row_filter=delete_filter, case_sensitive=case_sensitive) if branch is not None: file_scan = file_scan.use_ref(branch) - files = file_scan.plan_files() + # Each file is rewritten in full, so it needs every delete file that applies to it, + # not only those whose bounds can match the delete filter. + snapshot = file_scan.snapshot() + files = ( + ManifestGroupPlanner( + table_metadata=self.table_metadata, + io=self._table.io, + row_filter=delete_filter, + case_sensitive=case_sensitive, + prune_delete_files=False, + ).plan_files(snapshot.manifests(self._table.io)) + if snapshot is not None + else [] + ) commit_uuid = uuid.uuid4() counter = itertools.count(0) @@ -2863,12 +2876,20 @@ def __init__( row_filter: str | BooleanExpression = ALWAYS_TRUE, case_sensitive: bool = True, options: Properties = EMPTY_DICT, + prune_delete_files: bool = True, ): + """Create a planner for a group of manifests. + + Args: + prune_delete_files: Leave out delete files whose column bounds cannot match the row filter. + This is only correct when the planned data files are read with the row filter applied. + """ self.table_metadata = table_metadata self.io = io self.row_filter = _parse_row_filter(row_filter) self.case_sensitive = case_sensitive self.options = options + self.prune_delete_files = prune_delete_files @cached_property def partition_filters(self) -> KeyDefaultDict[int, BooleanExpression]: @@ -2906,7 +2927,7 @@ def plan_manifest_entries(self, manifests: Iterable[ManifestFile]) -> Iterator[l self.io, manifest, partition_evaluators[manifest.partition_spec_id], - metrics_evaluator, + metrics_evaluator if self.prune_delete_files or manifest.content == ManifestContent.DATA else lambda _: True, ) for manifest in manifests if self._check_sequence_number(min_sequence_number, manifest) diff --git a/tests/catalog/test_catalog_behaviors.py b/tests/catalog/test_catalog_behaviors.py index 6d694a9448..f35bf232e6 100644 --- a/tests/catalog/test_catalog_behaviors.py +++ b/tests/catalog/test_catalog_behaviors.py @@ -21,11 +21,14 @@ import json import os +import struct +import uuid from collections.abc import Callable, Generator from pathlib import Path from typing import Any import pyarrow as pa +import pyarrow.parquet as pq import pytest from pydantic_core import ValidationError from pytest_lazy_fixtures import lf @@ -41,14 +44,25 @@ TableAlreadyExistsError, ) from pyiceberg.io.pyarrow import _dataframe_to_data_files, schema_to_pyarrow +from pyiceberg.manifest import ( + DataFile, + DataFileContent, + FileFormat, + ManifestContent, + ManifestEntry, + ManifestEntryStatus, + ManifestWriterV2, + write_manifest_list, +) from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionField, PartitionSpec from pyiceberg.schema import Schema from pyiceberg.table import Table, TableProperties -from pyiceberg.table.snapshots import Operation +from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRefType +from pyiceberg.table.snapshots import Operation, Snapshot, Summary from pyiceberg.table.sorting import NullOrder, SortDirection, SortField, SortOrder -from pyiceberg.table.update import AddSchemaUpdate, SetCurrentSchemaUpdate +from pyiceberg.table.update import AddSchemaUpdate, AddSnapshotUpdate, SetCurrentSchemaUpdate, SetSnapshotRefUpdate from pyiceberg.transforms import IdentityTransform -from pyiceberg.typedef import Identifier +from pyiceberg.typedef import Identifier, Record from pyiceberg.types import BooleanType, DoubleType, IntegerType, LongType, NestedField, StringType @@ -1479,3 +1493,114 @@ def test_record_batch_reader_schema_mismatch_writes_no_files(catalog: Catalog) - # _append_snapshot_producer opens. assert tbl.metadata.current_snapshot() is None assert len(tbl.scan().to_arrow()) == 0 + + +def _commit_position_delete_with_row(tbl: Table, data_file_path: str, pos: int, row: dict[str, Any]) -> Table: + """Commit a position delete that stores the deleted row, which gives it bounds on the data columns.""" + + class DeleteManifestWriter(ManifestWriterV2): + def content(self) -> ManifestContent: + return ManifestContent.DELETES + + @property + def _meta(self) -> dict[str, str]: + return {**super()._meta, "content": "deletes"} + + def field(name: str, type_: pa.DataType, field_id: int) -> pa.Field: + return pa.field(name, type_, nullable=False, metadata={"PARQUET:field_id": str(field_id)}) + + row_type = pa.struct([field("k", pa.int64(), 1), field("v", pa.string(), 2)]) + deletes = pa.Table.from_pylist( + [{"file_path": data_file_path, "pos": pos, "row": row}], + schema=pa.schema( + [ + field("file_path", pa.string(), 2147483546), + field("pos", pa.int64(), 2147483545), + field("row", row_type, 2147483544), + ] + ), + ) + delete_path = f"{tbl.location()}/data/delete-{uuid.uuid4()}.parquet" + with tbl.io.new_output(delete_path).create() as output: + pq.write_table(deletes, output) + + bounds = { + 2147483546: data_file_path.encode(), + 2147483545: struct.pack("> 65 + sequence_number = tbl.metadata.next_sequence_number() + with DeleteManifestWriter( + spec=tbl.spec(), + schema=tbl.schema(), + output_file=tbl.io.new_output(f"{tbl.location()}/metadata/{uuid.uuid4()}-m0.avro"), + snapshot_id=snapshot_id, + avro_compression="deflate", + ) as writer: + writer.add(ManifestEntry.from_args(status=ManifestEntryStatus.ADDED, snapshot_id=snapshot_id, data_file=delete_file)) + manifest_list = f"{tbl.location()}/metadata/snap-{snapshot_id}.avro" + with write_manifest_list( + format_version=2, + output_file=tbl.io.new_output(manifest_list), + snapshot_id=snapshot_id, + parent_snapshot_id=parent.snapshot_id, + sequence_number=sequence_number, + avro_compression="deflate", + ) as list_writer: + list_writer.add_manifests([writer.to_manifest_file(), *parent.manifests(tbl.io)]) + + snapshot = Snapshot( + snapshot_id=snapshot_id, + parent_snapshot_id=parent.snapshot_id, + sequence_number=sequence_number, + manifest_list=manifest_list, + summary=Summary(Operation.DELETE), + schema_id=tbl.schema().schema_id, + ) + tbl.catalog.commit_table( + tbl, + (), + ( + AddSnapshotUpdate(snapshot=snapshot), + SetSnapshotRefUpdate(ref_name=MAIN_BRANCH, type=SnapshotRefType.BRANCH, snapshot_id=snapshot_id), + ), + ) + return tbl.catalog.load_table(tbl.name()) + + +def test_partial_delete_keeps_earlier_deletes_outside_the_filter(catalog: Catalog) -> None: + """Rewriting a file for a delete must apply every delete file of that file. A delete file whose + bounds cannot match the delete filter is skipped by a filtered scan, which is right for a read but + would bring its rows back in the rewritten file. + """ + catalog.create_namespace("default") + identifier = f"default.partial_delete_keeps_earlier_deletes_{catalog.name}" + iceberg_schema = Schema(NestedField(1, "k", LongType(), required=True), NestedField(2, "v", StringType(), required=True)) + tbl = catalog.create_table(identifier=identifier, schema=iceberg_schema, properties={"format-version": "2"}) + tbl.append(pa.Table.from_pydict({"k": [1, 2, 3], "v": ["a", "b", "c"]}, schema=schema_to_pyarrow(iceberg_schema))) + data_file_path = next(iter(tbl.scan().plan_files())).file.file_path + + # Delete (2, 'b'). The delete file has bounds k = 2, so a scan for k = 1 leaves it out. + tbl = _commit_position_delete_with_row(tbl, data_file_path, pos=1, row={"k": 2, "v": "b"}) + assert tbl.scan().to_arrow()["k"].to_pylist() == [1, 3] + assert [len(task.delete_files) for task in tbl.scan(row_filter="k = 1").plan_files()] == [0] + + tbl.delete("k = 1") + + assert tbl.scan().to_arrow()["k"].to_pylist() == [3] From d3343711337a8fb4bdc26b943b42649401e07eda Mon Sep 17 00:00:00 2001 From: ghoshp83 Date: Thu, 12 Nov 2026 20:32:20 +0000 Subject: [PATCH 2/3] Rename prune_delete_files to ignore_residuals --- pyiceberg/table/__init__.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 08a05f5dee..0e68842944 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -819,7 +819,7 @@ def delete( io=self._table.io, row_filter=delete_filter, case_sensitive=case_sensitive, - prune_delete_files=False, + ignore_residuals=True, ).plan_files(snapshot.manifests(self._table.io)) if snapshot is not None else [] @@ -2876,20 +2876,20 @@ def __init__( row_filter: str | BooleanExpression = ALWAYS_TRUE, case_sensitive: bool = True, options: Properties = EMPTY_DICT, - prune_delete_files: bool = True, + ignore_residuals: bool = False, ): """Create a planner for a group of manifests. Args: - prune_delete_files: Leave out delete files whose column bounds cannot match the row filter. - This is only correct when the planned data files are read with the row filter applied. + ignore_residuals: The planned data files are read in full, without the row filter, so keep + every delete file that applies to them instead of pruning by the row filter. """ self.table_metadata = table_metadata self.io = io self.row_filter = _parse_row_filter(row_filter) self.case_sensitive = case_sensitive self.options = options - self.prune_delete_files = prune_delete_files + self.ignore_residuals = ignore_residuals @cached_property def partition_filters(self) -> KeyDefaultDict[int, BooleanExpression]: @@ -2927,7 +2927,9 @@ def plan_manifest_entries(self, manifests: Iterable[ManifestFile]) -> Iterator[l self.io, manifest, partition_evaluators[manifest.partition_spec_id], - metrics_evaluator if self.prune_delete_files or manifest.content == ManifestContent.DATA else lambda _: True, + metrics_evaluator + if not self.ignore_residuals or manifest.content == ManifestContent.DATA + else lambda _: True, ) for manifest in manifests if self._check_sequence_number(min_sequence_number, manifest) From 9bf1f37d5176b2080f280955dc54b6c8e2a8f287 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Thu, 12 Nov 2026 12:51:08 -0800 Subject: [PATCH 3/3] Use a private _ignore_residuals and move the test to test_init.py Keep the flag private so the public API can be designed separately, and make residuals always true when it is set, as Java's ignoreResiduals does. Plan the rewrite from the branch head, which leaves Transaction._scan unused. Move the test next to the other Transaction write tests and reuse the existing position delete helper from test_commit_retry.py. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- pyiceberg/table/__init__.py | 44 +++----- tests/catalog/test_catalog_behaviors.py | 131 +----------------------- tests/table/test_commit_retry.py | 22 +++- tests/table/test_init.py | 20 ++++ 4 files changed, 55 insertions(+), 162 deletions(-) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 0e68842944..496c3bb53d 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -339,12 +339,6 @@ def _apply( return self - def _scan(self, row_filter: str | BooleanExpression = ALWAYS_TRUE, case_sensitive: bool = True) -> DataScan: - """Minimal data scan of the table with the current state of the transaction.""" - return DataScan( - table_metadata=self.table_metadata, io=self._table.io, row_filter=row_filter, case_sensitive=case_sensitive - ) - def upgrade_table_version(self, format_version: TableVersion) -> Transaction: """Set the table to a certain version. @@ -807,23 +801,17 @@ def delete( bound_delete_filter = bind(self.table_metadata.schema(), delete_filter, case_sensitive) preserve_row_filter = _expression_to_complementary_pyarrow(bound_delete_filter, self.table_metadata.schema()) - file_scan = self._scan(row_filter=delete_filter, case_sensitive=case_sensitive) - if branch is not None: - file_scan = file_scan.use_ref(branch) # Each file is rewritten in full, so it needs every delete file that applies to it, # not only those whose bounds can match the delete filter. - snapshot = file_scan.snapshot() - files = ( - ManifestGroupPlanner( - table_metadata=self.table_metadata, - io=self._table.io, - row_filter=delete_filter, - case_sensitive=case_sensitive, - ignore_residuals=True, - ).plan_files(snapshot.manifests(self._table.io)) - if snapshot is not None - else [] + planner = ManifestGroupPlanner( + table_metadata=self.table_metadata, + io=self._table.io, + row_filter=delete_filter, + case_sensitive=case_sensitive, + _ignore_residuals=True, ) + snapshot = self.table_metadata.snapshot_by_name(branch) + files = planner.plan_files(snapshot.manifests(self._table.io)) if snapshot is not None else [] commit_uuid = uuid.uuid4() counter = itertools.count(0) @@ -2876,20 +2864,16 @@ def __init__( row_filter: str | BooleanExpression = ALWAYS_TRUE, case_sensitive: bool = True, options: Properties = EMPTY_DICT, - ignore_residuals: bool = False, + _ignore_residuals: bool = False, ): - """Create a planner for a group of manifests. - - Args: - ignore_residuals: The planned data files are read in full, without the row filter, so keep - every delete file that applies to them instead of pruning by the row filter. - """ self.table_metadata = table_metadata self.io = io self.row_filter = _parse_row_filter(row_filter) self.case_sensitive = case_sensitive self.options = options - self.ignore_residuals = ignore_residuals + # Set when the planned data files are read in full, as in a rewrite. Residuals are then always true, and delete + # files are kept even if their bounds cannot match the row filter, as no residual drops the rows they delete. + self._ignore_residuals = _ignore_residuals @cached_property def partition_filters(self) -> KeyDefaultDict[int, BooleanExpression]: @@ -2928,7 +2912,7 @@ def plan_manifest_entries(self, manifests: Iterable[ManifestFile]) -> Iterator[l manifest, partition_evaluators[manifest.partition_spec_id], metrics_evaluator - if not self.ignore_residuals or manifest.content == ManifestContent.DATA + if manifest.content == ManifestContent.DATA or not self._ignore_residuals else lambda _: True, ) for manifest in manifests @@ -3008,7 +2992,7 @@ def residual_for(data_file: DataFile) -> BooleanExpression: data_entry.data_file, partition_key=data_entry.data_file.partition, ), - residual=residual_for(data_entry.data_file), + residual=ALWAYS_TRUE if self._ignore_residuals else residual_for(data_entry.data_file), ) for data_entry in data_entries ] diff --git a/tests/catalog/test_catalog_behaviors.py b/tests/catalog/test_catalog_behaviors.py index f35bf232e6..6d694a9448 100644 --- a/tests/catalog/test_catalog_behaviors.py +++ b/tests/catalog/test_catalog_behaviors.py @@ -21,14 +21,11 @@ import json import os -import struct -import uuid from collections.abc import Callable, Generator from pathlib import Path from typing import Any import pyarrow as pa -import pyarrow.parquet as pq import pytest from pydantic_core import ValidationError from pytest_lazy_fixtures import lf @@ -44,25 +41,14 @@ TableAlreadyExistsError, ) from pyiceberg.io.pyarrow import _dataframe_to_data_files, schema_to_pyarrow -from pyiceberg.manifest import ( - DataFile, - DataFileContent, - FileFormat, - ManifestContent, - ManifestEntry, - ManifestEntryStatus, - ManifestWriterV2, - write_manifest_list, -) from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionField, PartitionSpec from pyiceberg.schema import Schema from pyiceberg.table import Table, TableProperties -from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRefType -from pyiceberg.table.snapshots import Operation, Snapshot, Summary +from pyiceberg.table.snapshots import Operation from pyiceberg.table.sorting import NullOrder, SortDirection, SortField, SortOrder -from pyiceberg.table.update import AddSchemaUpdate, AddSnapshotUpdate, SetCurrentSchemaUpdate, SetSnapshotRefUpdate +from pyiceberg.table.update import AddSchemaUpdate, SetCurrentSchemaUpdate from pyiceberg.transforms import IdentityTransform -from pyiceberg.typedef import Identifier, Record +from pyiceberg.typedef import Identifier from pyiceberg.types import BooleanType, DoubleType, IntegerType, LongType, NestedField, StringType @@ -1493,114 +1479,3 @@ def test_record_batch_reader_schema_mismatch_writes_no_files(catalog: Catalog) - # _append_snapshot_producer opens. assert tbl.metadata.current_snapshot() is None assert len(tbl.scan().to_arrow()) == 0 - - -def _commit_position_delete_with_row(tbl: Table, data_file_path: str, pos: int, row: dict[str, Any]) -> Table: - """Commit a position delete that stores the deleted row, which gives it bounds on the data columns.""" - - class DeleteManifestWriter(ManifestWriterV2): - def content(self) -> ManifestContent: - return ManifestContent.DELETES - - @property - def _meta(self) -> dict[str, str]: - return {**super()._meta, "content": "deletes"} - - def field(name: str, type_: pa.DataType, field_id: int) -> pa.Field: - return pa.field(name, type_, nullable=False, metadata={"PARQUET:field_id": str(field_id)}) - - row_type = pa.struct([field("k", pa.int64(), 1), field("v", pa.string(), 2)]) - deletes = pa.Table.from_pylist( - [{"file_path": data_file_path, "pos": pos, "row": row}], - schema=pa.schema( - [ - field("file_path", pa.string(), 2147483546), - field("pos", pa.int64(), 2147483545), - field("row", row_type, 2147483544), - ] - ), - ) - delete_path = f"{tbl.location()}/data/delete-{uuid.uuid4()}.parquet" - with tbl.io.new_output(delete_path).create() as output: - pq.write_table(deletes, output) - - bounds = { - 2147483546: data_file_path.encode(), - 2147483545: struct.pack("> 65 - sequence_number = tbl.metadata.next_sequence_number() - with DeleteManifestWriter( - spec=tbl.spec(), - schema=tbl.schema(), - output_file=tbl.io.new_output(f"{tbl.location()}/metadata/{uuid.uuid4()}-m0.avro"), - snapshot_id=snapshot_id, - avro_compression="deflate", - ) as writer: - writer.add(ManifestEntry.from_args(status=ManifestEntryStatus.ADDED, snapshot_id=snapshot_id, data_file=delete_file)) - manifest_list = f"{tbl.location()}/metadata/snap-{snapshot_id}.avro" - with write_manifest_list( - format_version=2, - output_file=tbl.io.new_output(manifest_list), - snapshot_id=snapshot_id, - parent_snapshot_id=parent.snapshot_id, - sequence_number=sequence_number, - avro_compression="deflate", - ) as list_writer: - list_writer.add_manifests([writer.to_manifest_file(), *parent.manifests(tbl.io)]) - - snapshot = Snapshot( - snapshot_id=snapshot_id, - parent_snapshot_id=parent.snapshot_id, - sequence_number=sequence_number, - manifest_list=manifest_list, - summary=Summary(Operation.DELETE), - schema_id=tbl.schema().schema_id, - ) - tbl.catalog.commit_table( - tbl, - (), - ( - AddSnapshotUpdate(snapshot=snapshot), - SetSnapshotRefUpdate(ref_name=MAIN_BRANCH, type=SnapshotRefType.BRANCH, snapshot_id=snapshot_id), - ), - ) - return tbl.catalog.load_table(tbl.name()) - - -def test_partial_delete_keeps_earlier_deletes_outside_the_filter(catalog: Catalog) -> None: - """Rewriting a file for a delete must apply every delete file of that file. A delete file whose - bounds cannot match the delete filter is skipped by a filtered scan, which is right for a read but - would bring its rows back in the rewritten file. - """ - catalog.create_namespace("default") - identifier = f"default.partial_delete_keeps_earlier_deletes_{catalog.name}" - iceberg_schema = Schema(NestedField(1, "k", LongType(), required=True), NestedField(2, "v", StringType(), required=True)) - tbl = catalog.create_table(identifier=identifier, schema=iceberg_schema, properties={"format-version": "2"}) - tbl.append(pa.Table.from_pydict({"k": [1, 2, 3], "v": ["a", "b", "c"]}, schema=schema_to_pyarrow(iceberg_schema))) - data_file_path = next(iter(tbl.scan().plan_files())).file.file_path - - # Delete (2, 'b'). The delete file has bounds k = 2, so a scan for k = 1 leaves it out. - tbl = _commit_position_delete_with_row(tbl, data_file_path, pos=1, row={"k": 2, "v": "b"}) - assert tbl.scan().to_arrow()["k"].to_pylist() == [1, 3] - assert [len(task.delete_files) for task in tbl.scan(row_filter="k = 1").plan_files()] == [0] - - tbl.delete("k = 1") - - assert tbl.scan().to_arrow()["k"].to_pylist() == [3] diff --git a/tests/table/test_commit_retry.py b/tests/table/test_commit_retry.py index 4379e9ab40..d3168cfb56 100644 --- a/tests/table/test_commit_retry.py +++ b/tests/table/test_commit_retry.py @@ -453,11 +453,17 @@ def test_file_overwrite_validates_concurrent_file_delete( assert sorted(result["value"].to_pylist()) == expected_values -def _commit_partition_position_delete(table: Table, partition: Record, file_path: str, pos: int) -> None: - """PyIceberg cannot write delete files, so commit one without path bounds, as other engines write them.""" +def commit_partition_position_delete( + table: Table, partition: Record, file_path: str, pos: int, row: dict[str, Any] | None = None +) -> None: + """PyIceberg cannot write delete files, so commit one without path bounds, as other engines write them. + + If ``row`` is given, it is stored as the deleted row, which gives the delete file bounds on the data columns. + """ import pyarrow as pa import pyarrow.parquet as pq + from pyiceberg.conversions import to_bytes from pyiceberg.manifest import DataFile, DataFileContent, FileFormat, ManifestContent, ManifestWriter, ManifestWriterV2 from pyiceberg.partitioning import PartitionSpec from pyiceberg.table.update.snapshot import _OverwriteFiles @@ -475,9 +481,15 @@ def new_delete_manifest_writer(producer: _OverwriteFiles, spec: PartitionSpec) - spec, producer.schema(), producer.new_manifest_output(), producer._snapshot_id, producer._compression ) + columns: dict[str, list[Any]] = {"file_path": [file_path], "pos": [pos]} + bounds = None + if row is not None: + columns["row"] = [row] + bounds = {field.field_id: to_bytes(field.field_type, row[field.name]) for field in table.schema().fields} + delete_path = table.location_provider().new_data_location(f"pos-delete-{uuid.uuid4()}.parquet") with table.io.new_output(delete_path).create() as output: - pq.write_table(pa.table({"file_path": [file_path], "pos": [pos]}), output) + pq.write_table(pa.table(columns), output) delete_file = DataFile.from_args( content=DataFileContent.POSITION_DELETES, file_path=delete_path, @@ -485,6 +497,8 @@ def new_delete_manifest_writer(producer: _OverwriteFiles, spec: PartitionSpec) - partition=partition, record_count=1, file_size_in_bytes=len(table.io.new_input(delete_path)), + lower_bounds=bounds, + upper_bounds=bounds, spec_id=table.spec().spec_id, ) @@ -524,7 +538,7 @@ def test_file_overwrite_validates_concurrent_position_delete_in_nan_partition(ca overwrite.delete_data_file(file_to_replace) overwrite.append_data_file(replacement_file) - _commit_partition_position_delete(deleting_table, Record(float("nan")), file_to_replace.file_path, pos=0) + commit_partition_position_delete(deleting_table, Record(float("nan")), file_to_replace.file_path, pos=0) with pytest.raises(ValidationException, match="found new delete for replace data file"): replacing_transaction.commit_transaction() diff --git a/tests/table/test_init.py b/tests/table/test_init.py index 78a1199c01..4e254ac472 100644 --- a/tests/table/test_init.py +++ b/tests/table/test_init.py @@ -124,6 +124,7 @@ TimeType, UUIDType, ) +from tests.table.test_commit_retry import commit_partition_position_delete def test_schema(table_v2: Table) -> None: @@ -2390,6 +2391,25 @@ def test_overwrite_delete_data_file_on_bucket_partition(catalog: Catalog) -> Non assert table.scan().to_arrow().num_rows == 0 +def test_partial_delete_keeps_earlier_deletes_outside_the_filter(catalog: Catalog) -> None: + """Rewriting a file applies all of its delete files, not only those whose bounds can match the delete filter.""" + import pyarrow as pa + + catalog.create_namespace("default") + schema = Schema(NestedField(1, "k", LongType(), required=False), NestedField(2, "v", StringType(), required=False)) + table = catalog.create_table("default.partial_delete", schema=schema, properties={"format-version": "2"}) + table.append(pa.table({"k": [1, 2, 3], "v": ["a", "b", "c"]})) + data_file = next(iter(table.scan().plan_files())).file + + # The delete file stores the deleted row (2, 'b'), so a scan for k = 1 skips it. + commit_partition_position_delete(table, Record(), data_file.file_path, pos=1, row={"k": 2, "v": "b"}) + assert [task.delete_files for task in table.scan(row_filter="k = 1").plan_files()] == [set()] + + table.delete("k = 1") + + assert table.scan().to_arrow()["k"].to_pylist() == [3] + + def test_static_table_forwards_location_to_table_file_io(metadata_location: str, monkeypatch: pytest.MonkeyPatch) -> None: seen_locations: list[str | None] = [] real_load_file_io = load_file_io