Skip to content

Commit 7c78db8

Browse files
committed
Keep DV range identity internal to scan aggregation
Restore native scan task sets by routing deletion vectors to their referenced data files. Preserve REST range references and cover local and REST scans of packed DV fixtures.
1 parent c20271b commit 7c78db8

7 files changed

Lines changed: 258 additions & 184 deletions

File tree

‎pyiceberg/io/pyarrow.py‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,6 @@
149149
visit_with_partner,
150150
)
151151
from pyiceberg.table import DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE, TableProperties
152-
from pyiceberg.table.delete_file import DeleteFileSet
153152
from pyiceberg.table.deletion_vector import read_deletion_vectors
154153
from pyiceberg.table.locations import load_location_provider
155154
from pyiceberg.table.metadata import TableMetadata
@@ -1742,14 +1741,28 @@ def _task_to_record_batches(
17421741
)
17431742

17441743

1744+
@dataclass(frozen=True, slots=True)
1745+
class _DeleteFileKey:
1746+
"""Identity of a delete file or a referenced DV range."""
1747+
1748+
file_path: str
1749+
content_offset: int | None
1750+
content_size_in_bytes: int | None
1751+
1752+
17451753
def _read_all_delete_files(io: FileIO, tasks: Iterable[FileScanTask]) -> dict[str, list[ChunkedArray]]:
17461754
deletes_per_file: dict[str, list[ChunkedArray]] = {}
1747-
unique_deletes = DeleteFileSet(itertools.chain.from_iterable(task.delete_files for task in tasks))
1755+
# DataFile equality uses only the path; DVs from different tasks may share a file.
1756+
unique_deletes: dict[_DeleteFileKey, DataFile] = {}
1757+
for task in tasks:
1758+
for delete_file in task.delete_files:
1759+
key = _DeleteFileKey(delete_file.file_path, delete_file.content_offset, delete_file.content_size_in_bytes)
1760+
unique_deletes.setdefault(key, delete_file)
17481761
if len(unique_deletes) > 0:
17491762
executor = ExecutorFactory.get_or_create()
17501763
deletes_per_files: Iterator[dict[str, ChunkedArray]] = executor.map(
17511764
lambda args: _read_deletes(*args),
1752-
[(io, delete_file) for delete_file in unique_deletes],
1765+
[(io, delete_file) for delete_file in unique_deletes.values()],
17531766
)
17541767
for delete in deletes_per_files:
17551768
for file, arr in delete.items():

‎pyiceberg/table/__init__.py‎

Lines changed: 30 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -44,10 +44,17 @@
4444
manifest_evaluator,
4545
)
4646
from pyiceberg.io import FileIO, load_file_io
47-
from pyiceberg.manifest import DataFile, DataFileContent, ManifestContent, ManifestEntry, ManifestEntryStatus, ManifestFile
47+
from pyiceberg.manifest import (
48+
DataFile,
49+
DataFileContent,
50+
FileFormat,
51+
ManifestContent,
52+
ManifestEntry,
53+
ManifestEntryStatus,
54+
ManifestFile,
55+
)
4856
from pyiceberg.partitioning import PARTITION_FIELD_ID_START, UNPARTITIONED_PARTITION_SPEC, PartitionKey, PartitionSpec
4957
from pyiceberg.schema import Schema
50-
from pyiceberg.table.delete_file import DeleteFileSet
5158
from pyiceberg.table.delete_file_index import DeleteFileIndex
5259
from pyiceberg.table.inspect import InspectTable
5360
from pyiceberg.table.locations import LocationProvider, load_location_provider
@@ -2238,17 +2245,17 @@ class FileScanTask(ScanTask):
22382245
"""Task representing a data file and its corresponding delete files."""
22392246

22402247
file: DataFile
2241-
delete_files: DeleteFileSet
2248+
delete_files: set[DataFile]
22422249
residual: BooleanExpression
22432250

22442251
def __init__(
22452252
self,
22462253
data_file: DataFile,
2247-
delete_files: Iterable[DataFile] | None = None,
2254+
delete_files: set[DataFile] | None = None,
22482255
residual: BooleanExpression = ALWAYS_TRUE,
22492256
) -> None:
22502257
self.file = data_file
2251-
self.delete_files = DeleteFileSet(delete_files if delete_files is not None else [])
2258+
self.delete_files = delete_files or set()
22522259
self.residual = residual
22532260

22542261
@staticmethod
@@ -2272,13 +2279,13 @@ def from_rest_response(
22722279

22732280
data_file = _rest_file_to_data_file(rest_task.data_file)
22742281

2275-
resolved_deletes: list[DataFile] = []
2282+
resolved_deletes: set[DataFile] = set()
22762283
if rest_task.delete_file_references:
22772284
for idx in rest_task.delete_file_references:
22782285
delete_file = delete_files[idx]
22792286
if isinstance(delete_file, RESTEqualityDeleteFile):
22802287
raise NotImplementedError(f"PyIceberg does not yet support equality deletes: {delete_file.file_path}")
2281-
resolved_deletes.append(_rest_file_to_data_file(delete_file))
2288+
resolved_deletes.add(_rest_file_to_data_file(delete_file, default_referenced_data_file=data_file.file_path))
22822289

22832290
return FileScanTask(
22842291
data_file=data_file,
@@ -2287,9 +2294,9 @@ def from_rest_response(
22872294
)
22882295

22892296

2290-
def _rest_file_to_data_file(rest_file: RESTContentFile) -> DataFile:
2297+
def _rest_file_to_data_file(rest_file: RESTContentFile, default_referenced_data_file: str | None = None) -> DataFile:
22912298
"""Convert a REST content file to a manifest DataFile."""
2292-
from pyiceberg.catalog.rest.scan_planning import RESTDataFile
2299+
from pyiceberg.catalog.rest.scan_planning import RESTDataFile, RESTPositionDeleteFile
22932300

22942301
if isinstance(rest_file, RESTDataFile):
22952302
column_sizes = rest_file.column_sizes.to_dict() if rest_file.column_sizes else None
@@ -2302,6 +2309,17 @@ def _rest_file_to_data_file(rest_file: RESTContentFile) -> DataFile:
23022309
null_value_counts = None
23032310
nan_value_counts = None
23042311

2312+
referenced_data_file = None
2313+
if isinstance(rest_file, RESTPositionDeleteFile):
2314+
referenced_data_file = rest_file.referenced_data_file
2315+
# Infer the DV target from its task only for range reads, preserving whole-Puffin fallback.
2316+
if (
2317+
referenced_data_file is None
2318+
and rest_file.file_format == FileFormat.PUFFIN
2319+
and (rest_file.content_offset is not None or rest_file.content_size_in_bytes is not None)
2320+
):
2321+
referenced_data_file = default_referenced_data_file
2322+
23052323
data_file = DataFile.from_args(
23062324
content=DataFileContent.from_rest_type(rest_file.content),
23072325
file_path=rest_file.file_path,
@@ -2315,6 +2333,9 @@ def _rest_file_to_data_file(rest_file: RESTContentFile) -> DataFile:
23152333
nan_value_counts=nan_value_counts,
23162334
split_offsets=rest_file.split_offsets,
23172335
sort_order_id=rest_file.sort_order_id,
2336+
referenced_data_file=referenced_data_file,
2337+
content_offset=rest_file.content_offset if isinstance(rest_file, RESTPositionDeleteFile) else None,
2338+
content_size_in_bytes=rest_file.content_size_in_bytes if isinstance(rest_file, RESTPositionDeleteFile) else None,
23182339
)
23192340
data_file.spec_id = rest_file.spec_id
23202341
return data_file

‎pyiceberg/table/delete_file.py‎

Lines changed: 0 additions & 86 deletions
This file was deleted.

‎pyiceberg/table/delete_file_index.py‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
from pyiceberg.expressions import EqualTo
2222
from pyiceberg.expressions.visitors import _InclusiveMetricsEvaluator
2323
from pyiceberg.manifest import INITIAL_SEQUENCE_NUMBER, POSITIONAL_DELETE_SCHEMA, DataFile, ManifestEntry
24-
from pyiceberg.table.delete_file import DeleteFileSet
2524
from pyiceberg.typedef import Record
2625

2726
PATH_FIELD_ID = 2147483546
@@ -116,7 +115,7 @@ def is_empty(self) -> bool:
116115
def add_delete_file(self, manifest_entry: ManifestEntry, partition_key: Record | None = None) -> None:
117116
delete_file = manifest_entry.data_file
118117
seq = manifest_entry.sequence_number or INITIAL_SEQUENCE_NUMBER
119-
target_path = _referenced_data_file_path(delete_file)
118+
target_path = delete_file.referenced_data_file or _referenced_data_file_path(delete_file)
120119

121120
if target_path:
122121
deletes = self._by_path.setdefault(target_path, PositionDeletes())
@@ -126,25 +125,26 @@ def add_delete_file(self, manifest_entry: ManifestEntry, partition_key: Record |
126125
deletes = self._by_partition.setdefault(key, PositionDeletes())
127126
deletes.add(delete_file, seq)
128127

129-
def for_data_file(self, seq_num: int, data_file: DataFile, partition_key: Record | None = None) -> DeleteFileSet:
128+
def for_data_file(self, seq_num: int, data_file: DataFile, partition_key: Record | None = None) -> set[DataFile]:
130129
if self.is_empty():
131-
return DeleteFileSet()
130+
return set()
132131

133-
deletes: list[DataFile] = []
132+
# DVs are indexed by target path; a valid snapshot has at most one DV per data file.
133+
deletes: set[DataFile] = set()
134134
spec_id = data_file.spec_id or 0
135135

136136
key = _partition_key(spec_id, partition_key)
137137
partition_deletes = self._by_partition.get(key)
138138
if partition_deletes:
139139
for delete_file in partition_deletes.filter_by_seq(seq_num):
140140
if _applies_to_data_file(delete_file, data_file):
141-
deletes.append(delete_file)
141+
deletes.add(delete_file)
142142

143143
path_deletes = self._by_path.get(data_file.file_path)
144144
if path_deletes:
145-
deletes.extend(path_deletes.filter_by_seq(seq_num))
145+
deletes.update(path_deletes.filter_by_seq(seq_num))
146146

147-
return DeleteFileSet(deletes)
147+
return deletes
148148

149149
def referenced_delete_files(self) -> list[DataFile]:
150150
data_files: list[DataFile] = []

‎tests/catalog/test_scan_planning_models.py‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
)
4141
from pyiceberg.expressions import AlwaysTrue, EqualTo, Reference
4242
from pyiceberg.manifest import FileFormat
43+
from pyiceberg.table import FileScanTask
4344

4445
TEST_URI = "https://iceberg-test-catalog/"
4546

@@ -526,6 +527,80 @@ def test_plan_scan_with_delete_files(rest_scan_catalog: RestCatalog, requests_mo
526527
assert len(tasks[0].delete_files) == 1
527528

528529

530+
@pytest.mark.parametrize("include_referenced_data_file", [True, False])
531+
def test_plan_scan_with_shared_dv_file(
532+
rest_scan_catalog: RestCatalog, requests_mock: Mocker, include_referenced_data_file: bool
533+
) -> None:
534+
data_files = [_rest_data_file(file_path=f"s3://bucket/data-{index}.parquet") for index in range(2)]
535+
delete_files = [
536+
{
537+
**_rest_position_delete_file(
538+
file_path="s3://bucket/deletes.bin", file_format="puffin", file_size_in_bytes=400, content_offset=offset
539+
),
540+
**({"referenced-data-file": data_file["file-path"]} if include_referenced_data_file else {}),
541+
}
542+
for data_file, offset in zip(data_files, [0, 200], strict=True)
543+
]
544+
requests_mock.post(
545+
f"{TEST_URI}v1/namespaces/db/tables/tbl/plan",
546+
json={
547+
"status": "completed",
548+
"delete-files": delete_files,
549+
"file-scan-tasks": [
550+
{"data-file": data_file, "delete-file-references": [index, index]} for index, data_file in enumerate(data_files)
551+
],
552+
},
553+
)
554+
555+
tasks = rest_scan_catalog.plan_scan(("db", "tbl"), PlanTableScanRequest())
556+
557+
assert len(tasks) == 2
558+
assert all(len(task.delete_files) == 1 for task in tasks)
559+
deletes = [dv for task in tasks for dv in task.delete_files]
560+
assert len(deletes) == 2
561+
assert {(dv.referenced_data_file, dv.content_offset, dv.content_size_in_bytes) for dv in deletes} == {
562+
("s3://bucket/data-0.parquet", 0, 200),
563+
("s3://bucket/data-1.parquet", 200, 200),
564+
}
565+
566+
567+
@pytest.mark.parametrize("missing_field", ["content-offset", "content-size-in-bytes"])
568+
@pytest.mark.parametrize("include_referenced_data_file", [True, False])
569+
def test_rest_task_preserves_partial_dv_reference(missing_field: str, include_referenced_data_file: bool) -> None:
570+
fields = {
571+
**_rest_position_delete_file(file_format="puffin", content_offset=0),
572+
**({"referenced-data-file": "s3://bucket/data.parquet"} if include_referenced_data_file else {}),
573+
}
574+
fields.pop(missing_field)
575+
task = FileScanTask.from_rest_response(
576+
RESTFileScanTask.model_validate(
577+
{"data-file": _rest_data_file(file_path="s3://bucket/data.parquet"), "delete-file-references": [0]}
578+
),
579+
[RESTPositionDeleteFile.model_validate(fields)],
580+
)
581+
dv = next(iter(task.delete_files))
582+
583+
assert dv.referenced_data_file == "s3://bucket/data.parquet"
584+
assert dv.content_offset == fields.get("content-offset")
585+
assert dv.content_size_in_bytes == fields.get("content-size-in-bytes")
586+
587+
588+
@pytest.mark.parametrize("file_format", ["puffin", "parquet"])
589+
def test_rest_task_preserves_whole_delete_file_reference(file_format: str) -> None:
590+
fields = _rest_position_delete_file(file_format=file_format)
591+
fields.pop("content-offset")
592+
fields.pop("content-size-in-bytes")
593+
task = FileScanTask.from_rest_response(
594+
RESTFileScanTask.model_validate({"data-file": _rest_data_file(), "delete-file-references": [0]}),
595+
[RESTPositionDeleteFile.model_validate(fields)],
596+
)
597+
dv = next(iter(task.delete_files))
598+
599+
assert dv.referenced_data_file is None
600+
assert dv.content_offset is None
601+
assert dv.content_size_in_bytes is None
602+
603+
529604
def test_plan_scan_async_poll_completes(
530605
rest_scan_catalog: RestCatalog, requests_mock: Mocker, monkeypatch: pytest.MonkeyPatch
531606
) -> None:

0 commit comments

Comments
 (0)