Skip to content

Commit 3f32c12

Browse files
committed
perf(pyarrow): group delete positions by file_path in a single pass
_read_deletes built its result with a dict comprehension that filtered the whole delete table once per distinct file_path, making the cost O(rows x distinct paths). A delete file referencing many data files therefore degraded quadratically. The ORC branch had the same shape via unique(). Group by file_path instead, which is a single pass and lets both the Parquet and ORC branches share one code path. Dictionary encoding of file_path is retained, since delete files commonly repeat a small number of long paths across many rows; dictionaries are unified first because grouping cannot combine differing ones. Closes #3983
1 parent 1312c55 commit 3f32c12

2 files changed

Lines changed: 30 additions & 10 deletions

File tree

‎pyiceberg/io/pyarrow.py‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1134,20 +1134,10 @@ def _read_deletes(io: FileIO, data_file: DataFile) -> dict[str, pa.ChunkedArray]
11341134
data_file.file_format, dictionary_columns=("file_path",), pre_buffer=True, buffer_size=ONE_MEGABYTE
11351135
).make_fragment(fi)
11361136
table = ds.Scanner.from_fragment(fragment=delete_fragment).to_table()
1137-
table = table.unify_dictionaries()
1138-
return {
1139-
file.as_py(): table.filter(pc.field("file_path") == file).column("pos")
1140-
for file in table.column("file_path").chunks[0].dictionary
1141-
}
11421137
elif data_file.file_format == FileFormat.ORC:
11431138
with io.new_input(data_file.file_path).open() as fi:
11441139
delete_fragment = _get_file_format(data_file.file_format).make_fragment(fi)
11451140
table = ds.Scanner.from_fragment(fragment=delete_fragment).to_table()
1146-
# For ORC, file_path columns are not dictionary-encoded, so we use unique() directly
1147-
return {
1148-
path.as_py(): table.filter(pc.field("file_path") == path).column("pos")
1149-
for path in table.column("file_path").unique()
1150-
}
11511141
elif data_file.file_format == FileFormat.PUFFIN:
11521142
with io.new_input(data_file.file_path).open() as fi:
11531143
payload = fi.read()
@@ -1156,6 +1146,14 @@ def _read_deletes(io: FileIO, data_file: DataFile) -> dict[str, pa.ChunkedArray]
11561146
else:
11571147
raise ValueError(f"Delete file format not supported: {data_file.file_format}")
11581148

1149+
# Grouping is a single pass, where filtering once per distinct file_path is quadratic.
1150+
# Dictionaries have to be unified first, since grouping cannot combine differing ones.
1151+
grouped = table.unify_dictionaries().group_by("file_path", use_threads=False).aggregate([("pos", "list")])
1152+
return {
1153+
file_path.as_py(): pa.chunked_array([pos_list.values])
1154+
for file_path, pos_list in zip(grouped.column("file_path"), grouped.column("pos_list"), strict=True)
1155+
}
1156+
11591157

11601158
def _combine_positional_deletes(positional_deletes: list[pa.ChunkedArray], start_index: int, end_index: int) -> pa.Array:
11611159
if len(positional_deletes) == 1:

‎tests/io/test_pyarrow.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1822,6 +1822,28 @@ def test_read_deletes(deletes_file: str, request: pytest.FixtureRequest) -> None
18221822
assert list(deletes.values())[0] == pa.chunked_array([[1, 3, 5]])
18231823

18241824

1825+
@pytest.mark.parametrize("file_format", [FileFormat.PARQUET, FileFormat.ORC])
1826+
def test_read_deletes_many_distinct_file_paths(tmp_path: str, file_format: FileFormat) -> None:
1827+
"""Guard against quadratic behavior in _read_deletes: https://github.com/apache/iceberg-python/issues/3983."""
1828+
positions_per_file = {f"s3://bucket/data/file-{i}.parquet": [i, i + 1, i + 2] for i in range(200)}
1829+
file_path_column = [path for path, positions in positions_per_file.items() for _ in positions]
1830+
pos_column = [pos for positions in positions_per_file.values() for pos in positions]
1831+
table = pa.table({"file_path": file_path_column, "pos": pos_column})
1832+
1833+
if file_format == FileFormat.PARQUET:
1834+
deletes_file_path = f"{tmp_path}/deletes.parquet"
1835+
pq.write_table(table, deletes_file_path)
1836+
else:
1837+
deletes_file_path = f"{tmp_path}/deletes.orc"
1838+
orc.write_table(table, deletes_file_path)
1839+
1840+
deletes = _read_deletes(PyArrowFileIO(), DataFile.from_args(file_path=deletes_file_path, file_format=file_format))
1841+
1842+
assert set(deletes.keys()) == set(positions_per_file.keys())
1843+
for path, expected_positions in positions_per_file.items():
1844+
assert deletes[path] == pa.chunked_array([expected_positions])
1845+
1846+
18251847
def test_delete(deletes_file: str, request: pytest.FixtureRequest, table_schema_simple: Schema) -> None:
18261848
# Determine file format from the file extension
18271849
file_format = FileFormat.PARQUET if deletes_file.endswith(".parquet") else FileFormat.ORC

0 commit comments

Comments
 (0)