Skip to content

Commit 4e7b5ea

Browse files
committed
Normalize the equality-delete implementation
1 parent d9f2cbd commit 4e7b5ea

8 files changed

Lines changed: 31 additions & 74 deletions

File tree

‎pyiceberg/io/pyarrow.py‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1175,9 +1175,7 @@ def _read_equality_deletes(io: FileIO, delete_file: DataFile) -> pa.Table:
11751175
columns = projected_columns if len(projected_columns) == len(equality_ids) else None
11761176
# Delete files are already read concurrently by PyIceberg's executor;
11771177
# disable nested Arrow threading to avoid oversubscription per file.
1178-
return ds.Scanner.from_fragment(
1179-
fragment=fragment, schema=physical_schema, columns=columns, use_threads=False
1180-
).to_table()
1178+
return ds.Scanner.from_fragment(fragment=fragment, schema=physical_schema, columns=columns, use_threads=False).to_table()
11811179

11821180

11831181
def _column_name_for_field_id(table: pa.Table, field_id: int, schema: Schema) -> str | None:
@@ -1291,9 +1289,7 @@ def temporary_name(kind: str) -> str:
12911289
join_columns.append(nan_key)
12921290
temporary_columns.append(nan_key)
12931291

1294-
joined = data_join.join(
1295-
delete_join.select(join_columns), keys=join_columns, join_type="left anti", use_threads=False
1296-
)
1292+
joined = data_join.join(delete_join.select(join_columns), keys=join_columns, join_type="left anti", use_threads=False)
12971293
return joined.drop(temporary_columns)
12981294

12991295

@@ -3265,9 +3261,7 @@ def _dataframe_to_equality_delete_files(
32653261
raise ValueError("At least one equality field ID is required")
32663262

32673263
table_schema = table_metadata.schema()
3268-
missing_partition_sources = {
3269-
field.source_id for field in table_metadata.spec().fields if field.source_id not in equality_ids
3270-
}
3264+
missing_partition_sources = {field.source_id for field in table_metadata.spec().fields if field.source_id not in equality_ids}
32713265
if missing_partition_sources:
32723266
missing_names = sorted(table_schema.find_field(field_id).name for field_id in missing_partition_sources)
32733267
raise ValueError(
@@ -3290,6 +3284,7 @@ def _dataframe_to_equality_delete_files(
32903284
property_name=TableProperties.WRITE_TARGET_FILE_SIZE_BYTES,
32913285
default=TableProperties.WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT,
32923286
)
3287+
assert target_file_size is not None
32933288
name_mapping = table_schema.name_mapping
32943289

32953290
def write_partition(delete_rows: pa.Table, partition_key: PartitionKey | None) -> Iterable[DataFile]:

‎pyiceberg/table/delete_file_index.py‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -153,10 +153,9 @@ def _equality_delete_applies_to_data_file(delete_file: DataFile, data_file: Data
153153
and field_id in data_upper
154154
):
155155
field_type = field.field_type
156-
if (
157-
from_bytes(field_type, delete_upper[field_id]) < from_bytes(field_type, data_lower[field_id])
158-
or from_bytes(field_type, delete_lower[field_id]) > from_bytes(field_type, data_upper[field_id])
159-
):
156+
if from_bytes(field_type, delete_upper[field_id]) < from_bytes(field_type, data_lower[field_id]) or from_bytes(
157+
field_type, delete_lower[field_id]
158+
) > from_bytes(field_type, data_upper[field_id]):
160159
return False
161160

162161
return True

‎pyiceberg/table/update/snapshot.py‎

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -230,16 +230,12 @@ def _manifests(self) -> list[ManifestFile]:
230230
def _write_added_manifest() -> list[ManifestFile]:
231231
files_by_content: dict[ManifestContent, list[DataFile]] = defaultdict(list)
232232
for data_file in self._added_data_files:
233-
manifest_content = (
234-
ManifestContent.DATA if data_file.content == DataFileContent.DATA else ManifestContent.DELETES
235-
)
233+
manifest_content = ManifestContent.DATA if data_file.content == DataFileContent.DATA else ManifestContent.DELETES
236234
files_by_content[manifest_content].append(data_file)
237235

238236
manifests = []
239237
for manifest_content, data_files in files_by_content.items():
240-
with self.new_manifest_writer(
241-
spec=self._transaction.table_metadata.spec(), content=manifest_content
242-
) as writer:
238+
with self.new_manifest_writer(spec=self._transaction.table_metadata.spec(), content=manifest_content) as writer:
243239
for data_file in data_files:
244240
writer.add(
245241
ManifestEntry.from_args(
@@ -398,9 +394,7 @@ def schema(self) -> Schema:
398394
def spec(self, spec_id: int) -> PartitionSpec:
399395
return self._transaction.table_metadata.specs()[spec_id]
400396

401-
def new_manifest_writer(
402-
self, spec: PartitionSpec, content: ManifestContent = ManifestContent.DATA
403-
) -> ManifestWriter:
397+
def new_manifest_writer(self, spec: PartitionSpec, content: ManifestContent = ManifestContent.DATA) -> ManifestWriter:
404398
return write_manifest(
405399
format_version=self._transaction.table_metadata.format_version,
406400
spec=spec,
@@ -946,9 +940,7 @@ def __init__(
946940
self._merge_enabled = merge_enabled
947941
self._snapshot_producer = snapshot_producer
948942

949-
def _group_by_spec_and_content(
950-
self, manifests: list[ManifestFile]
951-
) -> dict[tuple[int, ManifestContent], list[ManifestFile]]:
943+
def _group_by_spec_and_content(self, manifests: list[ManifestFile]) -> dict[tuple[int, ManifestContent], list[ManifestFile]]:
952944
groups = defaultdict(list)
953945
for manifest in manifests:
954946
groups[(manifest.partition_spec_id, manifest.content)].append(manifest)

‎tests/integration/test_writes/test_equality_deletes.py‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,7 @@
2424

2525

2626
@pytest.mark.integration
27-
def test_large_composite_equality_upsert_is_visible_to_spark(
28-
session_catalog: Catalog, spark: SparkSession
29-
) -> None:
27+
def test_large_composite_equality_upsert_is_visible_to_spark(session_catalog: Catalog, spark: SparkSession) -> None:
3028
identifier = "default.large_composite_equality_upsert"
3129
if session_catalog.table_exists(identifier):
3230
session_catalog.drop_table(identifier)

‎tests/io/test_pyarrow.py‎

Lines changed: 7 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -2050,9 +2050,7 @@ def test_equality_delete_treats_uuid_nulls_as_equal(tmp_path: Path) -> None:
20502050
data_path = tmp_path / "data.parquet"
20512051
delete_path = tmp_path / "delete.parquet"
20522052
kept = uuid.UUID("00000000-0000-0000-0000-000000000001")
2053-
pq.write_table(
2054-
pa.table({"id": pa.array([kept, None], type=pa.uuid())}, schema=schema_to_pyarrow(schema)), data_path
2055-
)
2053+
pq.write_table(pa.table({"id": pa.array([kept, None], type=pa.uuid())}, schema=schema_to_pyarrow(schema)), data_path)
20562054
pq.write_table(pa.table({"id": pa.array([None], type=pa.uuid())}), delete_path)
20572055

20582056
result = _scan_equality_delete_files(schema, data_path, [delete_path], [1])
@@ -2065,9 +2063,7 @@ def test_equality_delete_aligns_renamed_fields_by_id(tmp_path: Path) -> None:
20652063
data_path = tmp_path / "data.parquet"
20662064
current_delete_path = tmp_path / "current-delete.parquet"
20672065
renamed_delete_path = tmp_path / "renamed-delete.parquet"
2068-
pq.write_table(
2069-
pa.table({"id": pa.array([1, 2, 3], type=pa.int32())}, schema=schema_to_pyarrow(current_schema)), data_path
2070-
)
2066+
pq.write_table(pa.table({"id": pa.array([1, 2, 3], type=pa.int32())}, schema=schema_to_pyarrow(current_schema)), data_path)
20712067
pq.write_table(
20722068
pa.table({"id": pa.array([2], type=pa.int32())}, schema=schema_to_pyarrow(current_schema)),
20732069
current_delete_path,
@@ -2077,9 +2073,7 @@ def test_equality_delete_aligns_renamed_fields_by_id(tmp_path: Path) -> None:
20772073
renamed_delete_path,
20782074
)
20792075

2080-
result = _scan_equality_delete_files(
2081-
current_schema, data_path, [current_delete_path, renamed_delete_path], [1]
2082-
)
2076+
result = _scan_equality_delete_files(current_schema, data_path, [current_delete_path, renamed_delete_path], [1])
20832077
assert result.column("id").to_pylist() == [1]
20842078

20852079

@@ -2092,9 +2086,7 @@ def test_equality_delete_dropped_field_does_not_match_a_partial_key(tmp_path: Pa
20922086
)
20932087
data_path = tmp_path / "data.parquet"
20942088
delete_path = tmp_path / "delete.parquet"
2095-
pq.write_table(
2096-
pa.table({"id": pa.array([1], type=pa.int32())}, schema=schema_to_pyarrow(current_schema)), data_path
2097-
)
2089+
pq.write_table(pa.table({"id": pa.array([1], type=pa.int32())}, schema=schema_to_pyarrow(current_schema)), data_path)
20982090
pq.write_table(
20992091
pa.table(
21002092
{"id": pa.array([1], type=pa.int32()), "dropped": ["not-null"]},
@@ -2111,9 +2103,7 @@ def test_orc_equality_delete(tmp_path: Path) -> None:
21112103
schema = Schema(NestedField(1, "id", IntegerType(), required=True), schema_id=1)
21122104
data_path = tmp_path / "data.orc"
21132105
delete_path = tmp_path / "delete.orc"
2114-
orc.write_table(
2115-
pa.table({"id": pa.array([1, 2, 3], type=pa.int32())}, schema=schema_to_pyarrow(schema)), data_path
2116-
)
2106+
orc.write_table(pa.table({"id": pa.array([1, 2, 3], type=pa.int32())}, schema=schema_to_pyarrow(schema)), data_path)
21172107
orc.write_table(pa.table({"id": pa.array([2], type=pa.int32())}, schema=schema_to_pyarrow(schema)), delete_path)
21182108

21192109
result = _scan_equality_delete_files(schema, data_path, [delete_path], [1], FileFormat.ORC)
@@ -2125,12 +2115,8 @@ def test_equality_delete_casts_promoted_key_types(tmp_path: Path) -> None:
21252115
old_schema = Schema(NestedField(1, "id", IntegerType(), required=True), schema_id=1)
21262116
data_path = tmp_path / "data.parquet"
21272117
delete_path = tmp_path / "delete.parquet"
2128-
pq.write_table(
2129-
pa.table({"id": pa.array([1, 2, 3], type=pa.int64())}, schema=schema_to_pyarrow(current_schema)), data_path
2130-
)
2131-
pq.write_table(
2132-
pa.table({"id": pa.array([2], type=pa.int32())}, schema=schema_to_pyarrow(old_schema)), delete_path
2133-
)
2118+
pq.write_table(pa.table({"id": pa.array([1, 2, 3], type=pa.int64())}, schema=schema_to_pyarrow(current_schema)), data_path)
2119+
pq.write_table(pa.table({"id": pa.array([2], type=pa.int32())}, schema=schema_to_pyarrow(old_schema)), delete_path)
21342120

21352121
result = _scan_equality_delete_files(current_schema, data_path, [delete_path], [1])
21362122
assert result.column("id").to_pylist() == [1, 3]

‎tests/table/test_delete_file_index.py‎

Lines changed: 7 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -292,15 +292,9 @@ def test_equality_delete_metrics_filtering() -> None:
292292
)
293293
index.add_delete_file(equality_delete)
294294

295-
before = _create_data_file(
296-
lower_bounds={1: to_bytes(IntegerType(), 0)}, upper_bounds={1: to_bytes(IntegerType(), 5)}
297-
)
298-
overlap = _create_data_file(
299-
lower_bounds={1: to_bytes(IntegerType(), 15)}, upper_bounds={1: to_bytes(IntegerType(), 25)}
300-
)
301-
after = _create_data_file(
302-
lower_bounds={1: to_bytes(IntegerType(), 25)}, upper_bounds={1: to_bytes(IntegerType(), 30)}
303-
)
295+
before = _create_data_file(lower_bounds={1: to_bytes(IntegerType(), 0)}, upper_bounds={1: to_bytes(IntegerType(), 5)})
296+
overlap = _create_data_file(lower_bounds={1: to_bytes(IntegerType(), 15)}, upper_bounds={1: to_bytes(IntegerType(), 25)})
297+
after = _create_data_file(lower_bounds={1: to_bytes(IntegerType(), 25)}, upper_bounds={1: to_bytes(IntegerType(), 30)})
304298
assert index.for_data_file(1, before) == set()
305299
assert index.for_data_file(1, overlap) == {equality_delete.data_file}
306300
assert index.for_data_file(1, after) == set()
@@ -310,9 +304,7 @@ def test_equality_delete_metrics_filtering() -> None:
310304
("delete_nulls", "data_nulls"),
311305
[((10, 10), (0, 100)), ((0, 10), (100, 100))],
312306
)
313-
def test_equality_delete_prunes_disjoint_null_populations(
314-
delete_nulls: tuple[int, int], data_nulls: tuple[int, int]
315-
) -> None:
307+
def test_equality_delete_prunes_disjoint_null_populations(delete_nulls: tuple[int, int], data_nulls: tuple[int, int]) -> None:
316308
index = DeleteFileIndex(Schema(NestedField(1, "id", IntegerType(), required=False)))
317309
equality_delete = _create_equality_delete(
318310
sequence_number=10,
@@ -334,12 +326,8 @@ def test_equality_delete_metrics_after_int_to_long_promotion() -> None:
334326
upper_bounds={1: to_bytes(IntegerType(), 20)},
335327
)
336328
index.add_delete_file(equality_delete)
337-
before = _create_data_file(
338-
lower_bounds={1: to_bytes(IntegerType(), 0)}, upper_bounds={1: to_bytes(IntegerType(), 5)}
339-
)
340-
overlap = _create_data_file(
341-
lower_bounds={1: to_bytes(IntegerType(), 15)}, upper_bounds={1: to_bytes(IntegerType(), 25)}
342-
)
329+
before = _create_data_file(lower_bounds={1: to_bytes(IntegerType(), 0)}, upper_bounds={1: to_bytes(IntegerType(), 5)})
330+
overlap = _create_data_file(lower_bounds={1: to_bytes(IntegerType(), 15)}, upper_bounds={1: to_bytes(IntegerType(), 25)})
343331
assert index.for_data_file(1, before) == set()
344332
assert index.for_data_file(1, overlap) == {equality_delete.data_file}
345333

@@ -353,9 +341,7 @@ def test_equality_delete_dropped_field_is_not_pruned() -> None:
353341
upper_bounds={1: to_bytes(IntegerType(), 20)},
354342
)
355343
index.add_delete_file(equality_delete)
356-
data_file = _create_data_file(
357-
lower_bounds={1: to_bytes(IntegerType(), 15)}, upper_bounds={1: to_bytes(IntegerType(), 25)}
358-
)
344+
data_file = _create_data_file(lower_bounds={1: to_bytes(IntegerType(), 15)}, upper_bounds={1: to_bytes(IntegerType(), 25)})
359345
assert index.for_data_file(1, data_file) == {equality_delete.data_file}
360346

361347

‎tests/table/test_upsert.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -167,9 +167,7 @@ def test_composite_equality_delete_upsert_15k_rows(catalog: InMemoryCatalog) ->
167167
assert delete_entries[0].data_file.equality_ids == [1, 2]
168168

169169

170-
def test_equality_delete_upsert_is_metadata_atomic_on_commit_failure(
171-
catalog: InMemoryCatalog, mocker: MockerFixture
172-
) -> None:
170+
def test_equality_delete_upsert_is_metadata_atomic_on_commit_failure(catalog: InMemoryCatalog, mocker: MockerFixture) -> None:
173171
initial = pa.table({"id": pa.array([1, 2], type=pa.int64()), "value": ["one", "two"]})
174172
source = pa.table({"id": pa.array([2, 3], type=pa.int64()), "value": ["updated", "three"]})
175173
table = catalog.create_table(
@@ -186,7 +184,9 @@ def test_equality_delete_upsert_is_metadata_atomic_on_commit_failure(
186184
table.upsert_by_equality_delete(source, join_cols=["id"])
187185

188186
table.refresh()
189-
assert table.current_snapshot().snapshot_id == original_snapshot.snapshot_id # type: ignore[union-attr]
187+
current_snapshot = table.current_snapshot()
188+
assert current_snapshot is not None
189+
assert current_snapshot.snapshot_id == original_snapshot.snapshot_id
190190
assert table.scan().to_arrow().sort_by("id").to_pydict() == {"id": [1, 2], "value": ["one", "two"]}
191191

192192

‎tests/utils/test_manifest.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ def reset_global_manifests_cache() -> None:
5959
@pytest.mark.parametrize("format_version", [2, 3])
6060
def test_equality_ids_use_iceberg_int_wire_type(format_version: int) -> None:
6161
equality_ids = DATA_FILE_TYPE[format_version].field_by_name("equality_ids")
62+
assert equality_ids is not None
6263
assert isinstance(equality_ids.field_type, ListType)
6364
assert isinstance(equality_ids.field_type.element_type, IntegerType)
6465

0 commit comments

Comments
 (0)