Skip to content

Commit 61c116f

Browse files
committed
Match NaN partition values with IsNaN in partition predicates
`_build_partition_predicate` emitted `EqualTo` for every non-null partition value. Expression literals reject NaN, so deleting, overwriting or upserting a file in a NaN identity partition raised, as did a dynamic partition overwrite with NaN in the incoming data. Closes #4277
1 parent e9f7f6b commit 61c116f

4 files changed

Lines changed: 147 additions & 11 deletions

File tree

‎pyiceberg/table/__init__.py‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
import itertools
2020
import logging
21+
import math
2122
import random
2223
import time
2324
import uuid
@@ -35,7 +36,7 @@
3536

3637
from pyiceberg.encryption.kms import KeyManagementClient
3738
from pyiceberg.exceptions import CommitFailedException, ValidationException
38-
from pyiceberg.expressions import AlwaysFalse, AlwaysTrue, And, BooleanExpression, EqualTo, IsNull, Or, Reference
39+
from pyiceberg.expressions import AlwaysFalse, AlwaysTrue, And, BooleanExpression, EqualTo, IsNaN, IsNull, Or, Reference
3940
from pyiceberg.expressions.visitors import (
4041
ResidualEvaluator,
4142
_InclusiveMetricsEvaluator,
@@ -422,12 +423,16 @@ def _build_partition_predicate(self, partition_records: set[Record], partition_f
422423

423424
per_record_exprs: list[BooleanExpression] = []
424425
for partition_record in partition_records:
425-
predicates: list[BooleanExpression] = [
426-
EqualTo(Reference(partition_field), partition_record[pos])
427-
if partition_record[pos] is not None
428-
else IsNull(Reference(partition_field))
429-
for pos, partition_field in enumerate(partition_fields)
430-
]
426+
predicates: list[BooleanExpression] = []
427+
for pos, partition_field in enumerate(partition_fields):
428+
ref = Reference(partition_field)
429+
value = partition_record[pos]
430+
if value is None:
431+
predicates.append(IsNull(ref))
432+
elif isinstance(value, float) and math.isnan(value):
433+
predicates.append(IsNaN(ref))
434+
else:
435+
predicates.append(EqualTo(ref, value))
431436
per_record_exprs.append(And(*predicates) if len(predicates) > 1 else predicates[0])
432437

433438
return Or(*per_record_exprs) if len(per_record_exprs) > 1 else per_record_exprs[0]

‎tests/catalog/test_catalog_behaviors.py‎

Lines changed: 41 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121

2222
import json
2323
import os
24-
from collections.abc import Generator
24+
from collections.abc import Callable, Generator
2525
from pathlib import Path
2626
from typing import Any
2727

@@ -43,7 +43,7 @@
4343
from pyiceberg.io.pyarrow import _dataframe_to_data_files, schema_to_pyarrow
4444
from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionField, PartitionSpec
4545
from pyiceberg.schema import Schema
46-
from pyiceberg.table import TableProperties
46+
from pyiceberg.table import Table, TableProperties
4747
from pyiceberg.table.snapshots import Operation
4848
from pyiceberg.table.sorting import NullOrder, SortDirection, SortField, SortOrder
4949
from pyiceberg.table.update import AddSchemaUpdate, SetCurrentSchemaUpdate
@@ -1376,6 +1376,45 @@ def test_append_nan_to_identity_partitioned_table(catalog: Catalog) -> None:
13761376
assert sorted(tbl.scan(row_filter="value is nan").to_arrow()["id"].to_pylist()) == [2, 4]
13771377

13781378

1379+
def _nan_partitioned_table(catalog: Catalog, name: str) -> Table:
1380+
iceberg_schema = Schema(
1381+
NestedField(1, "id", IntegerType(), required=False),
1382+
NestedField(2, "value", DoubleType(), required=False),
1383+
)
1384+
partition_spec = PartitionSpec(
1385+
PartitionField(source_id=2, field_id=1000, transform=IdentityTransform(), name="value"),
1386+
)
1387+
tbl = catalog.create_table(f"default.{name}_{catalog.name}", schema=iceberg_schema, partition_spec=partition_spec)
1388+
tbl.append(_id_value_rows(tbl, [1, 2, 3, 4], [1.0, 1.0, float("nan"), float("nan")]))
1389+
return tbl
1390+
1391+
1392+
def _id_value_rows(tbl: Table, ids: list[int], values: list[float]) -> pa.Table:
1393+
return pa.Table.from_pydict({"id": ids, "value": values}, schema=schema_to_pyarrow(tbl.schema()))
1394+
1395+
1396+
def test_replace_files_in_nan_identity_partition(catalog: Catalog) -> None:
1397+
catalog.create_namespace("default")
1398+
cases: list[tuple[str, Callable[[Table], None], list[int], list[int]]] = [
1399+
("delete_nan_file", lambda t: t.delete("value is nan"), [1, 2], []),
1400+
("rewrite_nan_file", lambda t: t.delete("id = 3"), [1, 2, 4], [4]),
1401+
("overwrite", lambda t: t.overwrite(_id_value_rows(t, [9], [2.0])), [9], []),
1402+
(
1403+
"dynamic_overwrite",
1404+
lambda t: t.dynamic_partition_overwrite(_id_value_rows(t, [9], [float("nan")])),
1405+
[1, 2, 9],
1406+
[9],
1407+
),
1408+
]
1409+
for name, operation, expected_ids, expected_nan_ids in cases:
1410+
tbl = _nan_partitioned_table(catalog, name)
1411+
1412+
operation(tbl)
1413+
1414+
assert sorted(tbl.scan().to_arrow()["id"].to_pylist()) == expected_ids, name
1415+
assert sorted(tbl.scan(row_filter="value is nan").to_arrow()["id"].to_pylist()) == expected_nan_ids, name
1416+
1417+
13791418
def test_scan_integer_column_with_decimal_literal(catalog: Catalog) -> None:
13801419
catalog.create_namespace("default")
13811420
identifier = f"default.scan_integer_decimal_literal_{catalog.name}"

‎tests/table/test_commit_retry.py‎

Lines changed: 82 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,10 @@
2323
from pyiceberg.catalog import Catalog
2424
from pyiceberg.exceptions import CommitFailedException, CommitStateUnknownException, ValidationException
2525
from pyiceberg.schema import Schema
26-
from pyiceberg.table import TableProperties, Transaction
26+
from pyiceberg.table import Table, TableProperties, Transaction
2727
from pyiceberg.table.snapshots import IsolationLevel, Operation
28-
from pyiceberg.types import LongType, NestedField, StringType
28+
from pyiceberg.typedef import Record
29+
from pyiceberg.types import DoubleType, LongType, NestedField, StringType
2930

3031

3132
def test_isolation_level_enum() -> None:
@@ -395,6 +396,85 @@ def test_file_overwrite_validates_concurrent_file_delete(
395396
assert sorted(result["value"].to_pylist()) == expected_values
396397

397398

399+
def _commit_partition_position_delete(table: Table, partition: Record, file_path: str, pos: int) -> None:
400+
"""PyIceberg cannot write delete files, so commit one without path bounds, as other engines write them."""
401+
import pyarrow as pa
402+
import pyarrow.parquet as pq
403+
404+
from pyiceberg.manifest import DataFile, DataFileContent, FileFormat, ManifestContent, ManifestWriter, ManifestWriterV2
405+
from pyiceberg.partitioning import PartitionSpec
406+
from pyiceberg.table.update.snapshot import _OverwriteFiles
407+
408+
class DeleteManifestWriter(ManifestWriterV2):
409+
def content(self) -> ManifestContent:
410+
return ManifestContent.DELETES
411+
412+
@property
413+
def _meta(self) -> dict[str, str]:
414+
return {**super()._meta, "content": "deletes"}
415+
416+
def new_delete_manifest_writer(producer: _OverwriteFiles, spec: PartitionSpec) -> ManifestWriter:
417+
return DeleteManifestWriter(
418+
spec, producer.schema(), producer.new_manifest_output(), producer._snapshot_id, producer._compression
419+
)
420+
421+
delete_path = table.location_provider().new_data_location(f"pos-delete-{uuid.uuid4()}.parquet")
422+
with table.io.new_output(delete_path).create() as output:
423+
pq.write_table(pa.table({"file_path": [file_path], "pos": [pos]}), output)
424+
delete_file = DataFile.from_args(
425+
content=DataFileContent.POSITION_DELETES,
426+
file_path=delete_path,
427+
file_format=FileFormat.PARQUET,
428+
partition=partition,
429+
record_count=1,
430+
file_size_in_bytes=len(table.io.new_input(delete_path)),
431+
spec_id=table.spec().spec_id,
432+
)
433+
434+
with patch.object(_OverwriteFiles, "new_manifest_writer", new_delete_manifest_writer):
435+
with table.transaction() as transaction, transaction.update_snapshot().overwrite() as overwrite:
436+
overwrite.append_data_file(delete_file)
437+
438+
439+
def test_file_overwrite_validates_concurrent_position_delete_in_nan_partition(catalog: Catalog) -> None:
440+
"""A file replacement in a NaN partition must fail when a delete for that file was committed concurrently."""
441+
import pyarrow as pa
442+
443+
from pyiceberg.io.pyarrow import _dataframe_to_data_files
444+
from pyiceberg.partitioning import PartitionField, PartitionSpec
445+
from pyiceberg.transforms import IdentityTransform
446+
447+
catalog.create_namespace("default")
448+
schema = Schema(
449+
NestedField(1, "id", LongType(), required=False),
450+
NestedField(2, "value", DoubleType(), required=False),
451+
)
452+
spec = PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=IdentityTransform(), name="value"))
453+
identifier = "default.concurrent_nan_position_delete"
454+
table = catalog.create_table(identifier, schema=schema, partition_spec=spec, properties={"format-version": "2"})
455+
rows = pa.table({"id": pa.array([1, 2], pa.int64()), "value": [float("nan"), float("nan")]})
456+
table.append(rows)
457+
file_to_replace = next(task.file for task in table.scan().plan_files())
458+
459+
replacing_table = catalog.load_table(identifier)
460+
deleting_table = catalog.load_table(identifier)
461+
462+
replacement_file = list(
463+
_dataframe_to_data_files(table_metadata=replacing_table.metadata, df=rows, io=replacing_table.io, write_uuid=uuid.uuid4())
464+
)[0]
465+
replacing_transaction = replacing_table.transaction()
466+
with replacing_transaction.update_snapshot().overwrite() as overwrite:
467+
overwrite.delete_data_file(file_to_replace)
468+
overwrite.append_data_file(replacement_file)
469+
470+
_commit_partition_position_delete(deleting_table, Record(float("nan")), file_to_replace.file_path, pos=0)
471+
472+
with pytest.raises(ValidationException, match="found new delete for replace data file"):
473+
replacing_transaction.commit_transaction()
474+
475+
assert catalog.load_table(identifier).scan().to_arrow()["id"].to_pylist() == [2]
476+
477+
398478
def test_concurrent_overwrite_append_retries_successfully(catalog: Catalog) -> None:
399479
"""Append after a concurrent overwrite should succeed via retry."""
400480
catalog.create_namespace("default")

‎tests/table/test_init.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@
3333
And,
3434
EqualTo,
3535
In,
36+
IsNaN,
37+
IsNull,
3638
)
3739
from pyiceberg.expressions.visitors import bind
3840
from pyiceberg.io import PY_IO_IMPL, FileIO, load_file_io
@@ -2267,6 +2269,16 @@ def test_build_large_partition_predicate(table_v2: Table) -> None:
22672269
bind(table_v2.metadata.schema(), expr, case_sensitive=True)
22682270

22692271

2272+
def test_build_partition_predicate_with_nan_and_null(table_v2: Table) -> None:
2273+
with table_v2.transaction() as tx:
2274+
expr = tx._build_partition_predicate(
2275+
partition_records={Record(float("nan"), None, 1.0)},
2276+
partition_fields=["a", "b", "c"],
2277+
)
2278+
2279+
assert expr == And(IsNaN("a"), IsNull("b"), EqualTo("c", 1.0))
2280+
2281+
22702282
def test_overwrite_delete_data_file_on_bucket_partition(catalog: Catalog) -> None:
22712283
"""Delete a data file without projecting its already-transformed partition value again."""
22722284
import pyarrow as pa

0 commit comments

Comments
 (0)