Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions pyiceberg/io/pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -3360,6 +3360,10 @@ def _determine_partitions(spec: PartitionSpec, schema: Schema, arrow_table: pa.T
# PyArrow has no group by or equality kernels for extension types such as arrow.uuid, so use their storage type
if isinstance(partition_array.type, pa.BaseExtensionType):
partition_array = partition_array.cast(partition_array.type.storage_type)
# Arrow groups floats by their bits, but NaNs with any sign or payload are one partition value
if pa.types.is_floating(partition_array.type):
nan = pa.scalar(math.nan, type=partition_array.type)
partition_array = pc.if_else(pc.is_nan(partition_array), nan, partition_array)
arrow_table = arrow_table.append_column(name, partition_array)

unique_partition_fields = arrow_table.select(partition_fields).group_by(partition_fields).aggregate([])
Expand Down
3 changes: 2 additions & 1 deletion tests/catalog/test_catalog_behaviors.py
Original file line number Diff line number Diff line change
Expand Up @@ -1493,7 +1493,8 @@ def test_append_nan_to_identity_partitioned_table(catalog: Catalog) -> None:

tbl.append(
pa.Table.from_pydict(
{"id": [1, 2, 3, 4], "value": [1.0, float("nan"), None, float("nan")]},
# -float("nan") has the sign bit set, but both NaNs are one partition value
{"id": [1, 2, 3, 4], "value": [1.0, float("nan"), None, -float("nan")]},
schema=schema_to_pyarrow(iceberg_schema),
)
)
Expand Down
28 changes: 28 additions & 0 deletions tests/io/test_pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -3475,6 +3475,34 @@ def test_partition_for_deep_nested_field() -> None:
assert partition_values == {"data-1", "data-2"}


def test_determine_partitions_identity_nan_sign_bits() -> None:
for field_type in (DoubleType(), FloatType()):
schema = Schema(
NestedField(id=1, name="id", field_type=IntegerType(), required=False),
NestedField(id=2, name="value", field_type=field_type, required=False),
)
spec = PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=IdentityTransform(), name="value"))
arrow_table = pa.Table.from_pydict(
{"id": [1, 2, 3, 4], "value": [float("nan"), -float("nan"), 1.0, None]},
schema=schema.as_arrow(),
)

partitions = list(_determine_partitions(spec, schema, arrow_table))

# Both NaNs are one partition value, so each row is in exactly one partition
rows_by_partition = sorted(
(repr(p.partition_key.partition[0]), sorted(p.arrow_table_partition["id"].to_pylist())) for p in partitions
)
assert rows_by_partition == [("1.0", [3]), ("None", [4]), ("nan", [1, 2])], field_type

# Only the partition value is normalized, the data keeps the sign bit of each NaN
nan_partition = next(p for p in partitions if repr(p.partition_key.partition[0]) == "nan")
assert [struct.pack(">d", v).hex() for v in nan_partition.arrow_table_partition["value"].to_pylist()] == [
"7ff8000000000000",
"fff8000000000000",
], field_type


def test_determine_partitions_identity_nan() -> None:
schema = Schema(
NestedField(id=1, name="id", field_type=IntegerType(), required=False),
Expand Down
Loading