From 0a4c0a7611b5f6d588d2c2800b10e12e0e8362cb Mon Sep 17 00:00:00 2001 From: rahulsmahadev Date: Thu, 19 Nov 2026 17:11:29 +0000 Subject: [PATCH] Group NaNs with different sign bits into one identity partition _determine_partitions groups identity values by raw bits, so NaNs with different sign bits form separate partitions. Each partition selects its rows with is_nan, which matches every NaN row, so each NaN row was written into every NaN partition. Canonicalize NaN in the partition helper column before grouping. Data is unchanged. Co-authored-by: Isaac --- pyiceberg/io/pyarrow.py | 4 ++++ tests/catalog/test_catalog_behaviors.py | 3 ++- tests/io/test_pyarrow.py | 28 +++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 56dbdf5c20..391a737dfd 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -3360,6 +3360,10 @@ def _determine_partitions(spec: PartitionSpec, schema: Schema, arrow_table: pa.T # PyArrow has no group by or equality kernels for extension types such as arrow.uuid, so use their storage type if isinstance(partition_array.type, pa.BaseExtensionType): partition_array = partition_array.cast(partition_array.type.storage_type) + # Arrow groups floats by their bits, but NaNs with any sign or payload are one partition value + if pa.types.is_floating(partition_array.type): + nan = pa.scalar(math.nan, type=partition_array.type) + partition_array = pc.if_else(pc.is_nan(partition_array), nan, partition_array) arrow_table = arrow_table.append_column(name, partition_array) unique_partition_fields = arrow_table.select(partition_fields).group_by(partition_fields).aggregate([]) diff --git a/tests/catalog/test_catalog_behaviors.py b/tests/catalog/test_catalog_behaviors.py index a5099e4e6b..2d1c8e1e9c 100644 --- a/tests/catalog/test_catalog_behaviors.py +++ b/tests/catalog/test_catalog_behaviors.py @@ -1493,7 +1493,8 @@ def test_append_nan_to_identity_partitioned_table(catalog: Catalog) -> None: tbl.append( pa.Table.from_pydict( - {"id": [1, 2, 3, 4], "value": [1.0, float("nan"), None, float("nan")]}, + # -float("nan") has the sign bit set, but both NaNs are one partition value + {"id": [1, 2, 3, 4], "value": [1.0, float("nan"), None, -float("nan")]}, schema=schema_to_pyarrow(iceberg_schema), ) ) diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index ca21fd05f0..9a4683aff2 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -3475,6 +3475,34 @@ def test_partition_for_deep_nested_field() -> None: assert partition_values == {"data-1", "data-2"} +def test_determine_partitions_identity_nan_sign_bits() -> None: + for field_type in (DoubleType(), FloatType()): + schema = Schema( + NestedField(id=1, name="id", field_type=IntegerType(), required=False), + NestedField(id=2, name="value", field_type=field_type, required=False), + ) + spec = PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=IdentityTransform(), name="value")) + arrow_table = pa.Table.from_pydict( + {"id": [1, 2, 3, 4], "value": [float("nan"), -float("nan"), 1.0, None]}, + schema=schema.as_arrow(), + ) + + partitions = list(_determine_partitions(spec, schema, arrow_table)) + + # Both NaNs are one partition value, so each row is in exactly one partition + rows_by_partition = sorted( + (repr(p.partition_key.partition[0]), sorted(p.arrow_table_partition["id"].to_pylist())) for p in partitions + ) + assert rows_by_partition == [("1.0", [3]), ("None", [4]), ("nan", [1, 2])], field_type + + # Only the partition value is normalized, the data keeps the sign bit of each NaN + nan_partition = next(p for p in partitions if repr(p.partition_key.partition[0]) == "nan") + assert [struct.pack(">d", v).hex() for v in nan_partition.arrow_table_partition["value"].to_pylist()] == [ + "7ff8000000000000", + "fff8000000000000", + ], field_type + + def test_determine_partitions_identity_nan() -> None: schema = Schema( NestedField(id=1, name="id", field_type=IntegerType(), required=False),