From 7178d82a38685b205c911e2675f204be1a0cb553 Mon Sep 17 00:00:00 2001 From: arash andishgar Date: Mon, 14 Sep 2026 10:09:41 +0330 Subject: [PATCH 1/2] Enable NaN --- cpp/src/arrow/array/array_test.cc | 28 ++++++++++++++++++++++++++ cpp/src/arrow/array/builder_run_end.cc | 4 +++- cpp/src/arrow/array/builder_run_end.h | 5 ++++- 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/cpp/src/arrow/array/array_test.cc b/cpp/src/arrow/array/array_test.cc index 6e1bfed0d81e..584e21c8b3f6 100644 --- a/cpp/src/arrow/array/array_test.cc +++ b/cpp/src/arrow/array/array_test.cc @@ -2582,6 +2582,34 @@ TEST(TestPrimitiveAdHoc, FloatingUlpDistanceEquality) { CheckFloatApproxEqualsWithUlpDistance(); } +TEST(TestRunEncodedBuilder, MergeConsecutiveNaNs) { + auto run_end_encoded_type = run_end_encoded(int32(), float32()); + std::shared_ptr result; + auto run_end_builder = std::make_shared(); + auto value_builder = std::make_shared(); + RunEndEncodedBuilder builder(default_memory_pool(), run_end_builder, value_builder, + run_end_encoded_type); + ASSERT_OK(builder.AppendScalar(**MakeScalar(float32(), 1), 15)); + ASSERT_OK(builder.AppendScalar(**MakeScalar(float32(), 1), 10)); + ASSERT_OK(builder.AppendScalar(**MakeScalar(float32(), 2), 20)); + ASSERT_OK(builder.AppendScalar( + **MakeScalar(float32(), std::numeric_limits::quiet_NaN()), 25)); + ASSERT_OK(builder.AppendScalar( + **MakeScalar(float32(), std::numeric_limits::signaling_NaN()), 5)); + ASSERT_OK(builder.AppendScalar(**MakeScalar(float32(), 3), 15)); + ASSERT_OK(builder.AppendScalar(**MakeScalar(float32(), std::nanf("")), 20)); + ASSERT_OK(builder.AppendScalar(**MakeScalar(float32(), 4), 25)); + ASSERT_OK(builder.Finish(&result)); + + std::shared_ptr expected_run_end_array; + std::shared_ptr expected_value_array; + ArrayFromVector({25, 45, 75, 90, 110, 135}, &expected_run_end_array); + ArrayFromVector({1.0, 2.0, std::nanf(""), 3.0, std::nanf(""), 4.0}, + &expected_value_array); + AssertArraysEqual(*expected_run_end_array, *result->run_ends()); + AssertArraysEqual(*expected_value_array, *result->values()); +} + // ---------------------------------------------------------------------- // FixedSizeBinary tests diff --git a/cpp/src/arrow/array/builder_run_end.cc b/cpp/src/arrow/array/builder_run_end.cc index 2edeaff504d2..8b38783eb898 100644 --- a/cpp/src/arrow/array/builder_run_end.cc +++ b/cpp/src/arrow/array/builder_run_end.cc @@ -23,6 +23,8 @@ #include #include +#include "arrow/array/array_run_end.h" +#include "arrow/array/util.h" #include "arrow/scalar.h" #include "arrow/util/checked_cast.h" #include "arrow/util/int_util_overflow.h" @@ -102,7 +104,7 @@ Status RunCompressorBuilder::AppendScalar(const Scalar& scalar, int64_t n_repeat current_value_ = scalar.is_valid ? scalar.shared_from_this() : NULLPTR; current_run_length_ = n_repeats; } else if ((current_value_ == NULLPTR && !scalar.is_valid) || - (current_value_ != NULLPTR && current_value_->Equals(scalar))) { + (current_value_ != NULLPTR && current_value_->Equals(scalar, options_))) { // Extend the currently open run current_run_length_ += n_repeats; } else { diff --git a/cpp/src/arrow/array/builder_run_end.h b/cpp/src/arrow/array/builder_run_end.h index ac92efbd0dbe..43e59df1c6ef 100644 --- a/cpp/src/arrow/array/builder_run_end.h +++ b/cpp/src/arrow/array/builder_run_end.h @@ -23,8 +23,10 @@ #include #include -#include "arrow/array.h" +#include "arrow/array/array_base.h" +#include "arrow/array/array_run_end.h" #include "arrow/array/builder_base.h" +#include "arrow/compare.h" namespace arrow { @@ -161,6 +163,7 @@ class RunCompressorBuilder : public ArrayBuilder { std::shared_ptr inner_builder_; std::shared_ptr current_value_ = NULLPTR; int64_t current_run_length_ = 0; + const EqualOptions options_ = EqualOptions::Defaults().nans_equal(true); }; } // namespace internal From 94aa4690530849d0a53a3c8cd0ee750d1fc471ac Mon Sep 17 00:00:00 2001 From: arash andishgar Date: Tue, 10 Nov 2026 06:58:57 +0330 Subject: [PATCH 2/2] add use_physical_rpr --- cpp/src/arrow/array/array_test.cc | 54 +++++++++++++++++-- cpp/src/arrow/array/builder_run_end.h | 3 +- cpp/src/arrow/compare.cc | 46 +++++++++++++--- cpp/src/arrow/compare.h | 14 +++++ .../compute/kernels/vector_run_end_encode.cc | 6 ++- .../kernels/vector_run_end_encode_test.cc | 32 +++++++++++ 6 files changed, 142 insertions(+), 13 deletions(-) diff --git a/cpp/src/arrow/array/array_test.cc b/cpp/src/arrow/array/array_test.cc index 584e21c8b3f6..26e28e67b965 100644 --- a/cpp/src/arrow/array/array_test.cc +++ b/cpp/src/arrow/array/array_test.cc @@ -2543,6 +2543,39 @@ void CheckFloatApproxEqualsWithUlpDistance() { ASSERT_FALSE(array_a->Equals(array_b, options.nans_equal(true).ulp_distance(1))); } +template +void CheckFloatEqualsWithPhysicalRepresentation() { + using CType = + std::conditional_t::value, Float16, typename Type::c_type>; + using UIntType = + std::conditional_t>; + + auto nan_value_1 = CType(NAN); + auto nan_value_2 = util::SafeCopy( + static_cast(util::SafeCopy(nan_value_1) + 1)); + + std::shared_ptr arr1; + std::shared_ptr arr2; + std::shared_ptr arr3; + + ArrayFromVector({CType(1), CType(2), CType(3), nan_value_1}, &arr1); + ArrayFromVector({CType(1), CType(2), CType(3), nan_value_2}, &arr2); + ArrayFromVector({CType(1), CType(2), CType(3), nan_value_1}, &arr3); + + auto opts = EqualOptions::Defaults(); + + ASSERT_FALSE(arr1->Equals(*arr2, opts)); + ASSERT_FALSE(arr1->Equals(*arr3, opts)); + + // nan_equal is false + ASSERT_FALSE(arr1->Equals(*arr2, opts.physical_repr_equal(true))); + ASSERT_TRUE(arr1->Equals(*arr3, opts.physical_repr_equal(true))); + + ASSERT_TRUE(arr1->Equals(*arr2, opts.physical_repr_equal(true).nans_equal(true))); + ASSERT_TRUE(arr1->Equals(*arr3, opts.physical_repr_equal(true).nans_equal(true))); +} + TEST(TestPrimitiveAdHoc, FloatingApproxEquals) { CheckApproxEquals(); CheckApproxEquals(); @@ -2582,6 +2615,12 @@ TEST(TestPrimitiveAdHoc, FloatingUlpDistanceEquality) { CheckFloatApproxEqualsWithUlpDistance(); } +TEST(TestPrimitiveAdHoc, FloatingPhysicalRepresentationEquality) { + CheckFloatEqualsWithPhysicalRepresentation(); + CheckFloatEqualsWithPhysicalRepresentation(); + CheckFloatEqualsWithPhysicalRepresentation(); +} + TEST(TestRunEncodedBuilder, MergeConsecutiveNaNs) { auto run_end_encoded_type = run_end_encoded(int32(), float32()); std::shared_ptr result; @@ -2599,15 +2638,24 @@ TEST(TestRunEncodedBuilder, MergeConsecutiveNaNs) { ASSERT_OK(builder.AppendScalar(**MakeScalar(float32(), 3), 15)); ASSERT_OK(builder.AppendScalar(**MakeScalar(float32(), std::nanf("")), 20)); ASSERT_OK(builder.AppendScalar(**MakeScalar(float32(), 4), 25)); + ASSERT_OK(builder.AppendScalar(**MakeScalar(float32(), +0.0), 10)); + ASSERT_OK(builder.AppendScalar(**MakeScalar(float32(), 5.0), 15)); + ASSERT_OK(builder.AppendScalar(**MakeScalar(float32(), -0.0), 25)); ASSERT_OK(builder.Finish(&result)); std::shared_ptr expected_run_end_array; std::shared_ptr expected_value_array; - ArrayFromVector({25, 45, 75, 90, 110, 135}, &expected_run_end_array); - ArrayFromVector({1.0, 2.0, std::nanf(""), 3.0, std::nanf(""), 4.0}, + ArrayFromVector({25, 45, 70, 75, 90, 110, 135, 145, 160, 185}, + &expected_run_end_array); + ArrayFromVector({1.0, 2.0, std::numeric_limits::quiet_NaN(), + std::numeric_limits::signaling_NaN(), 3.0, + std::nanf(""), 4.0, +0.0, 5, -0.0}, &expected_value_array); + AssertArraysEqual(*expected_run_end_array, *result->run_ends()); - AssertArraysEqual(*expected_value_array, *result->values()); + AssertArraysEqual( + *expected_value_array, *result->values(), false, + EqualOptions::Defaults().signed_zeros_equal(false).physical_repr_equal(true)); } // ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/array/builder_run_end.h b/cpp/src/arrow/array/builder_run_end.h index 43e59df1c6ef..057899746391 100644 --- a/cpp/src/arrow/array/builder_run_end.h +++ b/cpp/src/arrow/array/builder_run_end.h @@ -163,7 +163,8 @@ class RunCompressorBuilder : public ArrayBuilder { std::shared_ptr inner_builder_; std::shared_ptr current_value_ = NULLPTR; int64_t current_run_length_ = 0; - const EqualOptions options_ = EqualOptions::Defaults().nans_equal(true); + const EqualOptions options_ = + EqualOptions::Defaults().signed_zeros_equal(false).physical_repr_equal(true); }; } // namespace internal diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc index 06f1d14e73a3..c52368b93569 100644 --- a/cpp/src/arrow/compare.cc +++ b/cpp/src/arrow/compare.cc @@ -53,6 +53,7 @@ #include "arrow/util/macros.h" #include "arrow/util/memory_internal.h" #include "arrow/util/ree_util.h" +#include "arrow/util/ubsan.h" #include "arrow/util/ulp_distance_internal.h" #include "arrow/util/unreachable.h" #include "arrow/visit_scalar_inline.h" @@ -73,12 +74,13 @@ using util::Float16; namespace { template + bool UlpDistanceEqual, bool PhysicalValueEqual> struct FloatingEqualityFlags { static constexpr bool absolute_tolerance = AbsoluteTolerance; static constexpr bool nans_equal = NansEqual; static constexpr bool signed_zeros_equal = SignedZerosEqual; static constexpr bool ulp_distance_equal = UlpDistanceEqual; + static constexpr bool physical_Value_Equal = PhysicalValueEqual; }; template @@ -88,6 +90,17 @@ struct FloatingEquality { ulp_distance(options.ulp_distance().value_or(-1)) {} bool operator()(T x, T y) const { + if constexpr (Flags::physical_Value_Equal) { + using UIntType = std::conditional_t; + + auto uint_x = util::SafeCopy(x); + auto uint_y = util::SafeCopy(y); + + if (uint_x == uint_y) { + return true; + } + } + if (x == y) { return Flags::signed_zeros_equal || (std::signbit(x) == std::signbit(y)); } @@ -123,6 +136,13 @@ struct FloatingEquality { bool operator()(uint16_t x, uint16_t y) const { Float16 f_x = Float16::FromBits(x); Float16 f_y = Float16::FromBits(y); + + if constexpr (Flags::physical_Value_Equal) { + if (x == y) { + return true; + } + } + if (f_x == f_y) { return Flags::signed_zeros_equal || (f_x.signbit() == f_y.signbit()); } @@ -152,16 +172,28 @@ struct FloatingEqualityDispatcher { const EqualOptions& options; Visitor&& visit; + template + void DispatchL5() { + if (options.physical_repr_equal()) { + visit( + FloatingEquality>{ + options}); + } else { + visit( + FloatingEquality>{ + options}); + } + } + template void DispatchL4() { if (options.ulp_distance()) { - visit(FloatingEquality< - T, FloatingEqualityFlags>{ - options}); + DispatchL5(); } else { - visit(FloatingEquality< - T, FloatingEqualityFlags>{ - options}); + DispatchL5(); } } diff --git a/cpp/src/arrow/compare.h b/cpp/src/arrow/compare.h index 27b82c34c44a..500f7ad74f37 100644 --- a/cpp/src/arrow/compare.h +++ b/cpp/src/arrow/compare.h @@ -108,6 +108,19 @@ class EqualOptions { return res; } + /// Whether to compare floating-point values by their physical representation. + bool physical_repr_equal() const { return physical_repr_equal_; } + + /// Return a copy of this EqualOptions object with the "physical_repr_equal" + /// property set to the specified value. + /// If both "physical_repr_equal" and "nans_equal" are enabled, two values + /// are considered equal if either condition is satisfied. + EqualOptions physical_repr_equal(bool v) const { + auto res = EqualOptions(*this); + res.physical_repr_equal_ = v; + return res; + } + /// Whether the \ref arrow::Schema property is used in the comparison. /// /// This option only affects the Equals methods @@ -160,6 +173,7 @@ class EqualOptions { std::optional ulp_distance_ = std::nullopt; bool nans_equal_ = false; bool signed_zeros_equal_ = true; + bool physical_repr_equal_ = false; bool use_schema_ = true; bool use_metadata_ = false; diff --git a/cpp/src/arrow/compute/kernels/vector_run_end_encode.cc b/cpp/src/arrow/compute/kernels/vector_run_end_encode.cc index 9462ece25a9a..5f4e13c981e1 100644 --- a/cpp/src/arrow/compute/kernels/vector_run_end_encode.cc +++ b/cpp/src/arrow/compute/kernels/vector_run_end_encode.cc @@ -306,8 +306,10 @@ struct RunEndEncodeExec { auto input = input_array.ToArray(); // Avoid merging floating-point values whose representations may differ. Signed // zeros compare unequal, and NaNs remain in separate runs to preserve payloads. - const auto equal_options = - EqualOptions::Defaults().nans_equal(false).signed_zeros_equal(false); + const auto equal_options = EqualOptions::Defaults() + .nans_equal(false) + .signed_zeros_equal(false) + .physical_repr_equal(true); for (int64_t i = 1; i < input_length; ++i) { if (!ArrayRangeEquals(*input, *input, i - 1, i, i, equal_options)) { RETURN_NOT_OK(run_ends_builder.Append(static_cast(i))); diff --git a/cpp/src/arrow/compute/kernels/vector_run_end_encode_test.cc b/cpp/src/arrow/compute/kernels/vector_run_end_encode_test.cc index 68ab4543922f..91676dbfba07 100644 --- a/cpp/src/arrow/compute/kernels/vector_run_end_encode_test.cc +++ b/cpp/src/arrow/compute/kernels/vector_run_end_encode_test.cc @@ -22,6 +22,7 @@ #include "arrow/builder.h" #include "arrow/compute/api_vector.h" #include "arrow/datum.h" +#include "arrow/testing/builder.h" #include "arrow/testing/gtest_util.h" #include "arrow/type_fwd.h" #include "arrow/util/logging_internal.h" @@ -628,5 +629,36 @@ TEST(TestRunEndEncodeDecodeNested, Struct) { ArrayFromJSON(value_type, "[]")); } +TEST(TestRunEndEncodeDecodeNested, NaN) { + auto value_type = struct_({field("a", int32()), field("b", float32())}); + std::vector valid_bytes(9, 1); + float s_nan = std::numeric_limits::signaling_NaN(); + float q_nan = std::numeric_limits::quiet_NaN(); + + ASSERT_OK_AND_ASSIGN(auto raw_builder, MakeBuilder(value_type)); + auto struct_builder = + arrow::internal::checked_pointer_cast(std::move(raw_builder)); + + auto int_builder = arrow::internal::checked_pointer_cast( + struct_builder->child_builder(0)); + + auto float_builder = arrow::internal::checked_pointer_cast( + struct_builder->child_builder(1)); + + ASSERT_OK(int_builder->AppendValues({0, 0, 0, 1, 1, 1, 2, 2, 2})); + ASSERT_OK(float_builder->AppendValues( + {s_nan, s_nan, s_nan, 1.0, 1.0, 1.0, q_nan, q_nan, 1.0})); + ASSERT_OK(struct_builder->AppendValues(valid_bytes.size(), valid_bytes.data())); + ASSERT_OK_AND_ASSIGN(auto struct_array, struct_builder->Finish()); + + ASSERT_OK(struct_builder->AppendScalars( + {*struct_array->GetScalar(0), *struct_array->GetScalar(3), + *struct_array->GetScalar(6), *struct_array->GetScalar(8)})); + + ASSERT_OK_AND_ASSIGN(auto expected_values, struct_builder->Finish()); + + AssertNestedRunEndEncodeDecode(struct_array, "[3, 6, 8, 9]", expected_values); +} + } // namespace compute } // namespace arrow