Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 76 additions & 0 deletions cpp/src/arrow/array/array_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -2543,6 +2543,39 @@ void CheckFloatApproxEqualsWithUlpDistance() {
ASSERT_FALSE(array_a->Equals(array_b, options.nans_equal(true).ulp_distance(1)));
}

template <typename Type>
void CheckFloatEqualsWithPhysicalRepresentation() {
using CType =
std::conditional_t<is_half_float_type<Type>::value, Float16, typename Type::c_type>;
using UIntType =
std::conditional_t<sizeof(CType) == 2, uint16_t,
std::conditional_t<sizeof(CType) == 4, uint32_t, uint64_t>>;

auto nan_value_1 = CType(NAN);
auto nan_value_2 = util::SafeCopy<CType>(
static_cast<UIntType>(util::SafeCopy<UIntType>(nan_value_1) + 1));

std::shared_ptr<Array> arr1;
std::shared_ptr<Array> arr2;
std::shared_ptr<Array> arr3;

ArrayFromVector<Type, CType>({CType(1), CType(2), CType(3), nan_value_1}, &arr1);
ArrayFromVector<Type, CType>({CType(1), CType(2), CType(3), nan_value_2}, &arr2);
ArrayFromVector<Type, CType>({CType(1), CType(2), CType(3), nan_value_1}, &arr3);

auto opts = EqualOptions::Defaults();

ASSERT_FALSE(arr1->Equals(*arr2, opts));
ASSERT_FALSE(arr1->Equals(*arr3, opts));

// nan_equal is false
ASSERT_FALSE(arr1->Equals(*arr2, opts.physical_repr_equal(true)));
ASSERT_TRUE(arr1->Equals(*arr3, opts.physical_repr_equal(true)));

ASSERT_TRUE(arr1->Equals(*arr2, opts.physical_repr_equal(true).nans_equal(true)));
ASSERT_TRUE(arr1->Equals(*arr3, opts.physical_repr_equal(true).nans_equal(true)));
}

TEST(TestPrimitiveAdHoc, FloatingApproxEquals) {
CheckApproxEquals<FloatType>();
CheckApproxEquals<DoubleType>();
Expand Down Expand Up @@ -2582,6 +2615,49 @@ TEST(TestPrimitiveAdHoc, FloatingUlpDistanceEquality) {
CheckFloatApproxEqualsWithUlpDistance<DoubleType>();
}

TEST(TestPrimitiveAdHoc, FloatingPhysicalRepresentationEquality) {
CheckFloatEqualsWithPhysicalRepresentation<HalfFloatType>();
CheckFloatEqualsWithPhysicalRepresentation<FloatType>();
CheckFloatEqualsWithPhysicalRepresentation<DoubleType>();
}

TEST(TestRunEncodedBuilder, MergeConsecutiveNaNs) {
auto run_end_encoded_type = run_end_encoded(int32(), float32());
std::shared_ptr<RunEndEncodedArray> result;
auto run_end_builder = std::make_shared<Int32Builder>();
auto value_builder = std::make_shared<FloatBuilder>();
RunEndEncodedBuilder builder(default_memory_pool(), run_end_builder, value_builder,
run_end_encoded_type);
ASSERT_OK(builder.AppendScalar(**MakeScalar(float32(), 1), 15));
ASSERT_OK(builder.AppendScalar(**MakeScalar(float32(), 1), 10));
ASSERT_OK(builder.AppendScalar(**MakeScalar(float32(), 2), 20));
ASSERT_OK(builder.AppendScalar(
**MakeScalar(float32(), std::numeric_limits<float>::quiet_NaN()), 25));
ASSERT_OK(builder.AppendScalar(
**MakeScalar(float32(), std::numeric_limits<float>::signaling_NaN()), 5));
ASSERT_OK(builder.AppendScalar(**MakeScalar(float32(), 3), 15));
ASSERT_OK(builder.AppendScalar(**MakeScalar(float32(), std::nanf("")), 20));
ASSERT_OK(builder.AppendScalar(**MakeScalar(float32(), 4), 25));
ASSERT_OK(builder.AppendScalar(**MakeScalar(float32(), +0.0), 10));
ASSERT_OK(builder.AppendScalar(**MakeScalar(float32(), 5.0), 15));
ASSERT_OK(builder.AppendScalar(**MakeScalar(float32(), -0.0), 25));
Comment on lines +2634 to +2643

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could this test append the same NaN in two separate AppendScalar calls? The consecutive NaNs here are quiet versus signaling, which should remain separate. The other NaN is isolated, and +0 and -0 are separated by 5.

As a narrow negative control, reverting only the builder's Equals(scalar, options_) call to Equals(scalar), while keeping the rest of the current code and tests unchanged, still leaves this test passing. An independent repeated-NaN probe fails under that same reversion.

Two calls to AppendScalar(nan, 3) with identical NaN bits should produce run ends [6], not [3, 6]. Could we cover that case, two quiet NaNs with different payloads, and adjacent signed zeros, asserting both the run boundaries and exact decoded bits? This would protect the intended merge without allowing payload or sign loss.

ASSERT_OK(builder.Finish(&result));

std::shared_ptr<Array> expected_run_end_array;
std::shared_ptr<Array> expected_value_array;
ArrayFromVector<Int32Type>({25, 45, 70, 75, 90, 110, 135, 145, 160, 185},
&expected_run_end_array);
ArrayFromVector<FloatType>({1.0, 2.0, std::numeric_limits<float>::quiet_NaN(),
std::numeric_limits<float>::signaling_NaN(), 3.0,
std::nanf(""), 4.0, +0.0, 5, -0.0},
&expected_value_array);

AssertArraysEqual(*expected_run_end_array, *result->run_ends());
AssertArraysEqual(
*expected_value_array, *result->values(), false,
EqualOptions::Defaults().signed_zeros_equal(false).physical_repr_equal(true));
}

// ----------------------------------------------------------------------
// FixedSizeBinary tests

Expand Down
4 changes: 3 additions & 1 deletion cpp/src/arrow/array/builder_run_end.cc
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
#include <utility>
#include <vector>

#include "arrow/array/array_run_end.h"
#include "arrow/array/util.h"
#include "arrow/scalar.h"
#include "arrow/util/checked_cast.h"
#include "arrow/util/int_util_overflow.h"
Expand Down Expand Up @@ -102,7 +104,7 @@ Status RunCompressorBuilder::AppendScalar(const Scalar& scalar, int64_t n_repeat
current_value_ = scalar.is_valid ? scalar.shared_from_this() : NULLPTR;
current_run_length_ = n_repeats;
} else if ((current_value_ == NULLPTR && !scalar.is_valid) ||
(current_value_ != NULLPTR && current_value_->Equals(scalar))) {
(current_value_ != NULLPTR && current_value_->Equals(scalar, options_))) {
// Extend the currently open run
current_run_length_ += n_repeats;
} else {
Expand Down
6 changes: 5 additions & 1 deletion cpp/src/arrow/array/builder_run_end.h
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,10 @@
#include <utility>
#include <vector>

#include "arrow/array.h"
#include "arrow/array/array_base.h"
#include "arrow/array/array_run_end.h"
#include "arrow/array/builder_base.h"
#include "arrow/compare.h"

namespace arrow {

Expand Down Expand Up @@ -161,6 +163,8 @@ class RunCompressorBuilder : public ArrayBuilder {
std::shared_ptr<ArrayBuilder> inner_builder_;
std::shared_ptr<const Scalar> current_value_ = NULLPTR;
int64_t current_run_length_ = 0;
const EqualOptions options_ =
EqualOptions::Defaults().signed_zeros_equal(false).physical_repr_equal(true);
};

} // namespace internal
Expand Down
46 changes: 39 additions & 7 deletions cpp/src/arrow/compare.cc
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
#include "arrow/util/macros.h"
#include "arrow/util/memory_internal.h"
#include "arrow/util/ree_util.h"
#include "arrow/util/ubsan.h"
#include "arrow/util/ulp_distance_internal.h"
#include "arrow/util/unreachable.h"
#include "arrow/visit_scalar_inline.h"
Expand All @@ -73,12 +74,13 @@ using util::Float16;
namespace {

template <bool AbsoluteTolerance, bool NansEqual, bool SignedZerosEqual,
bool UlpDistanceEqual>
bool UlpDistanceEqual, bool PhysicalValueEqual>
struct FloatingEqualityFlags {
static constexpr bool absolute_tolerance = AbsoluteTolerance;
static constexpr bool nans_equal = NansEqual;
static constexpr bool signed_zeros_equal = SignedZerosEqual;
static constexpr bool ulp_distance_equal = UlpDistanceEqual;
static constexpr bool physical_Value_Equal = PhysicalValueEqual;
};

template <typename T, typename Flags>
Expand All @@ -88,6 +90,17 @@ struct FloatingEquality {
ulp_distance(options.ulp_distance().value_or(-1)) {}

bool operator()(T x, T y) const {
if constexpr (Flags::physical_Value_Equal) {
using UIntType = std::conditional_t<sizeof(T) == 4, uint32_t, uint64_t>;

auto uint_x = util::SafeCopy<UIntType>(x);
auto uint_y = util::SafeCopy<UIntType>(y);

if (uint_x == uint_y) {
return true;
}
}

if (x == y) {
return Flags::signed_zeros_equal || (std::signbit(x) == std::signbit(y));
}
Expand Down Expand Up @@ -123,6 +136,13 @@ struct FloatingEquality<uint16_t, Flags> {
bool operator()(uint16_t x, uint16_t y) const {
Float16 f_x = Float16::FromBits(x);
Float16 f_y = Float16::FromBits(y);

if constexpr (Flags::physical_Value_Equal) {
if (x == y) {
return true;
}
}

if (f_x == f_y) {
return Flags::signed_zeros_equal || (f_x.signbit() == f_y.signbit());
}
Expand Down Expand Up @@ -152,16 +172,28 @@ struct FloatingEqualityDispatcher {
const EqualOptions& options;
Visitor&& visit;

template <bool AbsoluteTolerance, bool NansEqual, bool SignedZero,
bool UlpDistanceEqual>
void DispatchL5() {
if (options.physical_repr_equal()) {
visit(
FloatingEquality<T, FloatingEqualityFlags<AbsoluteTolerance, NansEqual,
SignedZero, UlpDistanceEqual, true>>{
options});
} else {
visit(
FloatingEquality<T, FloatingEqualityFlags<AbsoluteTolerance, NansEqual,
SignedZero, UlpDistanceEqual, false>>{
options});
}
}

template <bool AbsoluteTolerance, bool NansEqual, bool SignedZero>
void DispatchL4() {
if (options.ulp_distance()) {
visit(FloatingEquality<
T, FloatingEqualityFlags<AbsoluteTolerance, NansEqual, SignedZero, true>>{
options});
DispatchL5<AbsoluteTolerance, NansEqual, SignedZero, true>();
} else {
visit(FloatingEquality<
T, FloatingEqualityFlags<AbsoluteTolerance, NansEqual, SignedZero, false>>{
options});
DispatchL5<AbsoluteTolerance, NansEqual, SignedZero, false>();
}
}

Expand Down
14 changes: 14 additions & 0 deletions cpp/src/arrow/compare.h
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,19 @@ class EqualOptions {
return res;
}

/// Whether to compare floating-point values by their physical representation.
bool physical_repr_equal() const { return physical_repr_equal_; }

/// Return a copy of this EqualOptions object with the "physical_repr_equal"
/// property set to the specified value.
/// If both "physical_repr_equal" and "nans_equal" are enabled, two values
/// are considered equal if either condition is satisfied.
EqualOptions physical_repr_equal(bool v) const {
Comment on lines +111 to +118

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we also propagate this new option to sparse floating-point equality? FloatSparseTensorDataEquals currently only checks nans_equal(), so it ignores physical_repr_equal().

For independently allocated float32 and float64 tensors containing identical NaN bits, EqualOptions::Defaults().signed_zeros_equal(false).physical_repr_equal(true) gives true for dense tensor equality but false after conversion to COO sparse tensors. Enabling nans_equal(true) is not equivalent, since that also accepts different NaN payloads.

Since both equality APIs accept EqualOptions and this new flag has no documented sparse exclusion, could the sparse implementation reuse the common floating-point equality visitor, with regression coverage for identical and different NaN payloads?

auto res = EqualOptions(*this);
res.physical_repr_equal_ = v;
return res;
}

/// Whether the \ref arrow::Schema property is used in the comparison.
///
/// This option only affects the Equals methods
Expand Down Expand Up @@ -160,6 +173,7 @@ class EqualOptions {
std::optional<int32_t> ulp_distance_ = std::nullopt;
bool nans_equal_ = false;
bool signed_zeros_equal_ = true;
bool physical_repr_equal_ = false;
bool use_schema_ = true;
bool use_metadata_ = false;

Expand Down
6 changes: 4 additions & 2 deletions cpp/src/arrow/compute/kernels/vector_run_end_encode.cc
Original file line number Diff line number Diff line change
Expand Up @@ -306,8 +306,10 @@ struct RunEndEncodeExec {
auto input = input_array.ToArray();
// Avoid merging floating-point values whose representations may differ. Signed
// zeros compare unequal, and NaNs remain in separate runs to preserve payloads.
const auto equal_options =
EqualOptions::Defaults().nans_equal(false).signed_zeros_equal(false);
const auto equal_options = EqualOptions::Defaults()
.nans_equal(false)
.signed_zeros_equal(false)
.physical_repr_equal(true);
for (int64_t i = 1; i < input_length; ++i) {
if (!ArrayRangeEquals(*input, *input, i - 1, i, i, equal_options)) {
RETURN_NOT_OK(run_ends_builder.Append(static_cast<RunEndCType>(i)));
Expand Down
32 changes: 32 additions & 0 deletions cpp/src/arrow/compute/kernels/vector_run_end_encode_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
#include "arrow/builder.h"
#include "arrow/compute/api_vector.h"
#include "arrow/datum.h"
#include "arrow/testing/builder.h"
#include "arrow/testing/gtest_util.h"
#include "arrow/type_fwd.h"
#include "arrow/util/logging_internal.h"
Expand Down Expand Up @@ -628,5 +629,36 @@ TEST(TestRunEndEncodeDecodeNested, Struct) {
ArrayFromJSON(value_type, "[]"));
}

TEST(TestRunEndEncodeDecodeNested, NaN) {
auto value_type = struct_({field("a", int32()), field("b", float32())});
std::vector<uint8_t> valid_bytes(9, 1);
float s_nan = std::numeric_limits<float>::signaling_NaN();
float q_nan = std::numeric_limits<float>::quiet_NaN();

ASSERT_OK_AND_ASSIGN(auto raw_builder, MakeBuilder(value_type));
auto struct_builder =
arrow::internal::checked_pointer_cast<StructBuilder>(std::move(raw_builder));

auto int_builder = arrow::internal::checked_pointer_cast<Int32Builder>(
struct_builder->child_builder(0));

auto float_builder = arrow::internal::checked_pointer_cast<FloatBuilder>(
struct_builder->child_builder(1));

ASSERT_OK(int_builder->AppendValues({0, 0, 0, 1, 1, 1, 2, 2, 2}));
ASSERT_OK(float_builder->AppendValues(
{s_nan, s_nan, s_nan, 1.0, 1.0, 1.0, q_nan, q_nan, 1.0}));
ASSERT_OK(struct_builder->AppendValues(valid_bytes.size(), valid_bytes.data()));
ASSERT_OK_AND_ASSIGN(auto struct_array, struct_builder->Finish());

ASSERT_OK(struct_builder->AppendScalars(
{*struct_array->GetScalar(0), *struct_array->GetScalar(3),
*struct_array->GetScalar(6), *struct_array->GetScalar(8)}));

ASSERT_OK_AND_ASSIGN(auto expected_values, struct_builder->Finish());

AssertNestedRunEndEncodeDecode(struct_array, "[3, 6, 8, 9]", expected_values);
}

} // namespace compute
} // namespace arrow
Loading