diff --git a/cpp/src/arrow/compute/kernels/scalar_cast_test.cc b/cpp/src/arrow/compute/kernels/scalar_cast_test.cc index 864ec4afce0c..6fc65665977e 100644 --- a/cpp/src/arrow/compute/kernels/scalar_cast_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_cast_test.cc @@ -3133,6 +3133,23 @@ TEST(Cast, StringToDecimal) { } } +TEST(Cast, StringToDecimalOverflow) { + // GH-55606: values that do not fit in the storage integer must not wrap around + const std::vector>> cases = { + {"18446744073709551621", decimal32(9, 0)}, // 2**64 + 5 + {"18446744073709551616", decimal64(18, 0)}, // 2**64 + {std::string(39, '9'), decimal128(38, 0)}, + {"340282366920938463463374607431768211455", decimal128(38, 0)}, // 2**128 - 1 + {"115792089237316195423570985008687907853269984665640564039457584007913129639943", + decimal256(76, 0)}, // 2**256 + 7 + }; + for (const auto& [value, decimal_type] : cases) { + ARROW_SCOPED_TRACE("value = ", value, ", type = ", *decimal_type); + CheckCastFails(ArrayFromJSON(utf8(), "[\"" + value + "\"]"), + CastOptions::Safe(decimal_type)); + } +} + TEST(Cast, StringToTimestamp) { for (auto string_type : {utf8(), large_utf8()}) { auto strings = ArrayFromJSON(string_type, R"(["1970-01-01", null, "2000-02-29"])"); diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc index a9d2fcb02d94..9042d845c84e 100644 --- a/cpp/src/arrow/util/decimal.cc +++ b/cpp/src/arrow/util/decimal.cc @@ -765,10 +765,26 @@ std::string Decimal128::ToString(int32_t scale) const { return str; } +// Computes out = out * multiple + addend, where out is an unsigned integer +// stored as out_size little-endian 64-bit words. +// Returns false if the result does not fit in out_size words. +static inline bool MultiplyAndAdd(uint64_t multiple, uint64_t addend, uint64_t out[], + size_t out_size) { + for (size_t i = 0; i < out_size; ++i) { + uint128_t tmp = out[i]; + tmp *= multiple; + tmp += addend; + out[i] = static_cast(tmp & 0xFFFFFFFFFFFFFFFFULL); + addend = static_cast(tmp >> 64); + } + return addend == 0; +} + // Iterates over input and for each group of kInt64DecimalDigits multiple out by // the appropriate power of 10 necessary to add source parsed as uint64 and // then adds the parsed value of source. -static inline void ShiftAndAdd(std::string_view input, uint64_t out[], size_t out_size) { +// Returns false if the result does not fit in out_size words. +static inline bool ShiftAndAdd(std::string_view input, uint64_t out[], size_t out_size) { for (size_t posn = 0; posn < input.size();) { const size_t group_size = std::min(kInt64DecimalDigits, input.size() - posn); const uint64_t multiple = kUInt64PowersOfTen[group_size]; @@ -776,15 +792,40 @@ static inline void ShiftAndAdd(std::string_view input, uint64_t out[], size_t ou ARROW_CHECK( internal::ParseValue(input.data() + posn, group_size, &chunk)); - for (size_t i = 0; i < out_size; ++i) { - uint128_t tmp = out[i]; - tmp *= multiple; - tmp += chunk; - out[i] = static_cast(tmp & 0xFFFFFFFFFFFFFFFFULL); - chunk = static_cast(tmp >> 64); + if (!MultiplyAndAdd(multiple, chunk, out, out_size)) { + return false; } posn += group_size; } + return true; +} + +// Multiplies out by 10^exponent. +// Returns false if the result does not fit in out_size words. +static inline bool MultiplyByPowerOfTen(int32_t exponent, uint64_t out[], + size_t out_size) { + while (exponent > 0) { + const auto step = std::min(static_cast(exponent), kInt64DecimalDigits); + if (!MultiplyAndAdd(kUInt64PowersOfTen[step], 0, out, out_size)) { + return false; + } + exponent -= static_cast(step); + } + return true; +} + +// Returns whether the absolute value stored as num_words little-endian 64-bit words +// is representable as a signed integer of the same width. +static inline bool FitsInSignedWords(const uint64_t words[], size_t num_words, + bool negative) { + const uint64_t high = words[num_words - 1]; + if ((high >> 63) == 0) { + return true; + } + // The only representable value whose absolute value has the top bit set + // is the minimum value -2^(bit width - 1) + return negative && high == (uint64_t{1} << 63) && + std::all_of(words, words + num_words - 1, [](uint64_t w) { return w == 0; }); } namespace { @@ -892,12 +933,24 @@ Status DecimalFromString(const char* type_name, std::string_view s, Decimal* out parsed_scale = static_cast(dec.fractional_digits.size()); } + // A negative scale is forced to zero below, to avoid negative scales (due to + // compatibility issues with external systems such as databases) + if (-parsed_scale > Decimal::kMaxScale) { + return Status::Invalid("The string '", s, "' cannot be represented as ", type_name); + } + if (out != nullptr) { static_assert(Decimal::kBitWidth % 64 == 0, "decimal bit-width not a multiple of 64"); std::array little_endian_array{}; - ShiftAndAdd(dec.whole_digits, little_endian_array.data(), little_endian_array.size()); - ShiftAndAdd(dec.fractional_digits, little_endian_array.data(), - little_endian_array.size()); + uint64_t* words = little_endian_array.data(); + const size_t num_words = little_endian_array.size(); + // Compute the absolute value, failing if the result is not representable + if (!ShiftAndAdd(dec.whole_digits, words, num_words) || + !ShiftAndAdd(dec.fractional_digits, words, num_words) || + !MultiplyByPowerOfTen(-parsed_scale, words, num_words) || + !FitsInSignedWords(words, num_words, dec.sign == '-')) { + return Status::Invalid("The string '", s, "' cannot be represented as ", type_name); + } *out = Decimal(bit_util::little_endian::ToNative(little_endian_array)); if (dec.sign == '-') { out->Negate(); @@ -905,14 +958,6 @@ Status DecimalFromString(const char* type_name, std::string_view s, Decimal* out } if (parsed_scale < 0) { - // Force the scale to zero, to avoid negative scales (due to compatibility issues - // with external systems such as databases) - if (-parsed_scale > Decimal::kMaxScale) { - return Status::Invalid("The string '", s, "' cannot be represented as ", type_name); - } - if (out != nullptr) { - *out *= Decimal::GetScaleMultiplier(-parsed_scale); - } parsed_precision -= parsed_scale; parsed_scale = 0; } @@ -960,11 +1005,19 @@ Status SimpleDecimalFromString(const char* type_name, std::string_view s, parsed_scale = static_cast(dec.fractional_digits.size()); } + // A negative scale is forced to zero below, to avoid negative scales (due to + // compatibility issues with external systems such as databases) + if (-parsed_scale > DecimalClass::kMaxScale) { + return Status::Invalid("The string '", s, "' cannot be represented as ", type_name); + } + if (out != nullptr) { + // Compute the absolute value, failing if the result is not representable uint64_t value{0}; - ShiftAndAdd(dec.whole_digits, &value, 1); - ShiftAndAdd(dec.fractional_digits, &value, 1); - if (value > static_cast( + if (!ShiftAndAdd(dec.whole_digits, &value, 1) || + !ShiftAndAdd(dec.fractional_digits, &value, 1) || + !MultiplyByPowerOfTen(-parsed_scale, &value, 1) || + value > static_cast( std::numeric_limits::max())) { return Status::Invalid("The string '", s, "' cannot be represented as ", type_name); } @@ -976,14 +1029,6 @@ Status SimpleDecimalFromString(const char* type_name, std::string_view s, } if (parsed_scale < 0) { - // Force the scale to zero, to avoid negative scales (due to compatibility issues - // with external systems such as databases) - if (-parsed_scale > DecimalClass::kMaxScale) { - return Status::Invalid("The string '", s, "' cannot be represented as ", type_name); - } - if (out != nullptr) { - *out *= DecimalClass::GetScaleMultiplier(-parsed_scale); - } parsed_precision -= parsed_scale; parsed_scale = 0; } diff --git a/cpp/src/arrow/util/decimal_test.cc b/cpp/src/arrow/util/decimal_test.cc index 7022c8117802..999c63401256 100644 --- a/cpp/src/arrow/util/decimal_test.cc +++ b/cpp/src/arrow/util/decimal_test.cc @@ -241,6 +241,25 @@ TEST(Decimal64Test, TestIntMinFitsPrecision) { ASSERT_FALSE(d.FitsInPrecision(18)); } +TEST(Decimal32Test, FromStringOverflow) { + AssertDecimalFromString("2147483647", Decimal32(INT32_MAX), 10, 0); + ASSERT_RAISES(Invalid, Decimal32::FromString("2147483648")); + // 2**64 + 5 + ASSERT_RAISES(Invalid, Decimal32::FromString("18446744073709551621")); + ASSERT_RAISES(Invalid, Decimal32::FromString("-18446744073709551621")); + ASSERT_RAISES(Invalid, Decimal32::FromString("9e9")); +} + +TEST(Decimal64Test, FromStringOverflow) { + AssertDecimalFromString("9223372036854775807", Decimal64(INT64_MAX), 19, 0); + ASSERT_RAISES(Invalid, Decimal64::FromString("9223372036854775808")); + // 2**64 + ASSERT_RAISES(Invalid, Decimal64::FromString("18446744073709551616")); + ASSERT_RAISES(Invalid, Decimal64::FromString("-18446744073709551616")); + ASSERT_RAISES(Invalid, Decimal64::FromString("9.3e18")); + ASSERT_RAISES(Invalid, Decimal64::FromString("99e18")); +} + TYPED_TEST_SUITE(DecimalFromStringTest, DecimalTypes); TYPED_TEST(DecimalFromStringTest, Basics) { this->TestBasics(); } @@ -435,15 +454,33 @@ TEST(Decimal128Test, FromStringLimits) { ASSERT_RAISES(Invalid, Decimal128::FromString("-9e39")); ASSERT_RAISES(Invalid, Decimal128::FromString("9.9e40")); ASSERT_RAISES(Invalid, Decimal128::FromString("-9.9e40")); - // XXX conversion overflows are currently not detected - // ASSERT_RAISES(Invalid, Decimal128::FromString("99e38")); - // ASSERT_RAISES(Invalid, Decimal128::FromString("-99e38")); - // ASSERT_RAISES(Invalid, - // Decimal128::FromString("999999999999999999999999999999999999999e1")); - // ASSERT_RAISES(Invalid, - // Decimal128::FromString("-999999999999999999999999999999999999999e1")); - // ASSERT_RAISES(Invalid, - // Decimal128::FromString("999999999999999999999999999999999999999")); + // Conversion overflows (GH-55606) + ASSERT_RAISES(Invalid, Decimal128::FromString("99e38")); + ASSERT_RAISES(Invalid, Decimal128::FromString("-99e38")); + ASSERT_RAISES(Invalid, + Decimal128::FromString("999999999999999999999999999999999999999e1")); + ASSERT_RAISES(Invalid, + Decimal128::FromString("-999999999999999999999999999999999999999e1")); + ASSERT_RAISES(Invalid, + Decimal128::FromString("999999999999999999999999999999999999999")); + // 2**127 - 1 is the largest representable value + AssertDecimalFromString( + "170141183460469231731687303715884105727", + Decimal128FromLE({18446744073709551615ULL, 9223372036854775807ULL}), 39, 0); + // 2**127 + ASSERT_RAISES(Invalid, + Decimal128::FromString("170141183460469231731687303715884105728")); + // -2**127 is the smallest representable value + AssertDecimalFromString("-170141183460469231731687303715884105728", + Decimal128FromLE({0, 9223372036854775808ULL}), 39, 0); + // -2**127 - 1 + ASSERT_RAISES(Invalid, + Decimal128::FromString("-170141183460469231731687303715884105729")); + // 2**128 + ASSERT_RAISES(Invalid, + Decimal128::FromString("340282366920938463463374607431768211456")); + ASSERT_RAISES(Invalid, + Decimal128::FromString("-340282366920938463463374607431768211456")); // No exponent, many fractional digits AssertDecimalFromString("9.9999999999999999999999999999999999999", dec38times9pos, 38, @@ -541,15 +578,30 @@ TEST(Decimal256Test, FromStringLimits) { ASSERT_RAISES(Invalid, Decimal256::FromString("9.9e78")); ASSERT_RAISES(Invalid, Decimal256::FromString("-9.9e78")); - // XXX conversion overflows are currently not detected - // ASSERT_RAISES(Invalid, Decimal256::FromString("99e76")); - // ASSERT_RAISES(Invalid, Decimal256::FromString("-99e76")); - // ASSERT_RAISES(Invalid, - // Decimal256::FromString("9999999999999999999999999999999999999999999999999999999999999999999999999999e1")); - // ASSERT_RAISES(Invalid, - // Decimal256::FromString("-9999999999999999999999999999999999999999999999999999999999999999999999999999e1")); - // ASSERT_RAISES(Invalid, - // Decimal256::FromString("99999999999999999999999999999999999999999999999999999999999999999999999999999")); + // Conversion overflows (GH-55606) + ASSERT_RAISES(Invalid, Decimal256::FromString("99e76")); + ASSERT_RAISES(Invalid, Decimal256::FromString("-99e76")); + ASSERT_RAISES(Invalid, + Decimal256::FromString("9999999999999999999999999999999999999999" + "999999999999999999999999999999999999e1")); + ASSERT_RAISES(Invalid, + Decimal256::FromString("-999999999999999999999999999999999999999" + "9999999999999999999999999999999999999e1")); + ASSERT_RAISES(Invalid, Decimal256::FromString("9999999999999999999999999999999999999999" + "9999999999999999999999999999999999999")); + // 2**255 - 1 is the largest representable value + AssertDecimalFromString( + "57896044618658097711785492504343953926634992332820282019728792003956564819967", + Decimal256FromLE({18446744073709551615ULL, 18446744073709551615ULL, + 18446744073709551615ULL, 9223372036854775807ULL}), + 77, 0); + // 2**255 + ASSERT_RAISES(Invalid, Decimal256::FromString("5789604461865809771178549250434395392663" + "4992332820282019728792003956564819968")); + // 2**256 + 7 + ASSERT_RAISES(Invalid, + Decimal256::FromString("1157920892373161954235709850086879078532" + "69984665640564039457584007913129639943")); // No exponent, many fractional digits AssertDecimalFromString(