Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions cpp/src/arrow/compute/kernels/scalar_cast_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -3133,6 +3133,23 @@ TEST(Cast, StringToDecimal) {
}
}

TEST(Cast, StringToDecimalOverflow) {
// GH-55606: values that do not fit in the storage integer must not wrap around
const std::vector<std::pair<std::string, std::shared_ptr<DataType>>> cases = {
{"18446744073709551621", decimal32(9, 0)}, // 2**64 + 5
{"18446744073709551616", decimal64(18, 0)}, // 2**64
{std::string(39, '9'), decimal128(38, 0)},
{"340282366920938463463374607431768211455", decimal128(38, 0)}, // 2**128 - 1
{"115792089237316195423570985008687907853269984665640564039457584007913129639943",
decimal256(76, 0)}, // 2**256 + 7
};
for (const auto& [value, decimal_type] : cases) {
ARROW_SCOPED_TRACE("value = ", value, ", type = ", *decimal_type);
CheckCastFails(ArrayFromJSON(utf8(), "[\"" + value + "\"]"),
CastOptions::Safe(decimal_type));
}
}

TEST(Cast, StringToTimestamp) {
for (auto string_type : {utf8(), large_utf8()}) {
auto strings = ArrayFromJSON(string_type, R"(["1970-01-01", null, "2000-02-29"])");
Expand Down
103 changes: 74 additions & 29 deletions cpp/src/arrow/util/decimal.cc
Original file line number Diff line number Diff line change
Expand Up @@ -765,26 +765,67 @@ std::string Decimal128::ToString(int32_t scale) const {
return str;
}

// Computes out = out * multiple + addend, where out is an unsigned integer
// stored as out_size little-endian 64-bit words.
// Returns false if the result does not fit in out_size words.
static inline bool MultiplyAndAdd(uint64_t multiple, uint64_t addend, uint64_t out[],
size_t out_size) {
for (size_t i = 0; i < out_size; ++i) {
uint128_t tmp = out[i];
tmp *= multiple;
tmp += addend;
out[i] = static_cast<uint64_t>(tmp & 0xFFFFFFFFFFFFFFFFULL);
addend = static_cast<uint64_t>(tmp >> 64);
}
return addend == 0;
}

// Iterates over input and for each group of kInt64DecimalDigits multiple out by
// the appropriate power of 10 necessary to add source parsed as uint64 and
// then adds the parsed value of source.
static inline void ShiftAndAdd(std::string_view input, uint64_t out[], size_t out_size) {
// Returns false if the result does not fit in out_size words.
static inline bool ShiftAndAdd(std::string_view input, uint64_t out[], size_t out_size) {
for (size_t posn = 0; posn < input.size();) {
const size_t group_size = std::min(kInt64DecimalDigits, input.size() - posn);
const uint64_t multiple = kUInt64PowersOfTen[group_size];
uint64_t chunk = 0;
ARROW_CHECK(
internal::ParseValue<UInt64Type>(input.data() + posn, group_size, &chunk));

for (size_t i = 0; i < out_size; ++i) {
uint128_t tmp = out[i];
tmp *= multiple;
tmp += chunk;
out[i] = static_cast<uint64_t>(tmp & 0xFFFFFFFFFFFFFFFFULL);
chunk = static_cast<uint64_t>(tmp >> 64);
if (!MultiplyAndAdd(multiple, chunk, out, out_size)) {
return false;
}
posn += group_size;
}
return true;
}

// Multiplies out by 10^exponent.
// Returns false if the result does not fit in out_size words.
static inline bool MultiplyByPowerOfTen(int32_t exponent, uint64_t out[],
size_t out_size) {
while (exponent > 0) {
const auto step = std::min(static_cast<size_t>(exponent), kInt64DecimalDigits);
if (!MultiplyAndAdd(kUInt64PowersOfTen[step], 0, out, out_size)) {
return false;
}
exponent -= static_cast<int32_t>(step);
}
return true;
}

// Returns whether the absolute value stored as num_words little-endian 64-bit words
// is representable as a signed integer of the same width.
static inline bool FitsInSignedWords(const uint64_t words[], size_t num_words,
bool negative) {
const uint64_t high = words[num_words - 1];
if ((high >> 63) == 0) {
return true;
}
// The only representable value whose absolute value has the top bit set
// is the minimum value -2^(bit width - 1)
return negative && high == (uint64_t{1} << 63) &&
std::all_of(words, words + num_words - 1, [](uint64_t w) { return w == 0; });
}

namespace {
Expand Down Expand Up @@ -892,27 +933,31 @@ Status DecimalFromString(const char* type_name, std::string_view s, Decimal* out
parsed_scale = static_cast<int32_t>(dec.fractional_digits.size());
}

// A negative scale is forced to zero below, to avoid negative scales (due to
// compatibility issues with external systems such as databases)
if (-parsed_scale > Decimal::kMaxScale) {
return Status::Invalid("The string '", s, "' cannot be represented as ", type_name);
}

if (out != nullptr) {
static_assert(Decimal::kBitWidth % 64 == 0, "decimal bit-width not a multiple of 64");
std::array<uint64_t, Decimal::kBitWidth / 64> little_endian_array{};
ShiftAndAdd(dec.whole_digits, little_endian_array.data(), little_endian_array.size());
ShiftAndAdd(dec.fractional_digits, little_endian_array.data(),
little_endian_array.size());
uint64_t* words = little_endian_array.data();
const size_t num_words = little_endian_array.size();
// Compute the absolute value, failing if the result is not representable
if (!ShiftAndAdd(dec.whole_digits, words, num_words) ||
!ShiftAndAdd(dec.fractional_digits, words, num_words) ||
!MultiplyByPowerOfTen(-parsed_scale, words, num_words) ||
!FitsInSignedWords(words, num_words, dec.sign == '-')) {
return Status::Invalid("The string '", s, "' cannot be represented as ", type_name);
}
*out = Decimal(bit_util::little_endian::ToNative(little_endian_array));
if (dec.sign == '-') {
out->Negate();
}
}

if (parsed_scale < 0) {
// Force the scale to zero, to avoid negative scales (due to compatibility issues
// with external systems such as databases)
if (-parsed_scale > Decimal::kMaxScale) {
return Status::Invalid("The string '", s, "' cannot be represented as ", type_name);
}
if (out != nullptr) {
*out *= Decimal::GetScaleMultiplier(-parsed_scale);
}
parsed_precision -= parsed_scale;
parsed_scale = 0;
}
Expand Down Expand Up @@ -960,11 +1005,19 @@ Status SimpleDecimalFromString(const char* type_name, std::string_view s,
parsed_scale = static_cast<int32_t>(dec.fractional_digits.size());
}

// A negative scale is forced to zero below, to avoid negative scales (due to
// compatibility issues with external systems such as databases)
if (-parsed_scale > DecimalClass::kMaxScale) {
return Status::Invalid("The string '", s, "' cannot be represented as ", type_name);
}

if (out != nullptr) {
// Compute the absolute value, failing if the result is not representable
uint64_t value{0};
ShiftAndAdd(dec.whole_digits, &value, 1);
ShiftAndAdd(dec.fractional_digits, &value, 1);
if (value > static_cast<uint64_t>(
if (!ShiftAndAdd(dec.whole_digits, &value, 1) ||
!ShiftAndAdd(dec.fractional_digits, &value, 1) ||
!MultiplyByPowerOfTen(-parsed_scale, &value, 1) ||
value > static_cast<uint64_t>(
std::numeric_limits<typename DecimalClass::ValueType>::max())) {
return Status::Invalid("The string '", s, "' cannot be represented as ", type_name);
}
Expand All @@ -976,14 +1029,6 @@ Status SimpleDecimalFromString(const char* type_name, std::string_view s,
}

if (parsed_scale < 0) {
// Force the scale to zero, to avoid negative scales (due to compatibility issues
// with external systems such as databases)
if (-parsed_scale > DecimalClass::kMaxScale) {
return Status::Invalid("The string '", s, "' cannot be represented as ", type_name);
}
if (out != nullptr) {
*out *= DecimalClass::GetScaleMultiplier(-parsed_scale);
}
parsed_precision -= parsed_scale;
parsed_scale = 0;
}
Expand Down
88 changes: 70 additions & 18 deletions cpp/src/arrow/util/decimal_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,25 @@ TEST(Decimal64Test, TestIntMinFitsPrecision) {
ASSERT_FALSE(d.FitsInPrecision(18));
}

TEST(Decimal32Test, FromStringOverflow) {
AssertDecimalFromString("2147483647", Decimal32(INT32_MAX), 10, 0);
ASSERT_RAISES(Invalid, Decimal32::FromString("2147483648"));
// 2**64 + 5
ASSERT_RAISES(Invalid, Decimal32::FromString("18446744073709551621"));
ASSERT_RAISES(Invalid, Decimal32::FromString("-18446744073709551621"));
ASSERT_RAISES(Invalid, Decimal32::FromString("9e9"));
}

TEST(Decimal64Test, FromStringOverflow) {
AssertDecimalFromString("9223372036854775807", Decimal64(INT64_MAX), 19, 0);
ASSERT_RAISES(Invalid, Decimal64::FromString("9223372036854775808"));
// 2**64
ASSERT_RAISES(Invalid, Decimal64::FromString("18446744073709551616"));
ASSERT_RAISES(Invalid, Decimal64::FromString("-18446744073709551616"));
ASSERT_RAISES(Invalid, Decimal64::FromString("9.3e18"));
ASSERT_RAISES(Invalid, Decimal64::FromString("99e18"));
}

TYPED_TEST_SUITE(DecimalFromStringTest, DecimalTypes);

TYPED_TEST(DecimalFromStringTest, Basics) { this->TestBasics(); }
Expand Down Expand Up @@ -435,15 +454,33 @@ TEST(Decimal128Test, FromStringLimits) {
ASSERT_RAISES(Invalid, Decimal128::FromString("-9e39"));
ASSERT_RAISES(Invalid, Decimal128::FromString("9.9e40"));
ASSERT_RAISES(Invalid, Decimal128::FromString("-9.9e40"));
// XXX conversion overflows are currently not detected
// ASSERT_RAISES(Invalid, Decimal128::FromString("99e38"));
// ASSERT_RAISES(Invalid, Decimal128::FromString("-99e38"));
// ASSERT_RAISES(Invalid,
// Decimal128::FromString("999999999999999999999999999999999999999e1"));
// ASSERT_RAISES(Invalid,
// Decimal128::FromString("-999999999999999999999999999999999999999e1"));
// ASSERT_RAISES(Invalid,
// Decimal128::FromString("999999999999999999999999999999999999999"));
// Conversion overflows (GH-55606)
ASSERT_RAISES(Invalid, Decimal128::FromString("99e38"));
ASSERT_RAISES(Invalid, Decimal128::FromString("-99e38"));
ASSERT_RAISES(Invalid,
Decimal128::FromString("999999999999999999999999999999999999999e1"));
ASSERT_RAISES(Invalid,
Decimal128::FromString("-999999999999999999999999999999999999999e1"));
ASSERT_RAISES(Invalid,
Decimal128::FromString("999999999999999999999999999999999999999"));
// 2**127 - 1 is the largest representable value
AssertDecimalFromString(
"170141183460469231731687303715884105727",
Decimal128FromLE({18446744073709551615ULL, 9223372036854775807ULL}), 39, 0);
// 2**127
ASSERT_RAISES(Invalid,
Decimal128::FromString("170141183460469231731687303715884105728"));
// -2**127 is the smallest representable value
AssertDecimalFromString("-170141183460469231731687303715884105728",
Decimal128FromLE({0, 9223372036854775808ULL}), 39, 0);
// -2**127 - 1
ASSERT_RAISES(Invalid,
Decimal128::FromString("-170141183460469231731687303715884105729"));
// 2**128
ASSERT_RAISES(Invalid,
Decimal128::FromString("340282366920938463463374607431768211456"));
ASSERT_RAISES(Invalid,
Decimal128::FromString("-340282366920938463463374607431768211456"));

// No exponent, many fractional digits
AssertDecimalFromString("9.9999999999999999999999999999999999999", dec38times9pos, 38,
Expand Down Expand Up @@ -541,15 +578,30 @@ TEST(Decimal256Test, FromStringLimits) {
ASSERT_RAISES(Invalid, Decimal256::FromString("9.9e78"));
ASSERT_RAISES(Invalid, Decimal256::FromString("-9.9e78"));

// XXX conversion overflows are currently not detected
// ASSERT_RAISES(Invalid, Decimal256::FromString("99e76"));
// ASSERT_RAISES(Invalid, Decimal256::FromString("-99e76"));
// ASSERT_RAISES(Invalid,
// Decimal256::FromString("9999999999999999999999999999999999999999999999999999999999999999999999999999e1"));
// ASSERT_RAISES(Invalid,
// Decimal256::FromString("-9999999999999999999999999999999999999999999999999999999999999999999999999999e1"));
// ASSERT_RAISES(Invalid,
// Decimal256::FromString("99999999999999999999999999999999999999999999999999999999999999999999999999999"));
// Conversion overflows (GH-55606)
ASSERT_RAISES(Invalid, Decimal256::FromString("99e76"));
ASSERT_RAISES(Invalid, Decimal256::FromString("-99e76"));
ASSERT_RAISES(Invalid,
Decimal256::FromString("9999999999999999999999999999999999999999"
"999999999999999999999999999999999999e1"));
ASSERT_RAISES(Invalid,
Decimal256::FromString("-999999999999999999999999999999999999999"
"9999999999999999999999999999999999999e1"));
ASSERT_RAISES(Invalid, Decimal256::FromString("9999999999999999999999999999999999999999"
"9999999999999999999999999999999999999"));
// 2**255 - 1 is the largest representable value
AssertDecimalFromString(
"57896044618658097711785492504343953926634992332820282019728792003956564819967",
Decimal256FromLE({18446744073709551615ULL, 18446744073709551615ULL,
18446744073709551615ULL, 9223372036854775807ULL}),
77, 0);
// 2**255
ASSERT_RAISES(Invalid, Decimal256::FromString("5789604461865809771178549250434395392663"
"4992332820282019728792003956564819968"));
// 2**256 + 7
ASSERT_RAISES(Invalid,
Decimal256::FromString("1157920892373161954235709850086879078532"
"69984665640564039457584007913129639943"));

// No exponent, many fractional digits
AssertDecimalFromString(
Expand Down
Loading