Skip to content

Commit 60e67fa

Browse files
authored
refactor: guard oversized bucket hash input in BucketIndex (#1023)
BucketUtils::HashBytes still rejects inputs larger than INT32_MAX with an IcebergError, but BucketIndex now checks the byte length of string, binary, and fixed literals before hashing and returns InvalidArgument instead, so the Result-returning entry point no longer throws. This replaces the internal HashBytes(const uint8_t*, size_t) overload and the bucket_util_internal.h header, which existed only so a test could reach the guard without materializing an oversized value. That overload, the header, and the HashBytesRejectsOversizedInput test are all removed: the guard cannot be exercised from a test without either a 2 GiB allocation or an invalid std::span.
1 parent 768ac65 commit 60e67fa

4 files changed

Lines changed: 37 additions & 82 deletions

File tree

‎src/iceberg/test/bucket_util_test.cc‎

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -20,16 +20,11 @@
2020
#include "iceberg/util/bucket_util.h"
2121

2222
#include <chrono>
23-
#include <cstdint>
24-
#include <limits>
2523

26-
#include <gmock/gmock.h>
2724
#include <gtest/gtest.h>
2825

29-
#include "iceberg/exception.h"
3026
#include "iceberg/expression/literal.h"
3127
#include "iceberg/test/temporal_test_helper.h"
32-
#include "iceberg/util/bucket_util_internal.h"
3328
#include "iceberg/util/decimal.h"
3429
#include "iceberg/util/uuid.h"
3530

@@ -113,19 +108,6 @@ TEST(BucketUtilsTest, HashHelper) {
113108
EXPECT_EQ(BucketUtils::HashBytes(fixed), -188683207);
114109
}
115110

116-
TEST(BucketUtilsTest, HashBytesRejectsOversizedInput) {
117-
// Lengths above INT32_MAX would wrap the 32-bit signed length parameter of
118-
// MurmurHash3_x86_32 and read out of bounds. The guard rejects them before the
119-
// data is touched, so pass a null pointer instead of building a std::span over
120-
// more bytes than the object provides (which is undefined behavior) or
121-
// allocating 2 GiB.
122-
const auto oversized_length =
123-
static_cast<size_t>(std::numeric_limits<int32_t>::max()) + 1;
124-
ASSERT_THAT([&]() { internal::HashBytes(nullptr, oversized_length); },
125-
::testing::ThrowsMessage<IcebergError>(
126-
::testing::HasSubstr("exceeds the maximum supported length")));
127-
}
128-
129111
TEST(BucketUtilsTest, BucketTimestampNanosMatchesMicros) {
130112
constexpr int32_t kNumBuckets = 1000;
131113
const auto ts_micros = TemporalTestHelper::CreateTimestamp({.year = 2017,

‎src/iceberg/util/bucket_util.cc‎

Lines changed: 37 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -20,18 +20,41 @@
2020
#include "iceberg/util/bucket_util.h"
2121

2222
#include <limits>
23+
#include <optional>
2324
#include <utility>
2425

2526
#include "iceberg/exception.h"
2627
#include "iceberg/expression/literal.h"
27-
#include "iceberg/util/bucket_util_internal.h"
2828
#include "iceberg/util/endian.h"
2929
#include "iceberg/util/murmurhash3_internal.h"
3030
#include "iceberg/util/temporal_util.h"
3131

3232
namespace iceberg {
3333

3434
namespace {
35+
36+
/// MurmurHash3_x86_32 takes a signed 32-bit length. Iceberg's bucket transform is
37+
/// defined on byte sequences of at most 2^31 - 1 bytes (Java arrays are int-indexed),
38+
/// and longer inputs would wrap the length negative and make the hash loop read far
39+
/// outside the buffer, so they are rejected instead.
40+
constexpr size_t kMaxHashBytes = static_cast<size_t>(std::numeric_limits<int32_t>::max());
41+
42+
/// \brief Returns the number of bytes `HashLiteral` will hash for `literal`.
43+
///
44+
/// Returns `std::nullopt` for values hashed as fixed-width integers, which have no
45+
/// length limit.
46+
std::optional<size_t> HashedByteSize(const Literal& literal) {
47+
switch (literal.type()->type_id()) {
48+
case TypeId::kString:
49+
return std::get<std::string>(literal.value()).size();
50+
case TypeId::kBinary:
51+
case TypeId::kFixed:
52+
return std::get<std::vector<uint8_t>>(literal.value()).size();
53+
default:
54+
return std::nullopt;
55+
}
56+
}
57+
3558
template <TypeId type_id>
3659
int32_t HashLiteral(const Literal& literal) {
3760
std::unreachable();
@@ -113,29 +136,16 @@ int32_t HashLiteral<TypeId::kFixed>(const Literal& literal) {
113136
} // namespace
114137

115138
int32_t BucketUtils::HashBytes(std::span<const uint8_t> bytes) {
116-
return internal::HashBytes(bytes.data(), bytes.size());
117-
}
118-
119-
namespace internal {
120-
121-
int32_t HashBytes(const uint8_t* data, size_t len) {
122-
// MurmurHash3_x86_32 takes a signed 32-bit length. Passing a larger size_t
123-
// would wrap it negative and make the hash loop read far outside `data`
124-
// (and silently hash the wrong prefix for even larger inputs). Iceberg's
125-
// bucket transform is defined on byte sequences of at most 2^31 - 1 bytes
126-
// (Java arrays), so reject anything longer before narrowing.
127139
ICEBERG_CHECK_OR_DIE(
128-
len <= static_cast<size_t>(std::numeric_limits<int32_t>::max()),
140+
bytes.size() <= kMaxHashBytes,
129141
"BucketUtils::HashBytes: input length {} exceeds the maximum supported "
130142
"length {}",
131-
len, std::numeric_limits<int32_t>::max());
143+
bytes.size(), kMaxHashBytes);
132144
int32_t hash_value = 0;
133-
MurmurHash3_x86_32(data, static_cast<int>(len), 0, &hash_value);
145+
MurmurHash3_x86_32(bytes.data(), static_cast<int>(bytes.size()), 0, &hash_value);
134146
return hash_value;
135147
}
136148

137-
} // namespace internal
138-
139149
int32_t BucketUtils::HashLong(int64_t value) {
140150
int32_t hash_value = 0;
141151
value = ToLittleEndian(value);
@@ -157,6 +167,16 @@ Result<int32_t> BucketUtils::BucketIndex(const Literal& literal, int32_t num_buc
157167
return NotSupported("Cannot compute bucket index for {}", literal.ToString());
158168
}
159169

170+
// HashBytes rejects oversized inputs by throwing, but this function reports failures
171+
// through its Result, so check the length up front.
172+
if (auto byte_size = HashedByteSize(literal);
173+
byte_size.has_value() && *byte_size > kMaxHashBytes) [[unlikely]] {
174+
return InvalidArgument(
175+
"Cannot compute bucket index for {}: value of {} bytes exceeds the maximum "
176+
"supported length {}",
177+
literal.type()->ToString(), *byte_size, kMaxHashBytes);
178+
}
179+
160180
int32_t hash_value = 0;
161181
switch (literal.type()->type_id()) {
162182
DISPATCH_HASH_LITERAL(TypeId::kInt)

‎src/iceberg/util/bucket_util.h‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,8 +62,6 @@ class ICEBERG_EXPORT BucketUtils {
6262
/// \param literal The input literal to hash.
6363
/// \param num_buckets The number of buckets to hash into.
6464
/// \return (murmur3_x86_32_hash(literal) & Integer.MAX_VALUE) % num_buckets
65-
/// \throws IcebergError if the literal is a string, binary, or fixed value whose
66-
/// byte length exceeds `INT32_MAX`; see HashBytes.
6765
static Result<int32_t> BucketIndex(const Literal& literal, int32_t num_buckets);
6866
};
6967

‎src/iceberg/util/bucket_util_internal.h‎

Lines changed: 0 additions & 45 deletions
This file was deleted.

0 commit comments

Comments
 (0)