Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 0 additions & 18 deletions src/iceberg/test/bucket_util_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,11 @@
#include "iceberg/util/bucket_util.h"

#include <chrono>
#include <cstdint>
#include <limits>

#include <gmock/gmock.h>
#include <gtest/gtest.h>

#include "iceberg/exception.h"
#include "iceberg/expression/literal.h"
#include "iceberg/test/temporal_test_helper.h"
#include "iceberg/util/bucket_util_internal.h"
#include "iceberg/util/decimal.h"
#include "iceberg/util/uuid.h"

Expand Down Expand Up @@ -113,19 +108,6 @@ TEST(BucketUtilsTest, HashHelper) {
EXPECT_EQ(BucketUtils::HashBytes(fixed), -188683207);
}

TEST(BucketUtilsTest, HashBytesRejectsOversizedInput) {
// Lengths above INT32_MAX would wrap the 32-bit signed length parameter of
// MurmurHash3_x86_32 and read out of bounds. The guard rejects them before the
// data is touched, so pass a null pointer instead of building a std::span over
// more bytes than the object provides (which is undefined behavior) or
// allocating 2 GiB.
const auto oversized_length =
static_cast<size_t>(std::numeric_limits<int32_t>::max()) + 1;
ASSERT_THAT([&]() { internal::HashBytes(nullptr, oversized_length); },
::testing::ThrowsMessage<IcebergError>(
::testing::HasSubstr("exceeds the maximum supported length")));
}

TEST(BucketUtilsTest, BucketTimestampNanosMatchesMicros) {
constexpr int32_t kNumBuckets = 1000;
const auto ts_micros = TemporalTestHelper::CreateTimestamp({.year = 2017,
Expand Down
54 changes: 37 additions & 17 deletions src/iceberg/util/bucket_util.cc
Original file line number Diff line number Diff line change
Expand Up @@ -20,18 +20,41 @@
#include "iceberg/util/bucket_util.h"

#include <limits>
#include <optional>
#include <utility>

#include "iceberg/exception.h"
#include "iceberg/expression/literal.h"
#include "iceberg/util/bucket_util_internal.h"
#include "iceberg/util/endian.h"
#include "iceberg/util/murmurhash3_internal.h"
#include "iceberg/util/temporal_util.h"

namespace iceberg {

namespace {

/// MurmurHash3_x86_32 takes a signed 32-bit length. Iceberg's bucket transform is
/// defined on byte sequences of at most 2^31 - 1 bytes (Java arrays are int-indexed),
/// and longer inputs would wrap the length negative and make the hash loop read far
/// outside the buffer, so they are rejected instead.
constexpr size_t kMaxHashBytes = static_cast<size_t>(std::numeric_limits<int32_t>::max());

/// \brief Returns the number of bytes `HashLiteral` will hash for `literal`.
///
/// Returns `std::nullopt` for values hashed as fixed-width integers, which have no
/// length limit.
std::optional<size_t> HashedByteSize(const Literal& literal) {
switch (literal.type()->type_id()) {
case TypeId::kString:
return std::get<std::string>(literal.value()).size();
case TypeId::kBinary:
case TypeId::kFixed:
return std::get<std::vector<uint8_t>>(literal.value()).size();
default:
return std::nullopt;
}
}

Comment thread
wgtmac marked this conversation as resolved.
Resolved
template <TypeId type_id>
int32_t HashLiteral(const Literal& literal) {
std::unreachable();
Expand Down Expand Up @@ -113,29 +136,16 @@ int32_t HashLiteral<TypeId::kFixed>(const Literal& literal) {
} // namespace

int32_t BucketUtils::HashBytes(std::span<const uint8_t> bytes) {
return internal::HashBytes(bytes.data(), bytes.size());
}

namespace internal {

int32_t HashBytes(const uint8_t* data, size_t len) {
// MurmurHash3_x86_32 takes a signed 32-bit length. Passing a larger size_t
// would wrap it negative and make the hash loop read far outside `data`
// (and silently hash the wrong prefix for even larger inputs). Iceberg's
// bucket transform is defined on byte sequences of at most 2^31 - 1 bytes
// (Java arrays), so reject anything longer before narrowing.
ICEBERG_CHECK_OR_DIE(
len <= static_cast<size_t>(std::numeric_limits<int32_t>::max()),
bytes.size() <= kMaxHashBytes,
"BucketUtils::HashBytes: input length {} exceeds the maximum supported "
"length {}",
len, std::numeric_limits<int32_t>::max());
bytes.size(), kMaxHashBytes);
Comment thread
wgtmac marked this conversation as resolved.
Resolved
int32_t hash_value = 0;
MurmurHash3_x86_32(data, static_cast<int>(len), 0, &hash_value);
MurmurHash3_x86_32(bytes.data(), static_cast<int>(bytes.size()), 0, &hash_value);
Comment thread
wgtmac marked this conversation as resolved.
Resolved
return hash_value;
}

} // namespace internal

int32_t BucketUtils::HashLong(int64_t value) {
int32_t hash_value = 0;
value = ToLittleEndian(value);
Expand All @@ -157,6 +167,16 @@ Result<int32_t> BucketUtils::BucketIndex(const Literal& literal, int32_t num_buc
return NotSupported("Cannot compute bucket index for {}", literal.ToString());
}

// HashBytes rejects oversized inputs by throwing, but this function reports failures
// through its Result, so check the length up front.
if (auto byte_size = HashedByteSize(literal);
byte_size.has_value() && *byte_size > kMaxHashBytes) [[unlikely]] {
return InvalidArgument(
"Cannot compute bucket index for {}: value of {} bytes exceeds the maximum "
"supported length {}",
literal.type()->ToString(), *byte_size, kMaxHashBytes);
}

int32_t hash_value = 0;
switch (literal.type()->type_id()) {
DISPATCH_HASH_LITERAL(TypeId::kInt)
Expand Down
2 changes: 0 additions & 2 deletions src/iceberg/util/bucket_util.h
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,6 @@ class ICEBERG_EXPORT BucketUtils {
/// \param literal The input literal to hash.
/// \param num_buckets The number of buckets to hash into.
/// \return (murmur3_x86_32_hash(literal) & Integer.MAX_VALUE) % num_buckets
/// \throws IcebergError if the literal is a string, binary, or fixed value whose
/// byte length exceeds `INT32_MAX`; see HashBytes.
static Result<int32_t> BucketIndex(const Literal& literal, int32_t num_buckets);
};

Expand Down
45 changes: 0 additions & 45 deletions src/iceberg/util/bucket_util_internal.h

This file was deleted.

Loading