Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 49 additions & 0 deletions native/spark-expr/src/hash_funcs/murmur3.rs
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,55 @@ mod tests {
test_hashes_with_nulls!(create_murmur3_hashes, T, values, expected, u32);
}

/// The length check lives in the shared `create_hashes_internal!`, so one test covers both
/// hashers.
#[test]
fn test_hashes_buffer_length_mismatch_is_error() {
use crate::hash_funcs::create_xxhash64_hashes;
use arrow::array::{BooleanArray, Decimal128Array, IntervalMonthDayNanoArray};
use arrow::datatypes::IntervalMonthDayNano;
// Every fixed-width kind the hash loops special-case, plus strings, with and without
// nulls.
let arrays: Vec<ArrayRef> = vec![
Arc::new(Int32Array::from(vec![1, 2, 3])),
Arc::new(Int64Array::from(vec![Some(1), None, Some(3)])),
Arc::new(Float64Array::from(vec![1.0, -0.0, f64::NAN])),
Arc::new(StringArray::from(vec!["a", "b", "c"])),
Arc::new(BooleanArray::from(vec![true, false, true])),
Arc::new(IntervalMonthDayNanoArray::from(vec![
IntervalMonthDayNano::new(1, 2, 3_000);
3
])),
Arc::new(
Decimal128Array::from(vec![1, 2, 3])
.with_precision_and_scale(10, 2)
.unwrap(),
),
Arc::new(Int32Array::from(Vec::<i32>::new())),
];
for array in arrays {
for len in [array.len().saturating_sub(1), array.len() + 1] {
if len == array.len() {
continue;
}
let columns = [Arc::clone(&array)];
let context = format!(
"{} array of length {} with {len} hashes",
array.data_type(),
array.len()
);
assert!(
create_murmur3_hashes(&columns, &mut vec![42u32; len]).is_err(),
"murmur3: {context}"
);
assert!(
create_xxhash64_hashes(&columns, &mut vec![42u64; len]).is_err(),
"xxhash64: {context}"
);
}
}
}

/// A dictionary array reached through a nested type arrives as the only column of its recursive
/// call, so deciding the dictionary fast path from column position alone treated it as a first
/// column and restarted from the seed, discarding the hash accumulated for earlier elements of
Expand Down
52 changes: 25 additions & 27 deletions native/spark-expr/src/hash_funcs/utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,8 @@ macro_rules! hash_array {
)
});
if array.null_count() == 0 {
// Fast path: no nulls, use direct indexing
for i in 0..$hashes.len() {
$hashes[i] = $hash_method(&array.value(i), $hashes[i]);
for (i, hash) in $hashes.iter_mut().enumerate() {
*hash = $hash_method(&array.value(i), *hash);
}
} else {
// Slow path: check nulls
Expand All @@ -62,12 +61,8 @@ macro_rules! hash_array_boolean {
)
});
if array.null_count() == 0 {
// Fast path: no nulls, use direct indexing
for i in 0..$hashes.len() {
$hashes[i] = $hash_method(
$hash_input_type::from(array.value(i)).to_le_bytes(),
$hashes[i],
);
for (hash, value) in $hashes.iter_mut().zip(array.values().iter()) {
*hash = $hash_method($hash_input_type::from(value).to_le_bytes(), *hash);
}
} else {
// Slow path: check nulls
Expand Down Expand Up @@ -99,9 +94,8 @@ macro_rules! hash_array_primitive {
let values = array.values();

if array.null_count() == 0 {
// Fast path: no nulls, use direct indexing
for i in 0..values.len() {
$hashes[i] = $hash_method((values[i] as $ty).to_le_bytes(), $hashes[i]);
for (hash, &value) in $hashes.iter_mut().zip(values.iter()) {
*hash = $hash_method((value as $ty).to_le_bytes(), *hash);
}
} else {
// Slow path: check nulls
Expand Down Expand Up @@ -130,10 +124,9 @@ macro_rules! hash_array_primitive_float {
let values = array.values();

if array.null_count() == 0 {
// Fast path: no nulls, use direct indexing
for i in 0..values.len() {
let value = $crate::float_semantics::hash_input(values[i]);
$hashes[i] = $hash_method(value.to_le_bytes(), $hashes[i]);
for (hash, &value) in $hashes.iter_mut().zip(values.iter()) {
let value = $crate::float_semantics::hash_input(value);
*hash = $hash_method(value.to_le_bytes(), *hash);
}
} else {
// Slow path: check nulls
Expand Down Expand Up @@ -168,17 +161,14 @@ macro_rules! hash_array_interval_month_day_nano {
// always an exact multiple of 1000 and out-of-range intervals throw at
// conversion time instead of reaching this hasher.
if array.null_count() == 0 {
// Fast path: no nulls, use direct indexing
for i in 0..$hashes.len() {
let value = array.value(i);
for (hash, value) in $hashes.iter_mut().zip(array.values().iter()) {
// Match Spark 4.2 generated code, which omits the days field:
// https://github.com/apache/spark/blob/v4.2.0/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala#L428-L431
// SPARK-58236 includes days starting in Spark 4.3; the version
// switch for that is tracked in
// https://github.com/apache/datafusion-comet/issues/5498.
$hashes[i] =
$hash_method((value.nanoseconds / 1_000).to_le_bytes(), $hashes[i]);
$hashes[i] = $hash_method(value.months.to_le_bytes(), $hashes[i]);
*hash = $hash_method((value.nanoseconds / 1_000).to_le_bytes(), *hash);
*hash = $hash_method(value.months.to_le_bytes(), *hash);
}
} else {
// Slow path: check nulls
Expand Down Expand Up @@ -209,13 +199,12 @@ macro_rules! hash_array_small_decimal {
});

if array.null_count() == 0 {
// Fast path: no nulls, use direct indexing
for i in 0..$hashes.len() {
$hashes[i] = $hash_method(
i64::try_from(array.value(i))
for (hash, &value) in $hashes.iter_mut().zip(array.values().iter()) {
*hash = $hash_method(
i64::try_from(value)
.map(|v| v.to_le_bytes())
.map_err(|e| DataFusionError::Execution(e.to_string()))?,
$hashes[i],
*hash,
);
}
} else {
Expand Down Expand Up @@ -875,6 +864,15 @@ macro_rules! create_hashes_internal {
use datafusion_comet_common::children_with_parent_nulls;

for (i, col) in $arrays.iter().enumerate() {
// The fast paths zip the hashes with the values, which would silently skip rows if
// the lengths differed.
if col.len() != $hashes_buffer.len() {
return Err(datafusion::common::DataFusionError::Internal(format!(
"Column {i} has {} rows but the hashes buffer has {}",
col.len(),
$hashes_buffer.len()
)));
}
// The dictionary fast path hashes each distinct dictionary value once and reuses that
// result for every key, which is only valid while every row carries the same incoming
// hash. Position in the column list is not a sufficient test: this macro also runs on
Expand Down