diff --git a/native/spark-expr/Cargo.toml b/native/spark-expr/Cargo.toml index a99a6891a59..b19228dbe98 100644 --- a/native/spark-expr/Cargo.toml +++ b/native/spark-expr/Cargo.toml @@ -379,6 +379,10 @@ harness = false name = "murmur3" harness = false +[[bench]] +name = "hash_nulls" +harness = false + [[bench]] name = "xxhash64" harness = false diff --git a/native/spark-expr/benches/hash_nulls.rs b/native/spark-expr/benches/hash_nulls.rs new file mode 100644 index 00000000000..a361983b0ff --- /dev/null +++ b/native/spark-expr/benches/hash_nulls.rs @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Benchmarks hashing fixed-width columns with nulls, the case shuffle hash partitioning hits on +//! nullable keys such as TPC-DS foreign keys. + +use arrow::array::{ArrayRef, Float64Array, Int32Array, Int64Array}; +use arrow::buffer::NullBuffer; +use criterion::{criterion_group, criterion_main, Criterion}; +use datafusion::common::ScalarValue; +use datafusion::physical_plan::ColumnarValue; +use datafusion_comet_spark_expr::murmur3::create_murmur3_hashes; +use datafusion_comet_spark_expr::spark_xxhash64; +use std::hint::black_box; +use std::sync::Arc; + +mod common; + +const NUM_ROWS: usize = 8192; + +/// Null percentages, from TPC-DS-like to nearly all null. +const SWEEP: [u32; 11] = [3, 10, 25, 50, 60, 70, 75, 80, 85, 90, 99]; + +/// Pseudo-random values from a fixed-seed xorshift, so runs are comparable. +fn xorshift(seed: u64) -> impl FnMut() -> u64 { + let mut x = seed; + move || { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + x + } +} + +/// Each row is null with probability `null_fraction`, independently. +fn random_nulls(null_fraction: f64) -> NullBuffer { + let mut next = xorshift(0x2545_f491_4f6c_dd1d); + NullBuffer::from( + (0..NUM_ROWS) + .map(|_| (next() as f64 / u64::MAX as f64) >= null_fraction) + .collect::>(), + ) +} + +fn int32(nulls: Option) -> ArrayRef { + let mut next = xorshift(1); + Arc::new(Int32Array::new( + (0..NUM_ROWS).map(|_| next() as i32).collect(), + nulls, + )) +} + +fn int64(nulls: Option) -> ArrayRef { + let mut next = xorshift(2); + Arc::new(Int64Array::new( + (0..NUM_ROWS).map(|_| next() as i64).collect(), + nulls, + )) +} + +fn float64(nulls: Option) -> ArrayRef { + let mut next = xorshift(3); + Arc::new(Float64Array::new( + (0..NUM_ROWS).map(|_| next() as f64 * 1e-6).collect(), + nulls, + )) +} + +fn criterion_benchmark(c: &mut Criterion) { + let mut murmur3 = |name: &str, array: ArrayRef| { + let arrays = [array]; + let mut hashes = vec![42u32; NUM_ROWS]; + c.bench_function(&format!("murmur3/{name}"), |b| { + b.iter(|| { + create_murmur3_hashes(black_box(&arrays), &mut hashes).unwrap(); + black_box(&hashes); + }) + }); + }; + // No nulls, which this path doesn't reach, to show the run's noise. + murmur3("int32/no_nulls", int32(None)); + // Every tenth row is null, a pattern the branch predictor learns. Murmur3 costs the same for + // every value, so these values don't need to be random. + murmur3( + "int32/stride_10%", + common::i32_array(NUM_ROWS, 0.1, |i| i as i32), + ); + // The rates from 25% to 85% leave about 48 down to 10 valid rows in a 64-row chunk, around + // `MIN_VALID_FOR_DENSE`. Rerun them when changing it. + for percent in SWEEP { + let nulls = random_nulls(percent as f64 / 100.0); + murmur3(&format!("int32/random_{percent}%"), int32(Some(nulls))); + } + for percent in [3, 50, 75] { + let nulls = random_nulls(percent as f64 / 100.0); + murmur3( + &format!("int64/random_{percent}%"), + int64(Some(nulls.clone())), + ); + murmur3(&format!("float64/random_{percent}%"), float64(Some(nulls))); + } + + // A seed other than 42 keeps xxhash64 on Comet's implementation. + for percent in SWEEP { + let nulls = random_nulls(percent as f64 / 100.0); + let args = vec![ + ColumnarValue::Array(int64(Some(nulls))), + ColumnarValue::Scalar(ScalarValue::Int64(Some(7))), + ]; + c.bench_function(&format!("xxhash64/int64/random_{percent}%"), |b| { + b.iter(|| black_box(spark_xxhash64(black_box(&args)).unwrap())) + }); + } +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/native/spark-expr/src/hash_funcs/murmur3.rs b/native/spark-expr/src/hash_funcs/murmur3.rs index 3d4142316c3..5288327f2ee 100644 --- a/native/spark-expr/src/hash_funcs/murmur3.rs +++ b/native/spark-expr/src/hash_funcs/murmur3.rs @@ -193,7 +193,7 @@ pub fn create_murmur3_hashes<'a>( } #[cfg(test)] -mod tests { +pub(crate) mod tests { /// Produced by the per-element `hash_list_array!` implementation. An empty list and a null /// list both leave the seed untouched. const EXPECTED_LIST_OF_STRUCT: [u32; 6] = @@ -212,6 +212,49 @@ mod tests { test_hashes_with_nulls!(create_murmur3_hashes, T, values, expected, u32); } + /// Arrays of every fixed-width kind the hash loops special-case, plus strings, with and + /// without nulls. + pub(crate) fn length_mismatch_arrays() -> Vec { + use arrow::array::{BooleanArray, Decimal128Array, IntervalMonthDayNanoArray}; + use arrow::datatypes::IntervalMonthDayNano; + vec![ + Arc::new(Int32Array::from(vec![1, 2, 3])), + Arc::new(Int64Array::from(vec![Some(1), None, Some(3)])), + Arc::new(Float64Array::from(vec![1.0, -0.0, f64::NAN])), + Arc::new(StringArray::from(vec!["a", "b", "c"])), + Arc::new(BooleanArray::from(vec![true, false, true])), + Arc::new(IntervalMonthDayNanoArray::from(vec![ + IntervalMonthDayNano::new(1, 2, 3_000); + 3 + ])), + Arc::new( + Decimal128Array::from(vec![1, 2, 3]) + .with_precision_and_scale(10, 2) + .unwrap(), + ), + Arc::new(Int32Array::from(Vec::::new())), + ] + } + + #[test] + fn test_hashes_buffer_length_mismatch_is_error() { + for array in length_mismatch_arrays() { + for len in [array.len().saturating_sub(1), array.len() + 1] { + if len == array.len() { + continue; + } + let mut hashes = vec![42u32; len]; + let result = create_murmur3_hashes(&[Arc::clone(&array)], &mut hashes); + assert!( + result.is_err(), + "{} array of length {} with {len} hashes", + array.data_type(), + array.len() + ); + } + } + } + /// A dictionary array reached through a nested type arrives as the only column of its recursive /// call, so deciding the dictionary fast path from column position alone treated it as a first /// column and restarted from the seed, discarding the hash accumulated for earlier elements of diff --git a/native/spark-expr/src/hash_funcs/utils.rs b/native/spark-expr/src/hash_funcs/utils.rs index 3c7a61ff10e..7625d0ff3e8 100644 --- a/native/spark-expr/src/hash_funcs/utils.rs +++ b/native/spark-expr/src/hash_funcs/utils.rs @@ -18,6 +18,72 @@ //! This includes utilities for hashing and murmur3 hashing. use arrow::array::Array; +use arrow::buffer::NullBuffer; +use datafusion::common::{internal_err, Result}; + +/// The positions of the set bits of `bits`, lowest first. +#[inline] +fn set_bits(mut bits: u64) -> impl Iterator { + std::iter::from_fn(move || { + (bits != 0).then(|| { + let i = bits.trailing_zeros() as usize; + bits &= bits - 1; + i + }) + }) +} + +/// The fewest valid rows in a 64-row chunk for which hashing every row of the chunk and restoring +/// its null rows costs less than hashing only its valid rows one at a time. +/// The null rates from 25% to 85% in the `hash_nulls` benchmark put random chunks around this +/// count, so rerun them when changing the value. +const MIN_VALID_FOR_DENSE: u32 = 16; + +/// Hashes the valid rows of a fixed-width column that has nulls, 64 rows at a time. Null rows keep +/// their previous hash. A chunk with at least [`MIN_VALID_FOR_DENSE`] valid rows is hashed in +/// full, which vectorizes, and its null rows then get their previous hashes back from a copy of the +/// chunk. Hashing a null slot is harmless because `hash` can't fail. A sparser chunk, and the last +/// `len % 64` rows, hash only their valid rows. +#[inline] +pub(crate) fn hash_fixed_width_with_nulls( + values: &[T], + nulls: &NullBuffer, + hashes: &mut [H], + hash: impl Fn(T, H) -> H, +) -> Result<()> { + // The chunks below zip the three inputs, which would skip rows if the lengths differed. + if nulls.len() != values.len() || hashes.len() != values.len() { + return internal_err!( + "{} values, {} validity bits, and {} hashes", + values.len(), + nulls.len(), + hashes.len() + ); + } + let chunks = nulls.inner().bit_chunks(); + let (value_chunks, value_tail) = values.as_chunks::<64>(); + let (hash_chunks, hash_tail) = hashes.as_chunks_mut::<64>(); + let rows = value_chunks.iter().zip(hash_chunks.iter_mut()); + for ((values, hashes), valid) in rows.zip(chunks.iter()) { + if valid.count_ones() < MIN_VALID_FOR_DENSE { + for i in set_bits(valid) { + hashes[i] = hash(values[i], hashes[i]); + } + continue; + } + let previous = *hashes; + for (h, &v) in hashes.iter_mut().zip(values) { + *h = hash(v, *h); + } + for i in set_bits(!valid) { + hashes[i] = previous[i]; + } + } + for i in set_bits(chunks.remainder_bits()) { + hash_tail[i] = hash(value_tail[i], hash_tail[i]); + } + Ok(()) +} #[macro_export] macro_rules! hash_array { @@ -33,9 +99,8 @@ macro_rules! hash_array { ) }); if array.null_count() == 0 { - // Fast path: no nulls, use direct indexing - for i in 0..$hashes.len() { - $hashes[i] = $hash_method(&array.value(i), $hashes[i]); + for (i, hash) in $hashes.iter_mut().enumerate() { + *hash = $hash_method(&array.value(i), *hash); } } else { // Slow path: check nulls @@ -62,12 +127,8 @@ macro_rules! hash_array_boolean { ) }); if array.null_count() == 0 { - // Fast path: no nulls, use direct indexing - for i in 0..$hashes.len() { - $hashes[i] = $hash_method( - $hash_input_type::from(array.value(i)).to_le_bytes(), - $hashes[i], - ); + for (hash, value) in $hashes.iter_mut().zip(array.values().iter()) { + *hash = $hash_method($hash_input_type::from(value).to_le_bytes(), *hash); } } else { // Slow path: check nulls @@ -99,17 +160,16 @@ macro_rules! hash_array_primitive { let values = array.values(); if array.null_count() == 0 { - // Fast path: no nulls, use direct indexing - for i in 0..values.len() { - $hashes[i] = $hash_method((values[i] as $ty).to_le_bytes(), $hashes[i]); - } - } else { - // Slow path: check nulls - for i in 0..values.len() { - if !array.is_null(i) { - $hashes[i] = $hash_method((values[i] as $ty).to_le_bytes(), $hashes[i]); - } + for (hash, &value) in $hashes.iter_mut().zip(values.iter()) { + *hash = $hash_method((value as $ty).to_le_bytes(), *hash); } + } else if let Some(nulls) = array.nulls() { + $crate::hash_funcs::utils::hash_fixed_width_with_nulls( + values, + nulls, + $hashes, + |value, hash| $hash_method((value as $ty).to_le_bytes(), hash), + )?; } }; } @@ -130,19 +190,20 @@ macro_rules! hash_array_primitive_float { let values = array.values(); if array.null_count() == 0 { - // Fast path: no nulls, use direct indexing - for i in 0..values.len() { - let value = $crate::float_semantics::hash_input(values[i]); - $hashes[i] = $hash_method(value.to_le_bytes(), $hashes[i]); - } - } else { - // Slow path: check nulls - for i in 0..values.len() { - if !array.is_null(i) { - let value = $crate::float_semantics::hash_input(values[i]); - $hashes[i] = $hash_method(value.to_le_bytes(), $hashes[i]); - } + for (hash, &value) in $hashes.iter_mut().zip(values.iter()) { + let value = $crate::float_semantics::hash_input(value); + *hash = $hash_method(value.to_le_bytes(), *hash); } + } else if let Some(nulls) = array.nulls() { + $crate::hash_funcs::utils::hash_fixed_width_with_nulls( + values, + nulls, + $hashes, + |value, hash| { + let value = $crate::float_semantics::hash_input(value); + $hash_method(value.to_le_bytes(), hash) + }, + )?; } }; } @@ -168,17 +229,14 @@ macro_rules! hash_array_interval_month_day_nano { // always an exact multiple of 1000 and out-of-range intervals throw at // conversion time instead of reaching this hasher. if array.null_count() == 0 { - // Fast path: no nulls, use direct indexing - for i in 0..$hashes.len() { - let value = array.value(i); + for (hash, value) in $hashes.iter_mut().zip(array.values().iter()) { // Match Spark 4.2 generated code, which omits the days field: // https://github.com/apache/spark/blob/v4.2.0/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala#L428-L431 // SPARK-58236 includes days starting in Spark 4.3; the version // switch for that is tracked in // https://github.com/apache/datafusion-comet/issues/5498. - $hashes[i] = - $hash_method((value.nanoseconds / 1_000).to_le_bytes(), $hashes[i]); - $hashes[i] = $hash_method(value.months.to_le_bytes(), $hashes[i]); + *hash = $hash_method((value.nanoseconds / 1_000).to_le_bytes(), *hash); + *hash = $hash_method(value.months.to_le_bytes(), *hash); } } else { // Slow path: check nulls @@ -209,13 +267,12 @@ macro_rules! hash_array_small_decimal { }); if array.null_count() == 0 { - // Fast path: no nulls, use direct indexing - for i in 0..$hashes.len() { - $hashes[i] = $hash_method( - i64::try_from(array.value(i)) + for (hash, &value) in $hashes.iter_mut().zip(array.values().iter()) { + *hash = $hash_method( + i64::try_from(value) .map(|v| v.to_le_bytes()) .map_err(|e| DataFusionError::Execution(e.to_string()))?, - $hashes[i], + *hash, ); } } else { @@ -875,6 +932,15 @@ macro_rules! create_hashes_internal { use datafusion_comet_common::children_with_parent_nulls; for (i, col) in $arrays.iter().enumerate() { + // The fast paths zip the hashes with the values, which would silently skip rows if + // the lengths differed. + if col.len() != $hashes_buffer.len() { + return Err(datafusion::common::DataFusionError::Internal(format!( + "Column {i} has {} rows but the hashes buffer has {}", + col.len(), + $hashes_buffer.len() + ))); + } // The dictionary fast path hashes each distinct dictionary value once and reuses that // result for every key, which is only valid while every row carries the same incoming // hash. Position in the column list is not a sufficient test: this macro also runs on @@ -1296,7 +1362,9 @@ pub(crate) mod test_utils { #[cfg(test)] mod tests { use super::super::{spark_murmur3_hash, spark_xxhash64}; + use super::{hash_fixed_width_with_nulls, MIN_VALID_FOR_DENSE}; use arrow::array::{Array, ArrayRef, BinaryArray, Decimal128Array, Int32Array}; + use arrow::buffer::NullBuffer; use datafusion::common::{Result, ScalarValue}; use datafusion::logical_expr::ColumnarValue; use num::BigInt; @@ -1498,4 +1566,74 @@ mod tests { } } } + + type Pattern = Box bool>; + + /// Validity by row index, chosen to reach every path through the chunk loop. + fn patterns(len: usize) -> Vec<(&'static str, Pattern)> { + let min_valid = MIN_VALID_FOR_DENSE as usize; + vec![ + ("one null", Box::new(move |i| i != len / 2)), + ( + "MIN_VALID_FOR_DENSE valid rows per chunk", + Box::new(move |i| i % 64 >= 64 - min_valid), + ), + ( + "one fewer valid row per chunk", + Box::new(move |i| i % 64 > 64 - min_valid), + ), + ("two thirds valid", Box::new(|i| i % 3 != 0)), + ( + "null run longer than a chunk", + Box::new(|i| !(1..70).contains(&i)), + ), + ("all null", Box::new(|_| false)), + ] + } + + #[test] + fn hash_fixed_width_with_nulls_matches_per_row() { + let hash = |value: u64, previous: u64| previous.wrapping_mul(0x9e37_79b9_7f4a_7c15) ^ value; + for len in [1, 63, 64, 65, 127, 128, 129, 200] { + for (name, valid) in patterns(len) { + // Offset 5 puts the validity bits off a byte boundary. + for offset in [0, 5] { + let nulls = NullBuffer::from((0..offset + len).map(&valid).collect::>()) + .slice(offset, len); + let values: Vec = (0..len as u64).map(|i| i * 7919 + 3).collect(); + // Distinct previous hashes, so a null row that loses its hash shows up. + let previous: Vec = (0..len as u64).map(|i| i * 31 + 1).collect(); + let expected: Vec = (0..len) + .map(|i| { + if nulls.is_valid(i) { + hash(values[i], previous[i]) + } else { + previous[i] + } + }) + .collect(); + let mut hashes = previous.clone(); + hash_fixed_width_with_nulls(&values, &nulls, &mut hashes, hash).unwrap(); + assert_eq!(hashes, expected, "{name}, len {len}, offset {offset}"); + } + } + } + } + + #[test] + fn hash_fixed_width_with_nulls_length_mismatch_is_error() { + let hash = |value: u64, previous: u64| previous ^ value; + let nulls = NullBuffer::from(vec![true, false, true]); + let values = [1u64, 2, 3]; + for len in [2, 4] { + let mut hashes = vec![0u64; len]; + assert!( + hash_fixed_width_with_nulls(&values, &nulls, &mut hashes, hash).is_err(), + "{len} hashes" + ); + } + let short_nulls = NullBuffer::from(vec![true, false]); + let mut hashes = vec![0u64; 3]; + assert!(hash_fixed_width_with_nulls(&values, &short_nulls, &mut hashes, hash).is_err()); + } } diff --git a/native/spark-expr/src/hash_funcs/xxhash64.rs b/native/spark-expr/src/hash_funcs/xxhash64.rs index 97db7fa7ff1..67638f788af 100644 --- a/native/spark-expr/src/hash_funcs/xxhash64.rs +++ b/native/spark-expr/src/hash_funcs/xxhash64.rs @@ -237,6 +237,25 @@ mod tests { test_hashes_with_nulls!(create_xxhash64_hashes, T, values, expected, u64); } + #[test] + fn test_hashes_buffer_length_mismatch_is_error() { + for array in crate::hash_funcs::murmur3::tests::length_mismatch_arrays() { + for len in [array.len().saturating_sub(1), array.len() + 1] { + if len == array.len() { + continue; + } + let mut hashes = vec![42u64; len]; + let result = create_xxhash64_hashes(&[Arc::clone(&array)], &mut hashes); + assert!( + result.is_err(), + "{} array of length {} with {len} hashes", + array.data_type(), + array.len() + ); + } + } + } + /// The dictionary fast path is shared in shape with murmur3, so it has the same requirement: /// a dictionary reached through a nested type must not restart from the seed. #[test] diff --git a/spark/src/test/resources/sql-tests/expressions/hash/hash_fixed_width_nulls.sql b/spark/src/test/resources/sql-tests/expressions/hash/hash_fixed_width_nulls.sql new file mode 100644 index 00000000000..fddbd2bb06e --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/hash/hash_fixed_width_nulls.sql @@ -0,0 +1,47 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Plain encoding keeps the columns off the dictionary path, so they reach the 64-row chunk loop +-- in `hash_fixed_width_with_nulls`. +-- Config: parquet.enable.dictionary=false + +-- One file of 1300 rows, so each block of 256 rows is four aligned 64-row chunks and the last 20 +-- rows are a partial chunk. Each block has its own validity pattern. Blocks 1 and 2 have 16 and +-- 15 valid rows per chunk, one at and one below `MIN_VALID_FOR_DENSE`, so they choose different +-- branches. Update them if that constant changes. +statement +CREATE TABLE test_hash_fixed_width_nulls USING parquet AS +SELECT /*+ COALESCE(1) */ id, IF(is_null, NULL, CAST(id * 7919 - 5000000 AS INT)) AS i, + IF(is_null, NULL, id * 1000000007 - 1000000000000) AS l, + IF(is_null, NULL, CASE id % 97 WHEN 0 THEN double('-0.0') WHEN 1 THEN double('NaN') + ELSE id / 7.0 - 50 END) AS d +FROM ( + SELECT id, CASE CAST(id DIV 256 AS INT) + WHEN 0 THEN id % 256 = 128 + WHEN 1 THEN id % 64 < 48 + WHEN 2 THEN id % 64 <= 48 + WHEN 3 THEN id % 3 = 0 + WHEN 4 THEN id % 256 BETWEEN 1 AND 69 + ELSE id % 2 = 0 + END AS is_null + FROM range(0, 1300, 1, 1) +) + +-- A float argument keeps xxhash64 on Comet's kernel at the default seed. +query expect_native(hash, xxhash64) +SELECT id, hash(i), hash(l), hash(d), xxhash64(d), xxhash64(i, d) +FROM test_hash_fixed_width_nulls ORDER BY id