From 1deaf5694059d236778a9d30ce313f80b40d9c11 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 12 Nov 2026 07:49:39 -0700 Subject: [PATCH 1/7] feat(spark): add Spark-compatible hash (Murmur3) function Port Comet's Murmur3 hash. Make the shared hash helpers generic over the hash state type so hash and xxhash64 reuse them. --- crates/spark/src/function/hash/mod.rs | 5 +- crates/spark/src/function/hash/murmur3.rs | 252 +++++++++++++++++++ crates/spark/src/function/hash/utils.rs | 18 +- crates/sqllogictest/test_files/hash/hash.slt | 203 +++++++++++++++ 4 files changed, 468 insertions(+), 10 deletions(-) create mode 100644 crates/spark/src/function/hash/murmur3.rs create mode 100644 crates/sqllogictest/test_files/hash/hash.slt diff --git a/crates/spark/src/function/hash/mod.rs b/crates/spark/src/function/hash/mod.rs index 927f1cbd..86cf68a5 100644 --- a/crates/spark/src/function/hash/mod.rs +++ b/crates/spark/src/function/hash/mod.rs @@ -3,6 +3,7 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements pub mod crc32; +pub mod murmur3; pub mod sha1; pub mod sha2; pub(crate) mod utils; @@ -13,6 +14,7 @@ use datafusion_functions::make_udf_function; use std::sync::Arc; make_udf_function!(crc32::SparkCrc32, crc32); +make_udf_function!(murmur3::SparkHash, hash); make_udf_function!(sha1::SparkSha1, sha1); make_udf_function!(sha2::SparkSha2, sha2); make_udf_function!(xxhash64::SparkXxhash64, xxhash64); @@ -21,6 +23,7 @@ pub mod expr_fn { use datafusion_functions::export_functions; export_functions!( (crc32, "crc32(expr) - Returns a cyclic redundancy check value of the expr as a bigint.", arg1), + (hash, "hash(expr1, expr2, ...) - Returns a hash value of the arguments using 32-bit Murmur3 with seed 42.", args), (sha1, "sha1(expr) - Returns a SHA-1 hash value of the expr as a hex string.", arg1), (sha2, "sha2(expr, bitLength) - Returns a checksum of SHA-2 family as a hex string of expr. SHA-224, SHA-256, SHA-384, and SHA-512 are supported. Bit length of 0 is equivalent to 256.", arg1 arg2), (xxhash64, "xxhash64(expr1, expr2, ...) - Returns a 64-bit hash value of the arguments using xxHash.", args) @@ -28,5 +31,5 @@ pub mod expr_fn { } pub fn functions() -> Vec> { - vec![crc32(), sha1(), sha2(), xxhash64()] + vec![crc32(), hash(), sha1(), sha2(), xxhash64()] } diff --git a/crates/spark/src/function/hash/murmur3.rs b/crates/spark/src/function/hash/murmur3.rs new file mode 100644 index 00000000..f7b630bd --- /dev/null +++ b/crates/spark/src/function/hash/murmur3.rs @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use std::sync::Arc; + +use arrow::array::{ + Array, ArrayRef, DictionaryArray, Int32Array, types::ArrowDictionaryKeyType, +}; +use arrow::buffer::{Buffer, ScalarBuffer}; +use arrow::compute::take; +use arrow::datatypes::{ArrowNativeType, DataType, Field, FieldRef}; +use datafusion_common::{Result, ScalarValue, internal_err}; +use datafusion_expr::{ + ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, + Volatility, +}; + +use crate::create_hashes_internal; + +const DEFAULT_SEED: u32 = 42; + +/// Spark-compatible `hash` function (32-bit Murmur3, seed 42). +/// +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkHash { + signature: Signature, +} + +impl Default for SparkHash { + fn default() -> Self { + Self::new() + } +} + +impl SparkHash { + pub fn new() -> Self { + Self { + signature: Signature::variadic_any(Volatility::Immutable), + } + } +} + +impl ScalarUDFImpl for SparkHash { + fn name(&self) -> &str { + "hash" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + internal_err!("return_field_from_args should be used instead") + } + + fn return_field_from_args(&self, _args: ReturnFieldArgs) -> Result { + // Spark's HashExpression overrides nullable to false: NULL inputs are + // skipped and the seed is used, so the result is never null. + Ok(Arc::new(Field::new(self.name(), DataType::Int32, false))) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + let num_rows = args.number_rows; + let mut hashes: Vec = vec![DEFAULT_SEED; num_rows]; + + let arrays = ColumnarValue::values_to_arrays(&args.args)?; + create_murmur3_hashes(&arrays, &mut hashes)?; + + if num_rows == 1 { + Ok(ColumnarValue::Scalar(ScalarValue::Int32(Some( + hashes[0] as i32, + )))) + } else { + // Reinterpret Vec as ScalarBuffer without copying. + let buffer = ScalarBuffer::::from(Buffer::from_vec(hashes)); + Ok(ColumnarValue::Array(Arc::new(Int32Array::new( + buffer, None, + )))) + } + } +} + +#[inline] +fn mix_k1(k1: u32) -> u32 { + k1.wrapping_mul(0xcc9e2d51) + .rotate_left(15) + .wrapping_mul(0x1b873593) +} + +#[inline] +fn mix_h1(h1: u32, k1: u32) -> u32 { + (h1 ^ k1) + .rotate_left(13) + .wrapping_mul(5) + .wrapping_add(0xe6546b64) +} + +#[inline] +fn fmix(mut h1: u32, len: u32) -> u32 { + h1 ^= len; + h1 ^= h1 >> 16; + h1 = h1.wrapping_mul(0x85ebca6b); + h1 ^= h1 >> 13; + h1 = h1.wrapping_mul(0xc2b2ae35); + h1 ^= h1 >> 16; + h1 +} + +/// Spark's `Murmur3_x86_32.hashUnsafeBytes`: whole 4-byte little-endian words +/// first, then each remaining tail byte is mixed separately as a sign-extended +/// word (this differs from the canonical MurmurHash3 tail handling). +#[inline] +fn spark_compatible_murmur3_hash>(data: T, seed: u32) -> u32 { + let data = data.as_ref(); + let mut chunks = data.chunks_exact(4); + let mut h1 = seed; + for chunk in &mut chunks { + let word = u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]); + h1 = mix_h1(h1, mix_k1(word)); + } + for &byte in chunks.remainder() { + h1 = mix_h1(h1, mix_k1(byte as i8 as i32 as u32)); + } + fmix(h1, data.len() as u32) +} + +/// Hash the values in a dictionary array using murmur3. +fn create_murmur3_hashes_dictionary( + array: &ArrayRef, + hashes_buffer: &mut [u32], + seeds_are_pristine: bool, +) -> Result<()> { + let dict_array = array.as_any().downcast_ref::>().unwrap(); + if !seeds_are_pristine { + let unpacked = take(dict_array.values().as_ref(), dict_array.keys(), None)?; + create_murmur3_hashes(&[unpacked], hashes_buffer)?; + } else { + // Hash each dictionary value once, then look up by key. + let dict_values = Arc::clone(dict_array.values()); + // Every row has the same running hash here, so start from it instead of 42 + let seed = hashes_buffer.first().copied().unwrap_or(DEFAULT_SEED); + let mut dict_hashes = vec![seed; dict_values.len()]; + create_murmur3_hashes(&[dict_values], &mut dict_hashes)?; + + for (hash, key) in hashes_buffer.iter_mut().zip(dict_array.keys().iter()) { + if let Some(key) = key { + *hash = dict_hashes[key.as_usize()] + } + // No update for Null keys, consistent with other types. + } + } + Ok(()) +} + +/// Create murmur3 hash values for every row, based on the values in the columns. +/// +/// The number of rows to hash is determined by `hashes_buffer.len()`. +/// `hashes_buffer` should be pre-sized appropriately and seeded with the +/// initial hash value (Spark uses `42`). +fn create_murmur3_hashes(arrays: &[ArrayRef], hashes_buffer: &mut [u32]) -> Result<()> { + create_hashes_internal!( + arrays, + hashes_buffer, + spark_compatible_murmur3_hash, + create_murmur3_hashes_dictionary, + create_murmur3_hashes + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{Float32Array, Float64Array, StringArray}; + use arrow::datatypes::Int32Type; + + fn hash_of(array: ArrayRef) -> Vec { + let mut hashes = vec![DEFAULT_SEED; array.len()]; + create_murmur3_hashes(&[array], &mut hashes).unwrap(); + hashes.into_iter().map(|h| h as i32).collect() + } + + #[test] + fn test_hash_nullability() -> Result<()> { + let func = SparkHash::new(); + let nullable: FieldRef = Arc::new(Field::new("a", DataType::Int32, true)); + let out = func.return_field_from_args(ReturnFieldArgs { + arg_fields: &[nullable], + scalar_arguments: &[None], + })?; + assert!(!out.is_nullable()); + assert_eq!(out.data_type(), &DataType::Int32); + Ok(()) + } + + // Expected values are taken from Comet's Spark-verified murmur3 tests. + #[test] + fn test_hash_i32() { + assert_eq!( + hash_of(Arc::new(Int32Array::from(vec![1, 0, -1]))), + vec![-559580957, 933211791, -1604776387] + ); + } + + #[test] + fn test_hash_str() { + let array: ArrayRef = Arc::new(StringArray::from(vec![ + "hello", "", "a", "ab", "abc", "abcd", "abcde", "天地", + ])); + // Covers every tail length (0-3 bytes) and multi-byte UTF-8. + assert_eq!( + hash_of(array), + [ + 3286402344u32, + 142593372, + 1485273170, + 0xfa37157b, + 1322437556, + 0xe860e5cc, + 814637928, + 2395000894 + ] + .map(|h| h as i32) + ); + } + + #[test] + fn test_hash_negative_zero() { + let f32s: ArrayRef = Arc::new(Float32Array::from(vec![0.0f32, -0.0f32])); + let h = hash_of(f32s); + assert_eq!(h[0], h[1]); + let f64s: ArrayRef = Arc::new(Float64Array::from(vec![0.0f64, -0.0f64])); + let h = hash_of(f64s); + assert_eq!(h[0], h[1]); + } + + #[test] + fn test_hash_null_keeps_seed() { + let array: ArrayRef = Arc::new(Int32Array::from(vec![None, Some(1)])); + assert_eq!(hash_of(array)[0], 42); + } + + #[test] + fn test_hash_dictionary_matches_decoded() { + let dict: DictionaryArray = + vec!["hello", "world", "hello"].into_iter().collect(); + let decoded: ArrayRef = + Arc::new(StringArray::from(vec!["hello", "world", "hello"])); + assert_eq!(hash_of(Arc::new(dict)), hash_of(decoded)); + } +} diff --git a/crates/spark/src/function/hash/utils.rs b/crates/spark/src/function/hash/utils.rs index 01f89cd2..640eaeb4 100644 --- a/crates/spark/src/function/hash/utils.rs +++ b/crates/spark/src/function/hash/utils.rs @@ -18,16 +18,16 @@ use datafusion_common::Result; /// Hash the values of a primitive array, calling `transform` to convert each /// value into a byte slice before feeding it to `hash_method`. #[inline] -pub(crate) fn hash_primitive_values( +pub(crate) fn hash_primitive_values( array: &PrimitiveArray

, - hashes: &mut [u64], + hashes: &mut [T], transform: F, hash_method: H, ) where P: ArrowPrimitiveType, F: Fn(P::Native) -> B, B: AsRef<[u8]>, - H: Fn(B, u64) -> u64, + H: Fn(B, T) -> T, { let values = array.values(); if array.null_count() == 0 { @@ -50,15 +50,15 @@ pub(crate) fn hash_primitive_values( /// and their Large/View variants) and for arrays whose values need a small /// transform (e.g. booleans widened to `i32`, decimals to `i128` bytes). #[inline] -pub(crate) fn hash_bytes_values( +pub(crate) fn hash_bytes_values( array: &A, - hashes: &mut [u64], + hashes: &mut [T], value_at: impl Fn(&A, usize) -> B, hash_method: H, ) where A: Array, B: AsRef<[u8]>, - H: Fn(B, u64) -> u64, + H: Fn(B, T) -> T, { if array.null_count() == 0 { for (i, hash) in hashes.iter_mut().enumerate() { @@ -76,16 +76,16 @@ pub(crate) fn hash_bytes_values( /// Fallible variant of [`hash_bytes_values`]: `value_at` may return an error /// (used by the small-decimal path, where values may not fit in `i64`). #[inline] -pub(crate) fn try_hash_bytes_values( +pub(crate) fn try_hash_bytes_values( array: &A, - hashes: &mut [u64], + hashes: &mut [T], value_at: impl Fn(&A, usize) -> Result, hash_method: H, ) -> Result<()> where A: Array, B: AsRef<[u8]>, - H: Fn(B, u64) -> u64, + H: Fn(B, T) -> T, { if array.null_count() == 0 { for (i, hash) in hashes.iter_mut().enumerate() { diff --git a/crates/sqllogictest/test_files/hash/hash.slt b/crates/sqllogictest/test_files/hash/hash.slt new file mode 100644 index 00000000..64613eac --- /dev/null +++ b/crates/sqllogictest/test_files/hash/hash.slt @@ -0,0 +1,203 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +# Tests for Spark-compatible hash (32-bit Murmur3, seed 42) +# Reference: https://spark.apache.org/docs/latest/api/sql/index.html#hash +# Expected values come from Comet's Spark-verified murmur3 tests. + +# Int32 + +query I +SELECT hash(arrow_cast(1, 'Int32')); +---- +-559580957 + +query I +SELECT hash(arrow_cast(0, 'Int32')); +---- +933211791 + +query I +SELECT hash(arrow_cast(-1, 'Int32')); +---- +-1604776387 + +query I +SELECT hash(arrow_cast(2147483647, 'Int32')); +---- +133916647 + +query I +SELECT hash(arrow_cast(-2147483648, 'Int32')); +---- +723455942 + +# Int64 + +query I +SELECT hash(arrow_cast(1, 'Int64')); +---- +-1712319331 + +query I +SELECT hash(arrow_cast(0, 'Int64')); +---- +-1670924195 + +query I +SELECT hash(arrow_cast(-1, 'Int64')); +---- +-939490007 + +query I +SELECT hash(arrow_cast(9223372036854775807, 'Int64')); +---- +-1604625029 + +query I +SELECT hash(arrow_cast(-9223372036854775808, 'Int64')); +---- +-853646085 + +# Float64: -0.0 hashes like 0.0 + +query I +SELECT hash(arrow_cast(1.0, 'Float64')); +---- +-460888942 + +query I +SELECT hash(arrow_cast(0.0, 'Float64')); +---- +-1670924195 + +query I +SELECT hash(arrow_cast(-0.0, 'Float64')); +---- +-1670924195 + +query I +SELECT hash(arrow_cast(-1.0, 'Float64')); +---- +332927831 + +# Float32: -0.0 hashes like 0.0 + +query I +SELECT hash(arrow_cast(1.0, 'Float32')); +---- +-466301895 + +query I +SELECT hash(arrow_cast(0.0, 'Float32')); +---- +933211791 + +query I +SELECT hash(arrow_cast(-0.0, 'Float32')); +---- +933211791 + +query I +SELECT hash(arrow_cast(-1.0, 'Float32')); +---- +-603084565 + +# Strings: every tail length, multi-byte UTF-8 + +query I +SELECT hash('hello'); +---- +-1008564952 + +query I +SELECT hash('bar'); +---- +-1808790533 + +query I +SELECT hash(''); +---- +142593372 + +query I +SELECT hash('a'); +---- +1485273170 + +query I +SELECT hash('ab'); +---- +-97053317 + +query I +SELECT hash('abc'); +---- +1322437556 + +query I +SELECT hash('abcd'); +---- +-396302900 + +query I +SELECT hash('abcde'); +---- +814637928 + +query I +SELECT hash('😁'); +---- +885025535 + +query I +SELECT hash('天地'); +---- +-1899966402 + +# NULL leaves the seed (42) unchanged; the result is never NULL +query I +SELECT hash(NULL); +---- +42 + +query I +SELECT hash(arrow_cast(NULL, 'Int32')); +---- +42 + +# NULL arguments are skipped when hashing several columns +query B +SELECT hash(1, NULL) = hash(1); +---- +true + +query B +SELECT hash(NULL, 1) = hash(1); +---- +true + +# Arguments chain: the hash of the previous argument seeds the next +query B +SELECT hash(1, 2) = hash(2, 1); +---- +false + +# Table data (VALUES integers are Int64, which hash as 8 bytes) +statement ok +CREATE TABLE hash_t AS VALUES (1, 'hello'), (NULL, 'bar'), (0, NULL); + +query I +SELECT hash(column1) FROM hash_t; +---- +-1712319331 +42 +-1670924195 + +query I +SELECT hash(column2) FROM hash_t; +---- +-1008564952 +-1808790533 +42 From 21f4310585de4b87b13b18bfa671cb4c1c0426f3 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 12 Nov 2026 07:52:35 -0700 Subject: [PATCH 2/7] refactor(spark): share dictionary hashing; reject hash() with no args Move the dictionary driver into utils::hash_dictionary_values so hash and xxhash64 share it. Add tests for the non-pristine dictionary path and the zero-argument error. --- crates/spark/src/function/hash/murmur3.rs | 72 ++++++++++++-------- crates/spark/src/function/hash/utils.rs | 41 ++++++++++- crates/spark/src/function/hash/xxhash64.rs | 37 +++------- crates/sqllogictest/test_files/hash/hash.slt | 4 ++ 4 files changed, 97 insertions(+), 57 deletions(-) diff --git a/crates/spark/src/function/hash/murmur3.rs b/crates/spark/src/function/hash/murmur3.rs index f7b630bd..1a7d99df 100644 --- a/crates/spark/src/function/hash/murmur3.rs +++ b/crates/spark/src/function/hash/murmur3.rs @@ -4,18 +4,16 @@ use std::sync::Arc; -use arrow::array::{ - Array, ArrayRef, DictionaryArray, Int32Array, types::ArrowDictionaryKeyType, -}; +use arrow::array::{ArrayRef, Int32Array, types::ArrowDictionaryKeyType}; use arrow::buffer::{Buffer, ScalarBuffer}; -use arrow::compute::take; -use arrow::datatypes::{ArrowNativeType, DataType, Field, FieldRef}; -use datafusion_common::{Result, ScalarValue, internal_err}; +use arrow::datatypes::{DataType, Field, FieldRef}; +use datafusion_common::{Result, ScalarValue, internal_err, plan_err}; use datafusion_expr::{ ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; +use super::utils::hash_dictionary_values; use crate::create_hashes_internal; const DEFAULT_SEED: u32 = 42; @@ -54,7 +52,10 @@ impl ScalarUDFImpl for SparkHash { internal_err!("return_field_from_args should be used instead") } - fn return_field_from_args(&self, _args: ReturnFieldArgs) -> Result { + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + if args.arg_fields.is_empty() { + return plan_err!("hash requires at least one argument"); + } // Spark's HashExpression overrides nullable to false: NULL inputs are // skipped and the seed is used, so the result is never null. Ok(Arc::new(Field::new(self.name(), DataType::Int32, false))) @@ -131,26 +132,12 @@ fn create_murmur3_hashes_dictionary( hashes_buffer: &mut [u32], seeds_are_pristine: bool, ) -> Result<()> { - let dict_array = array.as_any().downcast_ref::>().unwrap(); - if !seeds_are_pristine { - let unpacked = take(dict_array.values().as_ref(), dict_array.keys(), None)?; - create_murmur3_hashes(&[unpacked], hashes_buffer)?; - } else { - // Hash each dictionary value once, then look up by key. - let dict_values = Arc::clone(dict_array.values()); - // Every row has the same running hash here, so start from it instead of 42 - let seed = hashes_buffer.first().copied().unwrap_or(DEFAULT_SEED); - let mut dict_hashes = vec![seed; dict_values.len()]; - create_murmur3_hashes(&[dict_values], &mut dict_hashes)?; - - for (hash, key) in hashes_buffer.iter_mut().zip(dict_array.keys().iter()) { - if let Some(key) = key { - *hash = dict_hashes[key.as_usize()] - } - // No update for Null keys, consistent with other types. - } - } - Ok(()) + hash_dictionary_values::( + array, + hashes_buffer, + seeds_are_pristine, + create_murmur3_hashes, + ) } /// Create murmur3 hash values for every row, based on the values in the columns. @@ -172,7 +159,7 @@ fn create_murmur3_hashes(arrays: &[ArrayRef], hashes_buffer: &mut [u32]) -> Resu #[cfg(test)] mod tests { use super::*; - use arrow::array::{Float32Array, Float64Array, StringArray}; + use arrow::array::{DictionaryArray, Float32Array, Float64Array, StringArray}; use arrow::datatypes::Int32Type; fn hash_of(array: ArrayRef) -> Vec { @@ -235,6 +222,35 @@ mod tests { assert_eq!(h[0], h[1]); } + #[test] + fn test_hash_requires_an_argument() { + let err = SparkHash::new() + .return_field_from_args(ReturnFieldArgs { + arg_fields: &[], + scalar_arguments: &[], + }) + .unwrap_err(); + assert!(err.to_string().contains("at least one argument")); + } + + #[test] + fn test_hash_dictionary_after_other_column_matches_decoded() { + // The running hashes are no longer uniform once a first column is hashed, + // so the dictionary must be expanded rather than hashed per value. + let first: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); + let dict: ArrayRef = Arc::new( + vec!["a", "b", "a"] + .into_iter() + .collect::>(), + ); + let decoded: ArrayRef = Arc::new(StringArray::from(vec!["a", "b", "a"])); + let mut with_dict = vec![DEFAULT_SEED; 3]; + create_murmur3_hashes(&[Arc::clone(&first), dict], &mut with_dict).unwrap(); + let mut with_decoded = vec![DEFAULT_SEED; 3]; + create_murmur3_hashes(&[first, decoded], &mut with_decoded).unwrap(); + assert_eq!(with_dict, with_decoded); + } + #[test] fn test_hash_null_keeps_seed() { let array: ArrayRef = Arc::new(Int32Array::from(vec![None, Some(1)])); diff --git a/crates/spark/src/function/hash/utils.rs b/crates/spark/src/function/hash/utils.rs index 640eaeb4..2e1315f8 100644 --- a/crates/spark/src/function/hash/utils.rs +++ b/crates/spark/src/function/hash/utils.rs @@ -11,8 +11,11 @@ //! Ported from Apache DataFusion Comet: //! -use arrow::array::{Array, PrimitiveArray}; -use arrow::datatypes::ArrowPrimitiveType; +use std::sync::Arc; + +use arrow::array::{Array, ArrayRef, DictionaryArray, PrimitiveArray}; +use arrow::compute::take; +use arrow::datatypes::{ArrowDictionaryKeyType, ArrowNativeType, ArrowPrimitiveType}; use datafusion_common::Result; /// Hash the values of a primitive array, calling `transform` to convert each @@ -101,6 +104,40 @@ where Ok(()) } +/// Hash a dictionary array by delegating to `recurse`, the hash function's own +/// array-hashing entry point. +/// +/// When `seeds_are_pristine` (every row still carries the same running hash), +/// each dictionary value is hashed once and looked up per key. Otherwise the +/// dictionary is expanded so each row is hashed from its own running hash. +/// Null keys leave the running hash unchanged, like any other NULL. +pub(crate) fn hash_dictionary_values( + array: &ArrayRef, + hashes: &mut [T], + seeds_are_pristine: bool, + recurse: impl Fn(&[ArrayRef], &mut [T]) -> Result<()>, +) -> Result<()> { + let dict_array = array.as_any().downcast_ref::>().unwrap(); + if !seeds_are_pristine { + let unpacked = take(dict_array.values().as_ref(), dict_array.keys(), None)?; + return recurse(&[unpacked], hashes); + } + let dict_values = Arc::clone(dict_array.values()); + // Every row has the same running hash here, so seed the values with it. + // `hashes` is non-empty whenever there is a non-null key to look up. + let Some(&seed) = hashes.first() else { + return Ok(()); + }; + let mut dict_hashes = vec![seed; dict_values.len()]; + recurse(&[dict_values], &mut dict_hashes)?; + for (hash, key) in hashes.iter_mut().zip(dict_array.keys().iter()) { + if let Some(key) = key { + *hash = dict_hashes[key.as_usize()]; + } + } + Ok(()) +} + /// Downcast `$column` to `$array_type`, panicking with a useful error if the /// type does not match. #[macro_export] diff --git a/crates/spark/src/function/hash/xxhash64.rs b/crates/spark/src/function/hash/xxhash64.rs index 4a0158fa..743510b6 100644 --- a/crates/spark/src/function/hash/xxhash64.rs +++ b/crates/spark/src/function/hash/xxhash64.rs @@ -4,12 +4,9 @@ use std::sync::Arc; -use arrow::array::{ - Array, ArrayRef, DictionaryArray, Int64Array, types::ArrowDictionaryKeyType, -}; +use arrow::array::{ArrayRef, Int64Array, types::ArrowDictionaryKeyType}; use arrow::buffer::{Buffer, ScalarBuffer}; -use arrow::compute::take; -use arrow::datatypes::{ArrowNativeType, DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::{Result, ScalarValue, internal_err}; use datafusion_expr::{ ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, @@ -17,6 +14,7 @@ use datafusion_expr::{ }; use twox_hash::XxHash64; +use super::utils::hash_dictionary_values; use crate::create_hashes_internal; const DEFAULT_SEED: u64 = 42; @@ -94,27 +92,12 @@ fn create_xxhash64_hashes_dictionary( hashes_buffer: &mut [u64], seeds_are_pristine: bool, ) -> Result<()> { - let dict_array = array.as_any().downcast_ref::>().unwrap(); - if !seeds_are_pristine { - let unpacked = take(dict_array.values().as_ref(), dict_array.keys(), None)?; - create_xxhash64_hashes(&[unpacked], hashes_buffer)?; - } else { - // Hash each dictionary value once, then look up by key. This avoids - // redundant hashing of large dictionary entries (e.g. long strings). - let dict_values = Arc::clone(dict_array.values()); - // Every row has the same running hash here, so start from it instead of 42 - let seed = hashes_buffer.first().copied().unwrap_or(DEFAULT_SEED); - let mut dict_hashes = vec![seed; dict_values.len()]; - create_xxhash64_hashes(&[dict_values], &mut dict_hashes)?; - - for (hash, key) in hashes_buffer.iter_mut().zip(dict_array.keys().iter()) { - if let Some(key) = key { - *hash = dict_hashes[key.as_usize()] - } - // No update for Null keys, consistent with other types. - } - } - Ok(()) + hash_dictionary_values::( + array, + hashes_buffer, + seeds_are_pristine, + create_xxhash64_hashes, + ) } /// Create xxhash64 hash values for every row, based on the values in the columns. @@ -136,7 +119,7 @@ fn create_xxhash64_hashes(arrays: &[ArrayRef], hashes_buffer: &mut [u64]) -> Res #[cfg(test)] mod tests { use super::*; - use arrow::array::{FixedSizeBinaryArray, Int32Array, StringArray}; + use arrow::array::{DictionaryArray, FixedSizeBinaryArray, Int32Array, StringArray}; #[test] fn test_xxhash64_nullability() -> Result<()> { diff --git a/crates/sqllogictest/test_files/hash/hash.slt b/crates/sqllogictest/test_files/hash/hash.slt index 64613eac..7943403a 100644 --- a/crates/sqllogictest/test_files/hash/hash.slt +++ b/crates/sqllogictest/test_files/hash/hash.slt @@ -201,3 +201,7 @@ SELECT hash(column2) FROM hash_t; -1008564952 -1808790533 42 + +# Spark requires at least one argument +statement error +SELECT hash(); From 4e49569791fdcffbfe6d3cdf6c06f5ce04dcdea3 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 13 Nov 2026 09:55:44 -0700 Subject: [PATCH 3/7] fix(spark): match Spark for wide decimals and all-scalar args in hash/xxhash64 - Hash Decimal128 values with precision > 18 as the minimal big-endian two's complement bytes of the unscaled value (Java's BigInteger.toByteArray), as Spark does. Fixes both hash and xxhash64, including decimals nested in lists and structs. - Expand scalar arguments to number_rows, and hash a single row returning a scalar when every argument is a scalar, so an unfolded all-literal call no longer indexes past the end of the 1-row arrays. - Drop the unreachable zero-argument plan_err (the variadic_any signature already rejects it) and pin the planner message in hash.slt. - Add Spark 4.2.0-verified .slt cases for seed chaining, decimals, lists, structs, maps and dictionaries; move the value-only Rust tests to .slt. Co-Authored-By: Claude Opus 5.5 --- crates/spark/src/function/hash/murmur3.rs | 160 ++++++++------ crates/spark/src/function/hash/utils.rs | 37 +++- crates/spark/src/function/hash/xxhash64.rs | 59 ++++- crates/sqllogictest/test_files/hash/hash.slt | 209 +++++++++++++++++- .../sqllogictest/test_files/hash/xxhash64.slt | 97 ++++++++ 5 files changed, 478 insertions(+), 84 deletions(-) diff --git a/crates/spark/src/function/hash/murmur3.rs b/crates/spark/src/function/hash/murmur3.rs index 1a7d99df..cec128ae 100644 --- a/crates/spark/src/function/hash/murmur3.rs +++ b/crates/spark/src/function/hash/murmur3.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use arrow::array::{ArrayRef, Int32Array, types::ArrowDictionaryKeyType}; use arrow::buffer::{Buffer, ScalarBuffer}; use arrow::datatypes::{DataType, Field, FieldRef}; -use datafusion_common::{Result, ScalarValue, internal_err, plan_err}; +use datafusion_common::{Result, ScalarValue, internal_err}; use datafusion_expr::{ ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -52,23 +52,31 @@ impl ScalarUDFImpl for SparkHash { internal_err!("return_field_from_args should be used instead") } - fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - if args.arg_fields.is_empty() { - return plan_err!("hash requires at least one argument"); - } + fn return_field_from_args(&self, _args: ReturnFieldArgs) -> Result { // Spark's HashExpression overrides nullable to false: NULL inputs are // skipped and the seed is used, so the result is never null. Ok(Arc::new(Field::new(self.name(), DataType::Int32, false))) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { - let num_rows = args.number_rows; - let mut hashes: Vec = vec![DEFAULT_SEED; num_rows]; + // When every argument is a scalar the result is the same for every row, + // so hash a single row and return a scalar. Otherwise expand scalars to + // the batch length so every array has `number_rows` entries. + let all_scalars = args + .args + .iter() + .all(|arg| matches!(arg, ColumnarValue::Scalar(_))); + let num_rows = if all_scalars { 1 } else { args.number_rows }; + let arrays = args + .args + .iter() + .map(|arg| arg.to_array(num_rows)) + .collect::>>()?; - let arrays = ColumnarValue::values_to_arrays(&args.args)?; + let mut hashes: Vec = vec![DEFAULT_SEED; num_rows]; create_murmur3_hashes(&arrays, &mut hashes)?; - if num_rows == 1 { + if all_scalars { Ok(ColumnarValue::Scalar(ScalarValue::Int32(Some( hashes[0] as i32, )))) @@ -114,13 +122,12 @@ fn fmix(mut h1: u32, len: u32) -> u32 { #[inline] fn spark_compatible_murmur3_hash>(data: T, seed: u32) -> u32 { let data = data.as_ref(); - let mut chunks = data.chunks_exact(4); + let (words, tail) = data.as_chunks::<4>(); let mut h1 = seed; - for chunk in &mut chunks { - let word = u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]); - h1 = mix_h1(h1, mix_k1(word)); + for word in words { + h1 = mix_h1(h1, mix_k1(u32::from_le_bytes(*word))); } - for &byte in chunks.remainder() { + for &byte in tail { h1 = mix_h1(h1, mix_k1(byte as i8 as i32 as u32)); } fmix(h1, data.len() as u32) @@ -159,8 +166,9 @@ fn create_murmur3_hashes(arrays: &[ArrayRef], hashes_buffer: &mut [u32]) -> Resu #[cfg(test)] mod tests { use super::*; - use arrow::array::{DictionaryArray, Float32Array, Float64Array, StringArray}; + use arrow::array::{DictionaryArray, StringArray}; use arrow::datatypes::Int32Type; + use datafusion_common::config::ConfigOptions; fn hash_of(array: ArrayRef) -> Vec { let mut hashes = vec![DEFAULT_SEED; array.len()]; @@ -181,58 +189,6 @@ mod tests { Ok(()) } - // Expected values are taken from Comet's Spark-verified murmur3 tests. - #[test] - fn test_hash_i32() { - assert_eq!( - hash_of(Arc::new(Int32Array::from(vec![1, 0, -1]))), - vec![-559580957, 933211791, -1604776387] - ); - } - - #[test] - fn test_hash_str() { - let array: ArrayRef = Arc::new(StringArray::from(vec![ - "hello", "", "a", "ab", "abc", "abcd", "abcde", "天地", - ])); - // Covers every tail length (0-3 bytes) and multi-byte UTF-8. - assert_eq!( - hash_of(array), - [ - 3286402344u32, - 142593372, - 1485273170, - 0xfa37157b, - 1322437556, - 0xe860e5cc, - 814637928, - 2395000894 - ] - .map(|h| h as i32) - ); - } - - #[test] - fn test_hash_negative_zero() { - let f32s: ArrayRef = Arc::new(Float32Array::from(vec![0.0f32, -0.0f32])); - let h = hash_of(f32s); - assert_eq!(h[0], h[1]); - let f64s: ArrayRef = Arc::new(Float64Array::from(vec![0.0f64, -0.0f64])); - let h = hash_of(f64s); - assert_eq!(h[0], h[1]); - } - - #[test] - fn test_hash_requires_an_argument() { - let err = SparkHash::new() - .return_field_from_args(ReturnFieldArgs { - arg_fields: &[], - scalar_arguments: &[], - }) - .unwrap_err(); - assert!(err.to_string().contains("at least one argument")); - } - #[test] fn test_hash_dictionary_after_other_column_matches_decoded() { // The running hashes are no longer uniform once a first column is hashed, @@ -251,12 +207,6 @@ mod tests { assert_eq!(with_dict, with_decoded); } - #[test] - fn test_hash_null_keeps_seed() { - let array: ArrayRef = Arc::new(Int32Array::from(vec![None, Some(1)])); - assert_eq!(hash_of(array)[0], 42); - } - #[test] fn test_hash_dictionary_matches_decoded() { let dict: DictionaryArray = @@ -265,4 +215,68 @@ mod tests { Arc::new(StringArray::from(vec!["hello", "world", "hello"])); assert_eq!(hash_of(Arc::new(dict)), hash_of(decoded)); } + + fn invoke(args: Vec, number_rows: usize) -> Result { + let arg_fields = args + .iter() + .enumerate() + .map(|(i, arg)| Arc::new(Field::new(format!("a{i}"), arg.data_type(), true))) + .collect(); + SparkHash::new().invoke_with_args(ScalarFunctionArgs { + args, + arg_fields, + number_rows, + return_field: Arc::new(Field::new("hash", DataType::Int32, false)), + config_options: Arc::new(ConfigOptions::default()), + }) + } + + // An all-literal hash(...) that is not constant-folded (e.g. a physical + // expression built directly) is evaluated with the batch's row count, while + // every argument is a scalar. This cannot be reached from SQL. + #[test] + fn test_hash_all_scalar_args_with_many_rows() -> Result<()> { + let args = vec![ + ColumnarValue::Scalar(ScalarValue::Int32(Some(1))), + ColumnarValue::Scalar(ScalarValue::Utf8(Some("hello".to_string()))), + ]; + let mut expected = vec![DEFAULT_SEED]; + create_murmur3_hashes( + &[ + Arc::new(Int32Array::from(vec![1])), + Arc::new(StringArray::from(vec!["hello"])), + ], + &mut expected, + )?; + match invoke(args, 3)? { + ColumnarValue::Scalar(ScalarValue::Int32(Some(h))) => { + assert_eq!(h, expected[0] as i32) + } + other => panic!("expected an Int32 scalar, got {other:?}"), + } + Ok(()) + } + + #[test] + fn test_hash_scalar_and_array_args() -> Result<()> { + let args = vec![ + ColumnarValue::Scalar(ScalarValue::Int32(Some(1))), + ColumnarValue::Array(Arc::new(StringArray::from(vec!["a", "b", "c"]))), + ]; + let mut expected = vec![DEFAULT_SEED; 3]; + create_murmur3_hashes( + &[ + Arc::new(Int32Array::from(vec![1, 1, 1])), + Arc::new(StringArray::from(vec!["a", "b", "c"])), + ], + &mut expected, + )?; + let ColumnarValue::Array(out) = invoke(args, 3)? else { + panic!("expected an array"); + }; + let out = out.as_any().downcast_ref::().unwrap(); + let expected: Vec = expected.into_iter().map(|h| h as i32).collect(); + assert_eq!(out.values().to_vec(), expected); + Ok(()) + } } diff --git a/crates/spark/src/function/hash/utils.rs b/crates/spark/src/function/hash/utils.rs index 2e1315f8..771a7bf4 100644 --- a/crates/spark/src/function/hash/utils.rs +++ b/crates/spark/src/function/hash/utils.rs @@ -51,7 +51,7 @@ pub(crate) fn hash_primitive_values( /// Hash an array by calling `value_at(array, i)` to produce a byte slice for /// each non-null row. Used for byte-slice arrays (Utf8/Binary/FixedSizeBinary /// and their Large/View variants) and for arrays whose values need a small -/// transform (e.g. booleans widened to `i32`, decimals to `i128` bytes). +/// transform (e.g. booleans widened to `i32`). #[inline] pub(crate) fn hash_bytes_values( array: &A, @@ -138,6 +138,33 @@ pub(crate) fn hash_dictionary_values( Ok(()) } +/// The minimal big-endian two's complement bytes of an `i128`, matching Java's +/// `BigInteger.toByteArray()` (which Spark hashes for decimals with precision +/// greater than 18). +pub(crate) struct DecimalBytes { + bytes: [u8; 16], + start: usize, +} + +impl AsRef<[u8]> for DecimalBytes { + #[inline] + fn as_ref(&self) -> &[u8] { + &self.bytes[self.start..] + } +} + +/// Encode `value` like Java's `BigInteger.toByteArray()`: big-endian, dropping +/// leading bytes that only repeat the sign while keeping one sign bit. +#[inline] +pub(crate) fn decimal_unscaled_bytes(value: i128) -> DecimalBytes { + let sign_bits = value.leading_zeros().max(value.leading_ones()); + let start = (sign_bits.saturating_sub(1) / 8) as usize; + DecimalBytes { + bytes: value.to_be_bytes(), + start, + } +} + /// Downcast `$column` to `$array_type`, panicking with a useful error if the /// type does not match. #[macro_export] @@ -254,16 +281,18 @@ macro_rules! hash_array_small_decimal { }; } -/// Hash a wide decimal array using the raw 128-bit byte representation. +/// Hash a wide (precision > 18) decimal array the way Spark does: by hashing +/// `unscaledValue().toByteArray()`, the minimal big-endian two's complement +/// encoding of the unscaled value (see [`decimal_unscaled_bytes`]). #[macro_export] #[doc(hidden)] macro_rules! hash_array_decimal { ($array_type:ident, $column:ident, $hashes:ident, $hash_method:ident) => { let array = $crate::downcast_array!($column, $array_type); - $crate::function::hash::utils::hash_bytes_values( + $crate::function::hash::utils::hash_primitive_values( array, $hashes, - |a, i| a.value(i).to_le_bytes(), + $crate::function::hash::utils::decimal_unscaled_bytes, $hash_method, ); }; diff --git a/crates/spark/src/function/hash/xxhash64.rs b/crates/spark/src/function/hash/xxhash64.rs index 743510b6..00ef2adc 100644 --- a/crates/spark/src/function/hash/xxhash64.rs +++ b/crates/spark/src/function/hash/xxhash64.rs @@ -60,19 +60,29 @@ impl ScalarUDFImpl for SparkXxhash64 { } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { - let num_rows = args.number_rows; - let mut hashes: Vec = vec![DEFAULT_SEED; num_rows]; + // When every argument is a scalar the result is the same for every row, + // so hash a single row and return a scalar. Otherwise expand scalars to + // the batch length so every array has `number_rows` entries. + let all_scalars = args + .args + .iter() + .all(|arg| matches!(arg, ColumnarValue::Scalar(_))); + let num_rows = if all_scalars { 1 } else { args.number_rows }; + let arrays = args + .args + .iter() + .map(|arg| arg.to_array(num_rows)) + .collect::>>()?; - let arrays = ColumnarValue::values_to_arrays(&args.args)?; + let mut hashes: Vec = vec![DEFAULT_SEED; num_rows]; create_xxhash64_hashes(&arrays, &mut hashes)?; - if num_rows == 1 { + if all_scalars { Ok(ColumnarValue::Scalar(ScalarValue::Int64(Some( hashes[0] as i64, )))) } else { - // Reinterpret Vec as ScalarBuffer without copying — both - // types have identical layout, and `as i64` is a bitcast. + // Reinterpret Vec as ScalarBuffer without copying. let buffer = ScalarBuffer::::from(Buffer::from_vec(hashes)); Ok(ColumnarValue::Array(Arc::new(Int64Array::new( buffer, None, @@ -502,4 +512,41 @@ mod tests { } assert_ne!(hashes[0], hashes[1]); } + // An all-literal xxhash64(...) that is not constant-folded is evaluated with + // the batch's row count while every argument is a scalar. This cannot be + // reached from SQL. + #[test] + fn test_xxhash64_all_scalar_args_with_many_rows() -> Result<()> { + let args = vec![ + ColumnarValue::Scalar(ScalarValue::Int32(Some(1))), + ColumnarValue::Scalar(ScalarValue::Utf8(Some("hello".to_string()))), + ]; + let arg_fields = args + .iter() + .enumerate() + .map(|(i, arg)| Arc::new(Field::new(format!("a{i}"), arg.data_type(), true))) + .collect(); + let out = SparkXxhash64::new().invoke_with_args(ScalarFunctionArgs { + args, + arg_fields, + number_rows: 3, + return_field: Arc::new(Field::new("xxhash64", DataType::Int64, false)), + config_options: Arc::new(datafusion_common::config::ConfigOptions::default()), + })?; + let mut expected = vec![DEFAULT_SEED]; + create_xxhash64_hashes( + &[ + Arc::new(Int32Array::from(vec![1])), + Arc::new(StringArray::from(vec!["hello"])), + ], + &mut expected, + )?; + match out { + ColumnarValue::Scalar(ScalarValue::Int64(Some(h))) => { + assert_eq!(h, expected[0] as i64) + } + other => panic!("expected an Int64 scalar, got {other:?}"), + } + Ok(()) + } } diff --git a/crates/sqllogictest/test_files/hash/hash.slt b/crates/sqllogictest/test_files/hash/hash.slt index 7943403a..9efb947a 100644 --- a/crates/sqllogictest/test_files/hash/hash.slt +++ b/crates/sqllogictest/test_files/hash/hash.slt @@ -184,6 +184,213 @@ SELECT hash(1, 2) = hash(2, 1); ---- false +# Spark's documented example; DataFusion types 123 and 2 as BIGINT, Spark as INT +query I +SELECT hash('Spark', [CAST(123 AS INT)], CAST(2 AS INT)); +---- +-1321691492 + +query I +SELECT hash('Spark', [CAST(123 AS INT)]); +---- +1262264271 + +# Hashing several arguments equals hashing a struct of them +query I +SELECT hash(CAST(1 AS INT), 'x'); +---- +-1305545736 + +# Decimals with precision <= 18 hash as the unscaled long + +query I +SELECT hash(CAST(123.45 AS DECIMAL(10, 2))); +---- +1416086240 + +query I +SELECT hash(CAST(-123.45 AS DECIMAL(10, 2))); +---- +-1959512858 + +# Decimals with precision > 18 hash the minimal big-endian two's complement +# bytes of the unscaled value (Java's BigInteger.toByteArray) + +query I +SELECT hash(CAST(1 AS DECIMAL(20, 0))); +---- +-386724586 + +query I +SELECT hash(CAST(0 AS DECIMAL(20, 0))); +---- +-783713497 + +query I +SELECT hash(CAST(-1 AS DECIMAL(20, 0))); +---- +1398487324 + +# 127 and -128 fit in one byte; 128 and -129 need a leading sign byte +query I +SELECT hash(CAST(127 AS DECIMAL(20, 0))); +---- +1185089389 + +query I +SELECT hash(CAST(128 AS DECIMAL(20, 0))); +---- +-544401882 + +query I +SELECT hash(CAST(-128 AS DECIMAL(20, 0))); +---- +775851899 + +query I +SELECT hash(CAST(-129 AS DECIMAL(20, 0))); +---- +-771458971 + +query I +SELECT hash(CAST(1.5 AS DECIMAL(20, 1))); +---- +-357711666 + +query I +SELECT hash(CAST('12345678901234567890.1234567890' AS DECIMAL(38, 10))); +---- +269448356 + +query I +SELECT hash(CAST('-12345678901234567890.1234567890' AS DECIMAL(38, 10))); +---- +694017569 + +query I +SELECT hash(CAST('99999999999999999999999999999999999999' AS DECIMAL(38, 0))); +---- +-817514053 + +query I +SELECT hash(CAST('-99999999999999999999999999999999999999' AS DECIMAL(38, 0))); +---- +1400911110 + +query I +SELECT hash(CAST(NULL AS DECIMAL(20, 0))); +---- +42 + +# Lists hash each element in order, chaining the running hash; NULL elements +# are skipped and an empty or NULL list leaves the seed unchanged + +query I +SELECT hash([CAST(1 AS INT), CAST(2 AS INT), CAST(3 AS INT)]); +---- +-912918097 + +query I +SELECT hash([1, 2, 3]); +---- +1045631400 + +query I +SELECT hash(['a', 'b']); +---- +190734147 + +query I +SELECT hash([CAST(1 AS INT), NULL, CAST(3 AS INT)]); +---- +1066699649 + +query I +SELECT hash(arrow_cast([], 'List(Int32)')); +---- +42 + +query I +SELECT hash(arrow_cast(NULL, 'List(Int32)')); +---- +42 + +query I +SELECT hash([CAST(1 AS DECIMAL(20, 0)), CAST(-129 AS DECIMAL(20, 0))]); +---- +-265473650 + +query I +SELECT hash([CAST(123.45 AS DECIMAL(10, 2))]); +---- +1416086240 + +# Nested lists flatten into the same chain as a flat list +query I +SELECT hash([[CAST(1 AS INT)], [CAST(2 AS INT), CAST(3 AS INT)]]); +---- +-912918097 + +# Structs hash each field in order; NULL fields are skipped + +query I +SELECT hash(named_struct('a', CAST(1 AS INT), 'b', 'x')); +---- +-1305545736 + +query I +SELECT hash(named_struct('a', CAST(1 AS INT), 'b', CAST(NULL AS VARCHAR))); +---- +-559580957 + +query I +SELECT hash(named_struct('a', CAST(1 AS DECIMAL(20, 0)))); +---- +-386724586 + +# Maps hash each key then its value, entry by entry. Spark 4.2.0 rejects hash on +# maps by default (DATATYPE_MISMATCH.HASH_MAP_TYPE); these values are from Spark +# with spark.sql.legacy.allowHashOnMapType=true. + +query I +SELECT hash(map([CAST(1 AS INT), CAST(2 AS INT)], ['a', 'b'])); +---- +-1706464220 + +query I +SELECT hash(map(['a'], [CAST(1 AS INT)])); +---- +532252686 + +# Dictionary-encoded values hash like their decoded values + +query I +SELECT hash(arrow_cast('hello', 'Dictionary(Int32, Utf8)')); +---- +-1008564952 + +query I +SELECT hash(arrow_cast(CAST(1 AS INT), 'Dictionary(Int8, Int32)')); +---- +-559580957 + +query I +SELECT hash(arrow_cast(column1, 'Dictionary(Int32, Utf8)')) +FROM VALUES ('hello'), ('bar'), (NULL), ('hello'); +---- +-1008564952 +-1808790533 +42 +-1008564952 + +# A dictionary after another argument is seeded per row by the running hash +query I +SELECT hash(CAST(1 AS INT), arrow_cast(column1, 'Dictionary(Int32, Utf8)')) +FROM VALUES ('x'), ('x'); +---- +-1305545736 +-1305545736 + # Table data (VALUES integers are Int64, which hash as 8 bytes) statement ok CREATE TABLE hash_t AS VALUES (1, 'hello'), (NULL, 'bar'), (0, NULL); @@ -203,5 +410,5 @@ SELECT hash(column2) FROM hash_t; 42 # Spark requires at least one argument -statement error +statement error does not support zero arguments SELECT hash(); diff --git a/crates/sqllogictest/test_files/hash/xxhash64.slt b/crates/sqllogictest/test_files/hash/xxhash64.slt index bb44468d..4c7f601b 100644 --- a/crates/sqllogictest/test_files/hash/xxhash64.slt +++ b/crates/sqllogictest/test_files/hash/xxhash64.slt @@ -198,3 +198,100 @@ SELECT xxhash64(7, nullif(s, s)), xxhash64(7) FROM (SELECT named_struct('a', 1, 'b', 'x') AS s); ---- -2864211556195259663 -2864211556195259663 + +# Spark's documented example; DataFusion types 123 and 2 as BIGINT, Spark as INT +query I +SELECT xxhash64('Spark', [CAST(123 AS INT)], CAST(2 AS INT)); +---- +5602566077635097486 + +# Decimals with precision <= 18 hash as the unscaled long +query I +SELECT xxhash64(CAST(123.45 AS DECIMAL(10, 2))); +---- +8791244235932249694 + +query I +SELECT xxhash64(CAST(-123.45 AS DECIMAL(10, 2))); +---- +-4814648695243699264 + +# Decimals with precision > 18 hash the minimal big-endian two's complement +# bytes of the unscaled value (Java's BigInteger.toByteArray) + +query I +SELECT xxhash64(CAST(1 AS DECIMAL(20, 0))); +---- +6668291691252061002 + +query I +SELECT xxhash64(CAST(0 AS DECIMAL(20, 0))); +---- +-8959994473701255385 + +query I +SELECT xxhash64(CAST(-1 AS DECIMAL(20, 0))); +---- +-4006032525457443936 + +# 127 and -128 fit in one byte; 128 and -129 need a leading sign byte +query I +SELECT xxhash64(CAST(127 AS DECIMAL(20, 0))); +---- +-5998106174972408993 + +query I +SELECT xxhash64(CAST(128 AS DECIMAL(20, 0))); +---- +6715097930120473301 + +query I +SELECT xxhash64(CAST(-128 AS DECIMAL(20, 0))); +---- +-7898411661632158123 + +query I +SELECT xxhash64(CAST(-129 AS DECIMAL(20, 0))); +---- +-6459515886037897956 + +query I +SELECT xxhash64(CAST(1.5 AS DECIMAL(20, 1))); +---- +7491490015795856943 + +query I +SELECT xxhash64(CAST('12345678901234567890.1234567890' AS DECIMAL(38, 10))); +---- +7467050800690399880 + +query I +SELECT xxhash64(CAST('-12345678901234567890.1234567890' AS DECIMAL(38, 10))); +---- +-5128632804645768173 + +query I +SELECT xxhash64(CAST('99999999999999999999999999999999999999' AS DECIMAL(38, 0))); +---- +-47190729175993179 + +query I +SELECT xxhash64(CAST('-99999999999999999999999999999999999999' AS DECIMAL(38, 0))); +---- +-2254039905620870768 + +query I +SELECT xxhash64(CAST(NULL AS DECIMAL(20, 0))); +---- +42 + +# Wide decimals inside lists and structs +query I +SELECT xxhash64([CAST(1 AS DECIMAL(20, 0)), CAST(-129 AS DECIMAL(20, 0))]); +---- +7288194727865790865 + +query I +SELECT xxhash64(named_struct('a', CAST(1 AS DECIMAL(20, 0)))); +---- +6668291691252061002 From 7d86871119c67cf25fa76179a49b35715c1e0db5 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 13 Nov 2026 14:21:42 -0700 Subject: [PATCH 4/7] docs(spark): document hash with #[user_doc] Co-Authored-By: Claude Opus 5.5 --- crates/spark/src/function/hash/murmur3.rs | 45 +++++++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/crates/spark/src/function/hash/murmur3.rs b/crates/spark/src/function/hash/murmur3.rs index cec128ae..81467e88 100644 --- a/crates/spark/src/function/hash/murmur3.rs +++ b/crates/spark/src/function/hash/murmur3.rs @@ -9,17 +9,52 @@ use arrow::buffer::{Buffer, ScalarBuffer}; use arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::{Result, ScalarValue, internal_err}; use datafusion_expr::{ - ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, - Volatility, + ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, + Signature, Volatility, }; +use datafusion_macros::user_doc; use super::utils::hash_dictionary_values; use crate::create_hashes_internal; const DEFAULT_SEED: u32 = 42; -/// Spark-compatible `hash` function (32-bit Murmur3, seed 42). +/// Spark-compatible `hash` expression. +/// /// +#[user_doc( + doc_section(label = "Hashing Functions"), + description = "Returns the 32-bit Murmur3 hash (seed 42) of the arguments as an INT, matching Spark's `hash`. \ + DataFusion has no built-in equivalent. Arguments are hashed in order, and each argument's hash seeds the next, \ + so `hash(a, b)` equals `hash(named_struct('a', a, 'b', b))` and argument order matters. \ + NULL arguments, NULL list elements and NULL struct fields are skipped, so the result is never NULL: \ + `hash(NULL)` returns 42. The result depends on the input type, as in Spark: INT hashes 4 bytes and BIGINT 8, \ + and integer literals are BIGINT in DataFusion but INT in Spark, so cast them to INT to match Spark's results. \ + `-0.0` hashes like `0.0`. Decimals with precision up to 18 hash as their unscaled long, and wider decimals hash \ + the minimal big-endian two's complement bytes of the unscaled value. Lists hash each element in order \ + (nested lists flatten into the same chain, and an empty list leaves the seed unchanged), structs hash each field, \ + and maps hash each key then its value. Spark 4 rejects maps unless `spark.sql.legacy.allowHashOnMapType` is set. \ + Dictionary-encoded values hash like their decoded values. At least one argument is required.", + syntax_example = "hash(expr1[, ..., exprN])", + sql_example = r#"```sql +> SELECT hash('Spark', [CAST(123 AS INT)], CAST(2 AS INT)); +-1321691492 +> SELECT hash(CAST(1 AS INT)); +-559580957 +> SELECT hash(1); +-1712319331 +> SELECT hash(NULL); +42 +```"#, + argument( + name = "expr1", + description = "Expression of any type to hash. NULL is skipped." + ), + argument( + name = "exprN", + description = "Further expressions, hashed in order with the previous hash as the seed." + ) +)] #[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkHash { signature: Signature, @@ -58,6 +93,10 @@ impl ScalarUDFImpl for SparkHash { Ok(Arc::new(Field::new(self.name(), DataType::Int32, false))) } + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { // When every argument is a scalar the result is the same for every row, // so hash a single row and return a scalar. Otherwise expand scalars to From 05d1dbea6a01a977df851d715f1ca9b9f05af617 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 15 Nov 2026 11:30:15 -0700 Subject: [PATCH 5/7] fix(spark): hash timestamps as microseconds and canonicalize NaN in hash/xxhash64 - Timestamps in second, millisecond and nanosecond units are converted to microseconds before hashing (nanoseconds floored), and Date64 is hashed as Int32 days, matching Spark's TimestampType and DateType. - Every NaN hashes like the canonical NaN, and -0.0 like 0.0, through shared normalize_f32/normalize_f64 helpers used by the top-level and list paths. - Add hash.slt cases for booleans, TINYINT, SMALLINT, dates, binary, LargeUtf8/Utf8View, timestamps and NaN, and xxhash64.slt cases for timestamps, dates, NaN and zero arguments. All values checked against PySpark 4.2.0. - Reword the map note in the hash docs to describe this crate. Closes #32. Co-Authored-By: Claude Opus 5.5 --- crates/spark/src/function/hash/murmur3.rs | 42 ++++- crates/spark/src/function/hash/utils.rs | 135 ++++++++++---- crates/spark/src/function/hash/xxhash64.rs | 28 +++ crates/sqllogictest/test_files/hash/hash.slt | 174 +++++++++++++++++- .../sqllogictest/test_files/hash/xxhash64.slt | 93 ++++++++++ 5 files changed, 437 insertions(+), 35 deletions(-) diff --git a/crates/spark/src/function/hash/murmur3.rs b/crates/spark/src/function/hash/murmur3.rs index 81467e88..6e9c0a6a 100644 --- a/crates/spark/src/function/hash/murmur3.rs +++ b/crates/spark/src/function/hash/murmur3.rs @@ -30,10 +30,13 @@ const DEFAULT_SEED: u32 = 42; NULL arguments, NULL list elements and NULL struct fields are skipped, so the result is never NULL: \ `hash(NULL)` returns 42. The result depends on the input type, as in Spark: INT hashes 4 bytes and BIGINT 8, \ and integer literals are BIGINT in DataFusion but INT in Spark, so cast them to INT to match Spark's results. \ - `-0.0` hashes like `0.0`. Decimals with precision up to 18 hash as their unscaled long, and wider decimals hash \ + `-0.0` hashes like `0.0`, and every NaN hashes like the canonical NaN. Dates hash as INT days and \ + timestamps as BIGINT microseconds, whatever the Arrow unit (DataFusion's TIMESTAMP literals are \ + nanoseconds, floored to microseconds). Decimals with precision up to 18 hash as their unscaled long, and wider decimals hash \ the minimal big-endian two's complement bytes of the unscaled value. Lists hash each element in order \ (nested lists flatten into the same chain, and an empty list leaves the seed unchanged), structs hash each field, \ - and maps hash each key then its value. Spark 4 rejects maps unless `spark.sql.legacy.allowHashOnMapType` is set. \ + and maps hash each key then its value. This crate hashes maps as if Spark's `spark.sql.legacy.allowHashOnMapType` \ + were set, where Spark 4 rejects them by default (see https://github.com/apache/datafusion-spark/issues/69). \ Dictionary-encoded values hash like their decoded values. At least one argument is required.", syntax_example = "hash(expr1[, ..., exprN])", sql_example = r#"```sql @@ -246,6 +249,41 @@ mod tests { assert_eq!(with_dict, with_decoded); } + /// Non-canonical NaNs (sign bit or payload set) hash like the canonical NaN, + /// as in Spark, both at the top level and inside lists. + #[test] + fn test_hash_non_canonical_nan() { + use arrow::array::{Float32Array, Float64Array, ListArray}; + use arrow::buffer::OffsetBuffer; + + let nans = vec![ + f64::NAN, + -f64::NAN, + f64::from_bits(0x7ff0_0000_0000_0001), + f64::from_bits(0xfff8_dead_beef_0001), + ]; + // Spark: hash(CAST('NaN' AS DOUBLE)) + assert_eq!( + hash_of(Arc::new(Float64Array::from(nans.clone()))), + vec![-1281358385; 4] + ); + let list: ArrayRef = Arc::new(ListArray::new( + Arc::new(Field::new_list_field(DataType::Float64, false)), + OffsetBuffer::new(vec![0i32, 1, 2, 3, 4].into()), + Arc::new(Float64Array::from(nans)), + None, + )); + assert_eq!(hash_of(list), vec![-1281358385; 4]); + + let f32s: ArrayRef = Arc::new(Float32Array::from(vec![ + f32::NAN, + -f32::NAN, + f32::from_bits(0x7f80_0001), + ])); + // Spark: hash(CAST('NaN' AS FLOAT)) + assert_eq!(hash_of(f32s), vec![-349261430; 3]); + } + #[test] fn test_hash_dictionary_matches_decoded() { let dict: DictionaryArray = diff --git a/crates/spark/src/function/hash/utils.rs b/crates/spark/src/function/hash/utils.rs index 771a7bf4..5caeea5a 100644 --- a/crates/spark/src/function/hash/utils.rs +++ b/crates/spark/src/function/hash/utils.rs @@ -15,8 +15,10 @@ use std::sync::Arc; use arrow::array::{Array, ArrayRef, DictionaryArray, PrimitiveArray}; use arrow::compute::take; -use arrow::datatypes::{ArrowDictionaryKeyType, ArrowNativeType, ArrowPrimitiveType}; -use datafusion_common::Result; +use arrow::datatypes::{ + ArrowDictionaryKeyType, ArrowNativeType, ArrowPrimitiveType, TimeUnit, +}; +use datafusion_common::{DataFusionError, Result}; /// Hash the values of a primitive array, calling `transform` to convert each /// value into a byte slice before feeding it to `hash_method`. @@ -138,6 +140,60 @@ pub(crate) fn hash_dictionary_values( Ok(()) } +/// Normalize an `f32` the way Spark does before hashing: `-0.0` hashes like +/// `0.0`, and every NaN hashes like the canonical NaN (Java's +/// `Float.floatToIntBits`). +#[inline] +pub(crate) fn normalize_f32(v: f32) -> f32 { + if v == 0.0 { + 0.0 + } else if v.is_nan() { + f32::from_bits(0x7fc0_0000) + } else { + v + } +} + +/// Normalize an `f64` the way Spark does before hashing: `-0.0` hashes like +/// `0.0`, and every NaN hashes like the canonical NaN (Java's +/// `Double.doubleToLongBits`). +#[inline] +pub(crate) fn normalize_f64(v: f64) -> f64 { + if v == 0.0 { + 0.0 + } else if v.is_nan() { + f64::from_bits(0x7ff8_0000_0000_0000) + } else { + v + } +} + +/// Convert a timestamp in `unit` to microseconds, the unit Spark's +/// `TimestampType` hashes. Nanoseconds are floored, like Spark's conversion. +#[inline] +pub(crate) fn timestamp_to_micros(value: i64, unit: TimeUnit) -> Result { + let micros = match unit { + TimeUnit::Second => value.checked_mul(1_000_000), + TimeUnit::Millisecond => value.checked_mul(1_000), + TimeUnit::Microsecond => Some(value), + TimeUnit::Nanosecond => Some(value.div_euclid(1_000)), + }; + micros.ok_or_else(|| { + DataFusionError::Execution(format!( + "Timestamp {value} ({unit:?}) does not fit in microseconds" + )) + }) +} + +/// Convert a `Date64` (milliseconds) to days, the `Int32` that Spark's +/// `DateType` hashes. Partial days are floored. +#[inline] +pub(crate) fn date64_to_days(millis: i64) -> Result { + i32::try_from(millis.div_euclid(86_400_000)).map_err(|_| { + DataFusionError::Execution(format!("Date64 {millis} does not fit in Int32 days")) + }) +} + /// The minimal big-endian two's complement bytes of an `i128`, matching Java's /// `BigInteger.toByteArray()` (which Spark hashes for decimals with precision /// greater than 18). @@ -239,28 +295,41 @@ macro_rules! hash_array_primitive { }; } -/// Hash a floating-point primitive array, normalizing `-0.0` to `0.0` per Spark. +/// Hash a floating-point primitive array after `$normalize` (Spark hashes +/// `-0.0` like `0.0` and every NaN like the canonical NaN). #[macro_export] #[doc(hidden)] macro_rules! hash_array_primitive_float { - ($array_type:ident, $column:ident, $ty:ident, $ty2:ident, $hashes:ident, $hash_method:ident) => { + ($array_type:ident, $column:ident, $normalize:ident, $hashes:ident, $hash_method:ident) => { let array = $crate::downcast_array!($column, $array_type); $crate::function::hash::utils::hash_primitive_values( array, $hashes, - |v| { - // Spark uses 0 as hash for -0.0, see `Murmur3Hash` expression. - if v == 0.0 && v.is_sign_negative() { - (0 as $ty2).to_le_bytes() - } else { - (v as $ty).to_le_bytes() - } - }, + |v| $crate::function::hash::utils::$normalize(v).to_le_bytes(), $hash_method, ); }; } +/// Hash a timestamp array in a unit other than microseconds by converting each +/// value to microseconds first, the unit Spark's `TimestampType` hashes. +#[macro_export] +#[doc(hidden)] +macro_rules! hash_array_timestamp { + ($array_type:ident, $column:ident, $unit:expr, $hashes:ident, $hash_method:ident) => { + let array = $crate::downcast_array!($column, $array_type); + $crate::function::hash::utils::try_hash_bytes_values( + array, + $hashes, + |a, i| { + $crate::function::hash::utils::timestamp_to_micros(a.value(i), $unit) + .map(i64::to_le_bytes) + }, + $hash_method, + )?; + }; +} + /// Hash a small (precision <= 18) decimal array by reducing each value to `i64`. /// Errors via `?` if a value does not fit in `i64`. #[macro_export] @@ -388,12 +457,12 @@ macro_rules! hash_list_with_primitive_elements { DataType::Float32 => { let elem_array = $values.as_any().downcast_ref::().unwrap(); $crate::hash_list_primitive!(offsets: $offsets, $list_array, elem_array, $hashes_buffer, $hash_method, - |v: f32| if v == 0.0 && v.is_sign_negative() { (0_i32).to_le_bytes() } else { v.to_le_bytes() }); + |v: f32| $crate::function::hash::utils::normalize_f32(v).to_le_bytes()); } DataType::Float64 => { let elem_array = $values.as_any().downcast_ref::().unwrap(); $crate::hash_list_primitive!(offsets: $offsets, $list_array, elem_array, $hashes_buffer, $hash_method, - |v: f64| if v == 0.0 && v.is_sign_negative() { (0_i64).to_le_bytes() } else { v.to_le_bytes() }); + |v: f64| $crate::function::hash::utils::normalize_f64(v).to_le_bytes()); } DataType::Boolean => { let elem_array = $values.as_any().downcast_ref::().unwrap(); @@ -483,12 +552,12 @@ macro_rules! hash_list_with_primitive_elements { DataType::Float32 => { let elem_array = $values.as_any().downcast_ref::().unwrap(); $crate::hash_list_primitive!(fixed_size: $list_size, $list_array, elem_array, $hashes_buffer, $hash_method, - |v: f32| if v == 0.0 && v.is_sign_negative() { (0_i32).to_le_bytes() } else { v.to_le_bytes() }); + |v: f32| $crate::function::hash::utils::normalize_f32(v).to_le_bytes()); } DataType::Float64 => { let elem_array = $values.as_any().downcast_ref::().unwrap(); $crate::hash_list_primitive!(fixed_size: $list_size, $list_array, elem_array, $hashes_buffer, $hash_method, - |v: f64| if v == 0.0 && v.is_sign_negative() { (0_i64).to_le_bytes() } else { v.to_le_bytes() }); + |v: f64| $crate::function::hash::utils::normalize_f64(v).to_le_bytes()); } DataType::Boolean => { let elem_array = $values.as_any().downcast_ref::().unwrap(); @@ -849,8 +918,7 @@ macro_rules! create_hashes_internal { $crate::hash_array_primitive_float!( Float32Array, col, - f32, - i32, + normalize_f32, $hashes_buffer, $hash_method ); @@ -859,26 +927,25 @@ macro_rules! create_hashes_internal { $crate::hash_array_primitive_float!( Float64Array, col, - f64, - i64, + normalize_f64, $hashes_buffer, $hash_method ); } DataType::Timestamp(TimeUnit::Second, _) => { - $crate::hash_array_primitive!( + $crate::hash_array_timestamp!( TimestampSecondArray, col, - i64, + TimeUnit::Second, $hashes_buffer, $hash_method ); } DataType::Timestamp(TimeUnit::Millisecond, _) => { - $crate::hash_array_primitive!( + $crate::hash_array_timestamp!( TimestampMillisecondArray, col, - i64, + TimeUnit::Millisecond, $hashes_buffer, $hash_method ); @@ -893,10 +960,10 @@ macro_rules! create_hashes_internal { ); } DataType::Timestamp(TimeUnit::Nanosecond, _) => { - $crate::hash_array_primitive!( + $crate::hash_array_timestamp!( TimestampNanosecondArray, col, - i64, + TimeUnit::Nanosecond, $hashes_buffer, $hash_method ); @@ -911,13 +978,17 @@ macro_rules! create_hashes_internal { ); } DataType::Date64 => { - $crate::hash_array_primitive!( - Date64Array, - col, - i64, + // Spark's DateType is Int32 days; Date64 is milliseconds. + let array = $crate::downcast_array!(col, Date64Array); + $crate::function::hash::utils::try_hash_bytes_values( + array, $hashes_buffer, - $hash_method - ); + |a, i| { + $crate::function::hash::utils::date64_to_days(a.value(i)) + .map(i32::to_le_bytes) + }, + $hash_method, + )?; } DataType::Utf8 => { $crate::hash_array!(StringArray, col, $hashes_buffer, $hash_method); diff --git a/crates/spark/src/function/hash/xxhash64.rs b/crates/spark/src/function/hash/xxhash64.rs index 00ef2adc..5cf61dc3 100644 --- a/crates/spark/src/function/hash/xxhash64.rs +++ b/crates/spark/src/function/hash/xxhash64.rs @@ -512,6 +512,34 @@ mod tests { } assert_ne!(hashes[0], hashes[1]); } + + /// Non-canonical NaNs (sign bit or payload set) hash like the canonical NaN, + /// as in Spark. Not reachable from SQL except via negation. + #[test] + fn test_xxhash64_non_canonical_nan() { + use arrow::array::{Float32Array, Float64Array}; + let f64s: ArrayRef = Arc::new(Float64Array::from(vec![ + f64::NAN, + -f64::NAN, + f64::from_bits(0x7ff0_0000_0000_0001), + f64::from_bits(0xfff8_dead_beef_0001), + ])); + let mut hashes = vec![DEFAULT_SEED; 4]; + create_xxhash64_hashes(&[f64s], &mut hashes).unwrap(); + // Spark: xxhash64(CAST('NaN' AS DOUBLE)) + assert!(hashes.iter().all(|&h| h as i64 == -3127944061524951246)); + + let f32s: ArrayRef = Arc::new(Float32Array::from(vec![ + f32::NAN, + -f32::NAN, + f32::from_bits(0x7f80_0001), + ])); + let mut hashes = vec![DEFAULT_SEED; 3]; + create_xxhash64_hashes(&[f32s], &mut hashes).unwrap(); + // Spark: xxhash64(CAST('NaN' AS FLOAT)) + assert!(hashes.iter().all(|&h| h as i64 == 2692338816207849720)); + } + // An all-literal xxhash64(...) that is not constant-folded is evaluated with // the batch's row count while every argument is a scalar. This cannot be // reached from SQL. diff --git a/crates/sqllogictest/test_files/hash/hash.slt b/crates/sqllogictest/test_files/hash/hash.slt index 9efb947a..c78574a6 100644 --- a/crates/sqllogictest/test_files/hash/hash.slt +++ b/crates/sqllogictest/test_files/hash/hash.slt @@ -4,7 +4,7 @@ # Tests for Spark-compatible hash (32-bit Murmur3, seed 42) # Reference: https://spark.apache.org/docs/latest/api/sql/index.html#hash -# Expected values come from Comet's Spark-verified murmur3 tests. +# Expected values were checked against Spark 4.2.0 (PySpark). # Int32 @@ -391,6 +391,178 @@ FROM VALUES ('x'), ('x'); -1305545736 -1305545736 +# Booleans hash as INT 1 or 0 + +query I +SELECT hash(true); +---- +-559580957 + +query I +SELECT hash(false); +---- +933211791 + +# TINYINT and SMALLINT widen to INT before hashing + +query I +SELECT hash(CAST(1 AS TINYINT)); +---- +-559580957 + +query I +SELECT hash(CAST(0 AS TINYINT)); +---- +933211791 + +query I +SELECT hash(CAST(-1 AS TINYINT)); +---- +-1604776387 + +query I +SELECT hash(CAST(127 AS TINYINT)); +---- +1135925485 + +query I +SELECT hash(CAST(-128 AS TINYINT)); +---- +1110053733 + +query I +SELECT hash(CAST(1 AS SMALLINT)); +---- +-559580957 + +query I +SELECT hash(CAST(-1 AS SMALLINT)); +---- +-1604776387 + +query I +SELECT hash(CAST(32767 AS SMALLINT)); +---- +1249274084 + +query I +SELECT hash(CAST(-32768 AS SMALLINT)); +---- +-1871935946 + +# LargeUtf8 and Utf8View hash like Utf8 + +query I +SELECT hash(arrow_cast('hello', 'LargeUtf8')); +---- +-1008564952 + +query I +SELECT hash(arrow_cast('hello', 'Utf8View')); +---- +-1008564952 + +# Binary hashes its bytes + +query I +SELECT hash(X'0102'); +---- +1964783052 + +# Dates hash as INT days since the epoch; Date64 (milliseconds) is floored to days + +query I +SELECT hash(DATE '2020-01-01'); +---- +-725858435 + +query I +SELECT hash(arrow_cast(DATE '2020-01-01', 'Date64')); +---- +-725858435 + +query I +SELECT hash(arrow_cast(-1, 'Date64')); +---- +-1604776387 + +# Timestamps hash as BIGINT microseconds since the epoch, whatever the Arrow unit. +# DataFusion's TIMESTAMP literals are nanoseconds. Spark values are with +# spark.sql.session.timeZone=UTC. + +query I +SELECT hash(TIMESTAMP '2020-01-01 00:00:00'); +---- +-1486022740 + +query I +SELECT hash(arrow_cast(TIMESTAMP '2020-01-01 00:00:00', 'Timestamp(Second, None)')); +---- +-1486022740 + +query I +SELECT hash(arrow_cast(TIMESTAMP '2020-01-01 00:00:00', 'Timestamp(Millisecond, None)')); +---- +-1486022740 + +query I +SELECT hash(arrow_cast(TIMESTAMP '2020-01-01 00:00:00', 'Timestamp(Microsecond, None)')); +---- +-1486022740 + +query I +SELECT hash(arrow_cast(TIMESTAMP '2020-01-01 00:00:00', 'Timestamp(Nanosecond, Some("UTC"))')); +---- +-1486022740 + +query I +SELECT hash([TIMESTAMP '2020-01-01 00:00:00']); +---- +-1486022740 + +# -1 ns floors to -1 us (Spark: timestamp_micros(-1)); -1 s is -1000000 us +query I +SELECT hash(arrow_cast(-1, 'Timestamp(Nanosecond, None)')); +---- +-939490007 + +query I +SELECT hash(arrow_cast(-1, 'Timestamp(Second, None)')); +---- +346627249 + +# Every NaN hashes like the canonical NaN; negating NaN flips its sign bit + +query I +SELECT hash(CAST('NaN' AS DOUBLE)); +---- +-1281358385 + +query I +SELECT hash(-CAST('NaN' AS DOUBLE)); +---- +-1281358385 + +query I +SELECT hash(CAST('NaN' AS FLOAT)); +---- +-349261430 + +query I +SELECT hash(-CAST('NaN' AS FLOAT)); +---- +-349261430 + +query I +SELECT hash([CAST('NaN' AS DOUBLE), -CAST('NaN' AS DOUBLE)]); +---- +1050078426 + +query I +SELECT hash([-CAST('NaN' AS FLOAT)]); +---- +-349261430 + # Table data (VALUES integers are Int64, which hash as 8 bytes) statement ok CREATE TABLE hash_t AS VALUES (1, 'hello'), (NULL, 'bar'), (0, NULL); diff --git a/crates/sqllogictest/test_files/hash/xxhash64.slt b/crates/sqllogictest/test_files/hash/xxhash64.slt index 4c7f601b..78b48243 100644 --- a/crates/sqllogictest/test_files/hash/xxhash64.slt +++ b/crates/sqllogictest/test_files/hash/xxhash64.slt @@ -295,3 +295,96 @@ query I SELECT xxhash64(named_struct('a', CAST(1 AS DECIMAL(20, 0)))); ---- 6668291691252061002 + +# Timestamps hash as BIGINT microseconds since the epoch, whatever the Arrow unit. +# DataFusion's TIMESTAMP literals are nanoseconds. Spark values are with +# spark.sql.session.timeZone=UTC. + +query I +SELECT xxhash64(TIMESTAMP '2020-01-01 00:00:00'); +---- +7881863312764026046 + +query I +SELECT xxhash64(arrow_cast(TIMESTAMP '2020-01-01 00:00:00', 'Timestamp(Second, None)')); +---- +7881863312764026046 + +query I +SELECT xxhash64(arrow_cast(TIMESTAMP '2020-01-01 00:00:00', 'Timestamp(Millisecond, None)')); +---- +7881863312764026046 + +query I +SELECT xxhash64(arrow_cast(TIMESTAMP '2020-01-01 00:00:00', 'Timestamp(Microsecond, None)')); +---- +7881863312764026046 + +query I +SELECT xxhash64([TIMESTAMP '2020-01-01 00:00:00']); +---- +7881863312764026046 + +# -1 ns floors to -1 us (Spark: timestamp_micros(-1)); -1 s is -1000000 us +query I +SELECT xxhash64(arrow_cast(-1, 'Timestamp(Nanosecond, None)')); +---- +3858142552250413010 + +query I +SELECT xxhash64(arrow_cast(-1, 'Timestamp(Second, None)')); +---- +-3302326838082710953 + +# Dates hash as INT days since the epoch; Date64 (milliseconds) is floored to days + +query I +SELECT xxhash64(DATE '2020-01-01'); +---- +2299834470729022467 + +query I +SELECT xxhash64(arrow_cast(DATE '2020-01-01', 'Date64')); +---- +2299834470729022467 + +query I +SELECT xxhash64(arrow_cast(-1, 'Date64')); +---- +2017008487422258757 + +# Every NaN hashes like the canonical NaN; negating NaN flips its sign bit + +query I +SELECT xxhash64(CAST('NaN' AS DOUBLE)); +---- +-3127944061524951246 + +query I +SELECT xxhash64(-CAST('NaN' AS DOUBLE)); +---- +-3127944061524951246 + +query I +SELECT xxhash64(CAST('NaN' AS FLOAT)); +---- +2692338816207849720 + +query I +SELECT xxhash64(-CAST('NaN' AS FLOAT)); +---- +2692338816207849720 + +query I +SELECT xxhash64([CAST('NaN' AS DOUBLE), -CAST('NaN' AS DOUBLE)]); +---- +-1379520635868705102 + +query I +SELECT xxhash64([-CAST('NaN' AS FLOAT)]); +---- +2692338816207849720 + +# Spark requires at least one argument +statement error does not support zero arguments +SELECT xxhash64(); From e395b27f533161f64c0ddc28b7488e39da2c2d84 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 11:28:48 -0700 Subject: [PATCH 6/7] fix(spark): hash TIME and interval types, and port Comet's murmur3 tests Port Comet's hasher arms for Time64(Nanosecond), Interval(MonthDayNano), Interval(YearMonth) and Duration(Microsecond), which failed at execution. A calendar interval is hashed as Spark 4.2's generated code does, without the days. Port 27 of Comet's 29 murmur3 unit tests, and add .slt cases for list, struct and map columns with NULL rows and for the new types. --- crates/spark/src/function/hash/murmur3.rs | 1095 +++++++++++++++++ crates/spark/src/function/hash/utils.rs | 54 +- crates/sqllogictest/test_files/hash/hash.slt | 68 + .../sqllogictest/test_files/hash/xxhash64.slt | 8 + 4 files changed, 1224 insertions(+), 1 deletion(-) diff --git a/crates/spark/src/function/hash/murmur3.rs b/crates/spark/src/function/hash/murmur3.rs index 6e9c0a6a..c277b801 100644 --- a/crates/spark/src/function/hash/murmur3.rs +++ b/crates/spark/src/function/hash/murmur3.rs @@ -356,4 +356,1099 @@ mod tests { assert_eq!(out.values().to_vec(), expected); Ok(()) } + + /// Tests ported from Apache DataFusion Comet's + /// `native/spark-expr/src/hash_funcs/murmur3.rs` (commit ba16e1a), with their expected + /// values unchanged: Comet produced them and they match Spark. Comet's two tests of its + /// gather-eligibility fast-path selection are not ported, since this crate has no such + /// selection. + mod comet { + use std::sync::Arc; + + use arrow::array::{ + Array, ArrayRef, Float32Array, Float64Array, Int8Array, Int32Array, + Int64Array, StringArray, + }; + use datafusion_common::config::ConfigOptions; + use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl}; + + use super::super::create_murmur3_hashes; + use crate::function::hash::xxhash64::SparkXxhash64; + + /// Produced by the per-element `hash_list_array!` implementation. An empty list and a null + /// list both leave the seed untouched. + const EXPECTED_LIST_OF_STRUCT: [u32; 6] = + [262891156, 1206178823, 42, 42, 3798669693, 2032748937]; + + /// Stands in for Comet's `test_hashes_with_nulls!`: hashes `values` from seed 42 and + /// checks `expected`, then again with nulls inserted at the front and the middle, which + /// must leave the seed untouched. + fn test_murmur3_hash>> + 'static>( + values: Vec>, + expected: Vec, + ) { + let len = values.len(); + let mut hashes = vec![42u32; len]; + create_murmur3_hashes( + &[Arc::new(T::from(values.clone())) as ArrayRef], + &mut hashes, + ) + .unwrap(); + assert_eq!(hashes, expected); + + let median = len / 2; + let mut values_with_nulls = values; + let mut expected_with_nulls = expected; + values_with_nulls.insert(0, None); + values_with_nulls.insert(median, None); + expected_with_nulls.insert(0, 42); + expected_with_nulls.insert(median, 42); + let mut hashes = vec![42u32; len + 2]; + create_murmur3_hashes( + &[Arc::new(T::from(values_with_nulls)) as ArrayRef], + &mut hashes, + ) + .unwrap(); + assert_eq!(hashes, expected_with_nulls); + } + + /// Stands in for Comet's `create_xxhash64_hashes`, which is private to the `xxhash64` + /// module here, by evaluating the `xxhash64` UDF. That always starts from Spark's seed 42, + /// which is the only seed these tests use. + fn create_xxhash64_hashes( + arrays: &[ArrayRef], + hashes_buffer: &mut [u64], + ) -> datafusion_common::Result<()> { + assert!( + hashes_buffer.iter().all(|&h| h == 42), + "only seed 42 is supported" + ); + let args: Vec = arrays + .iter() + .map(|a| ColumnarValue::Array(Arc::clone(a))) + .collect(); + let arg_fields = arrays + .iter() + .enumerate() + .map(|(i, a)| { + Arc::new(arrow::datatypes::Field::new( + format!("a{i}"), + a.data_type().clone(), + true, + )) + }) + .collect(); + let out = SparkXxhash64::new().invoke_with_args(ScalarFunctionArgs { + args, + arg_fields, + number_rows: hashes_buffer.len(), + return_field: Arc::new(arrow::datatypes::Field::new( + "xxhash64", + arrow::datatypes::DataType::Int64, + false, + )), + config_options: Arc::new(ConfigOptions::default()), + })?; + let ColumnarValue::Array(out) = out else { + panic!("expected an array"); + }; + let out = out.as_any().downcast_ref::().unwrap(); + for (h, v) in hashes_buffer.iter_mut().zip(out.values()) { + *h = *v as u64; + } + Ok(()) + } + + #[test] + fn test_dictionary_element_in_list_matches_decoded() { + use arrow::array::{DictionaryArray, Int32Array, ListArray}; + use arrow::buffer::OffsetBuffer; + use arrow::datatypes::{Field, Int8Type}; + + let values: ArrayRef = Arc::new(Int32Array::from(vec![10, 20])); + let keys = Int8Array::from(vec![0i8, 1]); + let dict: ArrayRef = + Arc::new(DictionaryArray::::try_new(keys, values).unwrap()); + let decoded: ArrayRef = Arc::new(Int32Array::from(vec![10, 20])); + + // One row holding both elements, so the second element's hash chains onto the first. + let as_list = |elems: ArrayRef| -> ArrayRef { + Arc::new(ListArray::new( + Arc::new(Field::new("item", elems.data_type().clone(), true)), + OffsetBuffer::new(vec![0i32, 2].into()), + elems, + None, + )) + }; + + let mut from_dict = vec![42u32; 1]; + create_murmur3_hashes(&[as_list(dict)], &mut from_dict).unwrap(); + let mut from_decoded = vec![42u32; 1]; + create_murmur3_hashes(&[as_list(decoded)], &mut from_decoded).unwrap(); + + assert_eq!( + from_dict, from_decoded, + "a dictionary-encoded list element must hash like the decoded value" + ); + } + + /// The fast path must survive for a genuine first column, including one with a caller-supplied + /// seed that is not Spark's 42, since the test for it is that every row is seeded alike. + #[test] + fn test_top_level_dictionary_matches_decoded() { + use arrow::array::{DictionaryArray, Int32Array}; + use arrow::datatypes::Int8Type; + + let values: ArrayRef = Arc::new(Int32Array::from(vec![10, 20])); + let keys = Int8Array::from(vec![0i8, 1, 0]); + let dict: ArrayRef = + Arc::new(DictionaryArray::::try_new(keys, values).unwrap()); + let decoded: ArrayRef = Arc::new(Int32Array::from(vec![10, 20, 10])); + + for seed in [42u32, 7u32] { + let mut a = vec![seed; 3]; + create_murmur3_hashes(&[Arc::clone(&dict)], &mut a).unwrap(); + let mut b = vec![seed; 3]; + create_murmur3_hashes(&[Arc::clone(&decoded)], &mut b).unwrap(); + assert_eq!( + a, b, + "top-level dictionary must match decoded for seed {seed}" + ); + } + } + + /// The uniformity check is what makes the fast path safe, and a single-row case cannot pin it: + /// one row is trivially uniform. This hashes several rows whose incoming seeds all differ, so a + /// dictionary first column has to take the unpacking fallback. It also covers a null key and a + /// key pointing at a null dictionary value, since both skip the hash update. + #[test] + fn test_dictionary_with_nonuniform_seeds_matches_decoded() { + use arrow::array::{DictionaryArray, Int32Array}; + use arrow::datatypes::Int8Type; + + // values[2] is null, and one key is itself null + let values: ArrayRef = + Arc::new(Int32Array::from(vec![Some(10), Some(20), None])); + let keys = Int8Array::from(vec![Some(0), Some(1), Some(2), None, Some(0)]); + let dict: ArrayRef = + Arc::new(DictionaryArray::::try_new(keys, values).unwrap()); + // The same logical data with the dictionary resolved. + let decoded: ArrayRef = Arc::new(Int32Array::from(vec![ + Some(10), + Some(20), + None, + None, + Some(10), + ])); + + let seeds: Vec = vec![7, 38, 69, 100, 131]; + let mut from_dict = seeds.clone(); + create_murmur3_hashes(&[dict], &mut from_dict).unwrap(); + let mut from_decoded = seeds; + create_murmur3_hashes(&[decoded], &mut from_decoded).unwrap(); + + assert_eq!( + from_dict, from_decoded, + "with per-row seeds a dictionary must hash like the decoded array" + ); + } + + /// Arrow lets a `StructArray`'s children carry their own validity, so at a row where the struct + /// itself is null the child buffer can still hold a value. Spark hashes a null struct as the + /// seed, so those hidden child values must not reach the hash. + #[test] + fn test_null_struct_ignores_hidden_child_values() { + use arrow::array::{Int32Array, StructArray}; + use arrow::buffer::NullBuffer; + use arrow::datatypes::{DataType, Field, Fields}; + + let fields: Fields = + vec![Arc::new(Field::new("a", DataType::Int32, true))].into(); + // Row 1 is a null struct whose child still holds 999. + let child: ArrayRef = Arc::new(Int32Array::from(vec![Some(1), Some(999)])); + let nulls = NullBuffer::from(vec![true, false]); + let with_hidden: ArrayRef = Arc::new(StructArray::new( + fields.clone(), + vec![Arc::clone(&child)], + Some(nulls.clone()), + )); + // Same shape, but the hidden slot is null too. + let child_null: ArrayRef = Arc::new(Int32Array::from(vec![Some(1), None])); + let without_hidden: ArrayRef = + Arc::new(StructArray::new(fields, vec![child_null], Some(nulls))); + + let mut a = vec![42u32; 2]; + create_murmur3_hashes(&[with_hidden], &mut a).unwrap(); + let mut b = vec![42u32; 2]; + create_murmur3_hashes(&[without_hidden], &mut b).unwrap(); + + assert_eq!( + a, b, + "a null struct must hash the same regardless of what its child buffer holds" + ); + assert_eq!(a[1], 42, "a null struct must leave the seed untouched"); + } + + /// The struct branch is also reached once per element when hashing `array>`. The + /// test above hashes a struct directly, so it does not cover that route. + /// + /// Here the null is the list *element* itself, with valid elements either side so the chaining + /// is exercised. + #[test] + fn test_null_struct_element_of_list_ignores_hidden_child_values() { + use arrow::array::{Int32Array, ListArray, StructArray}; + use arrow::buffer::{NullBuffer, OffsetBuffer}; + use arrow::datatypes::{DataType, Field, Fields}; + + let fields: Fields = + vec![Arc::new(Field::new("a", DataType::Int32, true))].into(); + // Element 1 is a null struct whose child still holds 999; elements 0 and 2 are valid. + let element_nulls = NullBuffer::from(vec![true, false, true]); + let with_hidden: ArrayRef = Arc::new(StructArray::new( + fields.clone(), + vec![ + Arc::new(Int32Array::from(vec![Some(1), Some(999), Some(3)])) + as ArrayRef, + ], + Some(element_nulls.clone()), + )); + // The same shape with the hidden slot null as well. + let without_hidden: ArrayRef = Arc::new(StructArray::new( + fields, + vec![Arc::new(Int32Array::from(vec![Some(1), None, Some(3)])) as ArrayRef], + Some(element_nulls), + )); + + // One row holding all three elements, so element hashes chain in order. + let as_list = |elements: ArrayRef| -> ArrayRef { + Arc::new(ListArray::new( + Arc::new(Field::new("item", elements.data_type().clone(), true)), + OffsetBuffer::new(vec![0i32, 3].into()), + elements, + None, + )) + }; + + let mut from_hidden = vec![42u32; 1]; + create_murmur3_hashes(&[as_list(with_hidden)], &mut from_hidden).unwrap(); + let mut from_null = vec![42u32; 1]; + create_murmur3_hashes(&[as_list(without_hidden)], &mut from_null).unwrap(); + + assert_eq!( + from_hidden, from_null, + "a null struct element must hash the same regardless of its child buffer" + ); + } + + /// One `struct` element: `None` is a null struct, and the fields are + /// independently nullable. + type StructElem = Option<(Option, Option<&'static str>)>; + /// One `array>` row: `None` is a null list. + type ListRow = Option>; + + /// Builds `array>` from `rows`. + fn list_of_struct(rows: Vec) -> ArrayRef { + use arrow::array::{Int32Builder, ListBuilder, StringBuilder, StructBuilder}; + use arrow::datatypes::{DataType, Field, Fields}; + + let fields: Fields = vec![ + Arc::new(Field::new("a", DataType::Int32, true)), + Arc::new(Field::new("b", DataType::Utf8, true)), + ] + .into(); + let struct_builder = StructBuilder::new( + fields.clone(), + vec![ + Box::new(Int32Builder::new()), + Box::new(StringBuilder::new()), + ], + ); + let mut lb = ListBuilder::new(struct_builder); + for row in rows { + match row { + None => lb.append(false), + Some(elems) => { + for elem in elems { + let sb = lb.values(); + match elem { + None => { + sb.field_builder::(0) + .unwrap() + .append_null(); + sb.field_builder::(1) + .unwrap() + .append_null(); + sb.append(false); + } + Some((a, b)) => { + match a { + Some(v) => sb + .field_builder::(0) + .unwrap() + .append_value(v), + None => sb + .field_builder::(0) + .unwrap() + .append_null(), + } + match b { + Some(v) => sb + .field_builder::(1) + .unwrap() + .append_value(v), + None => sb + .field_builder::(1) + .unwrap() + .append_null(), + } + sb.append(true); + } + } + } + lb.append(true); + } + } + } + Arc::new(lb.finish()) + } + + fn hash_of(array: ArrayRef, num_rows: usize) -> Vec { + let mut hashes = vec![42u32; num_rows]; + create_murmur3_hashes(&[array], &mut hashes).unwrap(); + hashes + } + + /// `array>` is the shape that goes through `hash_list_array!`, the per-element path. + /// These values were produced by that implementation and pin it: the hash decides which + /// partition a row lands in, so any rewrite of that path has to reproduce them exactly. + #[test] + fn test_list_of_struct_hashes_are_stable() { + let rows = vec![ + Some(vec![Some((Some(1), Some("x"))), Some((Some(2), Some("y")))]), + Some(vec![Some((Some(3), Some("z")))]), + // empty list: contributes nothing, so the seed survives + Some(vec![]), + // null list + None, + // null struct element, and elements with null fields + Some(vec![None, Some((None, Some("w"))), Some((Some(4), None))]), + // repeated element values, to catch an implementation that dedupes or reorders + Some(vec![Some((Some(5), Some("s"))), Some((Some(5), Some("s")))]), + ]; + let n = rows.len(); + assert_eq!(hash_of(list_of_struct(rows), n), EXPECTED_LIST_OF_STRUCT); + } + + /// Element order must matter: Spark chains the element hashes in sequence. + #[test] + fn test_list_of_struct_is_order_sensitive() { + let forward = list_of_struct(vec![Some(vec![ + Some((Some(1), Some("a"))), + Some((Some(2), Some("b"))), + ])]); + let reversed = list_of_struct(vec![Some(vec![ + Some((Some(2), Some("b"))), + Some((Some(1), Some("a"))), + ])]); + assert_ne!( + hash_of(forward, 1), + hash_of(reversed, 1), + "element order must change the hash" + ); + } + + /// One list far longer than the rest. A skewed batch must agree with hashing each row on its + /// own. + #[test] + fn test_list_of_struct_skewed_lengths() { + let mut rows: Vec = vec![Some(vec![Some((Some(1), Some("a")))]); 8]; + // One row far longer than the rest. + rows.push(Some( + (0..64) + .map(|i| Some((Some(i), Some("long")))) + .collect::>(), + )); + rows.push(Some(vec![])); + + let batched = hash_of(list_of_struct(rows.clone()), rows.len()); + + // Hash each row as its own batch of one; the result must match position by position. + let per_row: Vec = rows + .into_iter() + .map(|row| hash_of(list_of_struct(vec![row]), 1)[0]) + .collect(); + assert_eq!(batched, per_row, "skewed batch must match per-row hashing"); + } + + /// `LargeList` has 64-bit offsets. The two list widths must agree on the same data. + #[test] + fn test_large_list_of_struct_matches_list() { + use arrow::array::{ + Int32Builder, LargeListBuilder, StringBuilder, StructBuilder, + }; + use arrow::datatypes::{DataType, Field, Fields}; + + let fields: Fields = vec![ + Arc::new(Field::new("a", DataType::Int32, true)), + Arc::new(Field::new("b", DataType::Utf8, true)), + ] + .into(); + let sb = StructBuilder::new( + fields.clone(), + vec![ + Box::new(Int32Builder::new()), + Box::new(StringBuilder::new()), + ], + ); + let mut lb = LargeListBuilder::new(sb); + for row in [vec![1, 2], vec![3], vec![]] { + for v in row { + let s = lb.values(); + s.field_builder::(0).unwrap().append_value(v); + s.field_builder::(1) + .unwrap() + .append_value(format!("s{v}")); + s.append(true); + } + lb.append(true); + } + let large: ArrayRef = Arc::new(lb.finish()); + + let small = list_of_struct(vec![ + Some(vec![ + Some((Some(1), Some("s1"))), + Some((Some(2), Some("s2"))), + ]), + Some(vec![Some((Some(3), Some("s3")))]), + Some(vec![]), + ]); + + assert_eq!( + hash_of(large, 3), + hash_of(small, 3), + "LargeList and List must hash identically" + ); + } + + /// `array>`: the element is a list rather than a struct, so it takes the same + /// non-primitive element path one level deeper. + #[test] + fn test_list_of_list_hashes_match_per_row() { + use arrow::array::{Int32Builder, ListBuilder}; + + let build = |rows: &[Vec>]| -> ArrayRef { + let mut lb = ListBuilder::new(ListBuilder::new(Int32Builder::new())); + for row in rows { + for inner in row { + for v in inner { + lb.values().values().append_value(*v); + } + lb.values().append(true); + } + lb.append(true); + } + Arc::new(lb.finish()) + }; + + let rows = vec![ + vec![vec![1, 2], vec![3]], + vec![vec![4]], + vec![], + vec![vec![5, 6, 7], vec![], vec![8]], + ]; + let batched = hash_of(build(&rows), rows.len()); + let per_row: Vec = rows + .iter() + .map(|row| hash_of(build(std::slice::from_ref(row)), 1)[0]) + .collect(); + assert_eq!( + batched, per_row, + "array> must match per-row hashing" + ); + } + + /// Rows that run out of elements at different positions, including in the middle of the + /// batch, must agree with hashing each row as its own batch. + #[test] + fn test_list_of_struct_rows_exhaust_on_different_passes() { + let rows: Vec = vec![ + Some((0..5).map(|i| Some((Some(i), Some("a")))).collect()), + Some(vec![Some((Some(9), Some("b")))]), + Some((0..3).map(|i| Some((Some(i), Some("c")))).collect()), + Some(vec![]), + Some((0..7).map(|i| Some((Some(i), Some("d")))).collect()), + None, + Some(vec![Some((Some(1), Some("e"))), Some((Some(2), Some("f")))]), + ]; + let batched = hash_of(list_of_struct(rows.clone()), rows.len()); + let per_row: Vec = rows + .into_iter() + .map(|row| hash_of(list_of_struct(vec![row]), 1)[0]) + .collect(); + assert_eq!( + batched, per_row, + "rows exhausting on different passes must agree" + ); + } + + /// A sliced list has a non-zero first offset, so an off-by-one in the offset handling would + /// only show up on a slice. + #[test] + fn test_sliced_list_of_struct_matches_unsliced() { + let rows: Vec = vec![ + Some(vec![Some((Some(1), Some("x")))]), + Some(vec![Some((Some(2), Some("y"))), Some((Some(3), Some("z")))]), + Some((0..4).map(|i| Some((Some(i), Some("w")))).collect()), + Some(vec![Some((Some(8), Some("v")))]), + ]; + let full = list_of_struct(rows.clone()); + + // Hash rows 1..3 through a slice, and the same rows built on their own. + let sliced = full.slice(1, 2); + let mut from_slice = vec![42u32; 2]; + create_murmur3_hashes(&[sliced], &mut from_slice).unwrap(); + + let standalone = list_of_struct(rows[1..3].to_vec()); + let mut from_standalone = vec![42u32; 2]; + create_murmur3_hashes(&[standalone], &mut from_standalone).unwrap(); + + assert_eq!( + from_slice, from_standalone, + "a sliced list must hash like the same rows built unsliced" + ); + } + + /// A null list can still cover a non-empty range of elements. Those elements must not be + /// hashed. + #[test] + fn test_null_list_with_populated_range_is_skipped() { + use arrow::array::{Int32Builder, ListArray, StringBuilder, StructBuilder}; + use arrow::buffer::{NullBuffer, OffsetBuffer}; + use arrow::datatypes::{DataType, Field, Fields}; + + let fields: Fields = vec![ + Arc::new(Field::new("a", DataType::Int32, true)), + Arc::new(Field::new("b", DataType::Utf8, true)), + ] + .into(); + let mut sb = StructBuilder::new( + fields.clone(), + vec![ + Box::new(Int32Builder::new()), + Box::new(StringBuilder::new()), + ], + ); + for v in 0..3 { + sb.field_builder::(0).unwrap().append_value(v); + sb.field_builder::(1) + .unwrap() + .append_value("hidden"); + sb.append(true); + } + let elements: ArrayRef = Arc::new(sb.finish()); + + // Row 0 covers elements 0..1, row 1 is null but still covers 1..3. + let list = ListArray::new( + Arc::new(Field::new("item", elements.data_type().clone(), true)), + OffsetBuffer::new(vec![0i32, 1, 3].into()), + elements, + Some(NullBuffer::from(vec![true, false])), + ); + let mut hashes = vec![42u32; 2]; + create_murmur3_hashes(&[Arc::new(list) as ArrayRef], &mut hashes).unwrap(); + assert_eq!(hashes[1], 42, "a null list must leave the seed untouched"); + + // And it must not disturb the visible row either. + let only_visible = + list_of_struct(vec![Some(vec![Some((Some(0), Some("hidden")))])]); + let mut expected = vec![42u32; 1]; + create_murmur3_hashes(&[only_visible], &mut expected).unwrap(); + assert_eq!(hashes[0], expected[0]); + } + + /// Lengths that step down, rather than being all equal or one long outlier. + #[test] + fn test_list_of_struct_descending_lengths() { + let rows: Vec = (1..=12) + .rev() + .map(|n| Some((0..n).map(|i| Some((Some(i), Some("s")))).collect())) + .collect(); + let batched = hash_of(list_of_struct(rows.clone()), rows.len()); + let per_row: Vec = rows + .into_iter() + .map(|row| hash_of(list_of_struct(vec![row]), 1)[0]) + .collect(); + assert_eq!(batched, per_row); + } + + /// A lone-row list must not stop the caller hashing the remaining columns. + #[test] + fn test_single_row_list_then_another_column() { + let l = list_of_struct(vec![Some(vec![ + Some((Some(1), Some("a"))), + Some((Some(2), Some("b"))), + ])]); + let other: ArrayRef = Arc::new(Int32Array::from(vec![7])); + + // both columns together + let mut both = vec![42u32; 1]; + create_murmur3_hashes(&[Arc::clone(&l), Arc::clone(&other)], &mut both) + .unwrap(); + + // chaining them by hand must agree + let mut step = vec![42u32; 1]; + create_murmur3_hashes(&[l], &mut step).unwrap(); + create_murmur3_hashes(&[other], &mut step).unwrap(); + + assert_eq!(both, step, "the second column must still be hashed"); + } + + #[test] + fn test_i8() { + test_murmur3_hash::( + vec![Some(1), Some(0), Some(-1), Some(i8::MAX), Some(i8::MIN)], + vec![0xdea578e3, 0x379fae8f, 0xa0590e3d, 0x43b4d8ed, 0x422a1365], + ); + } + + #[test] + fn test_i32() { + test_murmur3_hash::( + vec![Some(1), Some(0), Some(-1), Some(i32::MAX), Some(i32::MIN)], + vec![0xdea578e3, 0x379fae8f, 0xa0590e3d, 0x07fb67e7, 0x2b1f0fc6], + ); + } + + #[test] + fn test_i64() { + test_murmur3_hash::( + vec![Some(1), Some(0), Some(-1), Some(i64::MAX), Some(i64::MIN)], + vec![0x99f0149d, 0x9c67b85d, 0xc8008529, 0xa05b5d7b, 0xcd1e64fb], + ); + } + + /// Spark hashes `YearMonthIntervalType` as an int and `DayTimeIntervalType` as a long, which + /// Comet maps to `Interval(YearMonth)` and `Duration(Microsecond)`. + #[test] + fn test_ansi_intervals_hash_like_int_and_long() { + use arrow::array::{DurationMicrosecondArray, IntervalYearMonthArray}; + + test_murmur3_hash::( + vec![Some(1), Some(0), Some(-1), Some(i32::MAX), Some(i32::MIN)], + vec![0xdea578e3, 0x379fae8f, 0xa0590e3d, 0x07fb67e7, 0x2b1f0fc6], + ); + test_murmur3_hash::( + vec![Some(1), Some(0), Some(-1), Some(i64::MAX), Some(i64::MIN)], + vec![0x99f0149d, 0x9c67b85d, 0xc8008529, 0xa05b5d7b, 0xcd1e64fb], + ); + } + + /// A `NullType` value is always null, so it leaves the hash unchanged, as in Spark, both as a + /// column of its own and as a struct field, including a field of a null struct. + #[test] + fn test_null_type_leaves_hash_unchanged() { + use arrow::array::{NullArray, StructArray}; + use arrow::buffer::NullBuffer; + use arrow::datatypes::{DataType, Field}; + + let ints: ArrayRef = + Arc::new(Int32Array::from(vec![Some(1), None, Some(-1)])); + let nulls: ArrayRef = Arc::new(NullArray::new(3)); + + let mut expected = vec![42u32; 3]; + create_murmur3_hashes(&[Arc::clone(&ints)], &mut expected).unwrap(); + let mut hashes = vec![42u32; 3]; + create_murmur3_hashes(&[Arc::clone(&ints), Arc::clone(&nulls)], &mut hashes) + .unwrap(); + assert_eq!(hashes, expected); + + let struct_nulls = Some(NullBuffer::from(vec![true, true, false])); + let x = Arc::new(Field::new("x", DataType::Int32, true)); + let n = Arc::new(Field::new("n", DataType::Null, true)); + let with_null_field: ArrayRef = Arc::new(StructArray::new( + vec![Arc::clone(&x), n].into(), + vec![Arc::clone(&ints), nulls], + struct_nulls.clone(), + )); + let without_null_field: ArrayRef = + Arc::new(StructArray::new(vec![x].into(), vec![ints], struct_nulls)); + + let mut expected = vec![42u32; 3]; + create_murmur3_hashes(&[without_null_field], &mut expected).unwrap(); + let mut hashes = vec![42u32; 3]; + create_murmur3_hashes(&[with_null_field], &mut hashes).unwrap(); + assert_eq!(hashes, expected); + assert_eq!(hashes[2], 42, "a null struct must leave the seed untouched"); + } + + /// Lists whose elements are `NullType` or an ANSI interval type go through the recursive + /// list path into the `Null`, `Interval(YearMonth)` and `Duration(Microsecond)` arms. A list + /// of nulls leaves the hash unchanged, and a list of intervals hashes like the same list of + /// ints or longs. + #[test] + fn test_lists_of_null_type_and_intervals() { + use arrow::array::{ + DurationMicrosecondArray, IntervalYearMonthArray, ListArray, NullArray, + }; + use arrow::buffer::{NullBuffer, OffsetBuffer}; + use arrow::datatypes::Field; + + // Rows [v0, v1], [], null and [v2]. + let as_list = |values: ArrayRef| -> ArrayRef { + Arc::new(ListArray::new( + Arc::new(Field::new("item", values.data_type().clone(), true)), + OffsetBuffer::from_lengths([2, 0, 0, 1]), + values, + Some(NullBuffer::from(vec![true, true, false, true])), + )) + }; + let hash = |list: ArrayRef| { + let mut hashes = vec![42u32; 4]; + create_murmur3_hashes(&[list], &mut hashes).unwrap(); + hashes + }; + + assert_eq!(hash(as_list(Arc::new(NullArray::new(3)))), vec![42u32; 4]); + + let months = vec![Some(1), None, Some(i32::MIN)]; + assert_eq!( + hash(as_list(Arc::new(IntervalYearMonthArray::from( + months.clone() + )))), + hash(as_list(Arc::new(Int32Array::from(months)))) + ); + let micros = vec![Some(-1), Some(i64::MAX), None]; + assert_eq!( + hash(as_list(Arc::new(DurationMicrosecondArray::from( + micros.clone() + )))), + hash(as_list(Arc::new(Int64Array::from(micros)))) + ); + } + + #[test] + fn test_f32() { + test_murmur3_hash::( + vec![ + Some(1.0), + Some(0.0), + Some(-0.0), + Some(-1.0), + Some(99999999999.99999999999), + Some(-99999999999.99999999999), + // Every NaN hashes as the canonical NaN, whatever its sign or payload. + Some(f32::NAN), + Some(f32::from_bits(0xffc0_0000)), + Some(f32::from_bits(0x7f80_0001)), + ], + vec![ + 0xe434cc39, 0x379fae8f, 0x379fae8f, 0xdc0da8eb, 0xcbdc340f, + 0xc0361c86, 0xeb2eb18a, 0xeb2eb18a, 0xeb2eb18a, + ], + ); + } + + #[test] + fn test_f64() { + test_murmur3_hash::( + vec![ + Some(1.0), + Some(0.0), + Some(-0.0), + Some(-1.0), + Some(99999999999.99999999999), + Some(-99999999999.99999999999), + // Every NaN hashes as the canonical NaN, whatever its sign or payload. + Some(f64::NAN), + Some(f64::from_bits(0xfff8_0000_0000_0000)), + Some(f64::from_bits(0x7ff0_0000_0000_0001)), + ], + vec![ + 0xe4876492, 0x9c67b85d, 0x9c67b85d, 0x13d81357, 0xb87e1595, + 0xa0eef9f9, 0xb3a005cf, 0xb3a005cf, 0xb3a005cf, + ], + ); + } + + /// Spark hashes a float through `doubleToLongBits` or `floatToIntBits`, which canonicalize + /// NaN, so a NaN with the sign bit set or with a payload hashes like the canonical NaN inside + /// lists, structs, maps and dictionaries too. Checks both hashes, which expand the same macros. + #[test] + fn test_non_canonical_nan_hashes_like_canonical_nan() { + use arrow::array::{ + DictionaryArray, FixedSizeListArray, Int32Array, LargeListArray, + ListArray, MapArray, StructArray, + }; + use arrow::buffer::OffsetBuffer; + use arrow::datatypes::{DataType, Field, Fields}; + + let shapes = |values: ArrayRef| -> Vec { + let field = + Arc::new(Field::new("item", values.data_type().clone(), true)); + // One entry per row, with the float as both the key and the value. + let entry_fields = Fields::from(vec![ + Field::new("key", values.data_type().clone(), false), + Field::new("value", values.data_type().clone(), true), + ]); + let entries = StructArray::new( + entry_fields.clone(), + vec![Arc::clone(&values), Arc::clone(&values)], + None, + ); + vec![ + Arc::clone(&values), + Arc::new(ListArray::new( + Arc::clone(&field), + OffsetBuffer::from_lengths([1; 3]), + Arc::clone(&values), + None, + )), + Arc::new(LargeListArray::new( + Arc::clone(&field), + OffsetBuffer::from_lengths([1; 3]), + Arc::clone(&values), + None, + )), + Arc::new(FixedSizeListArray::new( + Arc::clone(&field), + 1, + Arc::clone(&values), + None, + )), + Arc::new(StructArray::new( + vec![field].into(), + vec![Arc::clone(&values)], + None, + )), + Arc::new(DictionaryArray::new( + Int32Array::from(vec![0, 1, 2]), + values, + )), + Arc::new(MapArray::new( + Arc::new(Field::new( + "entries", + DataType::Struct(entry_fields), + false, + )), + OffsetBuffer::from_lengths([1; 3]), + entries, + None, + false, + )), + ] + }; + let canonical64: ArrayRef = Arc::new(Float64Array::from(vec![f64::NAN; 3])); + let other64: ArrayRef = Arc::new(Float64Array::from(vec![ + f64::from_bits(0xfff8_0000_0000_0000), + f64::from_bits(0x7ff0_0000_0000_0001), + f64::from_bits(0xfff0_0000_0000_0001), + ])); + let canonical32: ArrayRef = Arc::new(Float32Array::from(vec![f32::NAN; 3])); + let other32: ArrayRef = Arc::new(Float32Array::from(vec![ + f32::from_bits(0xffc0_0000), + f32::from_bits(0x7f80_0001), + f32::from_bits(0xff80_0001), + ])); + for (canonical, other) in [(canonical64, other64), (canonical32, other32)] { + for (canonical, other) in shapes(canonical).into_iter().zip(shapes(other)) + { + let (mut expected, mut actual) = (vec![42u32; 3], vec![42u32; 3]); + create_murmur3_hashes(&[Arc::clone(&canonical)], &mut expected) + .unwrap(); + create_murmur3_hashes(&[Arc::clone(&other)], &mut actual).unwrap(); + assert_eq!(actual, expected, "murmur3 of {}", other.data_type()); + let (mut expected, mut actual) = (vec![42u64; 3], vec![42u64; 3]); + create_xxhash64_hashes(&[canonical], &mut expected).unwrap(); + create_xxhash64_hashes(&[Arc::clone(&other)], &mut actual).unwrap(); + assert_eq!(actual, expected, "xxhash64 of {}", other.data_type()); + } + } + } + + #[test] + fn test_str() { + let input = [ + "hello", "bar", "", "😁", "天地", "a", "ab", "abc", "abcd", "abcde", + ] + .iter() + .map(|s| Some(s.to_string())) + .collect::>>(); + let expected: Vec = vec![ + 3286402344, 2486176763, 142593372, 885025535, 2395000894, 1485273170, + 0xfa37157b, 1322437556, 0xe860e5cc, 814637928, + ]; + + test_murmur3_hash::(input.clone(), expected); + } + + // `create_hashes_internal!` specialises maps of (Utf8, Int32), (Int32, Utf8), (Utf8, Utf8) + // and (Int32, Int32), each with a loop for input without nulls and a null-aware loop. + // `sql-tests/expressions/hash/hash.sql` checks both loops of every pair against Spark. The + // tests below cover the shapes SQL cannot build by comparing with the same rows with that map row removed + + /// (key is Utf8, value is Utf8) for each specialised pair. + const MAP_PAIRS: [(bool, bool); 4] = + [(true, false), (false, true), (true, true), (false, false)]; + + /// Entries per row of every test map: row 2 owns entry 2. + const MAP_LENGTHS: [usize; 4] = [2, 0, 1, 3]; + /// Map validity per row. With every map valid the hash takes the loop for input without + /// nulls; row 2 being null sends it to the null-aware loop. + const MAP_ALL_VALID: [bool; 4] = [true; 4]; + const MAP_NULL_ROW: [bool; 4] = [true, true, false, true]; + + /// The seed every row starts from. Spark leaves it unchanged for a null map. + const SEED: u32 = 42; + + fn map_pair_label(key_utf8: bool, value_utf8: bool) -> String { + let name = |utf8: bool| if utf8 { "Utf8" } else { "Int32" }; + format!("({}, {})", name(key_utf8), name(value_utf8)) + } + + /// The first `len` entries of a pool, as a Utf8 or Int32 column. + fn map_entry_column( + utf8: bool, + strs: &[&str], + ints: &[i32], + len: usize, + ) -> ArrayRef { + if utf8 { + Arc::new(StringArray::from(strs[..len].to_vec())) + } else { + Arc::new(Int32Array::from(ints[..len].to_vec())) + } + } + + /// A map with `MAP_LENGTHS` entries per row and the given row validity. Every entry is + /// distinct, so hashing the wrong entry changes the hash. + fn map_of(key_utf8: bool, value_utf8: bool, valid: &[bool]) -> ArrayRef { + use arrow::array::{MapArray, StructArray}; + use arrow::buffer::{NullBuffer, OffsetBuffer}; + use arrow::datatypes::{DataType, Field, Fields}; + + const STR_KEYS: [&str; 6] = ["a", "b", "", "日本", "x", "yz"]; + const INT_KEYS: [i32; 6] = [1, -1, 0, i32::MAX, i32::MIN, 7]; + const STR_VALUES: [&str; 6] = ["v1", "", "é", "w", "long value", "z"]; + const INT_VALUES: [i32; 6] = [10, -20, 0, i32::MIN, i32::MAX, 3]; + + let len = MAP_LENGTHS.iter().sum(); + let keys = map_entry_column(key_utf8, &STR_KEYS, &INT_KEYS, len); + let values = map_entry_column(value_utf8, &STR_VALUES, &INT_VALUES, len); + let entry_fields = Fields::from(vec![ + Field::new("key", keys.data_type().clone(), false), + Field::new("value", values.data_type().clone(), true), + ]); + let entries = + StructArray::new(entry_fields.clone(), vec![keys, values], None); + Arc::new(MapArray::new( + Arc::new(Field::new("entries", DataType::Struct(entry_fields), false)), + OffsetBuffer::from_lengths(MAP_LENGTHS), + entries, + valid + .contains(&false) + .then(|| NullBuffer::from(valid.to_vec())), + false, + )) + } + + /// murmur3 and xxhash64 of every row, both starting from `SEED`. + fn map_hashes(array: &ArrayRef) -> (Vec, Vec) { + let mut murmur3 = vec![SEED; array.len()]; + create_murmur3_hashes(&[Arc::clone(array)], &mut murmur3).unwrap(); + let mut xxhash64 = vec![SEED as u64; array.len()]; + create_xxhash64_hashes(&[Arc::clone(array)], &mut xxhash64).unwrap(); + (murmur3, xxhash64) + } + + /// Whether the hash takes the null-aware loop. Like the loop selection itself, this counts + /// the nulls of the whole key and value arrays, not only the entries a slice covers. + fn map_takes_null_aware_loop(array: &ArrayRef) -> bool { + use arrow::array::{Array, MapArray}; + + let map = array.as_any().downcast_ref::().unwrap(); + map.null_count() + map.keys().null_count() + map.values().null_count() > 0 + } + + /// A null map can still own entries, which Spark never produces but Arrow allows. Those + /// entries must not be hashed: the null row keeps the seed, and the other rows hash as they + /// do once the null row, and with it the entry it owns, is filtered out. + #[test] + fn test_specialised_map_null_row_skips_owned_entries() { + use arrow::array::BooleanArray; + use arrow::compute::filter; + + let null_row = MAP_NULL_ROW.iter().position(|v| !v).unwrap(); + assert!(MAP_LENGTHS[null_row] > 0, "the null map must own an entry"); + let keep = BooleanArray::from(MAP_NULL_ROW.to_vec()); + + for (key_utf8, value_utf8) in MAP_PAIRS { + let label = map_pair_label(key_utf8, value_utf8); + let array = map_of(key_utf8, value_utf8, &MAP_NULL_ROW); + assert!(map_takes_null_aware_loop(&array), "{label}"); + + let (mut murmur3, mut xxhash64) = map_hashes(&array); + assert_eq!( + murmur3.remove(null_row), + SEED, + "murmur3 of the null map, {label}" + ); + assert_eq!( + xxhash64.remove(null_row), + SEED as u64, + "xxhash64 of the null map, {label}" + ); + + let without_null_row = filter(&array, &keep).unwrap(); + let (expected_murmur3, expected_xxhash64) = map_hashes(&without_null_row); + assert_eq!( + murmur3, expected_murmur3, + "murmur3 of the other rows, {label}" + ); + assert_eq!( + xxhash64, expected_xxhash64, + "xxhash64 of the other rows, {label}" + ); + } + } + + /// A sliced map keeps its unsliced keys and values, so its offsets do not start at 0. Each + /// row must hash as it does in the unsliced array, in both loops. + #[test] + fn test_specialised_map_slice_matches_unsliced() { + use arrow::array::{Array, MapArray}; + + for (key_utf8, value_utf8) in MAP_PAIRS { + for valid in [MAP_ALL_VALID, MAP_NULL_ROW] { + let has_nulls = valid.contains(&false); + let label = format!( + "{} {}", + map_pair_label(key_utf8, value_utf8), + if has_nulls { + "with a null map" + } else { + "without nulls" + } + ); + let array = map_of(key_utf8, value_utf8, &valid); + let sliced = array.slice(1, array.len() - 1); + + let offsets = sliced + .as_any() + .downcast_ref::() + .unwrap() + .offsets(); + assert_ne!( + offsets[0], 0, + "{label}: the slice must not start at entry 0" + ); + assert_eq!(map_takes_null_aware_loop(&sliced), has_nulls, "{label}"); + + let (murmur3, xxhash64) = map_hashes(&array); + let (sliced_murmur3, sliced_xxhash64) = map_hashes(&sliced); + assert_eq!(sliced_murmur3[..], murmur3[1..], "murmur3 of {label}"); + assert_eq!(sliced_xxhash64[..], xxhash64[1..], "xxhash64 of {label}"); + } + } + } + } } diff --git a/crates/spark/src/function/hash/utils.rs b/crates/spark/src/function/hash/utils.rs index 5caeea5a..4211bcea 100644 --- a/crates/spark/src/function/hash/utils.rs +++ b/crates/spark/src/function/hash/utils.rs @@ -863,7 +863,7 @@ macro_rules! hash_list_array { macro_rules! create_hashes_internal { ($arrays: ident, $hashes_buffer: ident, $hash_method: ident, $create_dictionary_hash_method: ident, $recursive_hash_method: ident) => { use arrow::array::{types::*, *}; - use arrow::datatypes::{DataType, TimeUnit}; + use arrow::datatypes::{DataType, IntervalUnit, TimeUnit}; use datafusion_common::DataFusionError; for (i, col) in $arrays.iter().enumerate() { @@ -990,6 +990,58 @@ macro_rules! create_hashes_internal { $hash_method, )?; } + DataType::Time64(TimeUnit::Nanosecond) => { + $crate::hash_array_primitive!( + Time64NanosecondArray, + col, + i64, + $hashes_buffer, + $hash_method + ); + } + DataType::Interval(IntervalUnit::MonthDayNano) => { + // Spark 4.2's generated code hashes the microseconds and then + // the months, and leaves out the days field: + // https://github.com/apache/spark/blob/v4.2.0/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/hash.scala#L428-L431 + // SPARK-58236 includes the days from Spark 4.3. Spark's + // CalendarInterval is in microseconds, so the nanoseconds are + // a whole number of microseconds. Hashing every row with the + // microseconds and then every row with the months chains the + // two per row, as Spark does. + let array = $crate::downcast_array!(col, IntervalMonthDayNanoArray); + $crate::function::hash::utils::hash_primitive_values( + array, + $hashes_buffer, + |v| (v.nanoseconds / 1_000).to_le_bytes(), + $hash_method, + ); + $crate::function::hash::utils::hash_primitive_values( + array, + $hashes_buffer, + |v| v.months.to_le_bytes(), + $hash_method, + ); + } + // Spark hashes YearMonthIntervalType as an INT and + // DayTimeIntervalType as a BIGINT. + DataType::Interval(IntervalUnit::YearMonth) => { + $crate::hash_array_primitive!( + IntervalYearMonthArray, + col, + i32, + $hashes_buffer, + $hash_method + ); + } + DataType::Duration(TimeUnit::Microsecond) => { + $crate::hash_array_primitive!( + DurationMicrosecondArray, + col, + i64, + $hashes_buffer, + $hash_method + ); + } DataType::Utf8 => { $crate::hash_array!(StringArray, col, $hashes_buffer, $hash_method); } diff --git a/crates/sqllogictest/test_files/hash/hash.slt b/crates/sqllogictest/test_files/hash/hash.slt index c78574a6..88f7c192 100644 --- a/crates/sqllogictest/test_files/hash/hash.slt +++ b/crates/sqllogictest/test_files/hash/hash.slt @@ -581,6 +581,74 @@ SELECT hash(column2) FROM hash_t; -1808790533 42 +# Column inputs with NULL rows, which take the null-aware loops +query I +SELECT hash(column1) FROM (VALUES ([CAST(1 AS INT), CAST(2 AS INT)]), (NULL), ([NULL, CAST(3 AS INT)]), (arrow_cast([], 'List(Int32)'))); +---- +-222940379 +42 +-1823081949 +42 + +query I +SELECT hash(column1) FROM (VALUES ([[CAST(1 AS INT), CAST(2 AS INT)], [CAST(3 AS INT)]]), ([arrow_cast(NULL, 'List(Int32)')]), (NULL)); +---- +-912918097 +42 +42 + +query I +SELECT hash(CASE WHEN i = 2 THEN NULL ELSE named_struct('a', i, 'b', CAST(i AS VARCHAR)) END) +FROM (SELECT CAST(column1 AS INT) AS i FROM (VALUES (1), (2), (3))); +---- +-1161632238 +42 +2015795020 + +query I +SELECT hash(CASE WHEN i = 2 THEN NULL ELSE map([i], [CAST(i AS VARCHAR)]) END) +FROM (SELECT CAST(column1 AS INT) AS i FROM (VALUES (1), (2), (3))); +---- +-1161632238 +42 +2015795020 + +# TIME is hashed as nanoseconds since midnight +query I +SELECT hash(TIME '12:00:00'); +---- +-1984942870 + +# Spark's day-time interval is Duration(Microsecond) in Arrow and is hashed as a +# BIGINT. DataFusion reads the literal INTERVAL '1' DAY as a calendar interval, +# so the Duration is built with arrow_cast. Spark 4.2.0: hash(INTERVAL '1' DAY). +query I +SELECT hash(arrow_cast(86400000000, 'Duration(Microsecond)')); +---- +892267096 + +# Spark's year-month interval is Interval(YearMonth) and is hashed as an INT. +# Spark 4.2.0: hash(INTERVAL '14' MONTH). +query I +SELECT hash(arrow_cast('1 year 2 months', 'Interval(YearMonth)')); +---- +335551368 + +# A calendar interval (Interval(MonthDayNano), from make_interval) is hashed as +# Spark 4.2.0's generated code does: the microseconds, then the months, without +# the days. So 0 and 1 days hash alike, as they do in Spark over a column, for +# example hash(make_interval(0, 0, 0, CAST(id AS INT))) FROM range(2). Spark's +# interpreted path, which it uses to constant-fold a literal, includes the days +# and gives other values. SPARK-58236 includes the days from Spark 4.3. Spark +# folds these VALUES rows with the interpreted path, so the PySpark runner skips +# this record. +skipif spark +query I +SELECT hash(make_interval(0, 0, 0, CAST(d AS INT))) FROM (VALUES (0), (1)) AS t(d); +---- +1081828689 +1081828689 + # Spark requires at least one argument statement error does not support zero arguments SELECT hash(); diff --git a/crates/sqllogictest/test_files/hash/xxhash64.slt b/crates/sqllogictest/test_files/hash/xxhash64.slt index 78b48243..7e26884b 100644 --- a/crates/sqllogictest/test_files/hash/xxhash64.slt +++ b/crates/sqllogictest/test_files/hash/xxhash64.slt @@ -388,3 +388,11 @@ SELECT xxhash64([-CAST('NaN' AS FLOAT)]); # Spark requires at least one argument statement error does not support zero arguments SELECT xxhash64(); + +# TIME and Spark's interval types. The intervals are built with arrow_cast, since +# DataFusion reads INTERVAL literals as calendar intervals. Spark 4.2.0: +# xxhash64(TIME '12:00:00'), xxhash64(INTERVAL '1' DAY), xxhash64(INTERVAL '14' MONTH). +query III +SELECT xxhash64(TIME '12:00:00'), xxhash64(arrow_cast(86400000000, 'Duration(Microsecond)')), xxhash64(arrow_cast('1 year 2 months', 'Interval(YearMonth)')); +---- +7141818012718557382 5650310968337874586 -3715488255824917081 From 24cf7ba22f3d1296fd2c90675607edd973eaae1b Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 16:54:07 -0700 Subject: [PATCH 7/7] docs(spark): document how hash treats TIME and interval types Describe in the hash docs how TIME, year-month, day-time and calendar intervals hash, and that DataFusion's INTERVAL '1' DAY literal is a calendar interval that hashes like a zero interval. Add a test for that literal. --- crates/spark/src/function/hash/murmur3.rs | 7 ++++++- crates/sqllogictest/test_files/hash/hash.slt | 10 ++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/crates/spark/src/function/hash/murmur3.rs b/crates/spark/src/function/hash/murmur3.rs index c277b801..85cd950a 100644 --- a/crates/spark/src/function/hash/murmur3.rs +++ b/crates/spark/src/function/hash/murmur3.rs @@ -32,7 +32,12 @@ const DEFAULT_SEED: u32 = 42; and integer literals are BIGINT in DataFusion but INT in Spark, so cast them to INT to match Spark's results. \ `-0.0` hashes like `0.0`, and every NaN hashes like the canonical NaN. Dates hash as INT days and \ timestamps as BIGINT microseconds, whatever the Arrow unit (DataFusion's TIMESTAMP literals are \ - nanoseconds, floored to microseconds). Decimals with precision up to 18 hash as their unscaled long, and wider decimals hash \ + nanoseconds, floored to microseconds). TIME hashes as BIGINT nanoseconds since midnight, a year-month \ + interval (`Interval(YearMonth)`) as INT months and a day-time interval (`Duration(Microsecond)`) as BIGINT \ + microseconds. A calendar interval (`Interval(MonthDayNano)`) hashes its microseconds and then its months, \ + and leaves out its days, as Spark 4.2's generated code does (Spark 4.3 includes the days, see SPARK-58236). \ + DataFusion types `INTERVAL '1' DAY` as a calendar interval, so it hashes like a zero interval, where Spark \ + hashes it as a day-time interval. Use `arrow_cast(86400000000, 'Duration(Microsecond)')` to match Spark. Decimals with precision up to 18 hash as their unscaled long, and wider decimals hash \ the minimal big-endian two's complement bytes of the unscaled value. Lists hash each element in order \ (nested lists flatten into the same chain, and an empty list leaves the seed unchanged), structs hash each field, \ and maps hash each key then its value. This crate hashes maps as if Spark's `spark.sql.legacy.allowHashOnMapType` \ diff --git a/crates/sqllogictest/test_files/hash/hash.slt b/crates/sqllogictest/test_files/hash/hash.slt index 88f7c192..fde5d6ed 100644 --- a/crates/sqllogictest/test_files/hash/hash.slt +++ b/crates/sqllogictest/test_files/hash/hash.slt @@ -649,6 +649,16 @@ SELECT hash(make_interval(0, 0, 0, CAST(d AS INT))) FROM (VALUES (0), (1)) AS t( 1081828689 1081828689 +# DataFusion types the literal INTERVAL '1' DAY as Interval(MonthDayNano), so it +# takes the calendar-interval path above and hashes like a zero interval. Spark +# 4.2.0 types it as a day-time interval and returns 892267096, the Duration case +# above, so the PySpark runner skips this record. +skipif spark +query II +SELECT hash(INTERVAL '1' DAY), hash(make_interval(0, 0, 0, 0)); +---- +1081828689 1081828689 + # Spark requires at least one argument statement error does not support zero arguments SELECT hash();