From ce8872eb458de1351bea0214a46e76f0d7dbf8ee Mon Sep 17 00:00:00 2001 From: ump45nose <52391318+ump45nose@users.noreply.github.com> Date: Sun, 25 Oct 2026 13:40:59 +0000 Subject: [PATCH] fix: hash NullType and ANSI interval values natively (#7084) * fix: hash NullType and ANSI interval values natively * test: cover NullType and ANSI interval hashing with SQL file tests Adds SQL file tests for the native paths this PR fixes: hash and xxhash64 with a NULL argument and with a NullType struct field, including under a null struct; reading a NullType field through a null struct with GetStructField and GetArrayStructFields; and native round-robin shuffle, with positional placement off, over a NULL column and both ANSI interval types. * test: hash lists of NullType and intervals; fold the NullType arm in struct_nulls Adds murmur3 and xxhash64 unit tests that hash a list of NullType, which must leave the hash unchanged, and lists of both ANSI interval types, which must hash like the same lists of ints and longs. Folds the NullType check in struct_nulls into the null-parent arm, and drops the unneeded localTableScan config from the round-robin SQL file test, noting why positional placement is pinned off. --------- Co-authored-by: ump45nose (cherry picked from commit f1778b986b76d14dcad4cbd902bf6493c134497b) --- native/common/src/struct_nulls.rs | 27 ++++- native/spark-expr/src/hash_funcs/murmur3.rs | 97 ++++++++++++++++ native/spark-expr/src/hash_funcs/utils.rs | 21 ++++ native/spark-expr/src/hash_funcs/xxhash64.rs | 109 ++++++++++++++++++ .../array/get_array_struct_fields.sql | 16 +++ .../expressions/hash/hash_null_type.sql | 40 +++++++ .../expressions/struct/get_struct_field.sql | 12 ++ .../round_robin_null_and_intervals.sql | 44 +++++++ 8 files changed, 362 insertions(+), 4 deletions(-) create mode 100644 spark/src/test/resources/sql-tests/expressions/hash/hash_null_type.sql create mode 100644 spark/src/test/resources/sql-tests/operators/round_robin_null_and_intervals.sql diff --git a/native/common/src/struct_nulls.rs b/native/common/src/struct_nulls.rs index f3390707702..64d8c4ece88 100644 --- a/native/common/src/struct_nulls.rs +++ b/native/common/src/struct_nulls.rs @@ -30,6 +30,7 @@ use arrow::array::{make_array, Array, ArrayRef, StructArray}; use arrow::buffer::NullBuffer; +use arrow::datatypes::DataType; use arrow::error::ArrowError; use std::sync::Arc; @@ -37,7 +38,8 @@ use std::sync::Arc; /// /// Skips the work when the parent has no null to contribute, which covers both no null buffer at /// all and an all-valid buffer — the latter is what slicing leaves behind. `NullBuffer` stores its -/// null count, so the test is O(1). +/// null count, so the test is O(1). A `NullType` child is returned unchanged too: it is null at +/// every row already, and a `NullArray` cannot carry a null buffer. /// /// The union only ever adds nulls, so the child's data buffers are untouched and are not /// revalidated. Callers on a hot path depend on that: the hash kernels reach this once per element @@ -49,7 +51,7 @@ pub fn child_with_parent_nulls( ) -> Result { let child = struct_array.column(ordinal); match struct_array.nulls() { - Some(parent) if parent.null_count() > 0 => { + Some(parent) if parent.null_count() > 0 && child.data_type() != &DataType::Null => { let combined = NullBuffer::union(Some(parent), child.nulls()); // SAFETY: only nulls are added; every data buffer and child array is carried over // unchanged, which is the same argument `StructArray::flatten` makes. @@ -77,8 +79,8 @@ pub fn children_with_parent_nulls(struct_array: &StructArray) -> Result) -> StructArray { let fields: Fields = vec![Arc::new(Field::new("a", DataType::Int32, true))].into(); @@ -160,4 +162,21 @@ mod tests { let sa = StructArray::new_empty_fields(2, Some(NullBuffer::from(vec![true, false]))); assert!(children_with_parent_nulls(&sa).unwrap().is_empty()); } + + /// A `NullType` field of a null struct is returned as is: it is null at every row already, and + /// a `NullArray` with a null buffer is invalid. + #[test] + fn null_type_child_is_returned_unchanged() { + let fields: Fields = vec![Arc::new(Field::new("n", DataType::Null, true))].into(); + let child: ArrayRef = Arc::new(NullArray::new(2)); + let sa = StructArray::new( + fields, + vec![Arc::clone(&child)], + Some(NullBuffer::from(vec![true, false])), + ); + + let out = child_with_parent_nulls(&sa, 0).unwrap(); + assert!(Arc::ptr_eq(&out, &child)); + assert_eq!(out.logical_null_count(), 2); + } } diff --git a/native/spark-expr/src/hash_funcs/murmur3.rs b/native/spark-expr/src/hash_funcs/murmur3.rs index f891087b246..fa1f28c826a 100644 --- a/native/spark-expr/src/hash_funcs/murmur3.rs +++ b/native/spark-expr/src/hash_funcs/murmur3.rs @@ -774,6 +774,103 @@ mod tests { ); } + /// Spark hashes `YearMonthIntervalType` as an int and `DayTimeIntervalType` as a long, which + /// Comet maps to `Interval(YearMonth)` and `Duration(Microsecond)`. + #[test] + fn test_ansi_intervals_hash_like_int_and_long() { + use arrow::array::{DurationMicrosecondArray, IntervalYearMonthArray}; + + test_murmur3_hash::( + vec![Some(1), Some(0), Some(-1), Some(i32::MAX), Some(i32::MIN)], + vec![0xdea578e3, 0x379fae8f, 0xa0590e3d, 0x07fb67e7, 0x2b1f0fc6], + ); + test_murmur3_hash::( + vec![Some(1), Some(0), Some(-1), Some(i64::MAX), Some(i64::MIN)], + vec![0x99f0149d, 0x9c67b85d, 0xc8008529, 0xa05b5d7b, 0xcd1e64fb], + ); + } + + /// A `NullType` value is always null, so it leaves the hash unchanged, as in Spark, both as a + /// column of its own and as a struct field, including a field of a null struct. + #[test] + fn test_null_type_leaves_hash_unchanged() { + use arrow::array::{NullArray, StructArray}; + use arrow::buffer::NullBuffer; + use arrow::datatypes::{DataType, Field}; + + let ints: ArrayRef = Arc::new(Int32Array::from(vec![Some(1), None, Some(-1)])); + let nulls: ArrayRef = Arc::new(NullArray::new(3)); + + let mut expected = vec![42u32; 3]; + create_murmur3_hashes(&[Arc::clone(&ints)], &mut expected).unwrap(); + let mut hashes = vec![42u32; 3]; + create_murmur3_hashes(&[Arc::clone(&ints), Arc::clone(&nulls)], &mut hashes).unwrap(); + assert_eq!(hashes, expected); + + let struct_nulls = Some(NullBuffer::from(vec![true, true, false])); + let x = Arc::new(Field::new("x", DataType::Int32, true)); + let n = Arc::new(Field::new("n", DataType::Null, true)); + let with_null_field: ArrayRef = Arc::new(StructArray::new( + vec![Arc::clone(&x), n].into(), + vec![Arc::clone(&ints), nulls], + struct_nulls.clone(), + )); + let without_null_field: ArrayRef = + Arc::new(StructArray::new(vec![x].into(), vec![ints], struct_nulls)); + + let mut expected = vec![42u32; 3]; + create_murmur3_hashes(&[without_null_field], &mut expected).unwrap(); + let mut hashes = vec![42u32; 3]; + create_murmur3_hashes(&[with_null_field], &mut hashes).unwrap(); + assert_eq!(hashes, expected); + assert_eq!(hashes[2], 42, "a null struct must leave the seed untouched"); + } + + /// Lists whose elements are `NullType` or an ANSI interval type go through the recursive + /// list path into the `Null`, `Interval(YearMonth)` and `Duration(Microsecond)` arms. A list + /// of nulls leaves the hash unchanged, and a list of intervals hashes like the same list of + /// ints or longs. + #[test] + fn test_lists_of_null_type_and_intervals() { + use arrow::array::{ + DurationMicrosecondArray, IntervalYearMonthArray, ListArray, NullArray, + }; + use arrow::buffer::{NullBuffer, OffsetBuffer}; + use arrow::datatypes::Field; + + // Rows [v0, v1], [], null and [v2]. + let as_list = |values: ArrayRef| -> ArrayRef { + Arc::new(ListArray::new( + Arc::new(Field::new("item", values.data_type().clone(), true)), + OffsetBuffer::from_lengths([2, 0, 0, 1]), + values, + Some(NullBuffer::from(vec![true, true, false, true])), + )) + }; + let hash = |list: ArrayRef| { + let mut hashes = vec![42u32; 4]; + create_murmur3_hashes(&[list], &mut hashes).unwrap(); + hashes + }; + + assert_eq!(hash(as_list(Arc::new(NullArray::new(3)))), vec![42u32; 4]); + + let months = vec![Some(1), None, Some(i32::MIN)]; + assert_eq!( + hash(as_list(Arc::new(IntervalYearMonthArray::from( + months.clone() + )))), + hash(as_list(Arc::new(Int32Array::from(months)))) + ); + let micros = vec![Some(-1), Some(i64::MAX), None]; + assert_eq!( + hash(as_list(Arc::new(DurationMicrosecondArray::from( + micros.clone() + )))), + hash(as_list(Arc::new(Int64Array::from(micros)))) + ); + } + #[test] fn test_f32() { test_murmur3_hash::( diff --git a/native/spark-expr/src/hash_funcs/utils.rs b/native/spark-expr/src/hash_funcs/utils.rs index a44102c5bc4..76f2f54011f 100644 --- a/native/spark-expr/src/hash_funcs/utils.rs +++ b/native/spark-expr/src/hash_funcs/utils.rs @@ -976,6 +976,27 @@ macro_rules! create_hashes_internal { $hash_method ); } + // Spark hashes YearMonthIntervalType as an int and DayTimeIntervalType as a long. + DataType::Interval(IntervalUnit::YearMonth) => { + $crate::hash_array_primitive!( + IntervalYearMonthArray, + col, + i32, + $hashes_buffer, + $hash_method + ); + } + DataType::Duration(TimeUnit::Microsecond) => { + $crate::hash_array_primitive!( + DurationMicrosecondArray, + col, + i64, + $hashes_buffer, + $hash_method + ); + } + // Every value of a NullType column is null, and a null leaves the hash unchanged. + DataType::Null => {} DataType::Utf8 => { $crate::hash_array!(StringArray, col, $hashes_buffer, $hash_method); } diff --git a/native/spark-expr/src/hash_funcs/xxhash64.rs b/native/spark-expr/src/hash_funcs/xxhash64.rs index 69f5ff966cc..97db7fa7ff1 100644 --- a/native/spark-expr/src/hash_funcs/xxhash64.rs +++ b/native/spark-expr/src/hash_funcs/xxhash64.rs @@ -399,6 +399,115 @@ mod tests { ); } + /// Spark hashes `YearMonthIntervalType` as an int and `DayTimeIntervalType` as a long, which + /// Comet maps to `Interval(YearMonth)` and `Duration(Microsecond)`. + #[test] + fn test_ansi_intervals_hash_like_int_and_long() { + use arrow::array::{DurationMicrosecondArray, IntervalYearMonthArray}; + + test_xxhash64_hash::( + vec![Some(1), Some(0), Some(-1), Some(i32::MAX), Some(i32::MIN)], + vec![ + 0xa309b38455455929, + 0x3229fbc4681e48f3, + 0x1bfdda8861c06e45, + 0x14f0ac009c21721c, + 0x1cc7cb8d034769cd, + ], + ); + test_xxhash64_hash::( + vec![Some(1), Some(0), Some(-1), Some(i64::MAX), Some(i64::MIN)], + vec![ + 0x9ed50fd59358d232, + 0xb71b47ebda15746c, + 0x358ae035bfb46fd2, + 0xd2f1c616ae7eb306, + 0x88608019c494c1f4, + ], + ); + } + + /// A `NullType` value is always null, so it leaves the hash unchanged, as in Spark, both as a + /// column of its own and as a struct field, including a field of a null struct. + #[test] + fn test_null_type_leaves_hash_unchanged() { + use arrow::array::{NullArray, StructArray}; + use arrow::buffer::NullBuffer; + use arrow::datatypes::{DataType, Field}; + + let ints: ArrayRef = Arc::new(Int32Array::from(vec![Some(1), None, Some(-1)])); + let nulls: ArrayRef = Arc::new(NullArray::new(3)); + + let mut expected = vec![42u64; 3]; + create_xxhash64_hashes(&[Arc::clone(&ints)], &mut expected).unwrap(); + let mut hashes = vec![42u64; 3]; + create_xxhash64_hashes(&[Arc::clone(&ints), Arc::clone(&nulls)], &mut hashes).unwrap(); + assert_eq!(hashes, expected); + + let struct_nulls = Some(NullBuffer::from(vec![true, true, false])); + let x = Arc::new(Field::new("x", DataType::Int32, true)); + let n = Arc::new(Field::new("n", DataType::Null, true)); + let with_null_field: ArrayRef = Arc::new(StructArray::new( + vec![Arc::clone(&x), n].into(), + vec![Arc::clone(&ints), nulls], + struct_nulls.clone(), + )); + let without_null_field: ArrayRef = + Arc::new(StructArray::new(vec![x].into(), vec![ints], struct_nulls)); + + let mut expected = vec![42u64; 3]; + create_xxhash64_hashes(&[without_null_field], &mut expected).unwrap(); + let mut hashes = vec![42u64; 3]; + create_xxhash64_hashes(&[with_null_field], &mut hashes).unwrap(); + assert_eq!(hashes, expected); + assert_eq!(hashes[2], 42, "a null struct must leave the seed untouched"); + } + + /// Lists whose elements are `NullType` or an ANSI interval type go through the recursive + /// list path into the `Null`, `Interval(YearMonth)` and `Duration(Microsecond)` arms. A list + /// of nulls leaves the hash unchanged, and a list of intervals hashes like the same list of + /// ints or longs. + #[test] + fn test_lists_of_null_type_and_intervals() { + use arrow::array::{ + DurationMicrosecondArray, IntervalYearMonthArray, ListArray, NullArray, + }; + use arrow::buffer::{NullBuffer, OffsetBuffer}; + use arrow::datatypes::Field; + + // Rows [v0, v1], [], null and [v2]. + let as_list = |values: ArrayRef| -> ArrayRef { + Arc::new(ListArray::new( + Arc::new(Field::new("item", values.data_type().clone(), true)), + OffsetBuffer::from_lengths([2, 0, 0, 1]), + values, + Some(NullBuffer::from(vec![true, true, false, true])), + )) + }; + let hash = |list: ArrayRef| { + let mut hashes = vec![42u64; 4]; + create_xxhash64_hashes(&[list], &mut hashes).unwrap(); + hashes + }; + + assert_eq!(hash(as_list(Arc::new(NullArray::new(3)))), vec![42u64; 4]); + + let months = vec![Some(1), None, Some(i32::MIN)]; + assert_eq!( + hash(as_list(Arc::new(IntervalYearMonthArray::from( + months.clone() + )))), + hash(as_list(Arc::new(Int32Array::from(months)))) + ); + let micros = vec![Some(-1), Some(i64::MAX), None]; + assert_eq!( + hash(as_list(Arc::new(DurationMicrosecondArray::from( + micros.clone() + )))), + hash(as_list(Arc::new(Int64Array::from(micros)))) + ); + } + #[test] fn test_f32() { test_xxhash64_hash::( diff --git a/spark/src/test/resources/sql-tests/expressions/array/get_array_struct_fields.sql b/spark/src/test/resources/sql-tests/expressions/array/get_array_struct_fields.sql index ded76f3d919..a2ab919ce76 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/get_array_struct_fields.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/get_array_struct_fields.sql @@ -15,6 +15,8 @@ -- specific language governing permissions and limitations -- under the License. +-- ExcludeRules: org.apache.spark.sql.catalyst.optimizer.SimplifyExtractValueOps + statement CREATE TABLE test_arr_struct(arr array>) USING parquet @@ -23,3 +25,17 @@ INSERT INTO test_arr_struct VALUES (array(named_struct('name', 'a', 'value', 1), query spark_answer_only SELECT arr.name, arr.value FROM test_arr_struct + +-- A NullType field of an array of structs reads as an array of NULLs, also for null structs, +-- whose nulls are pushed into the all-null field. SimplifyExtractValueOps is excluded (see the +-- top of the file) so the field access is not rewritten into the array constructor. +statement +CREATE TABLE test_arr_struct_null_field(a int) USING parquet + +statement +INSERT INTO test_arr_struct_null_field VALUES (1), (-1), (NULL) + +query +SELECT arr.x, arr.n FROM ( + SELECT array(IF(a > 0, named_struct('x', a, 'n', null), null), named_struct('x', 0, 'n', null)) AS arr + FROM test_arr_struct_null_field) diff --git a/spark/src/test/resources/sql-tests/expressions/hash/hash_null_type.sql b/spark/src/test/resources/sql-tests/expressions/hash/hash_null_type.sql new file mode 100644 index 00000000000..f9125c0d04e --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/hash/hash_null_type.sql @@ -0,0 +1,40 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Spark's HashExpression leaves the running hash unchanged for a NULL child, so a NullType +-- argument, or a NullType struct field, is skipped. The other inputs are read from Parquet so +-- the hashes run natively. +statement +CREATE TABLE test_hash_null_type(a int, s string) USING parquet + +statement +INSERT INTO test_hash_null_type VALUES (1, 'a'), (0, ''), (-1, NULL), (2147483647, 'Spark SQL'), (NULL, NULL) + +query expect_native(hash, xxhash64) +SELECT a, hash(a, null), xxhash64(a, null), hash(null, a), xxhash64(null, a), + hash(a, null, s), xxhash64(a, null, s) +FROM test_hash_null_type + +query expect_native(hash, xxhash64) +SELECT hash(named_struct('x', a, 'n', null)), xxhash64(named_struct('x', a, 'n', null)) +FROM test_hash_null_type + +-- A nullable struct with a NullType field: the struct's nulls are pushed into the all-null field. +query expect_native(hash, xxhash64) +SELECT hash(IF(a > 0, named_struct('x', a, 'n', null), null)), + xxhash64(IF(a > 0, named_struct('x', a, 'n', null), null)) +FROM test_hash_null_type diff --git a/spark/src/test/resources/sql-tests/expressions/struct/get_struct_field.sql b/spark/src/test/resources/sql-tests/expressions/struct/get_struct_field.sql index c5d9e64a634..6bfb2875000 100644 --- a/spark/src/test/resources/sql-tests/expressions/struct/get_struct_field.sql +++ b/spark/src/test/resources/sql-tests/expressions/struct/get_struct_field.sql @@ -26,3 +26,15 @@ SELECT s.name, s.age, s.score FROM test_struct query SELECT s.name, s.age + 1, s.score * 2 FROM test_struct + +-- A NullType field reads as NULL, also through a null struct, whose nulls are pushed into the +-- all-null field. +statement +CREATE TABLE test_struct_null_field(a int) USING parquet + +statement +INSERT INTO test_struct_null_field VALUES (1), (-1), (NULL) + +query +SELECT s.x, s.n FROM ( + SELECT IF(a > 0, named_struct('x', a, 'n', null), null) AS s FROM test_struct_null_field) diff --git a/spark/src/test/resources/sql-tests/operators/round_robin_null_and_intervals.sql b/spark/src/test/resources/sql-tests/operators/round_robin_null_and_intervals.sql new file mode 100644 index 00000000000..c0a2f10901a --- /dev/null +++ b/spark/src/test/resources/sql-tests/operators/round_robin_null_and_intervals.sql @@ -0,0 +1,44 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Config: spark.comet.shuffle.mode=native +-- Config: spark.comet.shuffle.native.partitioning.roundrobin.enabled=true +-- Positional placement is off by default, but pinned here on purpose: these queries test the +-- content-hash placement, which positional placement would skip. +-- Config: spark.comet.shuffle.native.partitioning.roundrobin.positional.enabled=false + +-- Without positional placement, native round-robin shuffle places each row by a murmur3 hash of +-- all its columns, so a NULL (NullType) column and ANSI interval columns go through the native +-- hash. The default query mode checks that the exchange is native and compares rows with Spark. + +statement +CREATE TABLE test_rr_null_intervals(id int, y int, m int, d int, h int) USING parquet + +statement +INSERT INTO test_rr_null_intervals VALUES + (1, 1, 2, 1, 2), (2, 0, 0, 0, 0), (3, -1, 6, -1, 23), (4, 5, -3, 3, -4), (5, NULL, NULL, NULL, NULL) + +query +SELECT /*+ REPARTITION(3) */ id, NULL AS n FROM test_rr_null_intervals + +query +SELECT /*+ REPARTITION(3) */ id, make_ym_interval(y, m) AS ym, make_dt_interval(d, h) AS dt +FROM test_rr_null_intervals + +query +SELECT /*+ REPARTITION(3) */ NULL AS n, make_ym_interval(y, m) AS ym, make_dt_interval(d, h) AS dt +FROM test_rr_null_intervals