From 4482ff91b6ca1c71fa12ef84df3fec4b38dd0f35 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 16 Nov 2026 06:32:12 -0700 Subject: [PATCH 1/8] feat(spark): add Spark-compatible array_max and array_min Ports array_max and array_min from Comet (array_funcs/array_extrema.rs). NULL elements are skipped, and a NULL, empty or all-NULL array returns NULL. Elements are compared with Spark's ordering: NaN is greater than every other value, -0.0 equals 0.0 (at any depth), strings and binary compare bytewise, and NULLs nested in arrays and structs sort first. On a tie the first element is returned unchanged. Map elements at any depth are rejected with INVALID_ORDERING_TYPE, and non-array input with UNEXPECTED_INPUT_TYPE. Supports List, LargeList and FixedSizeList. Closes #140. --- .../spark/src/function/array/array_extrema.rs | 574 ++++++++++++++++++ .../src/function/array/array_position.rs | 2 +- crates/spark/src/function/array/mod.rs | 17 +- .../test_files/array/array_extrema.slt | 207 +++++++ 4 files changed, 798 insertions(+), 2 deletions(-) create mode 100644 crates/spark/src/function/array/array_extrema.rs create mode 100644 crates/sqllogictest/test_files/array/array_extrema.slt diff --git a/crates/spark/src/function/array/array_extrema.rs b/crates/spark/src/function/array/array_extrema.rs new file mode 100644 index 00000000..01dec17c --- /dev/null +++ b/crates/spark/src/function/array/array_extrema.rs @@ -0,0 +1,574 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Spark-compatible `array_max` and `array_min`. + +use super::array_position::contains_map; +use super::float_normalize::normalize_array; +use arrow::array::{ + Array, ArrayRef, AsArray, GenericListArray, OffsetSizeTrait, UInt64Array, + make_comparator, new_null_array, +}; +use arrow::compute::{SortOptions, take}; +use arrow::datatypes::{DataType, Field}; +use datafusion_common::utils::take_function_args; +use datafusion_common::{Result, ScalarValue, exec_err, plan_err}; +use datafusion_expr::{ + ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, + Volatility, +}; +use datafusion_macros::user_doc; +use std::cmp::Ordering; +use std::sync::Arc; + +/// Spark-compatible `array_max` expression. +/// +/// +#[user_doc( + doc_section(label = "Array Functions"), + description = "Returns the maximum element of the array, using Spark's ordering. NULL \ + elements are skipped, and an empty array or an array of only NULL elements returns \ + NULL. For floating point values, also inside nested arrays and structs, NaN is \ + greater than every other value, including infinity, and -0.0 is equal to 0.0. On a \ + tie the first of the tied elements is returned as it is, so the sign of a zero and the \ + bits of a NaN are kept. Strings and binary values compare bytewise. Arrays compare \ + element by element and then by length, and structs compare field by field; NULLs \ + nested inside them sort before other values. Map elements, at any depth, are an \ + error, as in Spark.", + syntax_example = "array_max(array)", + sql_example = r#"```sql +> SELECT array_max(array(1, 20, NULL, 3)); +20 +```"#, + argument(name = "array", description = "Array expression. NULL returns NULL.") +)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkArrayMax { + signature: Signature, +} + +impl Default for SparkArrayMax { + fn default() -> Self { + Self::new() + } +} + +impl SparkArrayMax { + pub fn new() -> Self { + Self { + signature: Signature::user_defined(Volatility::Immutable), + } + } +} + +impl ScalarUDFImpl for SparkArrayMax { + fn name(&self) -> &str { + "array_max" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, arg_types: &[DataType]) -> Result { + element_type(self.name(), arg_types) + } + + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { + coerce_array(self.name(), arg_types) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + invoke(self.name(), args, Ordering::Greater) + } + + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } +} + +/// Spark-compatible `array_min` expression. +/// +/// +#[user_doc( + doc_section(label = "Array Functions"), + description = "Returns the minimum element of the array, using Spark's ordering. NULL \ + elements are skipped, and an empty array or an array of only NULL elements returns \ + NULL. For floating point values, also inside nested arrays and structs, NaN is \ + greater than every other value, including infinity, and -0.0 is equal to 0.0. On a \ + tie the first of the tied elements is returned as it is, so the sign of a zero and the \ + bits of a NaN are kept. Strings and binary values compare bytewise. Arrays compare \ + element by element and then by length, and structs compare field by field; NULLs \ + nested inside them sort before other values. Map elements, at any depth, are an \ + error, as in Spark.", + syntax_example = "array_min(array)", + sql_example = r#"```sql +> SELECT array_min(array(1, 20, NULL, 3)); +1 +```"#, + argument(name = "array", description = "Array expression. NULL returns NULL.") +)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkArrayMin { + signature: Signature, +} + +impl Default for SparkArrayMin { + fn default() -> Self { + Self::new() + } +} + +impl SparkArrayMin { + pub fn new() -> Self { + Self { + signature: Signature::user_defined(Volatility::Immutable), + } + } +} + +impl ScalarUDFImpl for SparkArrayMin { + fn name(&self) -> &str { + "array_min" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, arg_types: &[DataType]) -> Result { + element_type(self.name(), arg_types) + } + + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { + coerce_array(self.name(), arg_types) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + invoke(self.name(), args, Ordering::Less) + } + + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } +} + +/// The element type of the array argument. A NULL-typed argument, which Spark accepts, +/// returns the NULL type. +fn element_type(name: &str, arg_types: &[DataType]) -> Result { + let [array] = take_function_args(name, arg_types)?; + match array { + DataType::Null => Ok(DataType::Null), + DataType::List(field) + | DataType::LargeList(field) + | DataType::FixedSizeList(field, _) => Ok(field.data_type().clone()), + other => unexpected_input_type(name, other), + } +} + +/// Accepts the argument types that Spark's `ArrayMax` and `ArrayMin` accept: an array of an +/// orderable type, or NULL. Fixed-size lists are cast to lists. +fn coerce_array(name: &str, arg_types: &[DataType]) -> Result> { + let [array] = take_function_args(name, arg_types)?; + let coerced = match array { + DataType::Null | DataType::List(_) | DataType::LargeList(_) => array.clone(), + DataType::FixedSizeList(field, _) => DataType::List(Arc::new(Field::new( + field.name(), + field.data_type().clone(), + true, + ))), + other => return unexpected_input_type(name, other), + }; + let element = element_type(name, std::slice::from_ref(&coerced))?; + if contains_map(&element) { + return plan_err!( + "[DATATYPE_MISMATCH.INVALID_ORDERING_TYPE] The `{name}` does not support \ + ordering on type {element}" + ); + } + Ok(vec![coerced]) +} + +fn unexpected_input_type(name: &str, data_type: &DataType) -> Result { + plan_err!( + "[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] The first parameter of `{name}` \ + requires the \"ARRAY\" type, however it has the type {data_type}" + ) +} + +fn invoke( + name: &str, + args: ScalarFunctionArgs, + target: Ordering, +) -> Result { + let [arg] = take_function_args(name, &args.args)?; + let element_type = args.return_field.data_type(); + let array = match arg { + ColumnarValue::Array(array) => Arc::clone(array), + ColumnarValue::Scalar(scalar) => scalar.to_array()?, + }; + + let result = match array.data_type() { + DataType::Null => new_null_array(element_type, array.len()), + // All the elements are NULL, so every result is NULL. + _ if element_type.is_null() => new_null_array(element_type, array.len()), + DataType::List(_) => extrema(array.as_list::(), target)?, + DataType::LargeList(_) => extrema(array.as_list::(), target)?, + other => return exec_err!("{name} does not support type {other}"), + }; + + if matches!(arg, ColumnarValue::Scalar(_)) { + Ok(ColumnarValue::Scalar(ScalarValue::try_from_array( + &result, 0, + )?)) + } else { + Ok(ColumnarValue::Array(result)) + } +} + +/// Returns, for each list, its first non-NULL element that no other element orders +/// `target` of, or NULL when the list is NULL, empty, or holds only NULL elements. +fn extrema( + list: &GenericListArray, + target: Ordering, +) -> Result { + let offsets = list.offsets(); + // Only the values that this (possibly sliced) list refers to are compared. + let base = offsets[0].as_usize(); + let len = offsets[offsets.len() - 1].as_usize() - base; + let values = list.values().slice(base, len); + // Compare canonicalized floats, so that NaN is the greatest value and -0.0 equals 0.0 at + // any depth, but return the original element. + let normalized = normalize_array(&values)?; + // Spark orders NULLs nested inside arrays and structs before other values. + let cmp = make_comparator( + normalized.as_ref(), + normalized.as_ref(), + SortOptions::default(), + )?; + + let indices: UInt64Array = offsets + .windows(2) + .enumerate() + .map(|(row, w)| { + if list.is_null(row) { + return None; + } + let (start, end) = (w[0].as_usize() - base, w[1].as_usize() - base); + (start..end) + .filter(|&i| values.is_valid(i)) + .reduce(|best, i| if cmp(i, best) == target { i } else { best }) + .map(|i| i as u64) + }) + .collect(); + + Ok(take(values.as_ref(), &indices, None)?) +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{ + BinaryArray, Float32Array, Float64Array, Int32Array, LargeListArray, ListArray, + StringArray, StructArray, + }; + use arrow::buffer::{NullBuffer, OffsetBuffer}; + use arrow::datatypes::{Float32Type, Float64Type, Int32Type}; + use datafusion_common::config::ConfigOptions; + + fn run(udf: &dyn ScalarUDFImpl, input: ColumnarValue) -> ColumnarValue { + let input_type = input.data_type(); + let return_type = udf.return_type(std::slice::from_ref(&input_type)).unwrap(); + let number_rows = match &input { + ColumnarValue::Array(array) => array.len(), + ColumnarValue::Scalar(_) => 1, + }; + udf.invoke_with_args(ScalarFunctionArgs { + args: vec![input], + arg_fields: vec![Arc::new(Field::new("input", input_type, true))], + number_rows, + return_field: Arc::new(Field::new("result", return_type, true)), + config_options: Arc::new(ConfigOptions::default()), + }) + .unwrap() + } + + fn max(input: &dyn Array) -> ArrayRef { + run_array(&SparkArrayMax::new(), input) + } + + fn min(input: &dyn Array) -> ArrayRef { + run_array(&SparkArrayMin::new(), input) + } + + fn run_array(udf: &dyn ScalarUDFImpl, input: &dyn Array) -> ArrayRef { + let input = input.slice(0, input.len()); + match run(udf, ColumnarValue::Array(input)) { + ColumnarValue::Array(result) => result, + ColumnarValue::Scalar(_) => panic!("array input must produce array output"), + } + } + + fn list(values: ArrayRef, offsets: &[i32]) -> ListArray { + ListArray::new( + Arc::new(Field::new_list_field(values.data_type().clone(), true)), + OffsetBuffer::new(offsets.to_vec().into()), + values, + None, + ) + } + + fn f64_bits(array: &ArrayRef) -> Vec> { + array + .as_primitive::() + .iter() + .map(|v| v.map(f64::to_bits)) + .collect() + } + + #[test] + fn integers_skip_nulls() { + let input = ListArray::from_iter_primitive::(vec![ + Some(vec![Some(1), Some(20), None, Some(3)]), + Some(vec![]), + Some(vec![None, None]), + None, + Some(vec![Some(-5)]), + ]); + assert_eq!( + max(&input).as_ref(), + &Int32Array::from(vec![Some(20), None, None, None, Some(-5)]) + ); + assert_eq!( + min(&input).as_ref(), + &Int32Array::from(vec![Some(1), None, None, None, Some(-5)]) + ); + } + + macro_rules! float_tests { + ($name:ident, $arrow_type:ty, $array:ty, $native:ident, $positive:expr, $negative:expr) => { + #[test] + fn $name() { + let positive = $native::from_bits($positive); + let negative = $native::from_bits($negative); + let mut long = vec![Some(1.0); 67]; + long[7] = Some(negative); + long[8] = Some(positive); + // (input, min, max) + let cases = [ + (vec![Some(0.0), Some(-0.0)], Some(0.0), Some(0.0)), + (vec![None, Some(-0.0), Some(0.0)], Some(-0.0), Some(-0.0)), + ( + vec![Some(positive), Some(negative)], + Some(positive), + Some(positive), + ), + ( + vec![Some(negative), Some(positive)], + Some(negative), + Some(negative), + ), + ( + vec![ + Some(negative), + Some($native::INFINITY), + Some($native::NEG_INFINITY), + ], + Some($native::NEG_INFINITY), + Some(negative), + ), + (vec![Some(3.0), None, Some(-2.0)], Some(-2.0), Some(3.0)), + (long, Some(1.0), Some(negative)), + ]; + for (row, expected_min, expected_max) in cases { + let input = + ListArray::from_iter_primitive::<$arrow_type, _, _>([Some(row)]); + for (result, expected) in + [(min(&input), expected_min), (max(&input), expected_max)] + { + let result = result.as_any().downcast_ref::<$array>().unwrap(); + assert_eq!( + result.iter().next().unwrap().map($native::to_bits), + expected.map($native::to_bits), + ); + } + } + } + }; + } + + float_tests!( + float32_nan_and_signed_zero, + Float32Type, + Float32Array, + f32, + 0x7fc0_0001, + 0xffc0_0002 + ); + float_tests!( + float64_nan_and_signed_zero, + Float64Type, + Float64Array, + f64, + 0x7ff8_0000_0000_0001, + 0xfff8_0000_0000_0002 + ); + + #[test] + fn scalar_and_sliced_input() { + let input = ListArray::from_iter_primitive::([ + Some(vec![Some(99.0)]), + Some(vec![Some(-0.0), None, Some(0.0)]), + None, + ]) + .slice(1, 2); + for udf in [ + &SparkArrayMax::new() as &dyn ScalarUDFImpl, + &SparkArrayMin::new(), + ] { + assert_eq!( + f64_bits(&run_array(udf, &input)), + vec![Some((-0.0f64).to_bits()), None] + ); + let scalar = ScalarValue::try_from_array(&input, 0).unwrap(); + let ColumnarValue::Scalar(result) = run(udf, ColumnarValue::Scalar(scalar)) + else { + panic!("scalar input must produce scalar output") + }; + assert_eq!( + f64_bits(&result.to_array().unwrap()), + vec![Some((-0.0f64).to_bits())] + ); + } + } + + #[test] + fn strings_and_binary_compare_bytewise() { + let values: ArrayRef = + Arc::new(StringArray::from(vec!["a", "b", "B", "", "é", "z"])); + let input = list(values, &[0, 4, 6]); + assert_eq!(max(&input).as_ref(), &StringArray::from(vec!["b", "é"])); + assert_eq!(min(&input).as_ref(), &StringArray::from(vec!["", "z"])); + + let values: ArrayRef = Arc::new(BinaryArray::from(vec![ + &[0x01][..], + &[0xff][..], + &[0x01, 0x00][..], + ])); + let input = list(values, &[0, 3]); + assert_eq!(max(&input).as_ref(), &BinaryArray::from(vec![&[0xff][..]])); + assert_eq!(min(&input).as_ref(), &BinaryArray::from(vec![&[0x01][..]])); + } + + #[test] + fn large_list() { + let input = LargeListArray::from_iter_primitive::(vec![ + Some(vec![Some(3), None, Some(7)]), + None, + ]); + assert_eq!(max(&input).as_ref(), &Int32Array::from(vec![Some(7), None])); + assert_eq!(min(&input).as_ref(), &Int32Array::from(vec![Some(3), None])); + } + + #[test] + fn nested_lists_order_lexicographically_with_nulls_first() { + let children = ListArray::from_iter_primitive::([ + Some(vec![Some(1.0), Some(2.0)]), + Some(vec![Some(1.0), Some(2.0), Some(3.0)]), + Some(vec![Some(2.0)]), + Some(vec![]), + Some(vec![Some(1.0), None]), + Some(vec![Some(1.0), Some(2.0)]), + Some(vec![Some(f64::NAN)]), + Some(vec![Some(f64::INFINITY)]), + None, + Some(vec![Some(-0.0)]), + Some(vec![Some(0.0)]), + ]); + let input = list(Arc::new(children.clone()), &[0, 4, 6, 9, 11]); + let expected = |winners: [u64; 4]| -> ArrayRef { + take(&children, &UInt64Array::from(winners.to_vec()), None).unwrap() + }; + let result = max(&input); + assert_eq!(&result, &expected([2, 5, 6, 9])); + let result = min(&input); + assert_eq!(&result, &expected([3, 4, 7, 9])); + // The first tied winner keeps the sign of its zero. + let last = result.as_list::().value(3); + assert!( + last.as_primitive::() + .value(0) + .is_sign_negative() + ); + } + + #[test] + fn structs_compare_field_by_field() { + let columns: Vec = vec![ + Arc::new(Float64Array::from(vec![ + Some(-0.0), + Some(0.0), + Some(1.0), + None, + Some(5.0), + Some(5.0), + ])), + Arc::new(Int32Array::from(vec![2, 1, 1, 100, 1, 2])), + ]; + let fields = vec![ + Arc::new(Field::new("a", DataType::Float64, true)), + Arc::new(Field::new("b", DataType::Int32, true)), + ]; + let children = StructArray::new( + fields.into(), + columns, + Some(NullBuffer::from(vec![true, true, true, true, false, true])), + ); + let input = list(Arc::new(children.clone()), &[0, 2, 4, 6]); + let expected = |winners: [u64; 3]| { + take(&children, &UInt64Array::from(winners.to_vec()), None).unwrap() + }; + // -0.0 equals 0.0, so the second field decides. A NULL field sorts first, and a NULL + // struct is skipped. + assert_eq!(&max(&input), &expected([0, 2, 5])); + assert_eq!(&min(&input), &expected([1, 3, 5])); + } + + #[test] + fn null_input_and_null_elements() { + let input = new_null_array(&DataType::Null, 2); + assert_eq!(max(&input).as_ref(), &new_null_array(&DataType::Null, 2)); + let input = list(new_null_array(&DataType::Null, 3), &[0, 0, 3]); + assert_eq!(min(&input).as_ref(), &new_null_array(&DataType::Null, 2)); + } + + #[test] + fn rejects_unsupported_types() { + let udf = SparkArrayMax::new(); + let err = udf.coerce_types(&[DataType::Int32]).unwrap_err(); + assert!(err.to_string().contains("UNEXPECTED_INPUT_TYPE"), "{err}"); + let map = DataType::Map( + Arc::new(Field::new( + "entries", + DataType::Struct( + vec![ + Field::new("key", DataType::Int32, false), + Field::new("value", DataType::Int32, true), + ] + .into(), + ), + false, + )), + false, + ); + let array = DataType::new_list(DataType::new_list(map, true), true); + let err = udf.coerce_types(&[array]).unwrap_err(); + assert!(err.to_string().contains("INVALID_ORDERING_TYPE"), "{err}"); + let fixed = DataType::new_fixed_size_list(DataType::Int32, 2, true); + assert_eq!( + udf.coerce_types(&[fixed]).unwrap(), + vec![DataType::new_list(DataType::Int32, true)] + ); + } +} diff --git a/crates/spark/src/function/array/array_position.rs b/crates/spark/src/function/array/array_position.rs index 4ca52da9..fc8ce738 100644 --- a/crates/spark/src/function/array/array_position.rs +++ b/crates/spark/src/function/array/array_position.rs @@ -349,7 +349,7 @@ fn finer_unit(left: TimeUnit, right: TimeUnit) -> TimeUnit { } /// Spark cannot order maps, so it rejects them at any depth of the element type. -fn contains_map(data_type: &DataType) -> bool { +pub(super) fn contains_map(data_type: &DataType) -> bool { match data_type { DataType::Map(_, _) => true, DataType::List(field) diff --git a/crates/spark/src/function/array/mod.rs b/crates/spark/src/function/array/mod.rs index 7ff0f2aa..d3d72886 100644 --- a/crates/spark/src/function/array/mod.rs +++ b/crates/spark/src/function/array/mod.rs @@ -2,9 +2,10 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -//! Spark array functions, such as `array`, `array_contains`, `array_repeat`, `shuffle` and `slice`. +//! Spark array functions, such as `array`, `array_contains`, `array_max`, `array_repeat`, `shuffle` and `slice`. pub mod array_contains; +pub mod array_extrema; pub mod array_position; mod float_normalize; pub mod repeat; @@ -18,6 +19,8 @@ use std::sync::Arc; make_udf_function!(array_contains::SparkArrayContains, spark_array_contains); make_udf_function!(array_position::SparkArrayPosition, array_position); +make_udf_function!(array_extrema::SparkArrayMax, array_max); +make_udf_function!(array_extrema::SparkArrayMin, array_min); make_udf_function!(spark_array::SparkArray, array); make_udf_function!(shuffle::SparkShuffle, shuffle); make_udf_function!(repeat::SparkArrayRepeat, array_repeat); @@ -36,6 +39,16 @@ pub mod expr_fn { "Returns the 1-based position of the element in the array, or 0 if not found (Spark semantics).", array element )); + export_functions!(( + array_max, + "Returns the maximum non-NULL element of the array (Spark semantics).", + array + )); + export_functions!(( + array_min, + "Returns the minimum non-NULL element of the array (Spark semantics).", + array + )); export_functions!((array, "Returns an array with the given elements.", args)); export_functions!(( shuffle, @@ -58,6 +71,8 @@ pub fn functions() -> Vec> { vec![ spark_array_contains(), array_position(), + array_max(), + array_min(), array(), shuffle(), array_repeat(), diff --git a/crates/sqllogictest/test_files/array/array_extrema.slt b/crates/sqllogictest/test_files/array/array_extrema.slt new file mode 100644 index 00000000..661c4d5e --- /dev/null +++ b/crates/sqllogictest/test_files/array/array_extrema.slt @@ -0,0 +1,207 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +# Tests for Spark-compatible array_max and array_min. +# NULL elements are skipped; a NULL, empty or all-NULL array returns NULL. +# Expected values verified with pyspark==4.2.0. + +query II +SELECT array_max(array(1, 20, NULL, 3)), array_min(array(1, 20, NULL, 3)); +---- +20 1 + +query II +SELECT array_max(array(CAST(1 AS TINYINT), CAST(-1 AS TINYINT))), array_min(array(1L, 2L)); +---- +1 1 + +query II +SELECT array_max(CAST(NULL AS ARRAY)), array_min(CAST(NULL AS ARRAY)); +---- +NULL NULL + +query I +SELECT array_max(array(CAST(NULL AS INT))); +---- +NULL + +# Spark accepts an untyped NULL, an empty array and an array of untyped NULLs, and returns +# a NULL of the NULL type. +query ?? +SELECT array_max(NULL), array_min(NULL); +---- +NULL NULL + +query ?? +SELECT array_max(array()), array_min(array(NULL, NULL)); +---- +NULL NULL + +# NaN is greater than every other value, including infinity +query RR +SELECT array_max(array(CAST('NaN' AS DOUBLE), CAST(1.0 AS DOUBLE), CAST('Infinity' AS DOUBLE))), + array_min(array(CAST('NaN' AS DOUBLE), CAST(1.0 AS DOUBLE), CAST('-Infinity' AS DOUBLE))); +---- +NaN -Infinity + +query RR +SELECT array_max(array(CAST(-0.0 AS DOUBLE), CAST(0.0 AS DOUBLE), CAST('NaN' AS DOUBLE))), + array_min(array(CAST('NaN' AS DOUBLE), CAST('NaN' AS DOUBLE))); +---- +NaN NaN + +query RR +SELECT array_max(array(CAST('NaN' AS FLOAT), CAST(1.5 AS FLOAT))), array_min(array(CAST('NaN' AS FLOAT), CAST(1.5 AS FLOAT))); +---- +NaN 1.5 + +# -0.0 equals 0.0, so the first of them is returned, with its sign. The test runner prints +# both zeros as 0, so the sign is checked by casting the result to a string. +query RRRR +SELECT array_max(array(CAST(0.0 AS DOUBLE), CAST('-0.0' AS DOUBLE))), + array_max(array(CAST('-0.0' AS DOUBLE), CAST(0.0 AS DOUBLE))), + array_min(array(CAST(0.0 AS DOUBLE), CAST('-0.0' AS DOUBLE))), + array_min(array(CAST('-0.0' AS DOUBLE), CAST(0.0 AS DOUBLE))); +---- +0 0 0 0 + +query TTTT +SELECT CAST(array_max(array(CAST(0.0 AS DOUBLE), CAST('-0.0' AS DOUBLE))) AS STRING), + CAST(array_max(array(CAST('-0.0' AS DOUBLE), CAST(0.0 AS DOUBLE))) AS STRING), + CAST(array_min(array(CAST(0.0 AS DOUBLE), CAST('-0.0' AS DOUBLE))) AS STRING), + CAST(array_min(array(CAST('-0.0' AS DOUBLE), CAST(0.0 AS DOUBLE))) AS STRING); +---- +0.0 -0.0 0.0 -0.0 + +query TT +SELECT CAST(array_max(array(CAST('-0.0' AS FLOAT), CAST(0.0 AS FLOAT))) AS STRING), + CAST(array_min(array(CAST(NULL AS FLOAT), CAST(0.0 AS FLOAT), CAST('-0.0' AS FLOAT))) AS STRING); +---- +-0.0 0.0 + +# Strings compare bytewise: uppercase sorts before lowercase, and 'é' after 'z' +query TT +SELECT array_max(array('a', 'b', 'B', '')), array_min(array('a', 'b', 'B', '')); +---- +b (empty) + +query T +SELECT array_max(array('é', 'z')); +---- +é + +# Binary compares unsigned bytes, and a prefix sorts first +query ?? +SELECT array_max(array(X'01', X'FF', X'0100')), array_min(array(X'01', X'FF', X'0100')); +---- +ff 01 + +query BB +SELECT array_max(array(true, false)), array_min(array(true, false, NULL)); +---- +true false + +query R +SELECT array_max(array(CAST(1.5 AS DECIMAL(3, 2)), CAST(2.25 AS DECIMAL(3, 2)))); +---- +2.25 + +query D +SELECT array_max(array(DATE'2024-01-01', DATE'2023-12-31')); +---- +2024-01-01 + +query D +SELECT array_min(array(DATE'2024-01-01', DATE'2023-12-31', NULL)); +---- +2023-12-31 + +query P +SELECT array_max(array(TIMESTAMP'2024-01-01 00:00:00', TIMESTAMP'2023-12-31 00:00:00')); +---- +2024-01-01T00:00:00 + +# Nested arrays compare element by element and then by length +query ?? +SELECT array_max(array(array(1, 2), array(1, 2, 3), array(2))), array_min(array(array(1, 2), array(1, 2, 3), array(2), array())); +---- +[2] [] + +# A NULL nested inside an array sorts before other values +query ?? +SELECT array_min(array(array(1, NULL), array(1, 2))), array_max(array(array(1, NULL), array(1, 2))); +---- +[1, NULL] [1, 2] + +query ? +SELECT array_max(array(array(CAST('NaN' AS DOUBLE)), array(CAST('Infinity' AS DOUBLE)))); +---- +[NaN] + +# Structs compare field by field, and a NULL field sorts first +query ? +SELECT array_max(array(named_struct('a', 1, 'b', 'x'), named_struct('a', 1, 'b', 'y'))); +---- +{a: 1, b: y} + +query ? +SELECT array_min(array(named_struct('a', 1, 'b', 'x'), named_struct('a', CAST(NULL AS INT), 'b', 'y'))); +---- +{a: NULL, b: y} + +query ? +SELECT array_max(array(named_struct('a', CAST('-0.0' AS DOUBLE)), named_struct('a', CAST(0.0 AS DOUBLE)))); +---- +{a: -0.0} + +# LargeList +query II +SELECT array_max(arrow_cast(array(3, NULL, 7), 'LargeList(Int32)')), array_min(arrow_cast(array(3, NULL, 7), 'LargeList(Int32)')); +---- +7 3 + +# Column input, with NULL, empty and all-NULL rows +query II +SELECT array_max(a), array_min(a) FROM (VALUES + (array(1, 5, 3)), + (array(CAST(NULL AS INT), -2)), + (CAST(array() AS ARRAY)), + (array(CAST(NULL AS INT))), + (CAST(NULL AS ARRAY)) +) AS t(a); +---- +5 1 +-2 -2 +NULL NULL +NULL NULL +NULL NULL + +query RRTT +SELECT array_max(a) AS max, array_min(a) AS min, + CAST(array_max(a) AS STRING) AS max_str, CAST(array_min(a) AS STRING) AS min_str +FROM (VALUES + (array(CAST(1.5 AS DOUBLE), CAST('NaN' AS DOUBLE))), + (array(CAST('-0.0' AS DOUBLE), CAST(0.0 AS DOUBLE))), + (array(CAST(0.0 AS DOUBLE), CAST('-0.0' AS DOUBLE))) +) AS t(a); +---- +NaN 1.5 NaN 1.5 +0 0 -0.0 -0.0 +0 0 0.0 0.0 + +# Maps cannot be ordered, at any depth +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_max` does not support ordering on type +SELECT array_max(array(map(1, 2))); + +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_min` does not support ordering on type +SELECT array_min(array(array(map(1, 2)))); + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `array_max` requires the "ARRAY" type +SELECT array_max(1); + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `array_min` requires the "ARRAY" type +SELECT array_min('abc'); + +query error +SELECT array_min(array(1, 2), array(3)); From e849be360302e8ad864c1f0bdfdff911ee365283 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 16 Nov 2026 06:48:03 -0700 Subject: [PATCH 2/8] perf(spark): typed fast path for primitive elements in array_max and array_min Primitive element types scan the value buffer directly instead of copying the values, using arrow's type-erased comparator and taking the winners. Adds a benchmark and a randomized test that the typed float path matches the generic path. --- crates/spark/Cargo.toml | 4 + crates/spark/benches/array_extrema.rs | 113 +++++++++ .../spark/src/function/array/array_extrema.rs | 214 +++++++++++++++++- 3 files changed, 327 insertions(+), 4 deletions(-) create mode 100644 crates/spark/benches/array_extrema.rs diff --git a/crates/spark/Cargo.toml b/crates/spark/Cargo.toml index 49f9d467..e003e415 100644 --- a/crates/spark/Cargo.toml +++ b/crates/spark/Cargo.toml @@ -107,3 +107,7 @@ name = "from_utc_timestamp" [[bench]] harness = false name = "stack" + +[[bench]] +harness = false +name = "array_extrema" diff --git a/crates/spark/benches/array_extrema.rs b/crates/spark/benches/array_extrema.rs new file mode 100644 index 00000000..6325ce3a --- /dev/null +++ b/crates/spark/benches/array_extrema.rs @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use arrow::array::{ArrayRef, ListArray, PrimitiveArray, StringArray}; +use arrow::buffer::OffsetBuffer; +use arrow::datatypes::{Field, Float64Type, Int64Type}; +use criterion::{Criterion, criterion_group, criterion_main}; +use datafusion_common::config::ConfigOptions; +use datafusion_expr::{ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF}; +use datafusion_spark::function::array::{array_max, array_min}; +use rand::rngs::StdRng; +use rand::{Rng, SeedableRng}; +use std::hint::black_box; +use std::sync::Arc; + +const ROWS: usize = 8192; +const MAX_LIST_LEN: usize = 32; +const NULL_DENSITY: f32 = 0.1; + +fn list_of(values: ArrayRef, lengths: Vec) -> ColumnarValue { + let field = Arc::new(Field::new_list_field(values.data_type().clone(), true)); + ColumnarValue::Array(Arc::new(ListArray::new( + field, + OffsetBuffer::from_lengths(lengths), + values, + None, + ))) +} + +fn random_lengths(rng: &mut StdRng) -> Vec { + (0..ROWS) + .map(|_| rng.random_range(0..MAX_LIST_LEN)) + .collect() +} + +fn int64_lists(rng: &mut StdRng) -> ColumnarValue { + let lengths = random_lengths(rng); + let values: PrimitiveArray = (0..lengths.iter().sum()) + .map(|_| (rng.random::() >= NULL_DENSITY).then(|| rng.random())) + .collect(); + list_of(Arc::new(values), lengths) +} + +fn float64_lists(rng: &mut StdRng) -> ColumnarValue { + let lengths = random_lengths(rng); + let values: PrimitiveArray = (0..lengths.iter().sum()) + .map(|_| { + (rng.random::() >= NULL_DENSITY).then(|| { + match rng.random_range(0..100) { + 0 => f64::NAN, + 1 => -0.0, + _ => rng.random::() * 2.0 - 1.0, + } + }) + }) + .collect(); + list_of(Arc::new(values), lengths) +} + +fn string_lists(rng: &mut StdRng) -> ColumnarValue { + let lengths = random_lengths(rng); + let values: StringArray = (0..lengths.iter().sum()) + .map(|_| { + (rng.random::() >= NULL_DENSITY) + .then(|| format!("value-{}", rng.random::())) + }) + .collect(); + list_of(Arc::new(values), lengths) +} + +fn bench(c: &mut Criterion, udf: &ScalarUDF, label: &str, input: &ColumnarValue) { + let arg_fields = vec![Arc::new(Field::new("a", input.data_type(), true))]; + let return_field = udf + .return_field_from_args(ReturnFieldArgs { + arg_fields: &arg_fields, + scalar_arguments: &[None], + }) + .unwrap(); + let config = Arc::new(ConfigOptions::default()); + c.bench_function(&format!("{} {label}", udf.name()), |b| { + b.iter(|| { + black_box( + udf.invoke_with_args(ScalarFunctionArgs { + args: vec![input.clone()], + arg_fields: arg_fields.clone(), + number_rows: ROWS, + return_field: Arc::clone(&return_field), + config_options: Arc::clone(&config), + }) + .unwrap(), + ) + }) + }); +} + +fn criterion_benchmark(c: &mut Criterion) { + let rng = &mut StdRng::seed_from_u64(42); + let inputs = [ + ("int64", int64_lists(rng)), + ("float64", float64_lists(rng)), + ("utf8", string_lists(rng)), + ]; + let max = array_max(); + let min = array_min(); + for (label, input) in &inputs { + bench(c, &max, label, input); + } + bench(c, &min, "float64", &inputs[1].1); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/crates/spark/src/function/array/array_extrema.rs b/crates/spark/src/function/array/array_extrema.rs index 01dec17c..686bdb05 100644 --- a/crates/spark/src/function/array/array_extrema.rs +++ b/crates/spark/src/function/array/array_extrema.rs @@ -7,11 +7,16 @@ use super::array_position::contains_map; use super::float_normalize::normalize_array; use arrow::array::{ - Array, ArrayRef, AsArray, GenericListArray, OffsetSizeTrait, UInt64Array, - make_comparator, new_null_array, + Array, ArrayRef, ArrowPrimitiveType, AsArray, GenericListArray, OffsetSizeTrait, + PrimitiveArray, UInt64Array, make_comparator, new_null_array, }; use arrow::compute::{SortOptions, take}; -use arrow::datatypes::{DataType, Field}; +use arrow::datatypes::{ + DataType, Date32Type, Date64Type, Decimal128Type, Decimal256Type, Field, Float16Type, + Float32Type, Float64Type, Int8Type, Int16Type, Int32Type, Int64Type, TimeUnit, + TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType, + TimestampSecondType, UInt8Type, UInt16Type, UInt32Type, UInt64Type, +}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, exec_err, plan_err}; use datafusion_expr::{ @@ -238,9 +243,69 @@ fn extrema( let base = offsets[0].as_usize(); let len = offsets[offsets.len() - 1].as_usize() - base; let values = list.values().slice(base, len); + + // Primitive elements take a typed path: no copy of the values, no type-erased comparator + // and no `take`. + macro_rules! primitive { + ($t:ty, $cmp:expr) => { + return Ok(Arc::new(primitive_extrema::<$t, O>( + list, + values.as_primitive::<$t>(), + base, + target, + $cmp, + ))) + }; + } + macro_rules! ordered { + ($t:ty) => { + primitive!($t, |a, b| a.cmp(&b)) + }; + } + match values.data_type() { + DataType::Int8 => ordered!(Int8Type), + DataType::Int16 => ordered!(Int16Type), + DataType::Int32 => ordered!(Int32Type), + DataType::Int64 => ordered!(Int64Type), + DataType::UInt8 => ordered!(UInt8Type), + DataType::UInt16 => ordered!(UInt16Type), + DataType::UInt32 => ordered!(UInt32Type), + DataType::UInt64 => ordered!(UInt64Type), + DataType::Decimal128(_, _) => ordered!(Decimal128Type), + DataType::Decimal256(_, _) => ordered!(Decimal256Type), + DataType::Date32 => ordered!(Date32Type), + DataType::Date64 => ordered!(Date64Type), + DataType::Timestamp(TimeUnit::Second, _) => ordered!(TimestampSecondType), + DataType::Timestamp(TimeUnit::Millisecond, _) => { + ordered!(TimestampMillisecondType) + } + DataType::Timestamp(TimeUnit::Microsecond, _) => { + ordered!(TimestampMicrosecondType) + } + DataType::Timestamp(TimeUnit::Nanosecond, _) => { + ordered!(TimestampNanosecondType) + } + DataType::Float16 => primitive!(Float16Type, spark_float_cmp), + DataType::Float32 => primitive!(Float32Type, spark_float_cmp), + DataType::Float64 => primitive!(Float64Type, spark_float_cmp), + _ => {} + } + + generic_extrema(list, &values, base, target) +} + +/// [`extrema`] for any orderable element type. `values` is the list's values sliced to +/// start at offset `base`. +fn generic_extrema( + list: &GenericListArray, + values: &ArrayRef, + base: usize, + target: Ordering, +) -> Result { + let offsets = list.offsets(); // Compare canonicalized floats, so that NaN is the greatest value and -0.0 equals 0.0 at // any depth, but return the original element. - let normalized = normalize_array(&values)?; + let normalized = normalize_array(values)?; // Spark orders NULLs nested inside arrays and structs before other values. let cmp = make_comparator( normalized.as_ref(), @@ -266,6 +331,55 @@ fn extrema( Ok(take(values.as_ref(), &indices, None)?) } +/// Spark's float ordering: NaN is equal to NaN and greater than every other value, and +/// -0.0 is equal to 0.0 (as under `partial_cmp`). +fn spark_float_cmp(a: F, b: F) -> Ordering { + // `partial_cmp` fails only when a NaN is involved; `x != x` holds only for NaN. + #[expect(clippy::eq_op)] + a.partial_cmp(&b).unwrap_or_else(|| (a != a).cmp(&(b != b))) +} + +/// [`extrema`] for primitive elements. `values` is the list's values sliced to start at +/// offset `base`. The winning value is copied unchanged, so the sign of a zero and the bits +/// of a NaN are kept, and the result keeps the element type's precision, scale or time zone. +fn primitive_extrema( + list: &GenericListArray, + values: &PrimitiveArray, + base: usize, + target: Ordering, + cmp: impl Fn(T::Native, T::Native) -> Ordering, +) -> PrimitiveArray +where + T: ArrowPrimitiveType, + O: OffsetSizeTrait, +{ + let raw = values.values(); + let result: PrimitiveArray = list + .offsets() + .windows(2) + .enumerate() + .map(|(row, w)| { + if list.is_null(row) { + return None; + } + let (start, end) = (w[0].as_usize() - base, w[1].as_usize() - base); + let mut best: Option = None; + for i in start..end { + if values.is_null(i) { + continue; + } + let v = raw[i]; + best = match best { + Some(b) if cmp(v, b) != target => Some(b), + _ => Some(v), + }; + } + best + }) + .collect(); + result.with_data_type(values.data_type().clone()) +} + #[cfg(test)] mod tests { use super::*; @@ -571,4 +685,96 @@ mod tests { vec![DataType::new_list(DataType::Int32, true)] ); } + + /// Runs the typed path (through the UDF) and the generic path on the same input and + /// checks that they pick bit-identical values. + fn assert_paths_agree(input: &ListArray) { + let values = { + let offsets = input.offsets(); + let base = offsets[0] as usize; + let len = offsets[offsets.len() - 1] as usize - base; + (base, input.values().slice(base, len)) + }; + for (udf, target) in [ + ( + &SparkArrayMax::new() as &dyn ScalarUDFImpl, + Ordering::Greater, + ), + (&SparkArrayMin::new(), Ordering::Less), + ] { + let fast = run_array(udf, input); + let generic = generic_extrema(input, &values.1, values.0, target).unwrap(); + assert_eq!(f64_bits(&fast), f64_bits(&generic), "{}", udf.name()); + } + } + + #[test] + fn typed_float_path_matches_generic_path() { + use rand::rngs::StdRng; + use rand::{Rng, SeedableRng}; + let specials = [ + f64::NAN, + -f64::NAN, + f64::from_bits(0x7ff8_0000_0000_0001), + 0.0, + -0.0, + f64::INFINITY, + f64::NEG_INFINITY, + f64::MIN_POSITIVE, + 1.0, + -1.0, + ]; + let rng = &mut StdRng::seed_from_u64(7); + let rows: Vec>>> = (0..2000) + .map(|_| { + (rng.random_range(0..10) != 0).then(|| { + (0..rng.random_range(0..8)) + .map(|_| { + (rng.random_range(0..5) != 0) + .then(|| specials[rng.random_range(0..specials.len())]) + }) + .collect() + }) + }) + .collect(); + let input = ListArray::from_iter_primitive::(rows); + assert_paths_agree(&input); + assert_paths_agree(&input.slice(13, 1500)); + } + + #[test] + fn typed_path_keeps_element_type() { + use arrow::array::{Decimal128Array, TimestampMicrosecondArray}; + let decimal_type = DataType::Decimal128(20, 3); + let decimals: ArrayRef = Arc::new( + Decimal128Array::from(vec![Some(1_500), None, Some(-2_000), Some(1_499)]) + .with_data_type(decimal_type.clone()), + ); + let timestamp_type = + DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())); + let timestamps: ArrayRef = Arc::new( + TimestampMicrosecondArray::from(vec![Some(5), Some(9), None, Some(1)]) + .with_data_type(timestamp_type.clone()), + ); + for values in [decimals, timestamps] { + let data_type = values.data_type().clone(); + let input = ListArray::new( + Arc::new(Field::new_list_field(data_type.clone(), true)), + OffsetBuffer::from_lengths([4]), + values, + None, + ); + let (max, min) = (max(&input), min(&input)); + assert_eq!(max.data_type(), &data_type); + assert_eq!(min.data_type(), &data_type); + if data_type == decimal_type { + assert_eq!(max.as_primitive::().value(0), 1_500); + assert_eq!(min.as_primitive::().value(0), -2_000); + } else { + assert_eq!(data_type, timestamp_type); + assert_eq!(max.as_primitive::().value(0), 9); + assert_eq!(min.as_primitive::().value(0), 1); + } + } + } } From 01d8874a489db5b48b98f422b0c002c6825e9054 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 10:17:23 -0700 Subject: [PATCH 3/8] refactor(spark): share Spark's common-type rules between functions Move tightest_common_type from array_position into function/type_coercion, with an ANSI flag, a findTypeForComplex that widens map keys and values, Spark's isOrderable (which also rejects INTERVAL), and Spark type names for error messages. Move float_normalize up to function/ so functions outside array can use it. array_position now reports a map with different key or value types as INVALID_ORDERING_TYPE, rejects INTERVAL elements, and names Spark types and the SQLSTATE in its errors, as Spark does. --- .../src/function/array/array_contains.rs | 2 +- .../src/function/array/array_position.rs | 178 ++-------- crates/spark/src/function/array/mod.rs | 1 - .../function/{array => }/float_normalize.rs | 8 +- crates/spark/src/function/mod.rs | 2 + crates/spark/src/function/type_coercion.rs | 327 ++++++++++++++++++ .../test_files/array/array_position.slt | 15 +- 7 files changed, 382 insertions(+), 151 deletions(-) rename crates/spark/src/function/{array => }/float_normalize.rs (97%) create mode 100644 crates/spark/src/function/type_coercion.rs diff --git a/crates/spark/src/function/array/array_contains.rs b/crates/spark/src/function/array/array_contains.rs index 3802667e..029e6c09 100644 --- a/crates/spark/src/function/array/array_contains.rs +++ b/crates/spark/src/function/array/array_contains.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use super::float_normalize::normalize_floats; +use crate::function::float_normalize::normalize_floats; use arrow::array::{ Array, AsArray, BooleanArray, BooleanBufferBuilder, GenericListArray, OffsetSizeTrait, }; diff --git a/crates/spark/src/function/array/array_position.rs b/crates/spark/src/function/array/array_position.rs index 4ca52da9..d1a64701 100644 --- a/crates/spark/src/function/array/array_position.rs +++ b/crates/spark/src/function/array/array_position.rs @@ -2,13 +2,16 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use super::float_normalize::normalize_array; +use crate::function::float_normalize::normalize_array; +use crate::function::type_coercion::{ + is_orderable, spark_sql_type, tightest_common_type, +}; use arrow::array::{ Array, ArrayRef, AsArray, Int64Array, OffsetSizeTrait, make_comparator, }; use arrow::buffer::{NullBuffer, ScalarBuffer}; use arrow::compute::SortOptions; -use arrow::datatypes::{DataType, Field, Fields, TimeUnit}; +use arrow::datatypes::{DataType, Field}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, exec_err, plan_err}; use datafusion_expr::{ @@ -36,7 +39,7 @@ use std::sync::Arc; a date widens to a timestamp. Other combinations are an error, for example a string \ and a number, two different decimals, or structs whose field names or field types \ differ, as in Spark with ANSI mode off (with ANSI mode on, Spark widens struct field \ - types). A NULL-typed array and map elements are also an error.", + types). A NULL-typed array, and map or INTERVAL elements at any depth, are also an error.", syntax_example = "array_position(array, element)", sql_example = r#"```sql > SELECT array_position(array(3, 2, 1), 1); @@ -164,24 +167,31 @@ fn coerce_array_and_element( other => { return plan_err!( "[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] The first parameter of `{name}` \ - requires the \"ARRAY\" type, however it has the type {other}" + requires the \"ARRAY\" type, however it has the type \"{}\". SQLSTATE: 42K09", + spark_sql_type(other) ); } }; - let Some(common) = tightest_common_type(item.data_type(), element) else { + // Spark's `ArrayPosition` calls the non-ANSI `TypeCoercion.findTightestCommonType`. + let common = tightest_common_type(item.data_type(), element, false) + .filter(|_| struct_field_types_match(item.data_type(), element)); + let Some(common) = common else { return plan_err!( "[DATATYPE_MISMATCH.ARRAY_FUNCTION_DIFF_TYPES] Input to `{name}` should have \ been \"ARRAY\" followed by a value with same element type, but it's \ - [{array}, {element}]" + [\"{}\", \"{}\"]. SQLSTATE: 42K09", + spark_sql_type(array), + spark_sql_type(element) ); }; if common.is_null() { return null_type_err(name); } - if contains_map(&common) { + if !is_orderable(&common) { return plan_err!( "[DATATYPE_MISMATCH.INVALID_ORDERING_TYPE] The `{name}` does not support \ - ordering on type {common}" + ordering on type \"{}\". SQLSTATE: 42K09", + spark_sql_type(&common) ); } let item = Arc::new(Field::new(item.name(), common.clone(), true)); @@ -196,82 +206,30 @@ fn coerce_array_and_element( fn null_type_err(name: &str) -> Result { plan_err!( "[DATATYPE_MISMATCH.NULL_TYPE] Null typed values cannot be used as arguments of \ - `{name}`" + `{name}`. SQLSTATE: 42K09" ) } -/// Spark's `TypeCoercion.findTightestCommonType`, on the arrow types that Spark's types -/// map to. -fn tightest_common_type(left: &DataType, right: &DataType) -> Option { +/// With ANSI mode off, Spark cannot implicitly cast between structs whose field types +/// differ, so `array_position` rejects them although their tightest common type exists. +/// With ANSI mode on, Spark widens the field types. +fn struct_field_types_match(left: &DataType, right: &DataType) -> bool { use DataType::*; - if left == right { - return Some(left.clone()); - } - if let (Some(l), Some(r)) = (numeric_precedence(left), numeric_precedence(right)) { - return Some(if l >= r { left.clone() } else { right.clone() }); - } match (left, right) { - (Null, other) | (other, Null) => Some(other.clone()), - // Spark has a single string type and a single binary type. - (l, r) if is_string(l) && is_string(r) => Some(l.clone()), - (l, r) if is_binary(l) && is_binary(r) => Some(l.clone()), - // The decimal must be at least as wide as the integral type's DECIMAL(n, 0). - (int, Decimal128(p, s)) | (Decimal128(p, s), int) => { - let digits = integral_digits(int)?; - (*p as i16 - *s as i16 >= digits).then_some(Decimal128(*p, *s)) - } - (Date32, ts @ Timestamp(_, _)) | (ts @ Timestamp(_, _), Date32) => { - Some(ts.clone()) - } - // TIMESTAMP_NTZ and TIMESTAMP widen to TIMESTAMP. - (Timestamp(lu, ltz), Timestamp(ru, rtz)) => Some(Timestamp( - finer_unit(*lu, *ru), - ltz.clone().or_else(|| rtz.clone()), - )), ( - List(lf) | LargeList(lf) | FixedSizeList(lf, _), - List(rf) | LargeList(rf) | FixedSizeList(rf, _), - ) => { - let item = tightest_common_type(lf.data_type(), rf.data_type())?; - let field = Arc::new(Field::new(lf.name(), item, true)); - Some( - if matches!(left, LargeList(_)) || matches!(right, LargeList(_)) { - LargeList(field) - } else { - List(field) - }, - ) - } - // The fields must have the same names and types, ignoring nullability, in the same - // order. This follows Spark with ANSI mode off; with ANSI mode on, Spark also widens - // the field types. Spark compares the names ignoring case, but DataFusion cannot cast - // between structs whose field names differ only in case, so they must match exactly. - (Struct(lf), Struct(rf)) if lf.len() == rf.len() => { - let fields = lf + List(l) | LargeList(l) | FixedSizeList(l, _), + List(r) | LargeList(r) | FixedSizeList(r, _), + ) => struct_field_types_match(l.data_type(), r.data_type()), + // The entries are a struct of the key and the value, which are checked on their own. + (Map(l, _), Map(r, _)) => match (l.data_type(), r.data_type()) { + (Struct(l), Struct(r)) => l .iter() - .zip(rf.iter()) - .map(|(a, b)| { - if a.name() != b.name() || !same_type(a.data_type(), b.data_type()) { - return None; - } - let data_type = tightest_common_type(a.data_type(), b.data_type())?; - Some(Field::new( - a.name(), - data_type, - a.is_nullable() || b.is_nullable(), - )) - }) - .collect::>>()?; - Some(Struct(Fields::from(fields))) - } - (Map(lf, sorted), Map(rf, _)) => { - let entries = tightest_common_type(lf.data_type(), rf.data_type())?; - Some(Map( - Arc::new(Field::new(lf.name(), entries, false)), - *sorted, - )) - } - _ => None, + .zip(r.iter()) + .all(|(a, b)| struct_field_types_match(a.data_type(), b.data_type())), + _ => true, + }, + (Struct(_), Struct(_)) => same_type(left, right), + _ => true, } } @@ -296,72 +254,6 @@ fn same_type(left: &DataType, right: &DataType) -> bool { } } -fn is_string(data_type: &DataType) -> bool { - matches!( - data_type, - DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View - ) -} - -fn is_binary(data_type: &DataType) -> bool { - matches!( - data_type, - DataType::Binary | DataType::LargeBinary | DataType::BinaryView - ) -} - -/// The precision of Spark's `DecimalType.forType` for an integral type. -fn integral_digits(data_type: &DataType) -> Option { - match data_type { - DataType::Int8 => Some(3), - DataType::Int16 => Some(5), - DataType::Int32 => Some(10), - DataType::Int64 => Some(20), - _ => None, - } -} - -/// Spark's `numericPrecedence`, which leaves out decimals. -fn numeric_precedence(data_type: &DataType) -> Option { - match data_type { - DataType::Int8 => Some(0), - DataType::Int16 => Some(1), - DataType::Int32 => Some(2), - DataType::Int64 => Some(3), - DataType::Float32 => Some(4), - DataType::Float64 => Some(5), - _ => None, - } -} - -fn finer_unit(left: TimeUnit, right: TimeUnit) -> TimeUnit { - let rank = |unit: TimeUnit| match unit { - TimeUnit::Second => 0, - TimeUnit::Millisecond => 1, - TimeUnit::Microsecond => 2, - TimeUnit::Nanosecond => 3, - }; - if rank(left) >= rank(right) { - left - } else { - right - } -} - -/// Spark cannot order maps, so it rejects them at any depth of the element type. -fn contains_map(data_type: &DataType) -> bool { - match data_type { - DataType::Map(_, _) => true, - DataType::List(field) - | DataType::LargeList(field) - | DataType::FixedSizeList(field, _) => contains_map(field.data_type()), - DataType::Struct(fields) => { - fields.iter().any(|field| contains_map(field.data_type())) - } - _ => false, - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/spark/src/function/array/mod.rs b/crates/spark/src/function/array/mod.rs index 7ff0f2aa..36dcccc9 100644 --- a/crates/spark/src/function/array/mod.rs +++ b/crates/spark/src/function/array/mod.rs @@ -6,7 +6,6 @@ pub mod array_contains; pub mod array_position; -mod float_normalize; pub mod repeat; pub mod shuffle; pub mod slice; diff --git a/crates/spark/src/function/array/float_normalize.rs b/crates/spark/src/function/float_normalize.rs similarity index 97% rename from crates/spark/src/function/array/float_normalize.rs rename to crates/spark/src/function/float_normalize.rs index 07bbb28a..1a76b60d 100644 --- a/crates/spark/src/function/array/float_normalize.rs +++ b/crates/spark/src/function/float_normalize.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -//! Float normalization shared by the array functions that compare elements. +//! Float normalization shared by the functions that compare values. //! //! Spark compares floating point values with `-0.0 == 0.0` and `NaN == NaN`, while arrow //! compares them by their bits or by the IEEE total order. Rewriting `-0.0` to `0.0` and @@ -19,7 +19,7 @@ use datafusion_expr::ColumnarValue; use std::sync::Arc; /// Normalizes the floats in a [`ColumnarValue`], see [`normalize_array`]. -pub(super) fn normalize_floats(value: &ColumnarValue) -> Result { +pub(crate) fn normalize_floats(value: &ColumnarValue) -> Result { if !contains_float(&value.data_type()) { return Ok(value.clone()); } @@ -35,7 +35,7 @@ pub(super) fn normalize_floats(value: &ColumnarValue) -> Result { } /// Returns whether values of this type are, or hold at any depth, floating point numbers. -pub(super) fn contains_float(data_type: &DataType) -> bool { +pub(crate) fn contains_float(data_type: &DataType) -> bool { match data_type { DataType::Float32 | DataType::Float64 => true, DataType::List(field) @@ -51,7 +51,7 @@ pub(super) fn contains_float(data_type: &DataType) -> bool { /// Rewrites `-0.0` to `0.0` and every NaN to the canonical NaN at any depth of list, large /// list, fixed-size list, struct and map. Children without floats are reused as they are. -pub(super) fn normalize_array(array: &ArrayRef) -> Result { +pub(crate) fn normalize_array(array: &ArrayRef) -> Result { if !contains_float(array.data_type()) { return Ok(Arc::clone(array)); } diff --git a/crates/spark/src/function/mod.rs b/crates/spark/src/function/mod.rs index 483f9b64..4098d8e0 100644 --- a/crates/spark/src/function/mod.rs +++ b/crates/spark/src/function/mod.rs @@ -12,6 +12,7 @@ pub mod conversion; pub mod csv; pub mod datetime; pub mod error_utils; +pub(crate) mod float_normalize; pub mod functions_nested_utils; pub mod generator; pub mod hash; @@ -25,6 +26,7 @@ pub mod predicate; pub mod string; pub mod r#struct; pub mod table; +pub(crate) mod type_coercion; pub mod url; pub mod utils; pub mod window; diff --git a/crates/spark/src/function/type_coercion.rs b/crates/spark/src/function/type_coercion.rs new file mode 100644 index 00000000..035acbf6 --- /dev/null +++ b/crates/spark/src/function/type_coercion.rs @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Spark's rules for finding a common type of several arguments, from its `TypeCoercion` +//! and `AnsiTypeCoercion`, on the Arrow types that Spark's types map to. Also the checks +//! and type names that the errors for those arguments use. +//! +//! These are for functions that need one type for several arguments, such as +//! `array_position`. They are not Spark's implicit casts to a function's declared input +//! types, which the functions do through their [`datafusion_expr::Signature`]. + +use std::sync::Arc; + +use arrow::datatypes::{DataType, Field, Fields, IntervalUnit, TimeUnit}; + +/// Spark's `findTightestCommonType`. `ansi_mode` picks between `AnsiTypeCoercion` and +/// `TypeCoercion`, which differ only in that ANSI widens an integer and a FLOAT to DOUBLE. +/// Arrays, maps and structs are widened element by element with this function. +pub(crate) fn tightest_common_type( + left: &DataType, + right: &DataType, + ansi_mode: bool, +) -> Option { + use DataType::*; + if left == right { + return Some(left.clone()); + } + if let (Some(l), Some(r)) = (numeric_precedence(left), numeric_precedence(right)) { + let wider = if l >= r { left } else { right }; + // With ANSI mode on, an integer and a FLOAT widen to DOUBLE, which holds every INT + // exactly. + return Some(if ansi_mode && *wider == Float32 { + Float64 + } else { + wider.clone() + }); + } + match (left, right) { + (Null, other) | (other, Null) => Some(other.clone()), + // Spark has one string type and one binary type. + (l, r) if is_string(l) && is_string(r) => Some(wider_physical_type(l, r)), + (l, r) if is_binary(l) && is_binary(r) => Some(wider_physical_type(l, r)), + // The decimal must hold every value of the integral type. + (int, Decimal128(p, s)) | (Decimal128(p, s), int) => { + let digits = integral_digits(int)?; + (*p as i16 - *s as i16 >= digits).then_some(Decimal128(*p, *s)) + } + (Date32, ts @ Timestamp(_, _)) | (ts @ Timestamp(_, _), Date32) => { + Some(ts.clone()) + } + // TIMESTAMP_NTZ and TIMESTAMP widen to TIMESTAMP. + (Timestamp(lu, ltz), Timestamp(ru, rtz)) => Some(Timestamp( + finer_unit(*lu, *ru), + ltz.clone().or_else(|| rtz.clone()), + )), + _ => complex_type(left, right, &|l, r| tightest_common_type(l, r, ansi_mode)), + } +} + +/// Spark's `findTypeForComplex`: widens the elements of arrays, the keys and values of maps, +/// and the fields of structs with `find_type`. A nested field is nullable if it is on either +/// side. Struct fields must have the same names in the same order. Spark compares them +/// ignoring case, but DataFusion cannot cast between structs whose field names differ only +/// in case, so this requires an exact match. +pub(crate) fn complex_type( + left: &DataType, + right: &DataType, + find_type: &dyn Fn(&DataType, &DataType) -> Option, +) -> Option { + use DataType::*; + match (left, right) { + ( + List(lf) | LargeList(lf) | FixedSizeList(lf, _), + List(rf) | LargeList(rf) | FixedSizeList(rf, _), + ) => { + let item = find_type(lf.data_type(), rf.data_type())?; + let field = Arc::new(Field::new( + lf.name(), + item, + lf.is_nullable() || rf.is_nullable(), + )); + Some(match (left, right) { + (FixedSizeList(_, ln), FixedSizeList(_, rn)) if ln == rn => { + FixedSizeList(field, *ln) + } + (LargeList(_), _) | (_, LargeList(_)) => LargeList(field), + _ => List(field), + }) + } + (Struct(lf), Struct(rf)) if lf.len() == rf.len() => { + let fields = lf + .iter() + .zip(rf.iter()) + .map(|(a, b)| { + if a.name() != b.name() { + return None; + } + let data_type = find_type(a.data_type(), b.data_type())?; + Some(Field::new( + a.name(), + data_type, + a.is_nullable() || b.is_nullable(), + )) + }) + .collect::>>()?; + Some(Struct(Fields::from(fields))) + } + // The entries are a struct of the key and the value, so the struct arm widens both. + (Map(lf, sorted), Map(rf, _)) => { + let entries = complex_type(lf.data_type(), rf.data_type(), find_type)?; + Some(Map( + Arc::new(Field::new(lf.name(), entries, false)), + *sorted, + )) + } + _ => None, + } +} + +/// Spark's `OrderUtils.isOrderable`. Spark cannot order maps or `CalendarIntervalType`, which +/// is `Interval(MonthDayNano)` in Arrow, at any depth. +pub(crate) fn is_orderable(data_type: &DataType) -> bool { + match data_type { + DataType::Map(_, _) + | DataType::Union(_, _) + | DataType::Interval(IntervalUnit::MonthDayNano) => false, + DataType::List(field) + | DataType::LargeList(field) + | DataType::ListView(field) + | DataType::LargeListView(field) + | DataType::FixedSizeList(field, _) => is_orderable(field.data_type()), + DataType::Struct(fields) => fields.iter().all(|f| is_orderable(f.data_type())), + DataType::Dictionary(_, value) => is_orderable(value), + _ => true, + } +} + +/// The name Spark's `toSQLType` gives the Spark type that an Arrow type maps to, for error +/// messages. Arrow types without a Spark counterpart use their Arrow name. +pub(crate) fn spark_sql_type(data_type: &DataType) -> String { + use DataType::*; + match data_type { + Null => "VOID".to_string(), + Boolean => "BOOLEAN".to_string(), + Int8 => "TINYINT".to_string(), + Int16 => "SMALLINT".to_string(), + Int32 => "INT".to_string(), + Int64 => "BIGINT".to_string(), + Float32 => "FLOAT".to_string(), + Float64 => "DOUBLE".to_string(), + Decimal128(p, s) => format!("DECIMAL({p},{s})"), + Utf8 | LargeUtf8 | Utf8View => "STRING".to_string(), + Binary | LargeBinary | BinaryView => "BINARY".to_string(), + Date32 => "DATE".to_string(), + Timestamp(_, Some(_)) => "TIMESTAMP".to_string(), + Timestamp(_, None) => "TIMESTAMP_NTZ".to_string(), + Interval(IntervalUnit::MonthDayNano) => "INTERVAL".to_string(), + List(field) | LargeList(field) | FixedSizeList(field, _) => { + format!("ARRAY<{}>", spark_sql_type(field.data_type())) + } + Struct(fields) => format!( + "STRUCT<{}>", + fields + .iter() + .map(|f| format!( + "{}: {}{}", + f.name(), + spark_sql_type(f.data_type()), + if f.is_nullable() { "" } else { " NOT NULL" } + )) + .collect::>() + .join(", ") + ), + Map(entries, _) => match entries.data_type() { + Struct(kv) if kv.len() == 2 => format!( + "MAP<{}, {}>", + spark_sql_type(kv[0].data_type()), + spark_sql_type(kv[1].data_type()) + ), + other => format!("MAP<{other}>"), + }, + other => other.to_string(), + } +} + +fn is_string(data_type: &DataType) -> bool { + matches!( + data_type, + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View + ) +} + +fn is_binary(data_type: &DataType) -> bool { + matches!( + data_type, + DataType::Binary | DataType::LargeBinary | DataType::BinaryView + ) +} + +/// Of two physical layouts of the same Spark string or binary type, the one that can hold the +/// values of both. +fn wider_physical_type(left: &DataType, right: &DataType) -> DataType { + use DataType::*; + match (left, right) { + (LargeUtf8, _) | (_, LargeUtf8) => LargeUtf8, + (LargeBinary, _) | (_, LargeBinary) => LargeBinary, + (Utf8View, _) | (_, Utf8View) => Utf8View, + (BinaryView, _) | (_, BinaryView) => BinaryView, + _ => left.clone(), + } +} + +/// The precision of Spark's `DecimalType.forType` for an integral type. +fn integral_digits(data_type: &DataType) -> Option { + match data_type { + DataType::Int8 => Some(3), + DataType::Int16 => Some(5), + DataType::Int32 => Some(10), + DataType::Int64 => Some(20), + _ => None, + } +} + +/// Spark's `numericPrecedence`, which leaves out decimals. +fn numeric_precedence(data_type: &DataType) -> Option { + match data_type { + DataType::Int8 => Some(0), + DataType::Int16 => Some(1), + DataType::Int32 => Some(2), + DataType::Int64 => Some(3), + DataType::Float32 => Some(4), + DataType::Float64 => Some(5), + _ => None, + } +} + +fn finer_unit(left: TimeUnit, right: TimeUnit) -> TimeUnit { + let rank = |unit: TimeUnit| match unit { + TimeUnit::Second => 0, + TimeUnit::Millisecond => 1, + TimeUnit::Microsecond => 2, + TimeUnit::Nanosecond => 3, + }; + if rank(left) >= rank(right) { + left + } else { + right + } +} + +#[cfg(test)] +mod tests { + use super::*; + use DataType::*; + + fn list(item: DataType) -> DataType { + List(Arc::new(Field::new("item", item, true))) + } + + fn map(key: DataType, value: DataType) -> DataType { + let entries = Struct(Fields::from(vec![ + Field::new("key", key, false), + Field::new("value", value, true), + ])); + Map(Arc::new(Field::new("entries", entries, false)), false) + } + + #[test] + fn numbers_widen_and_ansi_widens_int_and_float_to_double() { + assert_eq!(tightest_common_type(&Int32, &Int64, false), Some(Int64)); + assert_eq!(tightest_common_type(&Int32, &Float32, false), Some(Float32)); + assert_eq!(tightest_common_type(&Int32, &Float32, true), Some(Float64)); + assert_eq!( + tightest_common_type(&Int32, &Decimal128(10, 0), false), + Some(Decimal128(10, 0)) + ); + assert_eq!( + tightest_common_type(&Int32, &Decimal128(10, 1), false), + None + ); + assert_eq!(tightest_common_type(&Int32, &Utf8, false), None); + } + + #[test] + fn string_layouts_widen_to_the_one_that_holds_both() { + assert_eq!( + tightest_common_type(&Utf8, &LargeUtf8, false), + Some(LargeUtf8) + ); + assert_eq!( + tightest_common_type(&LargeUtf8, &Utf8, false), + Some(LargeUtf8) + ); + assert_eq!( + tightest_common_type(&Utf8, &Utf8View, false), + Some(Utf8View) + ); + } + + #[test] + fn complex_types_widen_element_by_element() { + assert_eq!( + tightest_common_type(&list(Int32), &list(Int64), false), + Some(list(Int64)) + ); + assert_eq!( + tightest_common_type(&map(Int32, Int32), &map(Int64, Int16), false), + Some(map(Int64, Int32)) + ); + let a = Struct(Fields::from(vec![Field::new("a", Int32, false)])); + let b = Struct(Fields::from(vec![Field::new("b", Int32, false)])); + let wide = Struct(Fields::from(vec![Field::new("a", Int64, true)])); + assert_eq!(tightest_common_type(&a, &wide, false), Some(wide)); + assert_eq!(tightest_common_type(&a, &b, false), None); + } + + #[test] + fn maps_and_calendar_intervals_are_not_orderable() { + let interval = Interval(IntervalUnit::MonthDayNano); + assert!(!is_orderable(&map(Int32, Int32))); + assert!(!is_orderable(&interval)); + assert!(!is_orderable(&list(interval.clone()))); + assert!(is_orderable(&list(Int32))); + assert_eq!(spark_sql_type(&list(interval)), "ARRAY"); + } +} diff --git a/crates/sqllogictest/test_files/array/array_position.slt b/crates/sqllogictest/test_files/array/array_position.slt index cf9ef3e8..25ebfd45 100644 --- a/crates/sqllogictest/test_files/array/array_position.slt +++ b/crates/sqllogictest/test_files/array/array_position.slt @@ -274,8 +274,8 @@ SELECT array_position(array(CAST(1.0 AS DOUBLE)), CAST(1 AS DECIMAL(5,1))); query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] SELECT array_position(array(1), '1'); -query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] -SELECT array_position(array('1'), 1); +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] Input to `array_position` should have been "ARRAY" followed by a value with same element type, but it's \["ARRAY", "INT"\]\. SQLSTATE: 42K09 +SELECT array_position(array('1'), CAST(1 AS INT)); query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] SELECT array_position(array(1), CAST(NULL AS STRING)); @@ -332,6 +332,17 @@ SELECT array_position(array(named_struct('m', map(1, 2))), named_struct('m', map query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] SELECT array_position(array(array(map(1, 2))), array(map(1, 2))); +# Map keys and values are widened, and the common map type is then rejected +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_position` does not support ordering on type "MAP"\. SQLSTATE: 42K09 +SELECT array_position(array(map(CAST(1 AS INT), CAST(1 AS INT))), map(1L, 1L)); + +# Spark cannot order intervals +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_position` does not support ordering on type "INTERVAL"\. SQLSTATE: 42K09 +SELECT array_position(array(make_interval(1)), make_interval(2)); + +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_position` does not support ordering on type "ARRAY"\. SQLSTATE: 42K09 +SELECT array_position(array(array(make_interval(1))), array(make_interval(2))); + # Signed zero: -0.0 equals 0.0 query I SELECT array_position(array(CAST('-0.0' AS DOUBLE), CAST(1.0 AS DOUBLE)), CAST(0.0 AS DOUBLE)); From 3e9008cff980d3f1546656cbd8a71737c6bc22ac Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 10:19:52 -0700 Subject: [PATCH 4/8] refactor(spark): match map keys and values by position in complex_type Spark's MapType has no field names, so map entries named differently still have a common type. Keep contains_float private. --- crates/spark/src/function/float_normalize.rs | 2 +- crates/spark/src/function/type_coercion.rs | 29 ++++++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/crates/spark/src/function/float_normalize.rs b/crates/spark/src/function/float_normalize.rs index 1a76b60d..e166b362 100644 --- a/crates/spark/src/function/float_normalize.rs +++ b/crates/spark/src/function/float_normalize.rs @@ -35,7 +35,7 @@ pub(crate) fn normalize_floats(value: &ColumnarValue) -> Result { } /// Returns whether values of this type are, or hold at any depth, floating point numbers. -pub(crate) fn contains_float(data_type: &DataType) -> bool { +fn contains_float(data_type: &DataType) -> bool { match data_type { DataType::Float32 | DataType::Float64 => true, DataType::List(field) diff --git a/crates/spark/src/function/type_coercion.rs b/crates/spark/src/function/type_coercion.rs index 035acbf6..d4a8bbb5 100644 --- a/crates/spark/src/function/type_coercion.rs +++ b/crates/spark/src/function/type_coercion.rs @@ -106,9 +106,24 @@ pub(crate) fn complex_type( .collect::>>()?; Some(Struct(Fields::from(fields))) } - // The entries are a struct of the key and the value, so the struct arm widens both. + // The entries are a struct of the key and the value. Spark's `MapType` has no field + // names, so the key and the value are matched by position. (Map(lf, sorted), Map(rf, _)) => { - let entries = complex_type(lf.data_type(), rf.data_type(), find_type)?; + let (Struct(lkv), Struct(rkv)) = (lf.data_type(), rf.data_type()) else { + return None; + }; + let [lk, lv] = lkv.iter().collect::>()[..] else { + return None; + }; + let [rk, rv] = rkv.iter().collect::>()[..] else { + return None; + }; + let key = find_type(lk.data_type(), rk.data_type())?; + let value = find_type(lv.data_type(), rv.data_type())?; + let entries = Struct(Fields::from(vec![ + Field::new(lk.name(), key, false), + Field::new(lv.name(), value, lv.is_nullable() || rv.is_nullable()), + ])); Some(Map( Arc::new(Field::new(lf.name(), entries, false)), *sorted, @@ -310,6 +325,16 @@ mod tests { ); let a = Struct(Fields::from(vec![Field::new("a", Int32, false)])); let b = Struct(Fields::from(vec![Field::new("b", Int32, false)])); + // Map entries named differently are matched by position. + let renamed = Struct(Fields::from(vec![ + Field::new("keys", Int64, false), + Field::new("values", Int16, true), + ])); + let renamed = Map(Arc::new(Field::new("entries", renamed, false)), false); + assert_eq!( + tightest_common_type(&map(Int32, Int32), &renamed, false), + Some(map(Int64, Int32)) + ); let wide = Struct(Fields::from(vec![Field::new("a", Int64, true)])); assert_eq!(tightest_common_type(&a, &wide, false), Some(wide)); assert_eq!(tightest_common_type(&a, &b, false), None); From f5571e9d044f852c04e7c07035c16d30e19e33dd Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 10:22:21 -0700 Subject: [PATCH 5/8] refactor(spark): simplify the shared common-type rules Use DataType::is_string and TimeUnit's ordering, match map entries without collecting, keep complex_type private, and move the non-ANSI struct field check and same_type into type_coercion beside the rules they qualify. --- .../src/function/array/array_position.rs | 46 +------- crates/spark/src/function/type_coercion.rs | 106 ++++++++++++------ 2 files changed, 75 insertions(+), 77 deletions(-) diff --git a/crates/spark/src/function/array/array_position.rs b/crates/spark/src/function/array/array_position.rs index d1a64701..445a8b48 100644 --- a/crates/spark/src/function/array/array_position.rs +++ b/crates/spark/src/function/array/array_position.rs @@ -4,7 +4,7 @@ use crate::function::float_normalize::normalize_array; use crate::function::type_coercion::{ - is_orderable, spark_sql_type, tightest_common_type, + is_orderable, spark_sql_type, struct_field_types_match, tightest_common_type, }; use arrow::array::{ Array, ArrayRef, AsArray, Int64Array, OffsetSizeTrait, make_comparator, @@ -210,50 +210,6 @@ fn null_type_err(name: &str) -> Result { ) } -/// With ANSI mode off, Spark cannot implicitly cast between structs whose field types -/// differ, so `array_position` rejects them although their tightest common type exists. -/// With ANSI mode on, Spark widens the field types. -fn struct_field_types_match(left: &DataType, right: &DataType) -> bool { - use DataType::*; - match (left, right) { - ( - List(l) | LargeList(l) | FixedSizeList(l, _), - List(r) | LargeList(r) | FixedSizeList(r, _), - ) => struct_field_types_match(l.data_type(), r.data_type()), - // The entries are a struct of the key and the value, which are checked on their own. - (Map(l, _), Map(r, _)) => match (l.data_type(), r.data_type()) { - (Struct(l), Struct(r)) => l - .iter() - .zip(r.iter()) - .all(|(a, b)| struct_field_types_match(a.data_type(), b.data_type())), - _ => true, - }, - (Struct(_), Struct(_)) => same_type(left, right), - _ => true, - } -} - -/// Whether two types are equal when the nullability of nested fields is ignored, like -/// Spark's `DataTypeUtils.sameType`. -fn same_type(left: &DataType, right: &DataType) -> bool { - use DataType::*; - match (left, right) { - (List(l), List(r)) | (LargeList(l), LargeList(r)) | (Map(l, _), Map(r, _)) => { - same_type(l.data_type(), r.data_type()) - } - (FixedSizeList(l, ln), FixedSizeList(r, rn)) => { - ln == rn && same_type(l.data_type(), r.data_type()) - } - (Struct(l), Struct(r)) => { - l.len() == r.len() - && l.iter().zip(r.iter()).all(|(a, b)| { - a.name() == b.name() && same_type(a.data_type(), b.data_type()) - }) - } - _ => left == right, - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/spark/src/function/type_coercion.rs b/crates/spark/src/function/type_coercion.rs index d4a8bbb5..9c23939a 100644 --- a/crates/spark/src/function/type_coercion.rs +++ b/crates/spark/src/function/type_coercion.rs @@ -12,7 +12,7 @@ use std::sync::Arc; -use arrow::datatypes::{DataType, Field, Fields, IntervalUnit, TimeUnit}; +use arrow::datatypes::{DataType, Field, Fields, IntervalUnit}; /// Spark's `findTightestCommonType`. `ansi_mode` picks between `AnsiTypeCoercion` and /// `TypeCoercion`, which differ only in that ANSI widens an integer and a FLOAT to DOUBLE. @@ -39,7 +39,7 @@ pub(crate) fn tightest_common_type( match (left, right) { (Null, other) | (other, Null) => Some(other.clone()), // Spark has one string type and one binary type. - (l, r) if is_string(l) && is_string(r) => Some(wider_physical_type(l, r)), + (l, r) if l.is_string() && r.is_string() => Some(wider_physical_type(l, r)), (l, r) if is_binary(l) && is_binary(r) => Some(wider_physical_type(l, r)), // The decimal must hold every value of the integral type. (int, Decimal128(p, s)) | (Decimal128(p, s), int) => { @@ -51,7 +51,8 @@ pub(crate) fn tightest_common_type( } // TIMESTAMP_NTZ and TIMESTAMP widen to TIMESTAMP. (Timestamp(lu, ltz), Timestamp(ru, rtz)) => Some(Timestamp( - finer_unit(*lu, *ru), + // Arrow orders the units from Second to Nanosecond. + (*lu).max(*ru), ltz.clone().or_else(|| rtz.clone()), )), _ => complex_type(left, right, &|l, r| tightest_common_type(l, r, ansi_mode)), @@ -63,7 +64,7 @@ pub(crate) fn tightest_common_type( /// side. Struct fields must have the same names in the same order. Spark compares them /// ignoring case, but DataFusion cannot cast between structs whose field names differ only /// in case, so this requires an exact match. -pub(crate) fn complex_type( +fn complex_type( left: &DataType, right: &DataType, find_type: &dyn Fn(&DataType, &DataType) -> Option, @@ -112,10 +113,7 @@ pub(crate) fn complex_type( let (Struct(lkv), Struct(rkv)) = (lf.data_type(), rf.data_type()) else { return None; }; - let [lk, lv] = lkv.iter().collect::>()[..] else { - return None; - }; - let [rk, rv] = rkv.iter().collect::>()[..] else { + let ([lk, lv], [rk, rv]) = (&lkv[..], &rkv[..]) else { return None; }; let key = find_type(lk.data_type(), rk.data_type())?; @@ -151,6 +149,51 @@ pub(crate) fn is_orderable(data_type: &DataType) -> bool { } } +/// Whether the structs in two types, at any depth, have the same field types. With ANSI mode +/// off, Spark's implicit cast cannot change the field types of a struct, so a function that +/// casts its arguments to their tightest common type rejects such structs, as +/// `array_position` does. With ANSI mode on, Spark widens the field types. +pub(crate) fn struct_field_types_match(left: &DataType, right: &DataType) -> bool { + use DataType::*; + match (left, right) { + ( + List(l) | LargeList(l) | FixedSizeList(l, _), + List(r) | LargeList(r) | FixedSizeList(r, _), + ) => struct_field_types_match(l.data_type(), r.data_type()), + // The entries are a struct of the key and the value, which are checked on their own. + (Map(l, _), Map(r, _)) => match (l.data_type(), r.data_type()) { + (Struct(l), Struct(r)) => l + .iter() + .zip(r.iter()) + .all(|(a, b)| struct_field_types_match(a.data_type(), b.data_type())), + _ => true, + }, + (Struct(_), Struct(_)) => same_type(left, right), + _ => true, + } +} + +/// Whether two types are equal when the nullability of nested fields is ignored, like +/// Spark's `DataTypeUtils.sameType`. +fn same_type(left: &DataType, right: &DataType) -> bool { + use DataType::*; + match (left, right) { + (List(l), List(r)) | (LargeList(l), LargeList(r)) | (Map(l, _), Map(r, _)) => { + same_type(l.data_type(), r.data_type()) + } + (FixedSizeList(l, ln), FixedSizeList(r, rn)) => { + ln == rn && same_type(l.data_type(), r.data_type()) + } + (Struct(l), Struct(r)) => { + l.len() == r.len() + && l.iter().zip(r.iter()).all(|(a, b)| { + a.name() == b.name() && same_type(a.data_type(), b.data_type()) + }) + } + _ => left == right, + } +} + /// The name Spark's `toSQLType` gives the Spark type that an Arrow type maps to, for error /// messages. Arrow types without a Spark counterpart use their Arrow name. pub(crate) fn spark_sql_type(data_type: &DataType) -> String { @@ -188,10 +231,10 @@ pub(crate) fn spark_sql_type(data_type: &DataType) -> String { .join(", ") ), Map(entries, _) => match entries.data_type() { - Struct(kv) if kv.len() == 2 => format!( + Struct(kv) if let [k, v] = &kv[..] => format!( "MAP<{}, {}>", - spark_sql_type(kv[0].data_type()), - spark_sql_type(kv[1].data_type()) + spark_sql_type(k.data_type()), + spark_sql_type(v.data_type()) ), other => format!("MAP<{other}>"), }, @@ -199,13 +242,8 @@ pub(crate) fn spark_sql_type(data_type: &DataType) -> String { } } -fn is_string(data_type: &DataType) -> bool { - matches!( - data_type, - DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View - ) -} - +/// Unlike [`DataType::is_binary`], this leaves out `FixedSizeBinary`, which +/// [`wider_physical_type`] cannot widen. fn is_binary(data_type: &DataType) -> bool { matches!( data_type, @@ -250,20 +288,6 @@ fn numeric_precedence(data_type: &DataType) -> Option { } } -fn finer_unit(left: TimeUnit, right: TimeUnit) -> TimeUnit { - let rank = |unit: TimeUnit| match unit { - TimeUnit::Second => 0, - TimeUnit::Millisecond => 1, - TimeUnit::Microsecond => 2, - TimeUnit::Nanosecond => 3, - }; - if rank(left) >= rank(right) { - left - } else { - right - } -} - #[cfg(test)] mod tests { use super::*; @@ -340,6 +364,24 @@ mod tests { assert_eq!(tightest_common_type(&a, &b, false), None); } + #[test] + fn struct_field_types_must_match_at_any_depth() { + let int = Struct(Fields::from(vec![Field::new("a", Int32, false)])); + let nullable_int = Struct(Fields::from(vec![Field::new("a", Int32, true)])); + let long = Struct(Fields::from(vec![Field::new("a", Int64, false)])); + assert!(struct_field_types_match(&int, &nullable_int)); + assert!(!struct_field_types_match(&int, &long)); + assert!(!struct_field_types_match( + &list(int.clone()), + &list(long.clone()) + )); + assert!(!struct_field_types_match( + &map(Int32, int), + &map(Int64, long) + )); + assert!(struct_field_types_match(&list(Int32), &list(Int64))); + } + #[test] fn maps_and_calendar_intervals_are_not_orderable() { let interval = Interval(IntervalUnit::MonthDayNano); From 1728af8167ab055b1a8e399e1f7583fde1c0409b Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 10:34:19 -0700 Subject: [PATCH 6/8] fix(spark): reject INTERVAL and name Spark types in array_max and array_min errors Use the shared isOrderable, which also rejects INTERVAL, and render the error types the way Spark does, with its SQLSTATE. Say how the functions differ from DataFusion's built-ins and note the built-in result beside each test of a difference. Drop the typed arms for types Spark never produces. --- .../spark/src/function/array/array_extrema.rs | 36 +++++++++---------- .../test_files/array/array_extrema.slt | 24 +++++++++---- 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/crates/spark/src/function/array/array_extrema.rs b/crates/spark/src/function/array/array_extrema.rs index e573bdd3..d2832dc7 100644 --- a/crates/spark/src/function/array/array_extrema.rs +++ b/crates/spark/src/function/array/array_extrema.rs @@ -5,17 +5,16 @@ //! Spark-compatible `array_max` and `array_min`. use crate::function::float_normalize::normalize_array; -use crate::function::type_coercion::is_orderable; +use crate::function::type_coercion::{is_orderable, spark_sql_type}; use arrow::array::{ Array, ArrayRef, ArrowPrimitiveType, AsArray, GenericListArray, OffsetSizeTrait, PrimitiveArray, UInt64Array, make_comparator, new_null_array, }; use arrow::compute::{SortOptions, take}; use arrow::datatypes::{ - DataType, Date32Type, Date64Type, Decimal128Type, Decimal256Type, Field, Float16Type, - Float32Type, Float64Type, Int8Type, Int16Type, Int32Type, Int64Type, TimeUnit, - TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType, - TimestampSecondType, UInt8Type, UInt16Type, UInt32Type, UInt64Type, + DataType, Date32Type, Decimal128Type, Field, Float32Type, Float64Type, Int8Type, + Int16Type, Int32Type, Int64Type, TimeUnit, TimestampMicrosecondType, + TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, }; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, exec_err, plan_err}; @@ -39,8 +38,11 @@ use std::sync::Arc; tie the first of the tied elements is returned as it is, so the sign of a zero and the \ bits of a NaN are kept. Strings and binary values compare bytewise. Arrays compare \ element by element and then by length, and structs compare field by field; NULLs \ - nested inside them sort before other values. Map elements, at any depth, are an \ - error, as in Spark.", + nested inside them sort before other values. Map and INTERVAL elements, at any \ + depth, are an error, as in Spark. DataFusion's built-in of the same name differs: it \ + orders -0.0 before 0.0 rather than returning the first of them, sorts NULLs nested \ + inside arrays and structs after other values, rejects a NULL argument at planning \ + instead of returning NULL, and fails at execution on map elements.", syntax_example = "array_max(array)", sql_example = r#"```sql > SELECT array_max(array(1, 20, NULL, 3)); @@ -105,8 +107,11 @@ impl ScalarUDFImpl for SparkArrayMax { tie the first of the tied elements is returned as it is, so the sign of a zero and the \ bits of a NaN are kept. Strings and binary values compare bytewise. Arrays compare \ element by element and then by length, and structs compare field by field; NULLs \ - nested inside them sort before other values. Map elements, at any depth, are an \ - error, as in Spark.", + nested inside them sort before other values. Map and INTERVAL elements, at any \ + depth, are an error, as in Spark. DataFusion's built-in of the same name differs: it \ + orders -0.0 before 0.0 rather than returning the first of them, sorts NULLs nested \ + inside arrays and structs after other values, rejects a NULL argument at planning \ + instead of returning NULL, and fails at execution on map elements.", syntax_example = "array_min(array)", sql_example = r#"```sql > SELECT array_min(array(1, 20, NULL, 3)); @@ -189,7 +194,8 @@ fn coerce_array(name: &str, arg_types: &[DataType]) -> Result> { if !is_orderable(&element) { return plan_err!( "[DATATYPE_MISMATCH.INVALID_ORDERING_TYPE] The `{name}` does not support \ - ordering on type {element}" + ordering on type \"{}\". SQLSTATE: 42K09", + spark_sql_type(&element) ); } Ok(vec![coerced]) @@ -198,7 +204,8 @@ fn coerce_array(name: &str, arg_types: &[DataType]) -> Result> { fn unexpected_input_type(name: &str, data_type: &DataType) -> Result { plan_err!( "[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] The first parameter of `{name}` \ - requires the \"ARRAY\" type, however it has the type {data_type}" + requires the \"ARRAY\" type, however it has the type \"{}\". SQLSTATE: 42K09", + spark_sql_type(data_type) ) } @@ -267,14 +274,8 @@ fn extrema( DataType::Int16 => ordered!(Int16Type), DataType::Int32 => ordered!(Int32Type), DataType::Int64 => ordered!(Int64Type), - DataType::UInt8 => ordered!(UInt8Type), - DataType::UInt16 => ordered!(UInt16Type), - DataType::UInt32 => ordered!(UInt32Type), - DataType::UInt64 => ordered!(UInt64Type), DataType::Decimal128(_, _) => ordered!(Decimal128Type), - DataType::Decimal256(_, _) => ordered!(Decimal256Type), DataType::Date32 => ordered!(Date32Type), - DataType::Date64 => ordered!(Date64Type), DataType::Timestamp(TimeUnit::Second, _) => ordered!(TimestampSecondType), DataType::Timestamp(TimeUnit::Millisecond, _) => { ordered!(TimestampMillisecondType) @@ -285,7 +286,6 @@ fn extrema( DataType::Timestamp(TimeUnit::Nanosecond, _) => { ordered!(TimestampNanosecondType) } - DataType::Float16 => primitive!(Float16Type, spark_float_cmp), DataType::Float32 => primitive!(Float32Type, spark_float_cmp), DataType::Float64 => primitive!(Float64Type, spark_float_cmp), _ => {} diff --git a/crates/sqllogictest/test_files/array/array_extrema.slt b/crates/sqllogictest/test_files/array/array_extrema.slt index 661c4d5e..bc92fbb0 100644 --- a/crates/sqllogictest/test_files/array/array_extrema.slt +++ b/crates/sqllogictest/test_files/array/array_extrema.slt @@ -27,7 +27,7 @@ SELECT array_max(array(CAST(NULL AS INT))); NULL # Spark accepts an untyped NULL, an empty array and an array of untyped NULLs, and returns -# a NULL of the NULL type. +# a NULL of the NULL type. DataFusion's built-in raises a planning error for these. query ?? SELECT array_max(NULL), array_min(NULL); ---- @@ -56,7 +56,9 @@ SELECT array_max(array(CAST('NaN' AS FLOAT), CAST(1.5 AS FLOAT))), array_min(arr ---- NaN 1.5 -# -0.0 equals 0.0, so the first of them is returned, with its sign. The test runner prints +# -0.0 equals 0.0, so the first of them is returned, with its sign. DataFusion's built-in +# orders -0.0 before 0.0, so its array_max returns 0.0 and its array_min -0.0 for both +# orders. The test runner prints # both zeros as 0, so the sign is checked by casting the result to a string. query RRRR SELECT array_max(array(CAST(0.0 AS DOUBLE), CAST('-0.0' AS DOUBLE))), @@ -128,7 +130,8 @@ SELECT array_max(array(array(1, 2), array(1, 2, 3), array(2))), array_min(array( ---- [2] [] -# A NULL nested inside an array sorts before other values +# A NULL nested inside an array sorts before other values. DataFusion's built-in sorts it +# after them, and its array_max returns [1, NULL] here. query ?? SELECT array_min(array(array(1, NULL), array(1, 2))), array_max(array(array(1, NULL), array(1, 2))); ---- @@ -190,14 +193,21 @@ NaN 1.5 NaN 1.5 0 0 -0.0 -0.0 0 0 0.0 0.0 -# Maps cannot be ordered, at any depth -query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_max` does not support ordering on type +# Maps cannot be ordered, at any depth. DataFusion's built-in fails at execution instead. +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_max` does not support ordering on type "MAP"\. SQLSTATE: 42K09 SELECT array_max(array(map(1, 2))); -query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_min` does not support ordering on type +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_min` does not support ordering on type "ARRAY>"\. SQLSTATE: 42K09 SELECT array_min(array(array(map(1, 2)))); -query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `array_max` requires the "ARRAY" type +# Intervals cannot be ordered, at any depth +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_max` does not support ordering on type "INTERVAL"\. SQLSTATE: 42K09 +SELECT array_max(array(make_interval(1))); + +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_min` does not support ordering on type "STRUCT"\. SQLSTATE: 42K09 +SELECT array_min(array(named_struct('i', make_interval(1)))); + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `array_max` requires the "ARRAY" type, however it has the type "BIGINT"\. SQLSTATE: 42K09 SELECT array_max(1); query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `array_min` requires the "ARRAY" type From 4dcf74693cdb866c4ec7dfe6e8fcbaedff069fb7 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 16:55:10 -0700 Subject: [PATCH 7/8] fix(spark): compare dictionary-encoded elements as their values in array_max and array_min A list of dictionary-encoded floats kept its Dictionary element type, so it took the generic path, where normalize_array does not look inside dictionaries and -0.0 ordered below 0.0. coerce_types now casts dictionary types, at any depth of list and struct, to their value type. --- .../spark/src/function/array/array_extrema.rs | 41 ++++++++++++++++--- .../test_files/array/array_extrema.slt | 22 ++++++++++ 2 files changed, 58 insertions(+), 5 deletions(-) diff --git a/crates/spark/src/function/array/array_extrema.rs b/crates/spark/src/function/array/array_extrema.rs index d2832dc7..28cdbad4 100644 --- a/crates/spark/src/function/array/array_extrema.rs +++ b/crates/spark/src/function/array/array_extrema.rs @@ -12,8 +12,8 @@ use arrow::array::{ }; use arrow::compute::{SortOptions, take}; use arrow::datatypes::{ - DataType, Date32Type, Decimal128Type, Field, Float32Type, Float64Type, Int8Type, - Int16Type, Int32Type, Int64Type, TimeUnit, TimestampMicrosecondType, + DataType, Date32Type, Decimal128Type, Field, FieldRef, Float32Type, Float64Type, + Int8Type, Int16Type, Int32Type, Int64Type, TimeUnit, TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, }; use datafusion_common::utils::take_function_args; @@ -178,14 +178,17 @@ fn element_type(name: &str, arg_types: &[DataType]) -> Result { } /// Accepts the argument types that Spark's `ArrayMax` and `ArrayMin` accept: an array of an -/// orderable type, or NULL. Fixed-size lists are cast to lists. +/// orderable type, or NULL. Fixed-size lists are cast to lists, and dictionary-encoded +/// elements to their value type. fn coerce_array(name: &str, arg_types: &[DataType]) -> Result> { let [array] = take_function_args(name, arg_types)?; let coerced = match array { - DataType::Null | DataType::List(_) | DataType::LargeList(_) => array.clone(), + DataType::Null => DataType::Null, + DataType::List(field) => DataType::List(decode_field(field)), + DataType::LargeList(field) => DataType::LargeList(decode_field(field)), DataType::FixedSizeList(field, _) => DataType::List(Arc::new(Field::new( field.name(), - field.data_type().clone(), + decode_dictionaries(field.data_type()), true, ))), other => return unexpected_input_type(name, other), @@ -201,6 +204,34 @@ fn coerce_array(name: &str, arg_types: &[DataType]) -> Result> { Ok(vec![coerced]) } +/// Replaces every dictionary type, at any depth of list and struct, with its value type. +/// Spark has no dictionary types, but engines built on these functions pass +/// dictionary-encoded columns. The typed path and the float normalization of the generic +/// path do not look inside dictionaries, so `-0.0` would otherwise order below `0.0`. +fn decode_dictionaries(data_type: &DataType) -> DataType { + match data_type { + DataType::Dictionary(_, value) => decode_dictionaries(value), + DataType::List(field) => DataType::List(decode_field(field)), + DataType::LargeList(field) => DataType::LargeList(decode_field(field)), + DataType::FixedSizeList(field, size) => { + DataType::FixedSizeList(decode_field(field), *size) + } + DataType::Struct(fields) => { + DataType::Struct(fields.iter().map(decode_field).collect()) + } + other => other.clone(), + } +} + +fn decode_field(field: &FieldRef) -> FieldRef { + Arc::new( + field + .as_ref() + .clone() + .with_data_type(decode_dictionaries(field.data_type())), + ) +} + fn unexpected_input_type(name: &str, data_type: &DataType) -> Result { plan_err!( "[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] The first parameter of `{name}` \ diff --git a/crates/sqllogictest/test_files/array/array_extrema.slt b/crates/sqllogictest/test_files/array/array_extrema.slt index bc92fbb0..d358381b 100644 --- a/crates/sqllogictest/test_files/array/array_extrema.slt +++ b/crates/sqllogictest/test_files/array/array_extrema.slt @@ -193,6 +193,28 @@ NaN 1.5 NaN 1.5 0 0 -0.0 -0.0 0 0 0.0 0.0 +# Spark has no dictionary types. Dictionary-encoded elements, also inside nested arrays and +# structs, compare as their values, so -0.0 equals 0.0 and the first of them is returned. +query TTTT +SELECT CAST(array_max(arrow_cast(array(CAST('-0.0' AS DOUBLE), CAST(0.0 AS DOUBLE)), 'List(Dictionary(Int32, Float64))')) AS STRING), + CAST(array_min(arrow_cast(array(CAST(0.0 AS DOUBLE), CAST('-0.0' AS DOUBLE)), 'List(Dictionary(Int32, Float64))')) AS STRING), + CAST(array_max(arrow_cast(array(CAST(NULL AS DOUBLE), CAST('NaN' AS DOUBLE), CAST(1.0 AS DOUBLE)), 'LargeList(Dictionary(Int8, Float64))')) AS STRING), + CAST(array_min(arrow_cast(array(CAST(0.0 AS FLOAT), CAST('-0.0' AS FLOAT)), 'FixedSizeList(2, Dictionary(Int32, Float32))')) AS STRING); +---- +-0.0 0.0 NaN 0.0 + +query ?? +SELECT array_max(arrow_cast(array(array(CAST('-0.0' AS DOUBLE)), array(CAST(0.0 AS DOUBLE))), 'List(List(Dictionary(Int32, Float64)))')), + array_max(arrow_cast(array(named_struct('a', CAST('-0.0' AS DOUBLE)), named_struct('a', CAST(0.0 AS DOUBLE))), 'List(Struct("a": Dictionary(Int32, Float64)))')); +---- +[-0.0] {a: -0.0} + +query TT +SELECT array_max(arrow_cast(array('b', NULL, 'c', 'a'), 'List(Dictionary(Int32, Utf8))')), + array_min(arrow_cast(array('b', NULL, 'c', 'a'), 'List(Dictionary(Int32, Utf8))')); +---- +c a + # Maps cannot be ordered, at any depth. DataFusion's built-in fails at execution instead. query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_max` does not support ordering on type "MAP"\. SQLSTATE: 42K09 SELECT array_max(array(map(1, 2))); From 941de7238de8b65727250e6772a293067c2bbb04 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 18 Nov 2026 08:12:54 -0700 Subject: [PATCH 8/8] test(spark): expect the sign of a zero in the array_max and array_min results The runner prints floats at full precision since #333, including the sign of -0.0. --- crates/sqllogictest/test_files/array/array_extrema.slt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/sqllogictest/test_files/array/array_extrema.slt b/crates/sqllogictest/test_files/array/array_extrema.slt index d358381b..cbdb09b2 100644 --- a/crates/sqllogictest/test_files/array/array_extrema.slt +++ b/crates/sqllogictest/test_files/array/array_extrema.slt @@ -58,15 +58,14 @@ NaN 1.5 # -0.0 equals 0.0, so the first of them is returned, with its sign. DataFusion's built-in # orders -0.0 before 0.0, so its array_max returns 0.0 and its array_min -0.0 for both -# orders. The test runner prints -# both zeros as 0, so the sign is checked by casting the result to a string. +# orders. query RRRR SELECT array_max(array(CAST(0.0 AS DOUBLE), CAST('-0.0' AS DOUBLE))), array_max(array(CAST('-0.0' AS DOUBLE), CAST(0.0 AS DOUBLE))), array_min(array(CAST(0.0 AS DOUBLE), CAST('-0.0' AS DOUBLE))), array_min(array(CAST('-0.0' AS DOUBLE), CAST(0.0 AS DOUBLE))); ---- -0 0 0 0 +0 -0 0 -0 query TTTT SELECT CAST(array_max(array(CAST(0.0 AS DOUBLE), CAST('-0.0' AS DOUBLE))) AS STRING), @@ -190,7 +189,7 @@ FROM (VALUES ) AS t(a); ---- NaN 1.5 NaN 1.5 -0 0 -0.0 -0.0 +-0 -0 -0.0 -0.0 0 0 0.0 0.0 # Spark has no dictionary types. Dictionary-encoded elements, also inside nested arrays and