From b040e1590d0180d57f9a46247ffcc1326aabec15 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 16 Nov 2026 06:37:14 -0700 Subject: [PATCH 01/10] feat(spark): add Spark-compatible greatest and least Port greatest and least from Comet (math_funcs/greatest_least.rs) and extend them to cover Spark's type coercion and error classes. - NULL arguments are skipped; the result is NULL only if all are NULL. - Floats follow Spark's SQL ordering at any depth: NaN is greater than every other value and -0.0 equals 0.0. Of equal values the first argument wins. - A NULL inside an array or struct orders before every other value. - Arguments are coerced with Spark's findWiderTypeWithoutStringPromotion, with the ANSI difference for integer and FLOAT. - Fewer than two arguments, mismatched types and maps raise WRONG_NUM_ARGS, DATA_DIFF_TYPES and INVALID_ORDERING_TYPE. Closes #125. --- .../src/function/array/float_normalize.rs | 4 +- crates/spark/src/function/array/mod.rs | 2 +- .../spark/src/function/math/greatest_least.rs | 1173 +++++++++++++++++ crates/spark/src/function/math/mod.rs | 15 + .../sqllogictest/test_files/math/greatest.slt | 232 +++- crates/sqllogictest/test_files/math/least.slt | 186 ++- 6 files changed, 1605 insertions(+), 7 deletions(-) create mode 100644 crates/spark/src/function/math/greatest_least.rs diff --git a/crates/spark/src/function/array/float_normalize.rs b/crates/spark/src/function/array/float_normalize.rs index 07bbb28a..a321bc8d 100644 --- a/crates/spark/src/function/array/float_normalize.rs +++ b/crates/spark/src/function/array/float_normalize.rs @@ -35,7 +35,7 @@ pub(super) fn normalize_floats(value: &ColumnarValue) -> Result { } /// Returns whether values of this type are, or hold at any depth, floating point numbers. -pub(super) fn contains_float(data_type: &DataType) -> bool { +pub(crate) fn contains_float(data_type: &DataType) -> bool { match data_type { DataType::Float32 | DataType::Float64 => true, DataType::List(field) @@ -51,7 +51,7 @@ pub(super) fn contains_float(data_type: &DataType) -> bool { /// Rewrites `-0.0` to `0.0` and every NaN to the canonical NaN at any depth of list, large /// list, fixed-size list, struct and map. Children without floats are reused as they are. -pub(super) fn normalize_array(array: &ArrayRef) -> Result { +pub(crate) fn normalize_array(array: &ArrayRef) -> Result { if !contains_float(array.data_type()) { return Ok(Arc::clone(array)); } diff --git a/crates/spark/src/function/array/mod.rs b/crates/spark/src/function/array/mod.rs index 7ff0f2aa..a6abd771 100644 --- a/crates/spark/src/function/array/mod.rs +++ b/crates/spark/src/function/array/mod.rs @@ -6,7 +6,7 @@ pub mod array_contains; pub mod array_position; -mod float_normalize; +pub(crate) mod float_normalize; pub mod repeat; pub mod shuffle; pub mod slice; diff --git a/crates/spark/src/function/math/greatest_least.rs b/crates/spark/src/function/math/greatest_least.rs new file mode 100644 index 00000000..637f0b02 --- /dev/null +++ b/crates/spark/src/function/math/greatest_least.rs @@ -0,0 +1,1173 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Spark-compatible `greatest` and `least`. +//! +//! Spark evaluates the arguments in order, skips NULLs, and replaces its running result only +//! with a value that is strictly greater (or smaller) in its SQL ordering, so the result is NULL +//! only when every argument is NULL and the first of equal arguments wins. The SQL ordering +//! (`SQLOrderingUtil.compareDoubles`) treats `-0.0` as equal to `0.0` and NaN as greater than +//! every other value, at any depth of arrays and structs, and orders a NULL inside an array or +//! struct before every other value. +//! +//! DataFusion's built-in `greatest` and `least` differ: they compare floats by IEEE 754 total +//! order (so `-0.0 < 0.0`, and a NaN with the sign bit set is smaller than `-Infinity`), let a +//! later argument win a tie, compare the constant arguments before the columns, accept a single +//! argument, order a NULL inside an array or struct after every other value in `least`, and +//! coerce their arguments with DataFusion's comparison coercion, which for example compares a +//! number with a string. Spark rejects that. + +use crate::function::array::float_normalize::{contains_float, normalize_array}; +use arrow::array::{ + Array, ArrayRef, BooleanArray, Datum, Scalar, make_comparator, new_null_array, +}; +use arrow::buffer::BooleanBuffer; +use arrow::compute::kernels::cmp::{gt, lt}; +use arrow::compute::kernels::zip::zip; +use arrow::compute::{SortOptions, cast}; +use arrow::datatypes::{DataType, Field, FieldRef, Fields, TimeUnit}; +use datafusion_common::config::ConfigOptions; +use datafusion_common::{DataFusionError, Result, ScalarValue, plan_err}; +use datafusion_expr::{ + ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF, + ScalarUDFImpl, Signature, Volatility, +}; +use datafusion_macros::user_doc; +use std::sync::Arc; + +/// Spark-compatible `greatest` expression. +/// +/// +#[user_doc( + doc_section(label = "Math Functions"), + description = "Returns the greatest value of all arguments, skipping NULLs. The result is \ + NULL only if every argument is NULL. At least two arguments are required. The \ + arguments are cast to a common type as in Spark: numbers widen (an integer and a FLOAT \ + widen to DOUBLE with `datafusion.execution.enable_ansi_mode` on and to FLOAT with it \ + off), integers and decimals widen to a decimal that holds both, a decimal and a float \ + widen to DOUBLE, a date and a timestamp widen to the timestamp, and arrays and structs \ + widen element by element. Strings are never promoted: a string and a number, a string \ + and a date, or a boolean and a number are an error, as are maps, which cannot be \ + ordered. NaN is greater than every other value and `-0.0` equals `0.0`, also inside \ + arrays and structs, and of equal values the first argument wins, so \ + `greatest(-0.0, 0.0)` is `-0.0`. A NULL inside an array or struct is smaller than \ + every other value. DataFusion's built-in `greatest` orders floats by IEEE 754 total \ + order, lets a later argument win a tie, accepts a single argument, and compares \ + numbers with strings.", + syntax_example = "greatest(expr1, expr2[, ..., expr_n])", + sql_example = r#"```sql +> SELECT greatest(10, 9, 2, 4, 3); +10 +> SELECT greatest(1, NULL, 3); +3 +> SELECT greatest(1.0, 'NaN'::double); +NaN +```"#, + argument( + name = "expr1, expr2, ..., expr_n", + description = "Expressions of an orderable type that share a common type. NULLs are \ + skipped." + ) +)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkGreatest { + inner: GreatestLeast, +} + +impl Default for SparkGreatest { + fn default() -> Self { + Self::new() + } +} + +impl SparkGreatest { + pub fn new() -> Self { + Self { + inner: GreatestLeast::new(true, false), + } + } + + /// Creates a `greatest` that coerces its arguments as Spark does with + /// `datafusion.execution.enable_ansi_mode` set to `enable_ansi_mode`. + pub fn new_with_ansi_mode(enable_ansi_mode: bool) -> Self { + Self { + inner: GreatestLeast::new(true, enable_ansi_mode), + } + } +} + +/// Spark-compatible `least` expression. +/// +/// +#[user_doc( + doc_section(label = "Math Functions"), + description = "Returns the least value of all arguments, skipping NULLs. The result is \ + NULL only if every argument is NULL. At least two arguments are required. The \ + arguments are cast to a common type as in `greatest`. NaN is greater than every other \ + value and `-0.0` equals `0.0`, also inside arrays and structs, and of equal values the \ + first argument wins, so `least(0.0, -0.0)` is `0.0`. A NULL inside an array or struct \ + is smaller than every other value, so `least(array(1, NULL), array(1, 0))` is \ + `[1, NULL]`. DataFusion's built-in `least` orders floats by IEEE 754 total order, lets a \ + later argument win a tie, orders a NULL inside an array or struct after every other \ + value, accepts a single argument, and compares numbers with strings.", + syntax_example = "least(expr1, expr2[, ..., expr_n])", + sql_example = r#"```sql +> SELECT least(10, 9, 2, 4, 3); +2 +> SELECT least(NULL, 2, NULL); +2 +> SELECT least(1.0, 'NaN'::double); +1.0 +```"#, + argument( + name = "expr1, expr2, ..., expr_n", + description = "Expressions of an orderable type that share a common type. NULLs are \ + skipped." + ) +)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkLeast { + inner: GreatestLeast, +} + +impl Default for SparkLeast { + fn default() -> Self { + Self::new() + } +} + +impl SparkLeast { + pub fn new() -> Self { + Self { + inner: GreatestLeast::new(false, false), + } + } + + /// Creates a `least` that coerces its arguments as Spark does with + /// `datafusion.execution.enable_ansi_mode` set to `enable_ansi_mode`. + pub fn new_with_ansi_mode(enable_ansi_mode: bool) -> Self { + Self { + inner: GreatestLeast::new(false, enable_ansi_mode), + } + } +} + +macro_rules! impl_greatest_least_udf { + ($udf:ident) => { + impl ScalarUDFImpl for $udf { + fn name(&self) -> &str { + self.inner.name() + } + + fn signature(&self) -> &Signature { + &self.inner.signature + } + + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { + self.inner.coerce_types(arg_types) + } + + fn return_type(&self, arg_types: &[DataType]) -> Result { + self.inner.check_arg_count(arg_types.len(), true)?; + Ok(arg_types[0].clone()) + } + + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + self.inner.check_arg_count(args.arg_fields.len(), true)?; + // As in Spark, the result is nullable only if every argument is. + let nullable = args.arg_fields.iter().all(|f| f.is_nullable()); + Ok(Arc::new(Field::new( + self.name(), + args.arg_fields[0].data_type().clone(), + nullable, + ))) + } + + fn with_updated_config(&self, config: &ConfigOptions) -> Option { + Some(ScalarUDF::from(Self::new_with_ansi_mode( + config.execution.enable_ansi_mode, + ))) + } + + fn invoke_with_args( + &self, + args: ScalarFunctionArgs, + ) -> Result { + self.inner.invoke(args) + } + + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + } + }; +} + +impl_greatest_least_udf!(SparkGreatest); +impl_greatest_least_udf!(SparkLeast); + +/// The implementation shared by [`SparkGreatest`] and [`SparkLeast`]. +#[derive(Debug, PartialEq, Eq, Hash)] +struct GreatestLeast { + signature: Signature, + greatest: bool, + /// Whether to coerce as Spark's `AnsiTypeCoercion` does, rather than `TypeCoercion`. + ansi_mode: bool, +} + +/// Spark orders a NULL inside an array or struct before every other value. +const SORT_OPTIONS: SortOptions = SortOptions { + descending: false, + nulls_first: true, +}; + +impl GreatestLeast { + fn new(greatest: bool, ansi_mode: bool) -> Self { + Self { + signature: Signature::user_defined(Volatility::Immutable), + greatest, + ansi_mode, + } + } + + fn name(&self) -> &'static str { + if self.greatest { "greatest" } else { "least" } + } + + fn check_arg_count(&self, count: usize, planning: bool) -> Result<()> { + if count > 1 { + return Ok(()); + } + let message = format!( + "[WRONG_NUM_ARGS.WITHOUT_SUGGESTION] The `{}` requires > 1 parameters but the \ + actual number is {count}. Please, refer to \ + 'https://spark.apache.org/docs/latest/sql-ref-functions.html' for a fix.", + self.name() + ); + Err(if planning { + DataFusionError::Plan(message) + } else { + DataFusionError::Execution(message) + }) + } + + /// Spark's `TypeCoercion` rule for `Greatest` and `Least`: every argument is cast to + /// `findWiderTypeWithoutStringPromotion` of all the argument types, and the result must be + /// orderable. + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { + self.check_arg_count(arg_types.len(), true)?; + let common = arg_types + .iter() + .skip(1) + .try_fold(arg_types[0].clone(), |common, data_type| { + wider_type(&common, data_type, self.ansi_mode) + }); + let Some(common) = common else { + return plan_err!( + "[DATATYPE_MISMATCH.DATA_DIFF_TYPES] Input to `{}` should all be the same \ + type, but it's [{}]", + self.name(), + arg_types + .iter() + .map(|t| format!("\"{}\"", spark_sql_type(t))) + .collect::>() + .join(", ") + ); + }; + if !is_orderable(&common) { + return plan_err!( + "[DATATYPE_MISMATCH.INVALID_ORDERING_TYPE] The `{}` does not support ordering \ + on type \"{}\"", + self.name(), + spark_sql_type(&common) + ); + } + Ok(vec![common; arg_types.len()]) + } + + fn invoke(&self, args: ScalarFunctionArgs) -> Result { + self.check_arg_count(args.args.len(), false)?; + let data_type = args.return_field.data_type(); + let all_scalars = args + .args + .iter() + .all(|arg| matches!(arg, ColumnarValue::Scalar(_))); + let rows = if all_scalars { 1 } else { args.number_rows }; + + if data_type.is_null() { + return Ok(if all_scalars { + ColumnarValue::Scalar(ScalarValue::Null) + } else { + ColumnarValue::Array(new_null_array(&DataType::Null, rows)) + }); + } + + // The planner casts every argument to the common type, but a field name or the + // nullability of a nested field can still differ when the function is called directly, + // and `zip` needs identical types. + let to_array = |arg: &ColumnarValue, rows: usize| -> Result { + let array = arg.to_array(rows)?; + Ok(if array.data_type() == data_type { + array + } else { + cast(&array, data_type)? + }) + }; + // Floats are compared in a copy where `-0.0` is `0.0` and every NaN is the canonical + // NaN, where Arrow's total order agrees with Spark's. The result keeps the original. + let normalize = contains_float(data_type); + let comparable = |array: &ArrayRef| -> Result { + if normalize { + normalize_array(array) + } else { + Ok(Arc::clone(array)) + } + }; + + let mut result = to_array(&args.args[0], rows)?; + let mut result_cmp = comparable(&result)?; + for arg in &args.args[1..] { + let candidate = match arg { + // A NULL never replaces the result. + ColumnarValue::Scalar(scalar) if scalar.is_null() => continue, + ColumnarValue::Scalar(_) => { + let value = to_array(arg, 1)?; + let cmp = comparable(&value)?; + Candidate { + value, + cmp, + scalar: true, + } + } + ColumnarValue::Array(_) => { + let value = to_array(arg, rows)?; + let cmp = comparable(&value)?; + Candidate { + value, + cmp, + scalar: false, + } + } + }; + let take = self.replaces(&candidate, &result, &result_cmp)?; + let take = BooleanArray::new(take, None); + let new_result = zip(&take, candidate.value_datum().as_ref(), &result)?; + result_cmp = if normalize { + zip(&take, candidate.cmp_datum().as_ref(), &result_cmp)? + } else { + Arc::clone(&new_result) + }; + result = new_result; + } + + if all_scalars { + Ok(ColumnarValue::Scalar(ScalarValue::try_from_array( + &result, 0, + )?)) + } else { + Ok(ColumnarValue::Array(result)) + } + } + + /// For each row, whether `candidate` replaces the current result: it is not NULL, and + /// either the result is NULL or the candidate ranks strictly before it. + fn replaces( + &self, + candidate: &Candidate, + current: &ArrayRef, + current_cmp: &ArrayRef, + ) -> Result { + let len = current.len(); + let current_nulls = current.logical_nulls(); + let candidate_nulls = if candidate.scalar { + // A NULL constant is skipped before this is called. + None + } else { + candidate.value.logical_nulls() + }; + + if !current.data_type().is_nested() { + let candidate_cmp = candidate.cmp_datum(); + let ordered = if self.greatest { + gt(candidate_cmp.as_ref(), current_cmp)? + } else { + lt(candidate_cmp.as_ref(), current_cmp)? + }; + // Where either side is NULL the comparison's value is arbitrary, but such rows are + // decided by the null masks below. + let mut take = ordered.values().clone(); + if let Some(nulls) = ¤t_nulls { + take = &take | &!nulls.inner(); + } + if let Some(nulls) = &candidate_nulls { + take = &take & nulls.inner(); + } + return Ok(take); + } + + let compare = + make_comparator(candidate.cmp.as_ref(), current_cmp.as_ref(), SORT_OPTIONS)?; + let scalar = candidate.scalar; + Ok(BooleanBuffer::collect_bool(len, |row| { + let candidate_row = if scalar { 0 } else { row }; + if candidate_nulls + .as_ref() + .is_some_and(|nulls| nulls.is_null(candidate_row)) + { + return false; + } + if current_nulls + .as_ref() + .is_some_and(|nulls| nulls.is_null(row)) + { + return true; + } + let ordering = compare(candidate_row, row); + if self.greatest { + ordering.is_gt() + } else { + ordering.is_lt() + } + })) + } +} + +/// An argument after the first, with its values as they are and as they are compared. A +/// constant is a one-row array. +struct Candidate { + value: ArrayRef, + cmp: ArrayRef, + scalar: bool, +} + +impl Candidate { + fn datum(&self, array: &ArrayRef) -> Box { + if self.scalar { + Box::new(Scalar::new(Arc::clone(array))) + } else { + Box::new(Arc::clone(array)) + } + } + + fn value_datum(&self) -> Box { + self.datum(&self.value) + } + + fn cmp_datum(&self) -> Box { + self.datum(&self.cmp) + } +} + +/// Spark's `findWiderTypeWithoutStringPromotionForTwo`: the tightest common type, then a +/// decimal wide enough for both, then the same for the elements of arrays and the fields of +/// structs. Strings are never promoted. `ansi_mode` picks between `AnsiTypeCoercion` and +/// `TypeCoercion`, which differ only in that ANSI widens an integer and a FLOAT to DOUBLE. +fn wider_type(left: &DataType, right: &DataType, ansi_mode: bool) -> Option { + tightest_common_type(left, right, ansi_mode) + .or_else(|| wider_decimal_type(left, right)) + .or_else(|| complex_type(left, right, ansi_mode)) +} + +/// Spark's `findTightestCommonType`, on the Arrow types that Spark's types map to. Arrays and +/// structs are handled by [`complex_type`] with [`wider_type`], which gives the same result. +fn tightest_common_type( + left: &DataType, + right: &DataType, + ansi_mode: bool, +) -> Option { + use DataType::*; + if left == right { + return Some(left.clone()); + } + if let (Some(l), Some(r)) = (numeric_precedence(left), numeric_precedence(right)) { + let wider = if l >= r { left } else { right }; + // With ANSI mode on, an integer and a FLOAT widen to DOUBLE, which holds every INT + // exactly. + return Some(if ansi_mode && *wider == Float32 { + Float64 + } else { + wider.clone() + }); + } + match (left, right) { + (Null, other) | (other, Null) => Some(other.clone()), + // Spark has one string type and one binary type. + (l, r) if is_string(l) && is_string(r) => Some(wider_physical_type(l, r)), + (l, r) if is_binary(l) && is_binary(r) => Some(wider_physical_type(l, r)), + // The decimal must hold every value of the integral type. + (int, Decimal128(p, s)) | (Decimal128(p, s), int) => { + let digits = integral_digits(int)?; + (*p as i16 - *s as i16 >= digits).then_some(Decimal128(*p, *s)) + } + (Date32, ts @ Timestamp(_, _)) | (ts @ Timestamp(_, _), Date32) => { + Some(ts.clone()) + } + // TIMESTAMP_NTZ and TIMESTAMP widen to TIMESTAMP. + (Timestamp(lu, ltz), Timestamp(ru, rtz)) => Some(Timestamp( + finer_unit(*lu, *ru), + ltz.clone().or_else(|| rtz.clone()), + )), + _ => None, + } +} + +/// Spark's `findWiderTypeForDecimal`, with `DecimalType.boundedPreferIntegralDigits`, which +/// gives up fractional digits rather than integral ones when the precision exceeds 38. +fn wider_decimal_type(left: &DataType, right: &DataType) -> Option { + use DataType::*; + let (p1, s1, p2, s2) = match (left, right) { + (Decimal128(p1, s1), Decimal128(p2, s2)) => { + (*p1 as i16, *s1 as i16, *p2 as i16, *s2 as i16) + } + (int, Decimal128(p, s)) | (Decimal128(p, s), int) + if integral_digits(int).is_some() => + { + (integral_digits(int)?, 0, *p as i16, *s as i16) + } + (Float32 | Float64, Decimal128(_, _)) | (Decimal128(_, _), Float32 | Float64) => { + return Some(Float64); + } + _ => return None, + }; + let scale = s1.max(s2); + let range = (p1 - s1).max(p2 - s2); + let precision = scale + range; + const MAX_PRECISION: i16 = 38; + if precision <= MAX_PRECISION { + Some(Decimal128(precision as u8, scale as i8)) + } else { + let scale = (scale - (precision - MAX_PRECISION)).max(0); + Some(Decimal128(MAX_PRECISION as u8, scale as i8)) + } +} + +/// Spark's `findTypeForComplex` with [`wider_type`] for the elements and fields. A nested field +/// is nullable if it is on either side. Struct fields must have the same names in the same +/// order. Spark compares them ignoring case, but this requires an exact match. +fn complex_type(left: &DataType, right: &DataType, ansi_mode: bool) -> Option { + use DataType::*; + match (left, right) { + ( + List(lf) | LargeList(lf) | FixedSizeList(lf, _), + List(rf) | LargeList(rf) | FixedSizeList(rf, _), + ) => { + let item = wider_type(lf.data_type(), rf.data_type(), ansi_mode)?; + let field = Arc::new(Field::new( + lf.name(), + item, + lf.is_nullable() || rf.is_nullable(), + )); + Some(match (left, right) { + (FixedSizeList(_, ln), FixedSizeList(_, rn)) if ln == rn => { + FixedSizeList(field, *ln) + } + (LargeList(_), _) | (_, LargeList(_)) => LargeList(field), + _ => List(field), + }) + } + (Struct(lf), Struct(rf)) if lf.len() == rf.len() => { + let fields = lf + .iter() + .zip(rf.iter()) + .map(|(a, b)| { + if a.name() != b.name() { + return None; + } + let data_type = wider_type(a.data_type(), b.data_type(), ansi_mode)?; + Some(Field::new( + a.name(), + data_type, + a.is_nullable() || b.is_nullable(), + )) + }) + .collect::>>()?; + Some(Struct(Fields::from(fields))) + } + // Spark finds a common map type, which is then rejected as not orderable. + (Map(lf, sorted), Map(rf, _)) => { + let entries = wider_type(lf.data_type(), rf.data_type(), ansi_mode)?; + Some(Map( + Arc::new(Field::new(lf.name(), entries, false)), + *sorted, + )) + } + _ => None, + } +} + +/// Whether Spark can order values of this type. It cannot order maps, at any depth. +fn is_orderable(data_type: &DataType) -> bool { + match data_type { + DataType::Map(_, _) | DataType::Union(_, _) => false, + DataType::List(field) + | DataType::LargeList(field) + | DataType::ListView(field) + | DataType::LargeListView(field) + | DataType::FixedSizeList(field, _) => is_orderable(field.data_type()), + DataType::Struct(fields) => fields.iter().all(|f| is_orderable(f.data_type())), + DataType::Dictionary(_, value) => is_orderable(value), + _ => true, + } +} + +fn is_string(data_type: &DataType) -> bool { + matches!( + data_type, + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View + ) +} + +fn is_binary(data_type: &DataType) -> bool { + matches!( + data_type, + DataType::Binary | DataType::LargeBinary | DataType::BinaryView + ) +} + +/// Of two physical layouts of the same Spark string or binary type, the one that can hold the +/// values of both. +fn wider_physical_type(left: &DataType, right: &DataType) -> DataType { + use DataType::*; + match (left, right) { + (LargeUtf8, _) | (_, LargeUtf8) => LargeUtf8, + (LargeBinary, _) | (_, LargeBinary) => LargeBinary, + (Utf8View, _) | (_, Utf8View) => Utf8View, + (BinaryView, _) | (_, BinaryView) => BinaryView, + _ => left.clone(), + } +} + +/// The precision of Spark's `DecimalType.forType` for an integral type. +fn integral_digits(data_type: &DataType) -> Option { + match data_type { + DataType::Int8 => Some(3), + DataType::Int16 => Some(5), + DataType::Int32 => Some(10), + DataType::Int64 => Some(20), + _ => None, + } +} + +/// Spark's `numericPrecedence`, which leaves out decimals. +fn numeric_precedence(data_type: &DataType) -> Option { + match data_type { + DataType::Int8 => Some(0), + DataType::Int16 => Some(1), + DataType::Int32 => Some(2), + DataType::Int64 => Some(3), + DataType::Float32 => Some(4), + DataType::Float64 => Some(5), + _ => None, + } +} + +fn finer_unit(left: TimeUnit, right: TimeUnit) -> TimeUnit { + let rank = |unit: TimeUnit| match unit { + TimeUnit::Second => 0, + TimeUnit::Millisecond => 1, + TimeUnit::Microsecond => 2, + TimeUnit::Nanosecond => 3, + }; + if rank(left) >= rank(right) { + left + } else { + right + } +} + +/// The name Spark's `toSQLType` gives the Spark type that an Arrow type maps to, for error +/// messages. Arrow types without a Spark counterpart use their Arrow name. +fn spark_sql_type(data_type: &DataType) -> String { + use DataType::*; + match data_type { + Null => "VOID".to_string(), + Boolean => "BOOLEAN".to_string(), + Int8 => "TINYINT".to_string(), + Int16 => "SMALLINT".to_string(), + Int32 => "INT".to_string(), + Int64 => "BIGINT".to_string(), + Float32 => "FLOAT".to_string(), + Float64 => "DOUBLE".to_string(), + Decimal128(p, s) => format!("DECIMAL({p},{s})"), + Utf8 | LargeUtf8 | Utf8View => "STRING".to_string(), + Binary | LargeBinary | BinaryView => "BINARY".to_string(), + Date32 => "DATE".to_string(), + Timestamp(_, Some(_)) => "TIMESTAMP".to_string(), + Timestamp(_, None) => "TIMESTAMP_NTZ".to_string(), + List(field) | LargeList(field) | FixedSizeList(field, _) => { + format!("ARRAY<{}>", spark_sql_type(field.data_type())) + } + Struct(fields) => format!( + "STRUCT<{}>", + fields + .iter() + .map(|f| format!( + "{}: {}{}", + f.name(), + spark_sql_type(f.data_type()), + if f.is_nullable() { "" } else { " NOT NULL" } + )) + .collect::>() + .join(", ") + ), + Map(entries, _) => match entries.data_type() { + Struct(kv) if kv.len() == 2 => format!( + "MAP<{}, {}>", + spark_sql_type(kv[0].data_type()), + spark_sql_type(kv[1].data_type()) + ), + other => format!("MAP<{other}>"), + }, + other => other.to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{ + AsArray, Float32Array, Float64Array, Int32Array, ListArray, StringArray, + StructArray, + }; + use arrow::buffer::OffsetBuffer; + use arrow::datatypes::{Float64Type, Int32Type}; + + /// Values on which Spark's ordering and IEEE 754 total order disagree, with neighbors and a + /// NULL. `NEGATIVE_NAN` is what arithmetic produces on x86-64. + const NEGATIVE_NAN: f64 = f64::from_bits(0xfff8_0000_0000_0000); + const PAYLOAD_NAN: f64 = f64::from_bits(0x7ff0_0000_0000_0001); + const EDGE_VALUES: [Option; 10] = [ + Some(f64::NEG_INFINITY), + Some(-1.0), + Some(-0.0), + Some(0.0), + Some(1.0), + Some(f64::INFINITY), + Some(f64::NAN), + Some(NEGATIVE_NAN), + Some(PAYLOAD_NAN), + None, + ]; + + /// `SQLOrderingUtil.compareDoubles`. + fn compare_doubles(left: f64, right: f64) -> std::cmp::Ordering { + use std::cmp::Ordering::*; + match (left.is_nan(), right.is_nan()) { + (true, true) => Equal, + (true, false) => Greater, + (false, true) => Less, + _ => left.partial_cmp(&right).unwrap(), + } + } + + /// Spark's `Greatest.eval` and `Least.eval`, over doubles. + fn spark_extreme(values: &[Option], greatest: bool) -> Option { + values + .iter() + .flatten() + .fold(None, |best, &value| match best { + None => Some(value), + Some(best) => { + let ordering = compare_doubles(value, best); + let replace = if greatest { + ordering.is_gt() + } else { + ordering.is_lt() + }; + Some(if replace { value } else { best }) + } + }) + } + + fn udf(greatest: bool) -> GreatestLeast { + GreatestLeast::new(greatest, true) + } + + fn invoke( + greatest: bool, + args: Vec, + rows: usize, + return_type: DataType, + ) -> Result { + let return_field: FieldRef = Arc::new(Field::new("result", return_type, true)); + udf(greatest).invoke(ScalarFunctionArgs { + args, + arg_fields: vec![], + number_rows: rows, + return_field, + config_options: Arc::new(ConfigOptions::default()), + }) + } + + fn bits(array: &ArrayRef) -> Vec> { + let array = array.as_primitive::(); + (0..array.len()) + .map(|i| array.is_valid(i).then(|| array.value(i).to_bits())) + .collect() + } + + /// Every combination of three edge values, as columns and with the middle argument also as + /// a constant. The combinations without a NULL run again on their own. + #[test] + fn floats_follow_spark_order() -> Result<()> { + let all_rows: Vec<[Option; 3]> = EDGE_VALUES + .iter() + .flat_map(|&a| { + EDGE_VALUES + .iter() + .flat_map(move |&b| EDGE_VALUES.iter().map(move |&c| [a, b, c])) + }) + .collect(); + let without_nulls: Vec<_> = all_rows + .iter() + .copied() + .filter(|row| row.iter().all(Option::is_some)) + .collect(); + for rows in [all_rows, without_nulls] { + let column = |i: usize| -> ColumnarValue { + ColumnarValue::Array(Arc::new(Float64Array::from( + rows.iter().map(|row| row[i]).collect::>(), + ))) + }; + for greatest in [true, false] { + let args = vec![column(0), column(1), column(2)]; + let result = invoke(greatest, args, rows.len(), DataType::Float64)? + .into_array(rows.len())?; + let expected: Vec> = rows + .iter() + .map(|row| spark_extreme(row, greatest).map(f64::to_bits)) + .collect(); + assert_eq!(bits(&result), expected, "greatest={greatest}"); + + for middle in EDGE_VALUES { + let args = vec![ + column(0), + ColumnarValue::Scalar(ScalarValue::Float64(middle)), + column(2), + ]; + let result = invoke(greatest, args, rows.len(), DataType::Float64)? + .into_array(rows.len())?; + let expected: Vec> = rows + .iter() + .map(|row| { + spark_extreme(&[row[0], middle, row[2]], greatest) + .map(f64::to_bits) + }) + .collect(); + assert_eq!( + bits(&result), + expected, + "middle={middle:?} greatest={greatest}" + ); + } + } + } + Ok(()) + } + + #[test] + fn float32_ties_keep_the_first_argument() -> Result<()> { + let zero = ColumnarValue::Array(Arc::new(Float32Array::from(vec![0.0f32]))); + let negative_zero = + ColumnarValue::Array(Arc::new(Float32Array::from(vec![-0.0f32]))); + for greatest in [true, false] { + for (args, expected) in [ + (vec![negative_zero.clone(), zero.clone()], -0.0f32), + (vec![zero.clone(), negative_zero.clone()], 0.0f32), + ] { + let result = + invoke(greatest, args, 1, DataType::Float32)?.into_array(1)?; + let value = result + .as_primitive::() + .value(0); + assert_eq!(value.to_bits(), expected.to_bits(), "greatest={greatest}"); + } + } + Ok(()) + } + + #[test] + fn constants_only_return_a_constant() -> Result<()> { + let scalar = |v: Option| ColumnarValue::Scalar(ScalarValue::Float64(v)); + for (greatest, args, expected) in [ + ( + true, + vec![scalar(Some(-0.0)), scalar(Some(0.0))], + Some(-0.0), + ), + ( + false, + vec![scalar(Some(0.0)), scalar(Some(-0.0))], + Some(0.0), + ), + ( + true, + vec![ + scalar(None), + scalar(Some(f64::NAN)), + scalar(Some(f64::INFINITY)), + ], + Some(f64::NAN), + ), + (false, vec![scalar(None), scalar(None)], None), + ] { + match invoke(greatest, args, 5, DataType::Float64)? { + ColumnarValue::Scalar(ScalarValue::Float64(actual)) => { + assert_eq!(actual.map(f64::to_bits), expected.map(f64::to_bits)) + } + other => panic!("expected a double scalar, got {other:?}"), + } + } + Ok(()) + } + + #[test] + fn integers_and_strings_with_nulls() -> Result<()> { + let ints = + |v: Vec>| ColumnarValue::Array(Arc::new(Int32Array::from(v))); + let args = vec![ + ints(vec![Some(1), None, None, Some(5)]), + ColumnarValue::Scalar(ScalarValue::Int32(Some(3))), + ints(vec![Some(2), Some(7), None, None]), + ]; + let result = invoke(true, args.clone(), 4, DataType::Int32)?.into_array(4)?; + assert_eq!( + result.as_ref(), + &Int32Array::from(vec![Some(3), Some(7), Some(3), Some(5)]) + ); + let result = invoke(false, args, 4, DataType::Int32)?.into_array(4)?; + assert_eq!( + result.as_ref(), + &Int32Array::from(vec![Some(1), Some(3), Some(3), Some(3)]) + ); + + let strings = + |v: Vec>| ColumnarValue::Array(Arc::new(StringArray::from(v))); + let args = vec![ + strings(vec![Some("abc"), None, Some("")]), + strings(vec![Some("abd"), None, Some("a")]), + ]; + let result = invoke(true, args.clone(), 3, DataType::Utf8)?.into_array(3)?; + assert_eq!( + result.as_ref(), + &StringArray::from(vec![Some("abd"), None, Some("a")]) + ); + let result = invoke(false, args, 3, DataType::Utf8)?.into_array(3)?; + assert_eq!( + result.as_ref(), + &StringArray::from(vec![Some("abc"), None, Some("")]) + ); + Ok(()) + } + + /// A NULL inside a list ranks first, lists compare element by element and then by length, + /// and floats inside lists follow Spark's order. The second list's element field has a + /// different name and nullability, which the result type decides. + #[test] + fn nested_arguments() -> Result<()> { + let list = |values: Vec>>>| -> ColumnarValue { + ColumnarValue::Array(Arc::new(ListArray::from_iter_primitive::< + Int32Type, + _, + _, + >(values))) + }; + let args = vec![ + list(vec![ + Some(vec![Some(1), None]), + Some(vec![Some(1), Some(2)]), + None, + ]), + list(vec![ + Some(vec![Some(1), Some(0)]), + Some(vec![Some(1), Some(2), Some(3)]), + None, + ]), + ]; + let list_type = args[0].data_type(); + for (greatest, expected) in [ + ( + true, + list(vec![ + Some(vec![Some(1), Some(0)]), + Some(vec![Some(1), Some(2), Some(3)]), + None, + ]), + ), + ( + false, + list(vec![ + Some(vec![Some(1), None]), + Some(vec![Some(1), Some(2)]), + None, + ]), + ), + ] { + let result = + invoke(greatest, args.clone(), 3, list_type.clone())?.into_array(3)?; + assert_eq!(result.as_ref(), expected.into_array(3)?.as_ref()); + } + + let float_list = + |name: &str, nullable: bool, values: Vec| -> ColumnarValue { + let field = Arc::new(Field::new(name, DataType::Float64, nullable)); + ColumnarValue::Array(Arc::new(ListArray::new( + field, + OffsetBuffer::from_lengths(vec![1; values.len()]), + Arc::new(Float64Array::from(values)), + None, + ))) + }; + let return_type = + DataType::List(Arc::new(Field::new("item", DataType::Float64, true))); + for (greatest, expected) in + [(true, [-0.0, f64::NAN]), (false, [-0.0, f64::INFINITY])] + { + let args = vec![ + float_list("item", true, vec![-0.0, f64::INFINITY]), + float_list("element", false, vec![0.0, f64::NAN]), + ]; + let result = invoke(greatest, args, 2, return_type.clone())?.into_array(2)?; + assert_eq!(result.data_type(), &return_type); + let values: ArrayRef = Arc::clone(result.as_list::().values()); + let expected: Vec> = + expected.iter().map(|v| Some(v.to_bits())).collect(); + assert_eq!(bits(&values), expected, "greatest={greatest}"); + } + + // A constant struct against a column of structs. + let fields = Fields::from(vec![Field::new("a", DataType::Float64, true)]); + let structs = |values: Vec>| -> ArrayRef { + Arc::new(StructArray::new( + fields.clone(), + vec![Arc::new(Float64Array::from(values))], + None, + )) + }; + let constant = ScalarValue::try_from_array(&structs(vec![Some(f64::NAN)]), 0)?; + let args = vec![ + ColumnarValue::Array(structs(vec![Some(1.0), None])), + ColumnarValue::Scalar(constant), + ]; + let struct_type = DataType::Struct(fields.clone()); + let result = invoke(true, args.clone(), 2, struct_type.clone())?.into_array(2)?; + let a: ArrayRef = Arc::clone(result.as_struct().column(0)); + assert_eq!( + bits(&a), + vec![Some(f64::NAN.to_bits()), Some(f64::NAN.to_bits())] + ); + let result = invoke(false, args, 2, struct_type)?.into_array(2)?; + let a: ArrayRef = Arc::clone(result.as_struct().column(0)); + // A NULL field ranks first. + assert_eq!(bits(&a), vec![Some(1.0f64.to_bits()), None]); + Ok(()) + } + + #[test] + fn null_type() -> Result<()> { + let args = vec![ + ColumnarValue::Scalar(ScalarValue::Null), + ColumnarValue::Scalar(ScalarValue::Null), + ]; + assert!(matches!( + invoke(true, args, 3, DataType::Null)?, + ColumnarValue::Scalar(ScalarValue::Null) + )); + Ok(()) + } + + #[test] + fn rejects_fewer_than_two_arguments() { + let scalar = ColumnarValue::Scalar(ScalarValue::Float64(Some(1.0))); + for greatest in [true, false] { + for arg_types in [vec![], vec![DataType::Float64]] { + let error = udf(greatest).coerce_types(&arg_types).unwrap_err(); + assert!( + error + .to_string() + .contains("[WRONG_NUM_ARGS.WITHOUT_SUGGESTION]"), + "unexpected error: {error}" + ); + } + let error = + invoke(greatest, vec![scalar.clone()], 1, DataType::Float64).unwrap_err(); + assert!( + error.to_string().contains("requires > 1 parameters"), + "unexpected error: {error}" + ); + } + } + + #[test] + fn coercion() { + use DataType::*; + let coerce = |types: Vec, ansi: bool| { + GreatestLeast::new(true, ansi) + .coerce_types(&types) + .map(|t| t[0].clone()) + }; + let list = |t: DataType| List(Arc::new(Field::new("item", t, true))); + let ok = [ + (vec![Int32, Float64], true, Float64), + (vec![Int32, Float32], true, Float64), + (vec![Int64, Float32], false, Float32), + (vec![Int8, Int16], true, Int16), + (vec![Null, Int32, Null], true, Int32), + (vec![Null, Null], true, Null), + (vec![Decimal128(11, 1), Int32], true, Decimal128(11, 1)), + (vec![Decimal128(2, 1), Int32], true, Decimal128(11, 1)), + (vec![Decimal128(5, 2), Int64], true, Decimal128(22, 2)), + ( + vec![Decimal128(38, 10), Decimal128(38, 0)], + true, + Decimal128(38, 0), + ), + (vec![Decimal128(10, 2), Float32], true, Float64), + (vec![Utf8, Utf8View], true, Utf8View), + ( + vec![Date32, Timestamp(TimeUnit::Microsecond, Some("UTC".into()))], + true, + Timestamp(TimeUnit::Microsecond, Some("UTC".into())), + ), + (vec![list(Int32), list(Float64)], true, list(Float64)), + ]; + for (types, ansi, expected) in ok { + assert_eq!(coerce(types.clone(), ansi).unwrap(), expected, "{types:?}"); + } + let diff_types = [ + vec![Int32, Utf8], + vec![Date32, Utf8], + vec![Boolean, Int32], + vec![list(Int32), Int32], + ]; + for types in diff_types { + let error = coerce(types.clone(), true).unwrap_err().to_string(); + assert!( + error.contains("[DATATYPE_MISMATCH.DATA_DIFF_TYPES]"), + "{types:?}: {error}" + ); + } + let error = coerce(vec![Int32, Utf8], true).unwrap_err().to_string(); + assert!(error.contains(r#"but it's ["INT", "STRING"]"#), "{error}"); + + let map = Map( + Arc::new(Field::new( + "entries", + Struct(Fields::from(vec![ + Field::new("key", Int32, false), + Field::new("value", Int32, true), + ])), + false, + )), + false, + ); + let error = coerce(vec![map.clone(), map], true) + .unwrap_err() + .to_string(); + assert!( + error.contains(r#"[DATATYPE_MISMATCH.INVALID_ORDERING_TYPE] The `greatest` does not support ordering on type "MAP""#), + "{error}" + ); + } +} diff --git a/crates/spark/src/function/math/mod.rs b/crates/spark/src/function/math/mod.rs index 402a9d1a..2631d453 100644 --- a/crates/spark/src/function/math/mod.rs +++ b/crates/spark/src/function/math/mod.rs @@ -11,6 +11,7 @@ pub mod ceil; pub mod expm1; pub mod factorial; pub mod floor; +pub mod greatest_least; pub mod hex; pub mod hypot; pub mod modulus; @@ -32,6 +33,8 @@ make_udf_function!(ceil::SparkCeil, ceil); make_udf_function!(expm1::SparkExpm1, expm1); make_udf_function!(factorial::SparkFactorial, factorial); make_udf_function!(floor::SparkFloor, floor); +make_udf_function!(greatest_least::SparkGreatest, greatest); +make_udf_function!(greatest_least::SparkLeast, least); make_udf_function!(hex::SparkHex, hex); make_udf_function!(hypot::SparkHypot, hypot); make_udf_function!(modulus::SparkMod, modulus); @@ -59,6 +62,16 @@ pub mod expr_fn { arg1 )); export_functions!((floor, "Returns floor of expr.", arg1)); + export_functions!(( + greatest, + "Returns the greatest value of all arguments, skipping nulls.", + args, + )); + export_functions!(( + least, + "Returns the least value of all arguments, skipping nulls.", + args, + )); export_functions!((hex, "Computes hex value of the given column.", arg1)); export_functions!((hypot, "Returns sqrt(a^2 + b^2) without intermediate overflow or underflow.", arg1 arg2)); export_functions!((modulus, "Returns the remainder of division of the first argument by the second argument.", arg1 arg2)); @@ -102,6 +115,8 @@ pub fn functions() -> Vec> { expm1(), factorial(), floor(), + greatest(), + least(), hex(), hypot(), modulus(), diff --git a/crates/sqllogictest/test_files/math/greatest.slt b/crates/sqllogictest/test_files/math/greatest.slt index 48b11a8d..c6c5daba 100644 --- a/crates/sqllogictest/test_files/math/greatest.slt +++ b/crates/sqllogictest/test_files/math/greatest.slt @@ -10,5 +10,233 @@ ## Original Query: SELECT greatest(10, 9, 2, 4, 3); ## PySpark 3.5.5 Result: {'greatest(10, 9, 2, 4, 3)': 10, 'typeof(greatest(10, 9, 2, 4, 3))': 'int', 'typeof(10)': 'int', 'typeof(9)': 'int', 'typeof(2)': 'int', 'typeof(4)': 'int', 'typeof(3)': 'int'} -#query -#SELECT greatest(10::int, 9::int, 2::int, 4::int, 3::int); +query I +SELECT greatest(10::int, 9::int, 2::int, 4::int, 3::int); +---- +10 + +# NULLs are skipped; the result is NULL only if every argument is NULL +query III +SELECT greatest(1::int, NULL::int, 3::int), greatest(NULL::int, 2::int), greatest(NULL::int, NULL::int); +---- +3 2 NULL + +query ? +SELECT greatest(NULL, NULL); +---- +NULL + +# Integers widen to the widest integer type +query I +SELECT greatest(1::tinyint, 2::smallint); +---- +2 + +query T +SELECT arrow_typeof(greatest(1::tinyint, 2::smallint)); +---- +Int16 + +query I +SELECT greatest(1::int, 3000000000::bigint); +---- +3000000000 + +# An integer and a DOUBLE widen to DOUBLE +query R +SELECT greatest(1::int, 2.5::double); +---- +2.5 + +query T +SELECT arrow_typeof(greatest(1::int, 2.5::double)); +---- +Float64 + +# NaN is greater than every other value, including Infinity +query RRR +SELECT greatest('NaN'::double, 'Infinity'::double), greatest(1::double, 'NaN'::double), greatest('-Infinity'::double, NULL::double); +---- +NaN NaN -Infinity + +query R +SELECT greatest('NaN'::float, 1::float); +---- +NaN + +# -0.0 equals 0.0 and the first of equal values wins. The sign of a zero shows in its string +# form, which starts with '-' only for -0.0. +query BB +SELECT CAST(greatest('-0.0'::double, 0.0::double) AS STRING) LIKE '-%', CAST(greatest(0.0::double, '-0.0'::double) AS STRING) LIKE '-%'; +---- +true false + +# Decimals +query R +SELECT greatest(1.5::decimal(2,1), 2::int); +---- +2.0 + +query T +SELECT arrow_typeof(greatest(1.5::decimal(2,1), 2::int)); +---- +Decimal128(11, 1) + +query T +SELECT arrow_typeof(greatest(1::decimal(5,2), 1::bigint)); +---- +Decimal128(22, 2) + +query T +SELECT arrow_typeof(greatest(1::decimal(38,10), 1::decimal(38,0))); +---- +Decimal128(38, 0) + +# The fractional digits that do not fit are rounded away +query R +SELECT greatest(1.5::decimal(38,10), 1::decimal(38,0)); +---- +2 + +query R +SELECT greatest(1::decimal(38,10), '12345678901234567890123456789'::decimal(38,0)); +---- +12345678901234567890123456789 + +# A decimal and a DOUBLE widen to DOUBLE +query R +SELECT greatest(1.5::decimal(2,1), 2.5::double); +---- +2.5 + +query T +SELECT arrow_typeof(greatest(1.5::decimal(2,1), 2.5::double)); +---- +Float64 + +# Strings +query TTT +SELECT greatest('abc', 'abd', 'ab'), greatest('a', 'b', NULL), greatest('', 'a'); +---- +abd b a + +# Booleans +query B +SELECT greatest(true, false); +---- +true + +# Binary +query ? +SELECT greatest(X'01', X'0102'); +---- +0102 + +# Dates and timestamps; a date and a timestamp widen to the timestamp +query D +SELECT greatest(DATE '2024-01-01', DATE '2023-12-31', NULL::date); +---- +2024-01-01 + +query P +SELECT greatest(DATE '2024-01-01', TIMESTAMP '2023-01-01 10:00:00'); +---- +2024-01-01T00:00:00 + +query P +SELECT greatest(TIMESTAMP '2024-01-01 10:00:00', TIMESTAMP '2024-01-01 10:00:01'); +---- +2024-01-01T10:00:01 + +# Arrays compare element by element and then by length, a NULL element is smaller than any +# value, and elements widen +query ?? +SELECT greatest(array(1, 2), array(1, 2, 3)), greatest(array(1, NULL), array(1, 0)); +---- +[1, 2, 3] [1, 0] + +query ? +SELECT greatest(array(1), array(1.5::double)); +---- +[1.5] + +query ? +SELECT greatest(array('NaN'::double), array(1::double)); +---- +[NaN] + +# Structs compare field by field +query ? +SELECT greatest(named_struct('a', 1, 'b', 2), named_struct('a', 1, 'b', 3)); +---- +{a: 1, b: 3} + +# Columns +statement ok +CREATE TABLE t AS VALUES + (1::int, 2::int, 'NaN'::double, 1::double, 'b'), + (5::int, NULL::int, -1::double, NULL::double, 'a'), + (NULL::int, NULL::int, NULL::double, NULL::double, NULL), + (NULL::int, 7::int, 'Infinity'::double, 'NaN'::double, 'c'); + +query IIRRT +SELECT greatest(column1, column2), greatest(column1, 3::int, column2), greatest(column3, column4), greatest(column3, 0::double), greatest(column5, 'b') FROM t; +---- +2 3 NaN NaN b +5 5 -1 0 b +NULL 3 NULL 0 b +7 7 NaN Infinity c + +statement ok +DROP TABLE t; + +# Errors + +# At least two arguments +query error \[WRONG_NUM_ARGS\.WITHOUT_SUGGESTION\] The `greatest` requires > 1 parameters but the actual number is 1 +SELECT greatest(1::int); + +query error +SELECT greatest(); + +# Strings are not promoted +query error \[DATATYPE_MISMATCH\.DATA_DIFF_TYPES\] Input to `greatest` should all be the same type, but it's \["INT", "STRING"\] +SELECT greatest(1::int, 'a'); + +query error \[DATATYPE_MISMATCH\.DATA_DIFF_TYPES\] Input to `greatest` should all be the same type, but it's \["DATE", "STRING"\] +SELECT greatest(DATE '2024-01-01', '2025-01-01'); + +query error \[DATATYPE_MISMATCH\.DATA_DIFF_TYPES\] Input to `greatest` should all be the same type, but it's \["BOOLEAN", "INT"\] +SELECT greatest(true, 1::int); + +query error \[DATATYPE_MISMATCH\.DATA_DIFF_TYPES\] +SELECT greatest(array(1), 1::int); + +query error \[DATATYPE_MISMATCH\.DATA_DIFF_TYPES\] +SELECT greatest(named_struct('a', 1), named_struct('b', 1)); + +# Maps cannot be ordered +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `greatest` does not support ordering on type "MAP" +SELECT greatest(map(1::int, 2::int), map(1::int, 3::int)); + +# An integer and a FLOAT widen to FLOAT with ANSI mode off, and to DOUBLE with it on +query T +SELECT arrow_typeof(greatest(1::int, 2.5::float)); +---- +Float32 + +statement ok +set datafusion.execution.enable_ansi_mode = true; + +query T +SELECT arrow_typeof(greatest(1::int, 2.5::float)); +---- +Float64 + +query R +SELECT greatest(1::bigint, 2.5::float); +---- +2.5 + +statement ok +set datafusion.execution.enable_ansi_mode = false; diff --git a/crates/sqllogictest/test_files/math/least.slt b/crates/sqllogictest/test_files/math/least.slt index db0a890a..ec563ba6 100644 --- a/crates/sqllogictest/test_files/math/least.slt +++ b/crates/sqllogictest/test_files/math/least.slt @@ -10,5 +10,187 @@ ## Original Query: SELECT least(10, 9, 2, 4, 3); ## PySpark 3.5.5 Result: {'least(10, 9, 2, 4, 3)': 2, 'typeof(least(10, 9, 2, 4, 3))': 'int', 'typeof(10)': 'int', 'typeof(9)': 'int', 'typeof(2)': 'int', 'typeof(4)': 'int', 'typeof(3)': 'int'} -#query -#SELECT least(10::int, 9::int, 2::int, 4::int, 3::int); +query I +SELECT least(10::int, 9::int, 2::int, 4::int, 3::int); +---- +2 + +# NULLs are skipped; the result is NULL only if every argument is NULL +query III +SELECT least(NULL::int, 2::int, NULL::int), least(3::int, NULL::int, 1::int), least(NULL::int, NULL::int); +---- +2 1 NULL + +query ? +SELECT least(NULL, NULL); +---- +NULL + +# Integers widen to the widest integer type +query I +SELECT least(-1::tinyint, 2::bigint); +---- +-1 + +query T +SELECT arrow_typeof(least(-1::tinyint, 2::bigint)); +---- +Int64 + +# An integer and a DOUBLE widen to DOUBLE +query R +SELECT least(3::int, 2.5::double); +---- +2.5 + +# NaN is greater than every other value, so least returns it only if every argument is NaN or NULL +query RRRR +SELECT least('NaN'::double, 'Infinity'::double), least('NaN'::double, 1::double), least('NaN'::double, NULL::double), least('-Infinity'::double, 'NaN'::double); +---- +Infinity 1 NaN -Infinity + +query R +SELECT least('NaN'::float, 1::float); +---- +1 + +# -0.0 equals 0.0 and the first of equal values wins. The sign of a zero shows in its string +# form, which starts with '-' only for -0.0. +query BB +SELECT CAST(least('-0.0'::double, 0.0::double) AS STRING) LIKE '-%', CAST(least(0.0::double, '-0.0'::double) AS STRING) LIKE '-%'; +---- +true false + +# Decimals +query R +SELECT least(1.5::decimal(2,1), 2::int); +---- +1.5 + +query T +SELECT arrow_typeof(least(1.5::decimal(2,1), 2::int)); +---- +Decimal128(11, 1) + +query R +SELECT least(1.25::decimal(3,2), 1.3::decimal(2,1), NULL::decimal(10,0)); +---- +1.25 + +query R +SELECT least(1.5::decimal(2,1), 0.5::double); +---- +0.5 + +# Strings +query TTT +SELECT least('abc', 'abd', 'ab'), least('b', NULL, 'a'), least('', 'a'); +---- +ab a (empty) + +# Booleans +query B +SELECT least(true, false); +---- +false + +# Binary +query ? +SELECT least(X'01', X'0102'); +---- +01 + +# Dates and timestamps; a date and a timestamp widen to the timestamp +query D +SELECT least(DATE '2024-01-01', DATE '2023-12-31', NULL::date); +---- +2023-12-31 + +query P +SELECT least(DATE '2024-01-01', TIMESTAMP '2023-01-01 10:00:00'); +---- +2023-01-01T10:00:00 + +# Arrays compare element by element and then by length, and a NULL element is smaller than any +# value +query ?? +SELECT least(array(1, 2), array(1, 2, 3)), least(array(1, NULL), array(1, 0)); +---- +[1, 2] [1, NULL] + +query ? +SELECT least(array('NaN'::double), array(1::double)); +---- +[1.0] + +# Structs compare field by field, and a NULL field is smaller than any value +query ? +SELECT least(named_struct('a', 1, 'b', 2), named_struct('a', 1, 'b', 3)); +---- +{a: 1, b: 2} + +query ? +SELECT least(named_struct('a', 1, 'b', 2), named_struct('a', 1, 'b', NULL::int)); +---- +{a: 1, b: NULL} + +# Columns +statement ok +CREATE TABLE t AS VALUES + (1::int, 2::int, 'NaN'::double, 1::double, 'b'), + (5::int, NULL::int, -1::double, NULL::double, 'a'), + (NULL::int, NULL::int, NULL::double, NULL::double, NULL), + (NULL::int, 7::int, 'Infinity'::double, 'NaN'::double, 'c'); + +query IIRRT +SELECT least(column1, column2), least(column1, 3::int, column2), least(column3, column4), least(column3, 0::double), least(column5, 'b') FROM t; +---- +1 1 1 0 b +5 3 -1 -1 a +NULL 3 NULL 0 b +7 3 Infinity 0 b + +statement ok +DROP TABLE t; + +# Errors + +# At least two arguments +query error \[WRONG_NUM_ARGS\.WITHOUT_SUGGESTION\] The `least` requires > 1 parameters but the actual number is 1 +SELECT least(1::int); + +query error +SELECT least(); + +# Strings are not promoted +query error \[DATATYPE_MISMATCH\.DATA_DIFF_TYPES\] Input to `least` should all be the same type, but it's \["INT", "STRING"\] +SELECT least(1::int, '2'); + +query error \[DATATYPE_MISMATCH\.DATA_DIFF_TYPES\] +SELECT least(1::int, true); + +# Maps cannot be ordered +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `least` does not support ordering on type "MAP" +SELECT least(map(1::int, 2::int), map(1::int, 3::int)); + +# An integer and a FLOAT widen to FLOAT with ANSI mode off, and to DOUBLE with it on +query T +SELECT arrow_typeof(least(1::int, 2.5::float)); +---- +Float32 + +statement ok +set datafusion.execution.enable_ansi_mode = true; + +query T +SELECT arrow_typeof(least(1::int, 2.5::float)); +---- +Float64 + +query R +SELECT least(3::bigint, 2.5::float); +---- +2.5 + +statement ok +set datafusion.execution.enable_ansi_mode = false; From 01d8874a489db5b48b98f422b0c002c6825e9054 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 10:17:23 -0700 Subject: [PATCH 02/10] refactor(spark): share Spark's common-type rules between functions Move tightest_common_type from array_position into function/type_coercion, with an ANSI flag, a findTypeForComplex that widens map keys and values, Spark's isOrderable (which also rejects INTERVAL), and Spark type names for error messages. Move float_normalize up to function/ so functions outside array can use it. array_position now reports a map with different key or value types as INVALID_ORDERING_TYPE, rejects INTERVAL elements, and names Spark types and the SQLSTATE in its errors, as Spark does. --- .../src/function/array/array_contains.rs | 2 +- .../src/function/array/array_position.rs | 178 ++-------- crates/spark/src/function/array/mod.rs | 1 - .../function/{array => }/float_normalize.rs | 8 +- crates/spark/src/function/mod.rs | 2 + crates/spark/src/function/type_coercion.rs | 327 ++++++++++++++++++ .../test_files/array/array_position.slt | 15 +- 7 files changed, 382 insertions(+), 151 deletions(-) rename crates/spark/src/function/{array => }/float_normalize.rs (97%) create mode 100644 crates/spark/src/function/type_coercion.rs diff --git a/crates/spark/src/function/array/array_contains.rs b/crates/spark/src/function/array/array_contains.rs index 3802667e..029e6c09 100644 --- a/crates/spark/src/function/array/array_contains.rs +++ b/crates/spark/src/function/array/array_contains.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use super::float_normalize::normalize_floats; +use crate::function::float_normalize::normalize_floats; use arrow::array::{ Array, AsArray, BooleanArray, BooleanBufferBuilder, GenericListArray, OffsetSizeTrait, }; diff --git a/crates/spark/src/function/array/array_position.rs b/crates/spark/src/function/array/array_position.rs index 4ca52da9..d1a64701 100644 --- a/crates/spark/src/function/array/array_position.rs +++ b/crates/spark/src/function/array/array_position.rs @@ -2,13 +2,16 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use super::float_normalize::normalize_array; +use crate::function::float_normalize::normalize_array; +use crate::function::type_coercion::{ + is_orderable, spark_sql_type, tightest_common_type, +}; use arrow::array::{ Array, ArrayRef, AsArray, Int64Array, OffsetSizeTrait, make_comparator, }; use arrow::buffer::{NullBuffer, ScalarBuffer}; use arrow::compute::SortOptions; -use arrow::datatypes::{DataType, Field, Fields, TimeUnit}; +use arrow::datatypes::{DataType, Field}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, exec_err, plan_err}; use datafusion_expr::{ @@ -36,7 +39,7 @@ use std::sync::Arc; a date widens to a timestamp. Other combinations are an error, for example a string \ and a number, two different decimals, or structs whose field names or field types \ differ, as in Spark with ANSI mode off (with ANSI mode on, Spark widens struct field \ - types). A NULL-typed array and map elements are also an error.", + types). A NULL-typed array, and map or INTERVAL elements at any depth, are also an error.", syntax_example = "array_position(array, element)", sql_example = r#"```sql > SELECT array_position(array(3, 2, 1), 1); @@ -164,24 +167,31 @@ fn coerce_array_and_element( other => { return plan_err!( "[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] The first parameter of `{name}` \ - requires the \"ARRAY\" type, however it has the type {other}" + requires the \"ARRAY\" type, however it has the type \"{}\". SQLSTATE: 42K09", + spark_sql_type(other) ); } }; - let Some(common) = tightest_common_type(item.data_type(), element) else { + // Spark's `ArrayPosition` calls the non-ANSI `TypeCoercion.findTightestCommonType`. + let common = tightest_common_type(item.data_type(), element, false) + .filter(|_| struct_field_types_match(item.data_type(), element)); + let Some(common) = common else { return plan_err!( "[DATATYPE_MISMATCH.ARRAY_FUNCTION_DIFF_TYPES] Input to `{name}` should have \ been \"ARRAY\" followed by a value with same element type, but it's \ - [{array}, {element}]" + [\"{}\", \"{}\"]. SQLSTATE: 42K09", + spark_sql_type(array), + spark_sql_type(element) ); }; if common.is_null() { return null_type_err(name); } - if contains_map(&common) { + if !is_orderable(&common) { return plan_err!( "[DATATYPE_MISMATCH.INVALID_ORDERING_TYPE] The `{name}` does not support \ - ordering on type {common}" + ordering on type \"{}\". SQLSTATE: 42K09", + spark_sql_type(&common) ); } let item = Arc::new(Field::new(item.name(), common.clone(), true)); @@ -196,82 +206,30 @@ fn coerce_array_and_element( fn null_type_err(name: &str) -> Result { plan_err!( "[DATATYPE_MISMATCH.NULL_TYPE] Null typed values cannot be used as arguments of \ - `{name}`" + `{name}`. SQLSTATE: 42K09" ) } -/// Spark's `TypeCoercion.findTightestCommonType`, on the arrow types that Spark's types -/// map to. -fn tightest_common_type(left: &DataType, right: &DataType) -> Option { +/// With ANSI mode off, Spark cannot implicitly cast between structs whose field types +/// differ, so `array_position` rejects them although their tightest common type exists. +/// With ANSI mode on, Spark widens the field types. +fn struct_field_types_match(left: &DataType, right: &DataType) -> bool { use DataType::*; - if left == right { - return Some(left.clone()); - } - if let (Some(l), Some(r)) = (numeric_precedence(left), numeric_precedence(right)) { - return Some(if l >= r { left.clone() } else { right.clone() }); - } match (left, right) { - (Null, other) | (other, Null) => Some(other.clone()), - // Spark has a single string type and a single binary type. - (l, r) if is_string(l) && is_string(r) => Some(l.clone()), - (l, r) if is_binary(l) && is_binary(r) => Some(l.clone()), - // The decimal must be at least as wide as the integral type's DECIMAL(n, 0). - (int, Decimal128(p, s)) | (Decimal128(p, s), int) => { - let digits = integral_digits(int)?; - (*p as i16 - *s as i16 >= digits).then_some(Decimal128(*p, *s)) - } - (Date32, ts @ Timestamp(_, _)) | (ts @ Timestamp(_, _), Date32) => { - Some(ts.clone()) - } - // TIMESTAMP_NTZ and TIMESTAMP widen to TIMESTAMP. - (Timestamp(lu, ltz), Timestamp(ru, rtz)) => Some(Timestamp( - finer_unit(*lu, *ru), - ltz.clone().or_else(|| rtz.clone()), - )), ( - List(lf) | LargeList(lf) | FixedSizeList(lf, _), - List(rf) | LargeList(rf) | FixedSizeList(rf, _), - ) => { - let item = tightest_common_type(lf.data_type(), rf.data_type())?; - let field = Arc::new(Field::new(lf.name(), item, true)); - Some( - if matches!(left, LargeList(_)) || matches!(right, LargeList(_)) { - LargeList(field) - } else { - List(field) - }, - ) - } - // The fields must have the same names and types, ignoring nullability, in the same - // order. This follows Spark with ANSI mode off; with ANSI mode on, Spark also widens - // the field types. Spark compares the names ignoring case, but DataFusion cannot cast - // between structs whose field names differ only in case, so they must match exactly. - (Struct(lf), Struct(rf)) if lf.len() == rf.len() => { - let fields = lf + List(l) | LargeList(l) | FixedSizeList(l, _), + List(r) | LargeList(r) | FixedSizeList(r, _), + ) => struct_field_types_match(l.data_type(), r.data_type()), + // The entries are a struct of the key and the value, which are checked on their own. + (Map(l, _), Map(r, _)) => match (l.data_type(), r.data_type()) { + (Struct(l), Struct(r)) => l .iter() - .zip(rf.iter()) - .map(|(a, b)| { - if a.name() != b.name() || !same_type(a.data_type(), b.data_type()) { - return None; - } - let data_type = tightest_common_type(a.data_type(), b.data_type())?; - Some(Field::new( - a.name(), - data_type, - a.is_nullable() || b.is_nullable(), - )) - }) - .collect::>>()?; - Some(Struct(Fields::from(fields))) - } - (Map(lf, sorted), Map(rf, _)) => { - let entries = tightest_common_type(lf.data_type(), rf.data_type())?; - Some(Map( - Arc::new(Field::new(lf.name(), entries, false)), - *sorted, - )) - } - _ => None, + .zip(r.iter()) + .all(|(a, b)| struct_field_types_match(a.data_type(), b.data_type())), + _ => true, + }, + (Struct(_), Struct(_)) => same_type(left, right), + _ => true, } } @@ -296,72 +254,6 @@ fn same_type(left: &DataType, right: &DataType) -> bool { } } -fn is_string(data_type: &DataType) -> bool { - matches!( - data_type, - DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View - ) -} - -fn is_binary(data_type: &DataType) -> bool { - matches!( - data_type, - DataType::Binary | DataType::LargeBinary | DataType::BinaryView - ) -} - -/// The precision of Spark's `DecimalType.forType` for an integral type. -fn integral_digits(data_type: &DataType) -> Option { - match data_type { - DataType::Int8 => Some(3), - DataType::Int16 => Some(5), - DataType::Int32 => Some(10), - DataType::Int64 => Some(20), - _ => None, - } -} - -/// Spark's `numericPrecedence`, which leaves out decimals. -fn numeric_precedence(data_type: &DataType) -> Option { - match data_type { - DataType::Int8 => Some(0), - DataType::Int16 => Some(1), - DataType::Int32 => Some(2), - DataType::Int64 => Some(3), - DataType::Float32 => Some(4), - DataType::Float64 => Some(5), - _ => None, - } -} - -fn finer_unit(left: TimeUnit, right: TimeUnit) -> TimeUnit { - let rank = |unit: TimeUnit| match unit { - TimeUnit::Second => 0, - TimeUnit::Millisecond => 1, - TimeUnit::Microsecond => 2, - TimeUnit::Nanosecond => 3, - }; - if rank(left) >= rank(right) { - left - } else { - right - } -} - -/// Spark cannot order maps, so it rejects them at any depth of the element type. -fn contains_map(data_type: &DataType) -> bool { - match data_type { - DataType::Map(_, _) => true, - DataType::List(field) - | DataType::LargeList(field) - | DataType::FixedSizeList(field, _) => contains_map(field.data_type()), - DataType::Struct(fields) => { - fields.iter().any(|field| contains_map(field.data_type())) - } - _ => false, - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/spark/src/function/array/mod.rs b/crates/spark/src/function/array/mod.rs index 7ff0f2aa..36dcccc9 100644 --- a/crates/spark/src/function/array/mod.rs +++ b/crates/spark/src/function/array/mod.rs @@ -6,7 +6,6 @@ pub mod array_contains; pub mod array_position; -mod float_normalize; pub mod repeat; pub mod shuffle; pub mod slice; diff --git a/crates/spark/src/function/array/float_normalize.rs b/crates/spark/src/function/float_normalize.rs similarity index 97% rename from crates/spark/src/function/array/float_normalize.rs rename to crates/spark/src/function/float_normalize.rs index 07bbb28a..1a76b60d 100644 --- a/crates/spark/src/function/array/float_normalize.rs +++ b/crates/spark/src/function/float_normalize.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -//! Float normalization shared by the array functions that compare elements. +//! Float normalization shared by the functions that compare values. //! //! Spark compares floating point values with `-0.0 == 0.0` and `NaN == NaN`, while arrow //! compares them by their bits or by the IEEE total order. Rewriting `-0.0` to `0.0` and @@ -19,7 +19,7 @@ use datafusion_expr::ColumnarValue; use std::sync::Arc; /// Normalizes the floats in a [`ColumnarValue`], see [`normalize_array`]. -pub(super) fn normalize_floats(value: &ColumnarValue) -> Result { +pub(crate) fn normalize_floats(value: &ColumnarValue) -> Result { if !contains_float(&value.data_type()) { return Ok(value.clone()); } @@ -35,7 +35,7 @@ pub(super) fn normalize_floats(value: &ColumnarValue) -> Result { } /// Returns whether values of this type are, or hold at any depth, floating point numbers. -pub(super) fn contains_float(data_type: &DataType) -> bool { +pub(crate) fn contains_float(data_type: &DataType) -> bool { match data_type { DataType::Float32 | DataType::Float64 => true, DataType::List(field) @@ -51,7 +51,7 @@ pub(super) fn contains_float(data_type: &DataType) -> bool { /// Rewrites `-0.0` to `0.0` and every NaN to the canonical NaN at any depth of list, large /// list, fixed-size list, struct and map. Children without floats are reused as they are. -pub(super) fn normalize_array(array: &ArrayRef) -> Result { +pub(crate) fn normalize_array(array: &ArrayRef) -> Result { if !contains_float(array.data_type()) { return Ok(Arc::clone(array)); } diff --git a/crates/spark/src/function/mod.rs b/crates/spark/src/function/mod.rs index 483f9b64..4098d8e0 100644 --- a/crates/spark/src/function/mod.rs +++ b/crates/spark/src/function/mod.rs @@ -12,6 +12,7 @@ pub mod conversion; pub mod csv; pub mod datetime; pub mod error_utils; +pub(crate) mod float_normalize; pub mod functions_nested_utils; pub mod generator; pub mod hash; @@ -25,6 +26,7 @@ pub mod predicate; pub mod string; pub mod r#struct; pub mod table; +pub(crate) mod type_coercion; pub mod url; pub mod utils; pub mod window; diff --git a/crates/spark/src/function/type_coercion.rs b/crates/spark/src/function/type_coercion.rs new file mode 100644 index 00000000..035acbf6 --- /dev/null +++ b/crates/spark/src/function/type_coercion.rs @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Spark's rules for finding a common type of several arguments, from its `TypeCoercion` +//! and `AnsiTypeCoercion`, on the Arrow types that Spark's types map to. Also the checks +//! and type names that the errors for those arguments use. +//! +//! These are for functions that need one type for several arguments, such as +//! `array_position`. They are not Spark's implicit casts to a function's declared input +//! types, which the functions do through their [`datafusion_expr::Signature`]. + +use std::sync::Arc; + +use arrow::datatypes::{DataType, Field, Fields, IntervalUnit, TimeUnit}; + +/// Spark's `findTightestCommonType`. `ansi_mode` picks between `AnsiTypeCoercion` and +/// `TypeCoercion`, which differ only in that ANSI widens an integer and a FLOAT to DOUBLE. +/// Arrays, maps and structs are widened element by element with this function. +pub(crate) fn tightest_common_type( + left: &DataType, + right: &DataType, + ansi_mode: bool, +) -> Option { + use DataType::*; + if left == right { + return Some(left.clone()); + } + if let (Some(l), Some(r)) = (numeric_precedence(left), numeric_precedence(right)) { + let wider = if l >= r { left } else { right }; + // With ANSI mode on, an integer and a FLOAT widen to DOUBLE, which holds every INT + // exactly. + return Some(if ansi_mode && *wider == Float32 { + Float64 + } else { + wider.clone() + }); + } + match (left, right) { + (Null, other) | (other, Null) => Some(other.clone()), + // Spark has one string type and one binary type. + (l, r) if is_string(l) && is_string(r) => Some(wider_physical_type(l, r)), + (l, r) if is_binary(l) && is_binary(r) => Some(wider_physical_type(l, r)), + // The decimal must hold every value of the integral type. + (int, Decimal128(p, s)) | (Decimal128(p, s), int) => { + let digits = integral_digits(int)?; + (*p as i16 - *s as i16 >= digits).then_some(Decimal128(*p, *s)) + } + (Date32, ts @ Timestamp(_, _)) | (ts @ Timestamp(_, _), Date32) => { + Some(ts.clone()) + } + // TIMESTAMP_NTZ and TIMESTAMP widen to TIMESTAMP. + (Timestamp(lu, ltz), Timestamp(ru, rtz)) => Some(Timestamp( + finer_unit(*lu, *ru), + ltz.clone().or_else(|| rtz.clone()), + )), + _ => complex_type(left, right, &|l, r| tightest_common_type(l, r, ansi_mode)), + } +} + +/// Spark's `findTypeForComplex`: widens the elements of arrays, the keys and values of maps, +/// and the fields of structs with `find_type`. A nested field is nullable if it is on either +/// side. Struct fields must have the same names in the same order. Spark compares them +/// ignoring case, but DataFusion cannot cast between structs whose field names differ only +/// in case, so this requires an exact match. +pub(crate) fn complex_type( + left: &DataType, + right: &DataType, + find_type: &dyn Fn(&DataType, &DataType) -> Option, +) -> Option { + use DataType::*; + match (left, right) { + ( + List(lf) | LargeList(lf) | FixedSizeList(lf, _), + List(rf) | LargeList(rf) | FixedSizeList(rf, _), + ) => { + let item = find_type(lf.data_type(), rf.data_type())?; + let field = Arc::new(Field::new( + lf.name(), + item, + lf.is_nullable() || rf.is_nullable(), + )); + Some(match (left, right) { + (FixedSizeList(_, ln), FixedSizeList(_, rn)) if ln == rn => { + FixedSizeList(field, *ln) + } + (LargeList(_), _) | (_, LargeList(_)) => LargeList(field), + _ => List(field), + }) + } + (Struct(lf), Struct(rf)) if lf.len() == rf.len() => { + let fields = lf + .iter() + .zip(rf.iter()) + .map(|(a, b)| { + if a.name() != b.name() { + return None; + } + let data_type = find_type(a.data_type(), b.data_type())?; + Some(Field::new( + a.name(), + data_type, + a.is_nullable() || b.is_nullable(), + )) + }) + .collect::>>()?; + Some(Struct(Fields::from(fields))) + } + // The entries are a struct of the key and the value, so the struct arm widens both. + (Map(lf, sorted), Map(rf, _)) => { + let entries = complex_type(lf.data_type(), rf.data_type(), find_type)?; + Some(Map( + Arc::new(Field::new(lf.name(), entries, false)), + *sorted, + )) + } + _ => None, + } +} + +/// Spark's `OrderUtils.isOrderable`. Spark cannot order maps or `CalendarIntervalType`, which +/// is `Interval(MonthDayNano)` in Arrow, at any depth. +pub(crate) fn is_orderable(data_type: &DataType) -> bool { + match data_type { + DataType::Map(_, _) + | DataType::Union(_, _) + | DataType::Interval(IntervalUnit::MonthDayNano) => false, + DataType::List(field) + | DataType::LargeList(field) + | DataType::ListView(field) + | DataType::LargeListView(field) + | DataType::FixedSizeList(field, _) => is_orderable(field.data_type()), + DataType::Struct(fields) => fields.iter().all(|f| is_orderable(f.data_type())), + DataType::Dictionary(_, value) => is_orderable(value), + _ => true, + } +} + +/// The name Spark's `toSQLType` gives the Spark type that an Arrow type maps to, for error +/// messages. Arrow types without a Spark counterpart use their Arrow name. +pub(crate) fn spark_sql_type(data_type: &DataType) -> String { + use DataType::*; + match data_type { + Null => "VOID".to_string(), + Boolean => "BOOLEAN".to_string(), + Int8 => "TINYINT".to_string(), + Int16 => "SMALLINT".to_string(), + Int32 => "INT".to_string(), + Int64 => "BIGINT".to_string(), + Float32 => "FLOAT".to_string(), + Float64 => "DOUBLE".to_string(), + Decimal128(p, s) => format!("DECIMAL({p},{s})"), + Utf8 | LargeUtf8 | Utf8View => "STRING".to_string(), + Binary | LargeBinary | BinaryView => "BINARY".to_string(), + Date32 => "DATE".to_string(), + Timestamp(_, Some(_)) => "TIMESTAMP".to_string(), + Timestamp(_, None) => "TIMESTAMP_NTZ".to_string(), + Interval(IntervalUnit::MonthDayNano) => "INTERVAL".to_string(), + List(field) | LargeList(field) | FixedSizeList(field, _) => { + format!("ARRAY<{}>", spark_sql_type(field.data_type())) + } + Struct(fields) => format!( + "STRUCT<{}>", + fields + .iter() + .map(|f| format!( + "{}: {}{}", + f.name(), + spark_sql_type(f.data_type()), + if f.is_nullable() { "" } else { " NOT NULL" } + )) + .collect::>() + .join(", ") + ), + Map(entries, _) => match entries.data_type() { + Struct(kv) if kv.len() == 2 => format!( + "MAP<{}, {}>", + spark_sql_type(kv[0].data_type()), + spark_sql_type(kv[1].data_type()) + ), + other => format!("MAP<{other}>"), + }, + other => other.to_string(), + } +} + +fn is_string(data_type: &DataType) -> bool { + matches!( + data_type, + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View + ) +} + +fn is_binary(data_type: &DataType) -> bool { + matches!( + data_type, + DataType::Binary | DataType::LargeBinary | DataType::BinaryView + ) +} + +/// Of two physical layouts of the same Spark string or binary type, the one that can hold the +/// values of both. +fn wider_physical_type(left: &DataType, right: &DataType) -> DataType { + use DataType::*; + match (left, right) { + (LargeUtf8, _) | (_, LargeUtf8) => LargeUtf8, + (LargeBinary, _) | (_, LargeBinary) => LargeBinary, + (Utf8View, _) | (_, Utf8View) => Utf8View, + (BinaryView, _) | (_, BinaryView) => BinaryView, + _ => left.clone(), + } +} + +/// The precision of Spark's `DecimalType.forType` for an integral type. +fn integral_digits(data_type: &DataType) -> Option { + match data_type { + DataType::Int8 => Some(3), + DataType::Int16 => Some(5), + DataType::Int32 => Some(10), + DataType::Int64 => Some(20), + _ => None, + } +} + +/// Spark's `numericPrecedence`, which leaves out decimals. +fn numeric_precedence(data_type: &DataType) -> Option { + match data_type { + DataType::Int8 => Some(0), + DataType::Int16 => Some(1), + DataType::Int32 => Some(2), + DataType::Int64 => Some(3), + DataType::Float32 => Some(4), + DataType::Float64 => Some(5), + _ => None, + } +} + +fn finer_unit(left: TimeUnit, right: TimeUnit) -> TimeUnit { + let rank = |unit: TimeUnit| match unit { + TimeUnit::Second => 0, + TimeUnit::Millisecond => 1, + TimeUnit::Microsecond => 2, + TimeUnit::Nanosecond => 3, + }; + if rank(left) >= rank(right) { + left + } else { + right + } +} + +#[cfg(test)] +mod tests { + use super::*; + use DataType::*; + + fn list(item: DataType) -> DataType { + List(Arc::new(Field::new("item", item, true))) + } + + fn map(key: DataType, value: DataType) -> DataType { + let entries = Struct(Fields::from(vec![ + Field::new("key", key, false), + Field::new("value", value, true), + ])); + Map(Arc::new(Field::new("entries", entries, false)), false) + } + + #[test] + fn numbers_widen_and_ansi_widens_int_and_float_to_double() { + assert_eq!(tightest_common_type(&Int32, &Int64, false), Some(Int64)); + assert_eq!(tightest_common_type(&Int32, &Float32, false), Some(Float32)); + assert_eq!(tightest_common_type(&Int32, &Float32, true), Some(Float64)); + assert_eq!( + tightest_common_type(&Int32, &Decimal128(10, 0), false), + Some(Decimal128(10, 0)) + ); + assert_eq!( + tightest_common_type(&Int32, &Decimal128(10, 1), false), + None + ); + assert_eq!(tightest_common_type(&Int32, &Utf8, false), None); + } + + #[test] + fn string_layouts_widen_to_the_one_that_holds_both() { + assert_eq!( + tightest_common_type(&Utf8, &LargeUtf8, false), + Some(LargeUtf8) + ); + assert_eq!( + tightest_common_type(&LargeUtf8, &Utf8, false), + Some(LargeUtf8) + ); + assert_eq!( + tightest_common_type(&Utf8, &Utf8View, false), + Some(Utf8View) + ); + } + + #[test] + fn complex_types_widen_element_by_element() { + assert_eq!( + tightest_common_type(&list(Int32), &list(Int64), false), + Some(list(Int64)) + ); + assert_eq!( + tightest_common_type(&map(Int32, Int32), &map(Int64, Int16), false), + Some(map(Int64, Int32)) + ); + let a = Struct(Fields::from(vec![Field::new("a", Int32, false)])); + let b = Struct(Fields::from(vec![Field::new("b", Int32, false)])); + let wide = Struct(Fields::from(vec![Field::new("a", Int64, true)])); + assert_eq!(tightest_common_type(&a, &wide, false), Some(wide)); + assert_eq!(tightest_common_type(&a, &b, false), None); + } + + #[test] + fn maps_and_calendar_intervals_are_not_orderable() { + let interval = Interval(IntervalUnit::MonthDayNano); + assert!(!is_orderable(&map(Int32, Int32))); + assert!(!is_orderable(&interval)); + assert!(!is_orderable(&list(interval.clone()))); + assert!(is_orderable(&list(Int32))); + assert_eq!(spark_sql_type(&list(interval)), "ARRAY"); + } +} diff --git a/crates/sqllogictest/test_files/array/array_position.slt b/crates/sqllogictest/test_files/array/array_position.slt index cf9ef3e8..25ebfd45 100644 --- a/crates/sqllogictest/test_files/array/array_position.slt +++ b/crates/sqllogictest/test_files/array/array_position.slt @@ -274,8 +274,8 @@ SELECT array_position(array(CAST(1.0 AS DOUBLE)), CAST(1 AS DECIMAL(5,1))); query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] SELECT array_position(array(1), '1'); -query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] -SELECT array_position(array('1'), 1); +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] Input to `array_position` should have been "ARRAY" followed by a value with same element type, but it's \["ARRAY", "INT"\]\. SQLSTATE: 42K09 +SELECT array_position(array('1'), CAST(1 AS INT)); query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] SELECT array_position(array(1), CAST(NULL AS STRING)); @@ -332,6 +332,17 @@ SELECT array_position(array(named_struct('m', map(1, 2))), named_struct('m', map query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] SELECT array_position(array(array(map(1, 2))), array(map(1, 2))); +# Map keys and values are widened, and the common map type is then rejected +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_position` does not support ordering on type "MAP"\. SQLSTATE: 42K09 +SELECT array_position(array(map(CAST(1 AS INT), CAST(1 AS INT))), map(1L, 1L)); + +# Spark cannot order intervals +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_position` does not support ordering on type "INTERVAL"\. SQLSTATE: 42K09 +SELECT array_position(array(make_interval(1)), make_interval(2)); + +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_position` does not support ordering on type "ARRAY"\. SQLSTATE: 42K09 +SELECT array_position(array(array(make_interval(1))), array(make_interval(2))); + # Signed zero: -0.0 equals 0.0 query I SELECT array_position(array(CAST('-0.0' AS DOUBLE), CAST(1.0 AS DOUBLE)), CAST(0.0 AS DOUBLE)); From 3e9008cff980d3f1546656cbd8a71737c6bc22ac Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 10:19:52 -0700 Subject: [PATCH 03/10] refactor(spark): match map keys and values by position in complex_type Spark's MapType has no field names, so map entries named differently still have a common type. Keep contains_float private. --- crates/spark/src/function/float_normalize.rs | 2 +- crates/spark/src/function/type_coercion.rs | 29 ++++++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/crates/spark/src/function/float_normalize.rs b/crates/spark/src/function/float_normalize.rs index 1a76b60d..e166b362 100644 --- a/crates/spark/src/function/float_normalize.rs +++ b/crates/spark/src/function/float_normalize.rs @@ -35,7 +35,7 @@ pub(crate) fn normalize_floats(value: &ColumnarValue) -> Result { } /// Returns whether values of this type are, or hold at any depth, floating point numbers. -pub(crate) fn contains_float(data_type: &DataType) -> bool { +fn contains_float(data_type: &DataType) -> bool { match data_type { DataType::Float32 | DataType::Float64 => true, DataType::List(field) diff --git a/crates/spark/src/function/type_coercion.rs b/crates/spark/src/function/type_coercion.rs index 035acbf6..d4a8bbb5 100644 --- a/crates/spark/src/function/type_coercion.rs +++ b/crates/spark/src/function/type_coercion.rs @@ -106,9 +106,24 @@ pub(crate) fn complex_type( .collect::>>()?; Some(Struct(Fields::from(fields))) } - // The entries are a struct of the key and the value, so the struct arm widens both. + // The entries are a struct of the key and the value. Spark's `MapType` has no field + // names, so the key and the value are matched by position. (Map(lf, sorted), Map(rf, _)) => { - let entries = complex_type(lf.data_type(), rf.data_type(), find_type)?; + let (Struct(lkv), Struct(rkv)) = (lf.data_type(), rf.data_type()) else { + return None; + }; + let [lk, lv] = lkv.iter().collect::>()[..] else { + return None; + }; + let [rk, rv] = rkv.iter().collect::>()[..] else { + return None; + }; + let key = find_type(lk.data_type(), rk.data_type())?; + let value = find_type(lv.data_type(), rv.data_type())?; + let entries = Struct(Fields::from(vec![ + Field::new(lk.name(), key, false), + Field::new(lv.name(), value, lv.is_nullable() || rv.is_nullable()), + ])); Some(Map( Arc::new(Field::new(lf.name(), entries, false)), *sorted, @@ -310,6 +325,16 @@ mod tests { ); let a = Struct(Fields::from(vec![Field::new("a", Int32, false)])); let b = Struct(Fields::from(vec![Field::new("b", Int32, false)])); + // Map entries named differently are matched by position. + let renamed = Struct(Fields::from(vec![ + Field::new("keys", Int64, false), + Field::new("values", Int16, true), + ])); + let renamed = Map(Arc::new(Field::new("entries", renamed, false)), false); + assert_eq!( + tightest_common_type(&map(Int32, Int32), &renamed, false), + Some(map(Int64, Int32)) + ); let wide = Struct(Fields::from(vec![Field::new("a", Int64, true)])); assert_eq!(tightest_common_type(&a, &wide, false), Some(wide)); assert_eq!(tightest_common_type(&a, &b, false), None); From f5571e9d044f852c04e7c07035c16d30e19e33dd Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 10:22:21 -0700 Subject: [PATCH 04/10] refactor(spark): simplify the shared common-type rules Use DataType::is_string and TimeUnit's ordering, match map entries without collecting, keep complex_type private, and move the non-ANSI struct field check and same_type into type_coercion beside the rules they qualify. --- .../src/function/array/array_position.rs | 46 +------- crates/spark/src/function/type_coercion.rs | 106 ++++++++++++------ 2 files changed, 75 insertions(+), 77 deletions(-) diff --git a/crates/spark/src/function/array/array_position.rs b/crates/spark/src/function/array/array_position.rs index d1a64701..445a8b48 100644 --- a/crates/spark/src/function/array/array_position.rs +++ b/crates/spark/src/function/array/array_position.rs @@ -4,7 +4,7 @@ use crate::function::float_normalize::normalize_array; use crate::function::type_coercion::{ - is_orderable, spark_sql_type, tightest_common_type, + is_orderable, spark_sql_type, struct_field_types_match, tightest_common_type, }; use arrow::array::{ Array, ArrayRef, AsArray, Int64Array, OffsetSizeTrait, make_comparator, @@ -210,50 +210,6 @@ fn null_type_err(name: &str) -> Result { ) } -/// With ANSI mode off, Spark cannot implicitly cast between structs whose field types -/// differ, so `array_position` rejects them although their tightest common type exists. -/// With ANSI mode on, Spark widens the field types. -fn struct_field_types_match(left: &DataType, right: &DataType) -> bool { - use DataType::*; - match (left, right) { - ( - List(l) | LargeList(l) | FixedSizeList(l, _), - List(r) | LargeList(r) | FixedSizeList(r, _), - ) => struct_field_types_match(l.data_type(), r.data_type()), - // The entries are a struct of the key and the value, which are checked on their own. - (Map(l, _), Map(r, _)) => match (l.data_type(), r.data_type()) { - (Struct(l), Struct(r)) => l - .iter() - .zip(r.iter()) - .all(|(a, b)| struct_field_types_match(a.data_type(), b.data_type())), - _ => true, - }, - (Struct(_), Struct(_)) => same_type(left, right), - _ => true, - } -} - -/// Whether two types are equal when the nullability of nested fields is ignored, like -/// Spark's `DataTypeUtils.sameType`. -fn same_type(left: &DataType, right: &DataType) -> bool { - use DataType::*; - match (left, right) { - (List(l), List(r)) | (LargeList(l), LargeList(r)) | (Map(l, _), Map(r, _)) => { - same_type(l.data_type(), r.data_type()) - } - (FixedSizeList(l, ln), FixedSizeList(r, rn)) => { - ln == rn && same_type(l.data_type(), r.data_type()) - } - (Struct(l), Struct(r)) => { - l.len() == r.len() - && l.iter().zip(r.iter()).all(|(a, b)| { - a.name() == b.name() && same_type(a.data_type(), b.data_type()) - }) - } - _ => left == right, - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/spark/src/function/type_coercion.rs b/crates/spark/src/function/type_coercion.rs index d4a8bbb5..9c23939a 100644 --- a/crates/spark/src/function/type_coercion.rs +++ b/crates/spark/src/function/type_coercion.rs @@ -12,7 +12,7 @@ use std::sync::Arc; -use arrow::datatypes::{DataType, Field, Fields, IntervalUnit, TimeUnit}; +use arrow::datatypes::{DataType, Field, Fields, IntervalUnit}; /// Spark's `findTightestCommonType`. `ansi_mode` picks between `AnsiTypeCoercion` and /// `TypeCoercion`, which differ only in that ANSI widens an integer and a FLOAT to DOUBLE. @@ -39,7 +39,7 @@ pub(crate) fn tightest_common_type( match (left, right) { (Null, other) | (other, Null) => Some(other.clone()), // Spark has one string type and one binary type. - (l, r) if is_string(l) && is_string(r) => Some(wider_physical_type(l, r)), + (l, r) if l.is_string() && r.is_string() => Some(wider_physical_type(l, r)), (l, r) if is_binary(l) && is_binary(r) => Some(wider_physical_type(l, r)), // The decimal must hold every value of the integral type. (int, Decimal128(p, s)) | (Decimal128(p, s), int) => { @@ -51,7 +51,8 @@ pub(crate) fn tightest_common_type( } // TIMESTAMP_NTZ and TIMESTAMP widen to TIMESTAMP. (Timestamp(lu, ltz), Timestamp(ru, rtz)) => Some(Timestamp( - finer_unit(*lu, *ru), + // Arrow orders the units from Second to Nanosecond. + (*lu).max(*ru), ltz.clone().or_else(|| rtz.clone()), )), _ => complex_type(left, right, &|l, r| tightest_common_type(l, r, ansi_mode)), @@ -63,7 +64,7 @@ pub(crate) fn tightest_common_type( /// side. Struct fields must have the same names in the same order. Spark compares them /// ignoring case, but DataFusion cannot cast between structs whose field names differ only /// in case, so this requires an exact match. -pub(crate) fn complex_type( +fn complex_type( left: &DataType, right: &DataType, find_type: &dyn Fn(&DataType, &DataType) -> Option, @@ -112,10 +113,7 @@ pub(crate) fn complex_type( let (Struct(lkv), Struct(rkv)) = (lf.data_type(), rf.data_type()) else { return None; }; - let [lk, lv] = lkv.iter().collect::>()[..] else { - return None; - }; - let [rk, rv] = rkv.iter().collect::>()[..] else { + let ([lk, lv], [rk, rv]) = (&lkv[..], &rkv[..]) else { return None; }; let key = find_type(lk.data_type(), rk.data_type())?; @@ -151,6 +149,51 @@ pub(crate) fn is_orderable(data_type: &DataType) -> bool { } } +/// Whether the structs in two types, at any depth, have the same field types. With ANSI mode +/// off, Spark's implicit cast cannot change the field types of a struct, so a function that +/// casts its arguments to their tightest common type rejects such structs, as +/// `array_position` does. With ANSI mode on, Spark widens the field types. +pub(crate) fn struct_field_types_match(left: &DataType, right: &DataType) -> bool { + use DataType::*; + match (left, right) { + ( + List(l) | LargeList(l) | FixedSizeList(l, _), + List(r) | LargeList(r) | FixedSizeList(r, _), + ) => struct_field_types_match(l.data_type(), r.data_type()), + // The entries are a struct of the key and the value, which are checked on their own. + (Map(l, _), Map(r, _)) => match (l.data_type(), r.data_type()) { + (Struct(l), Struct(r)) => l + .iter() + .zip(r.iter()) + .all(|(a, b)| struct_field_types_match(a.data_type(), b.data_type())), + _ => true, + }, + (Struct(_), Struct(_)) => same_type(left, right), + _ => true, + } +} + +/// Whether two types are equal when the nullability of nested fields is ignored, like +/// Spark's `DataTypeUtils.sameType`. +fn same_type(left: &DataType, right: &DataType) -> bool { + use DataType::*; + match (left, right) { + (List(l), List(r)) | (LargeList(l), LargeList(r)) | (Map(l, _), Map(r, _)) => { + same_type(l.data_type(), r.data_type()) + } + (FixedSizeList(l, ln), FixedSizeList(r, rn)) => { + ln == rn && same_type(l.data_type(), r.data_type()) + } + (Struct(l), Struct(r)) => { + l.len() == r.len() + && l.iter().zip(r.iter()).all(|(a, b)| { + a.name() == b.name() && same_type(a.data_type(), b.data_type()) + }) + } + _ => left == right, + } +} + /// The name Spark's `toSQLType` gives the Spark type that an Arrow type maps to, for error /// messages. Arrow types without a Spark counterpart use their Arrow name. pub(crate) fn spark_sql_type(data_type: &DataType) -> String { @@ -188,10 +231,10 @@ pub(crate) fn spark_sql_type(data_type: &DataType) -> String { .join(", ") ), Map(entries, _) => match entries.data_type() { - Struct(kv) if kv.len() == 2 => format!( + Struct(kv) if let [k, v] = &kv[..] => format!( "MAP<{}, {}>", - spark_sql_type(kv[0].data_type()), - spark_sql_type(kv[1].data_type()) + spark_sql_type(k.data_type()), + spark_sql_type(v.data_type()) ), other => format!("MAP<{other}>"), }, @@ -199,13 +242,8 @@ pub(crate) fn spark_sql_type(data_type: &DataType) -> String { } } -fn is_string(data_type: &DataType) -> bool { - matches!( - data_type, - DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View - ) -} - +/// Unlike [`DataType::is_binary`], this leaves out `FixedSizeBinary`, which +/// [`wider_physical_type`] cannot widen. fn is_binary(data_type: &DataType) -> bool { matches!( data_type, @@ -250,20 +288,6 @@ fn numeric_precedence(data_type: &DataType) -> Option { } } -fn finer_unit(left: TimeUnit, right: TimeUnit) -> TimeUnit { - let rank = |unit: TimeUnit| match unit { - TimeUnit::Second => 0, - TimeUnit::Millisecond => 1, - TimeUnit::Microsecond => 2, - TimeUnit::Nanosecond => 3, - }; - if rank(left) >= rank(right) { - left - } else { - right - } -} - #[cfg(test)] mod tests { use super::*; @@ -340,6 +364,24 @@ mod tests { assert_eq!(tightest_common_type(&a, &b, false), None); } + #[test] + fn struct_field_types_must_match_at_any_depth() { + let int = Struct(Fields::from(vec![Field::new("a", Int32, false)])); + let nullable_int = Struct(Fields::from(vec![Field::new("a", Int32, true)])); + let long = Struct(Fields::from(vec![Field::new("a", Int64, false)])); + assert!(struct_field_types_match(&int, &nullable_int)); + assert!(!struct_field_types_match(&int, &long)); + assert!(!struct_field_types_match( + &list(int.clone()), + &list(long.clone()) + )); + assert!(!struct_field_types_match( + &map(Int32, int), + &map(Int64, long) + )); + assert!(struct_field_types_match(&list(Int32), &list(Int64))); + } + #[test] fn maps_and_calendar_intervals_are_not_orderable() { let interval = Interval(IntervalUnit::MonthDayNano); From 80d7b76f4cd7618edcc3ad63ee8c46b441fe6590 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 10:24:14 -0700 Subject: [PATCH 05/10] refactor(spark): use the shared common-type rules in greatest and least Move wider_type and wider_decimal_type into function/type_coercion and drop the local copies of the tightest common type, isOrderable and the Spark type names. isOrderable now also rejects INTERVAL. --- .../spark/src/function/math/greatest_least.rs | 270 +----------------- crates/spark/src/function/type_coercion.rs | 44 +++ 2 files changed, 48 insertions(+), 266 deletions(-) diff --git a/crates/spark/src/function/math/greatest_least.rs b/crates/spark/src/function/math/greatest_least.rs index 637f0b02..ac853be6 100644 --- a/crates/spark/src/function/math/greatest_least.rs +++ b/crates/spark/src/function/math/greatest_least.rs @@ -18,7 +18,8 @@ //! coerce their arguments with DataFusion's comparison coercion, which for example compares a //! number with a string. Spark rejects that. -use crate::function::array::float_normalize::{contains_float, normalize_array}; +use crate::function::float_normalize::{contains_float, normalize_array}; +use crate::function::type_coercion::{is_orderable, spark_sql_type, wider_type}; use arrow::array::{ Array, ArrayRef, BooleanArray, Datum, Scalar, make_comparator, new_null_array, }; @@ -26,7 +27,7 @@ use arrow::buffer::BooleanBuffer; use arrow::compute::kernels::cmp::{gt, lt}; use arrow::compute::kernels::zip::zip; use arrow::compute::{SortOptions, cast}; -use arrow::datatypes::{DataType, Field, FieldRef, Fields, TimeUnit}; +use arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::config::ConfigOptions; use datafusion_common::{DataFusionError, Result, ScalarValue, plan_err}; use datafusion_expr::{ @@ -459,270 +460,6 @@ impl Candidate { } } -/// Spark's `findWiderTypeWithoutStringPromotionForTwo`: the tightest common type, then a -/// decimal wide enough for both, then the same for the elements of arrays and the fields of -/// structs. Strings are never promoted. `ansi_mode` picks between `AnsiTypeCoercion` and -/// `TypeCoercion`, which differ only in that ANSI widens an integer and a FLOAT to DOUBLE. -fn wider_type(left: &DataType, right: &DataType, ansi_mode: bool) -> Option { - tightest_common_type(left, right, ansi_mode) - .or_else(|| wider_decimal_type(left, right)) - .or_else(|| complex_type(left, right, ansi_mode)) -} - -/// Spark's `findTightestCommonType`, on the Arrow types that Spark's types map to. Arrays and -/// structs are handled by [`complex_type`] with [`wider_type`], which gives the same result. -fn tightest_common_type( - left: &DataType, - right: &DataType, - ansi_mode: bool, -) -> Option { - use DataType::*; - if left == right { - return Some(left.clone()); - } - if let (Some(l), Some(r)) = (numeric_precedence(left), numeric_precedence(right)) { - let wider = if l >= r { left } else { right }; - // With ANSI mode on, an integer and a FLOAT widen to DOUBLE, which holds every INT - // exactly. - return Some(if ansi_mode && *wider == Float32 { - Float64 - } else { - wider.clone() - }); - } - match (left, right) { - (Null, other) | (other, Null) => Some(other.clone()), - // Spark has one string type and one binary type. - (l, r) if is_string(l) && is_string(r) => Some(wider_physical_type(l, r)), - (l, r) if is_binary(l) && is_binary(r) => Some(wider_physical_type(l, r)), - // The decimal must hold every value of the integral type. - (int, Decimal128(p, s)) | (Decimal128(p, s), int) => { - let digits = integral_digits(int)?; - (*p as i16 - *s as i16 >= digits).then_some(Decimal128(*p, *s)) - } - (Date32, ts @ Timestamp(_, _)) | (ts @ Timestamp(_, _), Date32) => { - Some(ts.clone()) - } - // TIMESTAMP_NTZ and TIMESTAMP widen to TIMESTAMP. - (Timestamp(lu, ltz), Timestamp(ru, rtz)) => Some(Timestamp( - finer_unit(*lu, *ru), - ltz.clone().or_else(|| rtz.clone()), - )), - _ => None, - } -} - -/// Spark's `findWiderTypeForDecimal`, with `DecimalType.boundedPreferIntegralDigits`, which -/// gives up fractional digits rather than integral ones when the precision exceeds 38. -fn wider_decimal_type(left: &DataType, right: &DataType) -> Option { - use DataType::*; - let (p1, s1, p2, s2) = match (left, right) { - (Decimal128(p1, s1), Decimal128(p2, s2)) => { - (*p1 as i16, *s1 as i16, *p2 as i16, *s2 as i16) - } - (int, Decimal128(p, s)) | (Decimal128(p, s), int) - if integral_digits(int).is_some() => - { - (integral_digits(int)?, 0, *p as i16, *s as i16) - } - (Float32 | Float64, Decimal128(_, _)) | (Decimal128(_, _), Float32 | Float64) => { - return Some(Float64); - } - _ => return None, - }; - let scale = s1.max(s2); - let range = (p1 - s1).max(p2 - s2); - let precision = scale + range; - const MAX_PRECISION: i16 = 38; - if precision <= MAX_PRECISION { - Some(Decimal128(precision as u8, scale as i8)) - } else { - let scale = (scale - (precision - MAX_PRECISION)).max(0); - Some(Decimal128(MAX_PRECISION as u8, scale as i8)) - } -} - -/// Spark's `findTypeForComplex` with [`wider_type`] for the elements and fields. A nested field -/// is nullable if it is on either side. Struct fields must have the same names in the same -/// order. Spark compares them ignoring case, but this requires an exact match. -fn complex_type(left: &DataType, right: &DataType, ansi_mode: bool) -> Option { - use DataType::*; - match (left, right) { - ( - List(lf) | LargeList(lf) | FixedSizeList(lf, _), - List(rf) | LargeList(rf) | FixedSizeList(rf, _), - ) => { - let item = wider_type(lf.data_type(), rf.data_type(), ansi_mode)?; - let field = Arc::new(Field::new( - lf.name(), - item, - lf.is_nullable() || rf.is_nullable(), - )); - Some(match (left, right) { - (FixedSizeList(_, ln), FixedSizeList(_, rn)) if ln == rn => { - FixedSizeList(field, *ln) - } - (LargeList(_), _) | (_, LargeList(_)) => LargeList(field), - _ => List(field), - }) - } - (Struct(lf), Struct(rf)) if lf.len() == rf.len() => { - let fields = lf - .iter() - .zip(rf.iter()) - .map(|(a, b)| { - if a.name() != b.name() { - return None; - } - let data_type = wider_type(a.data_type(), b.data_type(), ansi_mode)?; - Some(Field::new( - a.name(), - data_type, - a.is_nullable() || b.is_nullable(), - )) - }) - .collect::>>()?; - Some(Struct(Fields::from(fields))) - } - // Spark finds a common map type, which is then rejected as not orderable. - (Map(lf, sorted), Map(rf, _)) => { - let entries = wider_type(lf.data_type(), rf.data_type(), ansi_mode)?; - Some(Map( - Arc::new(Field::new(lf.name(), entries, false)), - *sorted, - )) - } - _ => None, - } -} - -/// Whether Spark can order values of this type. It cannot order maps, at any depth. -fn is_orderable(data_type: &DataType) -> bool { - match data_type { - DataType::Map(_, _) | DataType::Union(_, _) => false, - DataType::List(field) - | DataType::LargeList(field) - | DataType::ListView(field) - | DataType::LargeListView(field) - | DataType::FixedSizeList(field, _) => is_orderable(field.data_type()), - DataType::Struct(fields) => fields.iter().all(|f| is_orderable(f.data_type())), - DataType::Dictionary(_, value) => is_orderable(value), - _ => true, - } -} - -fn is_string(data_type: &DataType) -> bool { - matches!( - data_type, - DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View - ) -} - -fn is_binary(data_type: &DataType) -> bool { - matches!( - data_type, - DataType::Binary | DataType::LargeBinary | DataType::BinaryView - ) -} - -/// Of two physical layouts of the same Spark string or binary type, the one that can hold the -/// values of both. -fn wider_physical_type(left: &DataType, right: &DataType) -> DataType { - use DataType::*; - match (left, right) { - (LargeUtf8, _) | (_, LargeUtf8) => LargeUtf8, - (LargeBinary, _) | (_, LargeBinary) => LargeBinary, - (Utf8View, _) | (_, Utf8View) => Utf8View, - (BinaryView, _) | (_, BinaryView) => BinaryView, - _ => left.clone(), - } -} - -/// The precision of Spark's `DecimalType.forType` for an integral type. -fn integral_digits(data_type: &DataType) -> Option { - match data_type { - DataType::Int8 => Some(3), - DataType::Int16 => Some(5), - DataType::Int32 => Some(10), - DataType::Int64 => Some(20), - _ => None, - } -} - -/// Spark's `numericPrecedence`, which leaves out decimals. -fn numeric_precedence(data_type: &DataType) -> Option { - match data_type { - DataType::Int8 => Some(0), - DataType::Int16 => Some(1), - DataType::Int32 => Some(2), - DataType::Int64 => Some(3), - DataType::Float32 => Some(4), - DataType::Float64 => Some(5), - _ => None, - } -} - -fn finer_unit(left: TimeUnit, right: TimeUnit) -> TimeUnit { - let rank = |unit: TimeUnit| match unit { - TimeUnit::Second => 0, - TimeUnit::Millisecond => 1, - TimeUnit::Microsecond => 2, - TimeUnit::Nanosecond => 3, - }; - if rank(left) >= rank(right) { - left - } else { - right - } -} - -/// The name Spark's `toSQLType` gives the Spark type that an Arrow type maps to, for error -/// messages. Arrow types without a Spark counterpart use their Arrow name. -fn spark_sql_type(data_type: &DataType) -> String { - use DataType::*; - match data_type { - Null => "VOID".to_string(), - Boolean => "BOOLEAN".to_string(), - Int8 => "TINYINT".to_string(), - Int16 => "SMALLINT".to_string(), - Int32 => "INT".to_string(), - Int64 => "BIGINT".to_string(), - Float32 => "FLOAT".to_string(), - Float64 => "DOUBLE".to_string(), - Decimal128(p, s) => format!("DECIMAL({p},{s})"), - Utf8 | LargeUtf8 | Utf8View => "STRING".to_string(), - Binary | LargeBinary | BinaryView => "BINARY".to_string(), - Date32 => "DATE".to_string(), - Timestamp(_, Some(_)) => "TIMESTAMP".to_string(), - Timestamp(_, None) => "TIMESTAMP_NTZ".to_string(), - List(field) | LargeList(field) | FixedSizeList(field, _) => { - format!("ARRAY<{}>", spark_sql_type(field.data_type())) - } - Struct(fields) => format!( - "STRUCT<{}>", - fields - .iter() - .map(|f| format!( - "{}: {}{}", - f.name(), - spark_sql_type(f.data_type()), - if f.is_nullable() { "" } else { " NOT NULL" } - )) - .collect::>() - .join(", ") - ), - Map(entries, _) => match entries.data_type() { - Struct(kv) if kv.len() == 2 => format!( - "MAP<{}, {}>", - spark_sql_type(kv[0].data_type()), - spark_sql_type(kv[1].data_type()) - ), - other => format!("MAP<{other}>"), - }, - other => other.to_string(), - } -} - #[cfg(test)] mod tests { use super::*; @@ -731,6 +468,7 @@ mod tests { StructArray, }; use arrow::buffer::OffsetBuffer; + use arrow::datatypes::{Fields, TimeUnit}; use arrow::datatypes::{Float64Type, Int32Type}; /// Values on which Spark's ordering and IEEE 754 total order disagree, with neighbors and a diff --git a/crates/spark/src/function/type_coercion.rs b/crates/spark/src/function/type_coercion.rs index 9c23939a..3a83c058 100644 --- a/crates/spark/src/function/type_coercion.rs +++ b/crates/spark/src/function/type_coercion.rs @@ -59,6 +59,50 @@ pub(crate) fn tightest_common_type( } } +/// Spark's `findWiderTypeWithoutStringPromotionForTwo`: the tightest common type, then a +/// decimal wide enough for both, then the same for the elements of arrays and the fields of +/// structs. Strings are never promoted. `ansi_mode` picks between `AnsiTypeCoercion` and +/// `TypeCoercion`, which differ only in that ANSI widens an integer and a FLOAT to DOUBLE. +pub(crate) fn wider_type( + left: &DataType, + right: &DataType, + ansi_mode: bool, +) -> Option { + tightest_common_type(left, right, ansi_mode) + .or_else(|| wider_decimal_type(left, right)) + .or_else(|| complex_type(left, right, &|l, r| wider_type(l, r, ansi_mode))) +} + +/// Spark's `findWiderTypeForDecimal`, with `DecimalType.boundedPreferIntegralDigits`, which +/// gives up fractional digits rather than integral ones when the precision exceeds 38. +fn wider_decimal_type(left: &DataType, right: &DataType) -> Option { + use DataType::*; + let (p1, s1, p2, s2) = match (left, right) { + (Decimal128(p1, s1), Decimal128(p2, s2)) => { + (*p1 as i16, *s1 as i16, *p2 as i16, *s2 as i16) + } + (int, Decimal128(p, s)) | (Decimal128(p, s), int) + if integral_digits(int).is_some() => + { + (integral_digits(int)?, 0, *p as i16, *s as i16) + } + (Float32 | Float64, Decimal128(_, _)) | (Decimal128(_, _), Float32 | Float64) => { + return Some(Float64); + } + _ => return None, + }; + let scale = s1.max(s2); + let range = (p1 - s1).max(p2 - s2); + let precision = scale + range; + const MAX_PRECISION: i16 = 38; + if precision <= MAX_PRECISION { + Some(Decimal128(precision as u8, scale as i8)) + } else { + let scale = (scale - (precision - MAX_PRECISION)).max(0); + Some(Decimal128(MAX_PRECISION as u8, scale as i8)) + } +} + /// Spark's `findTypeForComplex`: widens the elements of arrays, the keys and values of maps, /// and the fields of structs with `find_type`. A nested field is nullable if it is on either /// side. Struct fields must have the same names in the same order. Spark compares them From 8e4756f5ce6939b3063571ee1513918c7de1cbcc Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 10:28:36 -0700 Subject: [PATCH 06/10] perf(spark): compare flat floats in place in greatest and least Compare Float32 and Float64 arguments with Spark's float order directly instead of normalizing a copy first, about 39% faster over Float64 columns. Add a criterion bench for greatest and least. Also make new_with_ansi_mode private, end the error messages with Spark's SQLSTATE, and test a constant first argument and INTERVAL arguments. --- crates/spark/Cargo.toml | 4 + crates/spark/benches/greatest_least.rs | 87 +++++++++++++++++++ .../spark/src/function/math/greatest_least.rs | 86 ++++++++++++++---- .../sqllogictest/test_files/math/greatest.slt | 22 ++++- crates/sqllogictest/test_files/math/least.slt | 22 ++++- 5 files changed, 197 insertions(+), 24 deletions(-) create mode 100644 crates/spark/benches/greatest_least.rs diff --git a/crates/spark/Cargo.toml b/crates/spark/Cargo.toml index 829ce8a1..2a7fb1d8 100644 --- a/crates/spark/Cargo.toml +++ b/crates/spark/Cargo.toml @@ -110,3 +110,7 @@ name = "from_utc_timestamp" [[bench]] harness = false name = "stack" + +[[bench]] +harness = false +name = "greatest_least" diff --git a/crates/spark/benches/greatest_least.rs b/crates/spark/benches/greatest_least.rs new file mode 100644 index 00000000..160517c9 --- /dev/null +++ b/crates/spark/benches/greatest_least.rs @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use arrow::array::PrimitiveArray; +use arrow::datatypes::{Field, Float64Type, Int64Type}; +use criterion::{Criterion, criterion_group, criterion_main}; +use datafusion_common::config::ConfigOptions; +use datafusion_expr::{ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF}; +use datafusion_spark::function::math::{greatest, least}; +use rand::rngs::StdRng; +use rand::{Rng, SeedableRng}; +use std::hint::black_box; +use std::sync::Arc; + +const ROWS: usize = 8192; +const NULL_DENSITY: f32 = 0.1; + +fn int64_column(rng: &mut StdRng) -> ColumnarValue { + let values: PrimitiveArray = (0..ROWS) + .map(|_| (rng.random::() >= NULL_DENSITY).then(|| rng.random())) + .collect(); + ColumnarValue::Array(Arc::new(values)) +} + +/// Includes NaN and -0.0, which Spark orders differently from IEEE 754. +fn float64_column(rng: &mut StdRng) -> ColumnarValue { + let values: PrimitiveArray = (0..ROWS) + .map(|_| { + (rng.random::() >= NULL_DENSITY).then(|| { + match rng.random_range(0..100) { + 0 => f64::NAN, + 1 => -0.0, + _ => rng.random::() * 2.0 - 1.0, + } + }) + }) + .collect(); + ColumnarValue::Array(Arc::new(values)) +} + +fn bench(c: &mut Criterion, udf: &ScalarUDF, label: &str, args: &[ColumnarValue]) { + let arg_fields: Vec<_> = args + .iter() + .map(|arg| Arc::new(Field::new("a", arg.data_type(), true))) + .collect(); + let scalar_arguments = vec![None; args.len()]; + let return_field = udf + .return_field_from_args(ReturnFieldArgs { + arg_fields: &arg_fields, + scalar_arguments: &scalar_arguments, + }) + .unwrap(); + let config = Arc::new(ConfigOptions::default()); + c.bench_function(&format!("{} {label}", udf.name()), |b| { + b.iter(|| { + black_box( + udf.invoke_with_args(ScalarFunctionArgs { + args: args.to_vec(), + arg_fields: arg_fields.clone(), + number_rows: ROWS, + return_field: Arc::clone(&return_field), + config_options: Arc::clone(&config), + }) + .unwrap(), + ) + }) + }); +} + +fn criterion_benchmark(c: &mut Criterion) { + let rng = &mut StdRng::seed_from_u64(42); + let ints = [int64_column(rng), int64_column(rng)]; + let floats = [ + float64_column(rng), + float64_column(rng), + float64_column(rng), + ]; + let greatest = greatest(); + bench(c, &greatest, "2 x int64", &ints); + bench(c, &greatest, "2 x float64", &floats[..2]); + bench(c, &greatest, "3 x float64", &floats); + bench(c, &least(), "3 x float64", &floats); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/crates/spark/src/function/math/greatest_least.rs b/crates/spark/src/function/math/greatest_least.rs index ac853be6..480843af 100644 --- a/crates/spark/src/function/math/greatest_least.rs +++ b/crates/spark/src/function/math/greatest_least.rs @@ -21,13 +21,14 @@ use crate::function::float_normalize::{contains_float, normalize_array}; use crate::function::type_coercion::{is_orderable, spark_sql_type, wider_type}; use arrow::array::{ - Array, ArrayRef, BooleanArray, Datum, Scalar, make_comparator, new_null_array, + Array, ArrayRef, ArrowPrimitiveType, AsArray, BooleanArray, Datum, Scalar, + make_comparator, new_null_array, }; use arrow::buffer::BooleanBuffer; use arrow::compute::kernels::cmp::{gt, lt}; use arrow::compute::kernels::zip::zip; use arrow::compute::{SortOptions, cast}; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, Field, FieldRef, Float32Type, Float64Type}; use datafusion_common::config::ConfigOptions; use datafusion_common::{DataFusionError, Result, ScalarValue, plan_err}; use datafusion_expr::{ @@ -35,6 +36,7 @@ use datafusion_expr::{ ScalarUDFImpl, Signature, Volatility, }; use datafusion_macros::user_doc; +use num_traits::Float; use std::sync::Arc; /// Spark-compatible `greatest` expression. @@ -91,7 +93,7 @@ impl SparkGreatest { /// Creates a `greatest` that coerces its arguments as Spark does with /// `datafusion.execution.enable_ansi_mode` set to `enable_ansi_mode`. - pub fn new_with_ansi_mode(enable_ansi_mode: bool) -> Self { + fn new_with_ansi_mode(enable_ansi_mode: bool) -> Self { Self { inner: GreatestLeast::new(true, enable_ansi_mode), } @@ -147,7 +149,7 @@ impl SparkLeast { /// Creates a `least` that coerces its arguments as Spark does with /// `datafusion.execution.enable_ansi_mode` set to `enable_ansi_mode`. - pub fn new_with_ansi_mode(enable_ansi_mode: bool) -> Self { + fn new_with_ansi_mode(enable_ansi_mode: bool) -> Self { Self { inner: GreatestLeast::new(false, enable_ansi_mode), } @@ -243,7 +245,8 @@ impl GreatestLeast { let message = format!( "[WRONG_NUM_ARGS.WITHOUT_SUGGESTION] The `{}` requires > 1 parameters but the \ actual number is {count}. Please, refer to \ - 'https://spark.apache.org/docs/latest/sql-ref-functions.html' for a fix.", + 'https://spark.apache.org/docs/latest/sql-ref-functions.html' for a fix. \ + SQLSTATE: 42605", self.name() ); Err(if planning { @@ -267,7 +270,7 @@ impl GreatestLeast { let Some(common) = common else { return plan_err!( "[DATATYPE_MISMATCH.DATA_DIFF_TYPES] Input to `{}` should all be the same \ - type, but it's [{}]", + type, but it's [{}]. SQLSTATE: 42K09", self.name(), arg_types .iter() @@ -279,7 +282,7 @@ impl GreatestLeast { if !is_orderable(&common) { return plan_err!( "[DATATYPE_MISMATCH.INVALID_ORDERING_TYPE] The `{}` does not support ordering \ - on type \"{}\"", + on type \"{}\". SQLSTATE: 42K09", self.name(), spark_sql_type(&common) ); @@ -315,9 +318,10 @@ impl GreatestLeast { cast(&array, data_type)? }) }; - // Floats are compared in a copy where `-0.0` is `0.0` and every NaN is the canonical - // NaN, where Arrow's total order agrees with Spark's. The result keeps the original. - let normalize = contains_float(data_type); + // Nested floats are compared in a copy where `-0.0` is `0.0` and every NaN is the + // canonical NaN, where Arrow's total order agrees with Spark's. The result keeps the + // original. Flat floats are compared in place by `spark_float_ordered`. + let normalize = data_type.is_nested() && contains_float(data_type); let comparable = |array: &ArrayRef| -> Result { if normalize { normalize_array(array) @@ -389,15 +393,31 @@ impl GreatestLeast { }; if !current.data_type().is_nested() { - let candidate_cmp = candidate.cmp_datum(); - let ordered = if self.greatest { - gt(candidate_cmp.as_ref(), current_cmp)? - } else { - lt(candidate_cmp.as_ref(), current_cmp)? - }; // Where either side is NULL the comparison's value is arbitrary, but such rows are // decided by the null masks below. - let mut take = ordered.values().clone(); + let mut take = match current.data_type() { + DataType::Float32 => spark_float_ordered::( + &candidate.cmp, + candidate.scalar, + current_cmp, + self.greatest, + ), + DataType::Float64 => spark_float_ordered::( + &candidate.cmp, + candidate.scalar, + current_cmp, + self.greatest, + ), + _ => { + let candidate_cmp = candidate.cmp_datum(); + let ordered = if self.greatest { + gt(candidate_cmp.as_ref(), current_cmp)? + } else { + lt(candidate_cmp.as_ref(), current_cmp)? + }; + ordered.values().clone() + } + }; if let Some(nulls) = ¤t_nulls { take = &take | &!nulls.inner(); } @@ -434,6 +454,36 @@ impl GreatestLeast { } } +/// For each row, whether `candidate` ranks strictly before `current` in Spark's float order, +/// where NaN is greater than every other value and `-0.0` equals `0.0`. A scalar candidate is +/// a one-row array. +fn spark_float_ordered( + candidate: &ArrayRef, + scalar: bool, + current: &ArrayRef, + greatest: bool, +) -> BooleanBuffer +where + T: ArrowPrimitiveType, + T::Native: Float, +{ + let candidate = candidate.as_primitive::().values(); + let current = current.as_primitive::().values(); + // IEEE 754 `>` already treats `-0.0` as equal to `0.0`, so only NaN needs handling. + let gt = |a: T::Native, b: T::Native| { + if a.is_nan() { + !b.is_nan() + } else { + !b.is_nan() && a > b + } + }; + BooleanBuffer::collect_bool(current.len(), |row| { + let c = candidate[if scalar { 0 } else { row }]; + let r = current[row]; + if greatest { gt(c, r) } else { gt(r, c) } + }) +} + /// An argument after the first, with its values as they are and as they are compared. A /// constant is a one-row array. struct Candidate { @@ -617,7 +667,7 @@ mod tests { let result = invoke(greatest, args, 1, DataType::Float32)?.into_array(1)?; let value = result - .as_primitive::() + .as_primitive::() .value(0); assert_eq!(value.to_bits(), expected.to_bits(), "greatest={greatest}"); } diff --git a/crates/sqllogictest/test_files/math/greatest.slt b/crates/sqllogictest/test_files/math/greatest.slt index c6c5daba..e4410a48 100644 --- a/crates/sqllogictest/test_files/math/greatest.slt +++ b/crates/sqllogictest/test_files/math/greatest.slt @@ -187,20 +187,29 @@ SELECT greatest(column1, column2), greatest(column1, 3::int, column2), greatest( NULL 3 NULL 0 b 7 7 NaN Infinity c +# A constant first argument, followed by columns. A NULL constant is skipped. +query IIIR +SELECT greatest(3::int, column1), greatest(NULL::int, column1), greatest(NULL::int, column1, column2), greatest(0::double, column3) FROM t; +---- +3 1 2 NaN +5 5 5 0 +3 NULL NULL 0 +3 NULL 7 Infinity + statement ok DROP TABLE t; # Errors # At least two arguments -query error \[WRONG_NUM_ARGS\.WITHOUT_SUGGESTION\] The `greatest` requires > 1 parameters but the actual number is 1 +query error \[WRONG_NUM_ARGS\.WITHOUT_SUGGESTION\] The `greatest` requires > 1 parameters but the actual number is 1\. Please, refer to 'https://spark\.apache\.org/docs/latest/sql-ref-functions\.html' for a fix\. SQLSTATE: 42605 SELECT greatest(1::int); query error SELECT greatest(); # Strings are not promoted -query error \[DATATYPE_MISMATCH\.DATA_DIFF_TYPES\] Input to `greatest` should all be the same type, but it's \["INT", "STRING"\] +query error \[DATATYPE_MISMATCH\.DATA_DIFF_TYPES\] Input to `greatest` should all be the same type, but it's \["INT", "STRING"\]\. SQLSTATE: 42K09 SELECT greatest(1::int, 'a'); query error \[DATATYPE_MISMATCH\.DATA_DIFF_TYPES\] Input to `greatest` should all be the same type, but it's \["DATE", "STRING"\] @@ -216,9 +225,16 @@ query error \[DATATYPE_MISMATCH\.DATA_DIFF_TYPES\] SELECT greatest(named_struct('a', 1), named_struct('b', 1)); # Maps cannot be ordered -query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `greatest` does not support ordering on type "MAP" +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `greatest` does not support ordering on type "MAP"\. SQLSTATE: 42K09 SELECT greatest(map(1::int, 2::int), map(1::int, 3::int)); +# Intervals cannot be ordered +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `greatest` does not support ordering on type "INTERVAL"\. SQLSTATE: 42K09 +SELECT greatest(make_interval(1), make_interval(2)); + +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `greatest` does not support ordering on type "ARRAY"\. SQLSTATE: 42K09 +SELECT greatest(array(make_interval(1)), array(make_interval(2))); + # An integer and a FLOAT widen to FLOAT with ANSI mode off, and to DOUBLE with it on query T SELECT arrow_typeof(greatest(1::int, 2.5::float)); diff --git a/crates/sqllogictest/test_files/math/least.slt b/crates/sqllogictest/test_files/math/least.slt index ec563ba6..dbc8d8fb 100644 --- a/crates/sqllogictest/test_files/math/least.slt +++ b/crates/sqllogictest/test_files/math/least.slt @@ -150,29 +150,45 @@ SELECT least(column1, column2), least(column1, 3::int, column2), least(column3, NULL 3 NULL 0 b 7 3 Infinity 0 b +# A constant first argument, followed by columns. A NULL constant is skipped. +query IIIR +SELECT least(3::int, column1), least(NULL::int, column1), least(NULL::int, column1, column2), least(0::double, column3) FROM t; +---- +1 1 1 0 +3 5 5 -1 +3 NULL NULL 0 +3 NULL 7 0 + statement ok DROP TABLE t; # Errors # At least two arguments -query error \[WRONG_NUM_ARGS\.WITHOUT_SUGGESTION\] The `least` requires > 1 parameters but the actual number is 1 +query error \[WRONG_NUM_ARGS\.WITHOUT_SUGGESTION\] The `least` requires > 1 parameters but the actual number is 1\. Please, refer to 'https://spark\.apache\.org/docs/latest/sql-ref-functions\.html' for a fix\. SQLSTATE: 42605 SELECT least(1::int); query error SELECT least(); # Strings are not promoted -query error \[DATATYPE_MISMATCH\.DATA_DIFF_TYPES\] Input to `least` should all be the same type, but it's \["INT", "STRING"\] +query error \[DATATYPE_MISMATCH\.DATA_DIFF_TYPES\] Input to `least` should all be the same type, but it's \["INT", "STRING"\]\. SQLSTATE: 42K09 SELECT least(1::int, '2'); query error \[DATATYPE_MISMATCH\.DATA_DIFF_TYPES\] SELECT least(1::int, true); # Maps cannot be ordered -query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `least` does not support ordering on type "MAP" +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `least` does not support ordering on type "MAP"\. SQLSTATE: 42K09 SELECT least(map(1::int, 2::int), map(1::int, 3::int)); +# Intervals cannot be ordered +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `least` does not support ordering on type "INTERVAL"\. SQLSTATE: 42K09 +SELECT least(make_interval(1), make_interval(2)); + +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `least` does not support ordering on type "ARRAY"\. SQLSTATE: 42K09 +SELECT least(array(make_interval(1)), array(make_interval(2))); + # An integer and a FLOAT widen to FLOAT with ANSI mode off, and to DOUBLE with it on query T SELECT arrow_typeof(least(1::int, 2.5::float)); From 96a80e17fda9c679634bb8078ede40a870bd31b0 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 16:55:13 -0700 Subject: [PATCH 07/10] style(spark): apply rustfmt to the greatest and least tests --- crates/spark/src/function/math/greatest_least.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/spark/src/function/math/greatest_least.rs b/crates/spark/src/function/math/greatest_least.rs index 480843af..54b2648d 100644 --- a/crates/spark/src/function/math/greatest_least.rs +++ b/crates/spark/src/function/math/greatest_least.rs @@ -666,9 +666,7 @@ mod tests { ] { let result = invoke(greatest, args, 1, DataType::Float32)?.into_array(1)?; - let value = result - .as_primitive::() - .value(0); + let value = result.as_primitive::().value(0); assert_eq!(value.to_bits(), expected.to_bits(), "greatest={greatest}"); } } From 78623c670c8fbdf8658704ece1e4c3b48678bcd0 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 16:55:13 -0700 Subject: [PATCH 08/10] perf(spark): compare flat floats without branches in greatest and least --- .../spark/src/function/math/greatest_least.rs | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/crates/spark/src/function/math/greatest_least.rs b/crates/spark/src/function/math/greatest_least.rs index 54b2648d..89ca33e5 100644 --- a/crates/spark/src/function/math/greatest_least.rs +++ b/crates/spark/src/function/math/greatest_least.rs @@ -467,21 +467,30 @@ where T: ArrowPrimitiveType, T::Native: Float, { - let candidate = candidate.as_primitive::().values(); - let current = current.as_primitive::().values(); - // IEEE 754 `>` already treats `-0.0` as equal to `0.0`, so only NaN needs handling. - let gt = |a: T::Native, b: T::Native| { - if a.is_nan() { - !b.is_nan() + let candidate: &[T::Native] = candidate.as_primitive::().values(); + let current: &[T::Native] = current.as_primitive::().values(); + // IEEE 754 `>` already treats `-0.0` as equal to `0.0` and is false when either side is + // NaN, so only a NaN `a` over a non-NaN `b` needs adding. `&` and `|` instead of `&&` and + // `||` keep the loop free of branches so it vectorizes. + let gt = |a: T::Native, b: T::Native| (a > b) | (a.is_nan() & !b.is_nan()); + let len = current.len(); + if scalar { + let c = candidate[0]; + if greatest { + BooleanBuffer::collect_bool(len, |row| gt(c, current[row])) } else { - !b.is_nan() && a > b + BooleanBuffer::collect_bool(len, |row| gt(current[row], c)) } - }; - BooleanBuffer::collect_bool(current.len(), |row| { - let c = candidate[if scalar { 0 } else { row }]; - let r = current[row]; - if greatest { gt(c, r) } else { gt(r, c) } - }) + } else { + // Slicing first tells the compiler both slices have `len` values, so it drops the + // bounds check on every row. + let candidate = &candidate[..len]; + if greatest { + BooleanBuffer::collect_bool(len, |row| gt(candidate[row], current[row])) + } else { + BooleanBuffer::collect_bool(len, |row| gt(current[row], candidate[row])) + } + } } /// An argument after the first, with its values as they are and as they are compared. A From cef776e2dab1f094bb5849f3c23ad30ed50a7443 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 19 Nov 2026 05:44:03 -0700 Subject: [PATCH 09/10] refactor(spark): drop the redundant slice in greatest/least's float compare The branchless compare is what lets the per-row loop vectorize, so the explicit slice to len and its comment add nothing. --- crates/spark/src/function/math/greatest_least.rs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/crates/spark/src/function/math/greatest_least.rs b/crates/spark/src/function/math/greatest_least.rs index 89ca33e5..ed8e445c 100644 --- a/crates/spark/src/function/math/greatest_least.rs +++ b/crates/spark/src/function/math/greatest_least.rs @@ -481,15 +481,10 @@ where } else { BooleanBuffer::collect_bool(len, |row| gt(current[row], c)) } + } else if greatest { + BooleanBuffer::collect_bool(len, |row| gt(candidate[row], current[row])) } else { - // Slicing first tells the compiler both slices have `len` values, so it drops the - // bounds check on every row. - let candidate = &candidate[..len]; - if greatest { - BooleanBuffer::collect_bool(len, |row| gt(candidate[row], current[row])) - } else { - BooleanBuffer::collect_bool(len, |row| gt(current[row], candidate[row])) - } + BooleanBuffer::collect_bool(len, |row| gt(current[row], candidate[row])) } } From 2339f588bea4f017e489ce6248fdb73a01590a58 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 19 Nov 2026 10:21:50 -0700 Subject: [PATCH 10/10] docs: regenerate the function reference for greatest and least --- docs/functions/README.md | 4 ++- docs/functions/scalar/math.md | 48 +++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/docs/functions/README.md b/docs/functions/README.md index ba6a387d..efb17586 100644 --- a/docs/functions/README.md +++ b/docs/functions/README.md @@ -17,7 +17,7 @@ function differs from the DataFusion built-in of the same name, if there is one. ## Scalar Functions -- [Math Functions](scalar/math.md) (25) +- [Math Functions](scalar/math.md) (27) - [Conditional Functions](scalar/conditional.md) (2) - [String Functions](scalar/string.md) (17) - [Binary String Functions](scalar/binary_string.md) (5) @@ -86,6 +86,7 @@ function differs from the DataFusion built-in of the same name, if there is one. - [format_string](scalar/string.md#format_string) - [from_utc_timestamp](scalar/time_and_date.md#from_utc_timestamp) - [getbit](scalar/math.md#getbit) +- [greatest](scalar/math.md#greatest) - [hash](scalar/hashing.md#hash) - [hex](scalar/math.md#hex) - [hour](scalar/time_and_date.md#hour) @@ -96,6 +97,7 @@ function differs from the DataFusion built-in of the same name, if there is one. - [json_array_length](scalar/json.md#json_array_length) - [json_tuple](scalar/json.md#json_tuple) - [last_day](scalar/time_and_date.md#last_day) +- [least](scalar/math.md#least) - [len](scalar/string.md#len) - [length](scalar/string.md#length) - [like](scalar/string.md#like) diff --git a/docs/functions/scalar/math.md b/docs/functions/scalar/math.md index 0a0a673d..ce76c2f9 100644 --- a/docs/functions/scalar/math.md +++ b/docs/functions/scalar/math.md @@ -28,8 +28,10 @@ Part of the [datafusion-spark function reference](../README.md). - [factorial](#factorial) - [floor](#floor) - [getbit](#getbit) +- [greatest](#greatest) - [hex](#hex) - [hypot](#hypot) +- [least](#least) - [negative](#negative) - [positive](#positive) - [pow](#pow) @@ -344,6 +346,29 @@ floor(expr) _Alias of [bit_get](#bit_get)._ +## `greatest` + +Returns the greatest value of all arguments, skipping NULLs. The result is NULL only if every argument is NULL. At least two arguments are required. The arguments are cast to a common type as in Spark: numbers widen (an integer and a FLOAT widen to DOUBLE with `datafusion.execution.enable_ansi_mode` on and to FLOAT with it off), integers and decimals widen to a decimal that holds both, a decimal and a float widen to DOUBLE, a date and a timestamp widen to the timestamp, and arrays and structs widen element by element. Strings are never promoted: a string and a number, a string and a date, or a boolean and a number are an error, as are maps, which cannot be ordered. NaN is greater than every other value and `-0.0` equals `0.0`, also inside arrays and structs, and of equal values the first argument wins, so `greatest(-0.0, 0.0)` is `-0.0`. A NULL inside an array or struct is smaller than every other value. DataFusion's built-in `greatest` orders floats by IEEE 754 total order, lets a later argument win a tie, accepts a single argument, and compares numbers with strings. + +```sql +greatest(expr1, expr2[, ..., expr_n]) +``` + +### Arguments + +- **expr1, expr2, ..., expr_n**: Expressions of an orderable type that share a common type. NULLs are skipped. + +### Example + +```sql +> SELECT greatest(10, 9, 2, 4, 3); +10 +> SELECT greatest(1, NULL, 3); +3 +> SELECT greatest(1.0, 'NaN'::double); +NaN +``` + ## `hex` Returns the hexadecimal representation of `expr` as an upper-case string. Numbers are cast to BIGINT and rendered with no leading zeros; negative values are rendered as their 64-bit two's complement (so -1 is `FFFFFFFFFFFFFFFF`). Strings are encoded as the hex of their UTF-8 bytes, and binary values as the hex of their bytes. DataFusion's `to_hex` only accepts integers and returns lower case. Dictionary-encoded input returns a dictionary-encoded string. Returns NULL if the input is NULL. @@ -389,6 +414,29 @@ Infinity NULL ``` +## `least` + +Returns the least value of all arguments, skipping NULLs. The result is NULL only if every argument is NULL. At least two arguments are required. The arguments are cast to a common type as in `greatest`. NaN is greater than every other value and `-0.0` equals `0.0`, also inside arrays and structs, and of equal values the first argument wins, so `least(0.0, -0.0)` is `0.0`. A NULL inside an array or struct is smaller than every other value, so `least(array(1, NULL), array(1, 0))` is `[1, NULL]`. DataFusion's built-in `least` orders floats by IEEE 754 total order, lets a later argument win a tie, orders a NULL inside an array or struct after every other value, accepts a single argument, and compares numbers with strings. + +```sql +least(expr1, expr2[, ..., expr_n]) +``` + +### Arguments + +- **expr1, expr2, ..., expr_n**: Expressions of an orderable type that share a common type. NULLs are skipped. + +### Example + +```sql +> SELECT least(10, 9, 2, 4, 3); +2 +> SELECT least(NULL, 2, NULL); +2 +> SELECT least(1.0, 'NaN'::double); +1.0 +``` + ## `negative` Returns the negation of `expr` (the same as unary minus), with the same type as the input. DataFusion has no built-in `negative`. Accepts integer, floating-point, decimal and interval values. Negating the minimum value of a signed integer type (for example `-2147483648` for INT) depends on ANSI mode: with `datafusion.execution.enable_ansi_mode` off it wraps around and returns the same value, and with it on the function fails with an overflow error. Returns NULL if the input is NULL and NaN if it is NaN.