diff --git a/crates/spark/Cargo.toml b/crates/spark/Cargo.toml index ef542cfb..fcd1357e 100644 --- a/crates/spark/Cargo.toml +++ b/crates/spark/Cargo.toml @@ -118,3 +118,7 @@ name = "array_extrema" [[bench]] harness = false name = "array_remove" + +[[bench]] +harness = false +name = "greatest_least" diff --git a/crates/spark/benches/greatest_least.rs b/crates/spark/benches/greatest_least.rs new file mode 100644 index 00000000..160517c9 --- /dev/null +++ b/crates/spark/benches/greatest_least.rs @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use arrow::array::PrimitiveArray; +use arrow::datatypes::{Field, Float64Type, Int64Type}; +use criterion::{Criterion, criterion_group, criterion_main}; +use datafusion_common::config::ConfigOptions; +use datafusion_expr::{ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF}; +use datafusion_spark::function::math::{greatest, least}; +use rand::rngs::StdRng; +use rand::{Rng, SeedableRng}; +use std::hint::black_box; +use std::sync::Arc; + +const ROWS: usize = 8192; +const NULL_DENSITY: f32 = 0.1; + +fn int64_column(rng: &mut StdRng) -> ColumnarValue { + let values: PrimitiveArray = (0..ROWS) + .map(|_| (rng.random::() >= NULL_DENSITY).then(|| rng.random())) + .collect(); + ColumnarValue::Array(Arc::new(values)) +} + +/// Includes NaN and -0.0, which Spark orders differently from IEEE 754. +fn float64_column(rng: &mut StdRng) -> ColumnarValue { + let values: PrimitiveArray = (0..ROWS) + .map(|_| { + (rng.random::() >= NULL_DENSITY).then(|| { + match rng.random_range(0..100) { + 0 => f64::NAN, + 1 => -0.0, + _ => rng.random::() * 2.0 - 1.0, + } + }) + }) + .collect(); + ColumnarValue::Array(Arc::new(values)) +} + +fn bench(c: &mut Criterion, udf: &ScalarUDF, label: &str, args: &[ColumnarValue]) { + let arg_fields: Vec<_> = args + .iter() + .map(|arg| Arc::new(Field::new("a", arg.data_type(), true))) + .collect(); + let scalar_arguments = vec![None; args.len()]; + let return_field = udf + .return_field_from_args(ReturnFieldArgs { + arg_fields: &arg_fields, + scalar_arguments: &scalar_arguments, + }) + .unwrap(); + let config = Arc::new(ConfigOptions::default()); + c.bench_function(&format!("{} {label}", udf.name()), |b| { + b.iter(|| { + black_box( + udf.invoke_with_args(ScalarFunctionArgs { + args: args.to_vec(), + arg_fields: arg_fields.clone(), + number_rows: ROWS, + return_field: Arc::clone(&return_field), + config_options: Arc::clone(&config), + }) + .unwrap(), + ) + }) + }); +} + +fn criterion_benchmark(c: &mut Criterion) { + let rng = &mut StdRng::seed_from_u64(42); + let ints = [int64_column(rng), int64_column(rng)]; + let floats = [ + float64_column(rng), + float64_column(rng), + float64_column(rng), + ]; + let greatest = greatest(); + bench(c, &greatest, "2 x int64", &ints); + bench(c, &greatest, "2 x float64", &floats[..2]); + bench(c, &greatest, "3 x float64", &floats); + bench(c, &least(), "3 x float64", &floats); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/crates/spark/src/function/float_normalize.rs b/crates/spark/src/function/float_normalize.rs index e166b362..1a76b60d 100644 --- a/crates/spark/src/function/float_normalize.rs +++ b/crates/spark/src/function/float_normalize.rs @@ -35,7 +35,7 @@ pub(crate) fn normalize_floats(value: &ColumnarValue) -> Result { } /// Returns whether values of this type are, or hold at any depth, floating point numbers. -fn contains_float(data_type: &DataType) -> bool { +pub(crate) fn contains_float(data_type: &DataType) -> bool { match data_type { DataType::Float32 | DataType::Float64 => true, DataType::List(field) diff --git a/crates/spark/src/function/math/greatest_least.rs b/crates/spark/src/function/math/greatest_least.rs new file mode 100644 index 00000000..ed8e445c --- /dev/null +++ b/crates/spark/src/function/math/greatest_least.rs @@ -0,0 +1,963 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Spark-compatible `greatest` and `least`. +//! +//! Spark evaluates the arguments in order, skips NULLs, and replaces its running result only +//! with a value that is strictly greater (or smaller) in its SQL ordering, so the result is NULL +//! only when every argument is NULL and the first of equal arguments wins. The SQL ordering +//! (`SQLOrderingUtil.compareDoubles`) treats `-0.0` as equal to `0.0` and NaN as greater than +//! every other value, at any depth of arrays and structs, and orders a NULL inside an array or +//! struct before every other value. +//! +//! DataFusion's built-in `greatest` and `least` differ: they compare floats by IEEE 754 total +//! order (so `-0.0 < 0.0`, and a NaN with the sign bit set is smaller than `-Infinity`), let a +//! later argument win a tie, compare the constant arguments before the columns, accept a single +//! argument, order a NULL inside an array or struct after every other value in `least`, and +//! coerce their arguments with DataFusion's comparison coercion, which for example compares a +//! number with a string. Spark rejects that. + +use crate::function::float_normalize::{contains_float, normalize_array}; +use crate::function::type_coercion::{is_orderable, spark_sql_type, wider_type}; +use arrow::array::{ + Array, ArrayRef, ArrowPrimitiveType, AsArray, BooleanArray, Datum, Scalar, + make_comparator, new_null_array, +}; +use arrow::buffer::BooleanBuffer; +use arrow::compute::kernels::cmp::{gt, lt}; +use arrow::compute::kernels::zip::zip; +use arrow::compute::{SortOptions, cast}; +use arrow::datatypes::{DataType, Field, FieldRef, Float32Type, Float64Type}; +use datafusion_common::config::ConfigOptions; +use datafusion_common::{DataFusionError, Result, ScalarValue, plan_err}; +use datafusion_expr::{ + ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF, + ScalarUDFImpl, Signature, Volatility, +}; +use datafusion_macros::user_doc; +use num_traits::Float; +use std::sync::Arc; + +/// Spark-compatible `greatest` expression. +/// +/// +#[user_doc( + doc_section(label = "Math Functions"), + description = "Returns the greatest value of all arguments, skipping NULLs. The result is \ + NULL only if every argument is NULL. At least two arguments are required. The \ + arguments are cast to a common type as in Spark: numbers widen (an integer and a FLOAT \ + widen to DOUBLE with `datafusion.execution.enable_ansi_mode` on and to FLOAT with it \ + off), integers and decimals widen to a decimal that holds both, a decimal and a float \ + widen to DOUBLE, a date and a timestamp widen to the timestamp, and arrays and structs \ + widen element by element. Strings are never promoted: a string and a number, a string \ + and a date, or a boolean and a number are an error, as are maps, which cannot be \ + ordered. NaN is greater than every other value and `-0.0` equals `0.0`, also inside \ + arrays and structs, and of equal values the first argument wins, so \ + `greatest(-0.0, 0.0)` is `-0.0`. A NULL inside an array or struct is smaller than \ + every other value. DataFusion's built-in `greatest` orders floats by IEEE 754 total \ + order, lets a later argument win a tie, accepts a single argument, and compares \ + numbers with strings.", + syntax_example = "greatest(expr1, expr2[, ..., expr_n])", + sql_example = r#"```sql +> SELECT greatest(10, 9, 2, 4, 3); +10 +> SELECT greatest(1, NULL, 3); +3 +> SELECT greatest(1.0, 'NaN'::double); +NaN +```"#, + argument( + name = "expr1, expr2, ..., expr_n", + description = "Expressions of an orderable type that share a common type. NULLs are \ + skipped." + ) +)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkGreatest { + inner: GreatestLeast, +} + +impl Default for SparkGreatest { + fn default() -> Self { + Self::new() + } +} + +impl SparkGreatest { + pub fn new() -> Self { + Self { + inner: GreatestLeast::new(true, false), + } + } + + /// Creates a `greatest` that coerces its arguments as Spark does with + /// `datafusion.execution.enable_ansi_mode` set to `enable_ansi_mode`. + fn new_with_ansi_mode(enable_ansi_mode: bool) -> Self { + Self { + inner: GreatestLeast::new(true, enable_ansi_mode), + } + } +} + +/// Spark-compatible `least` expression. +/// +/// +#[user_doc( + doc_section(label = "Math Functions"), + description = "Returns the least value of all arguments, skipping NULLs. The result is \ + NULL only if every argument is NULL. At least two arguments are required. The \ + arguments are cast to a common type as in `greatest`. NaN is greater than every other \ + value and `-0.0` equals `0.0`, also inside arrays and structs, and of equal values the \ + first argument wins, so `least(0.0, -0.0)` is `0.0`. A NULL inside an array or struct \ + is smaller than every other value, so `least(array(1, NULL), array(1, 0))` is \ + `[1, NULL]`. DataFusion's built-in `least` orders floats by IEEE 754 total order, lets a \ + later argument win a tie, orders a NULL inside an array or struct after every other \ + value, accepts a single argument, and compares numbers with strings.", + syntax_example = "least(expr1, expr2[, ..., expr_n])", + sql_example = r#"```sql +> SELECT least(10, 9, 2, 4, 3); +2 +> SELECT least(NULL, 2, NULL); +2 +> SELECT least(1.0, 'NaN'::double); +1.0 +```"#, + argument( + name = "expr1, expr2, ..., expr_n", + description = "Expressions of an orderable type that share a common type. NULLs are \ + skipped." + ) +)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkLeast { + inner: GreatestLeast, +} + +impl Default for SparkLeast { + fn default() -> Self { + Self::new() + } +} + +impl SparkLeast { + pub fn new() -> Self { + Self { + inner: GreatestLeast::new(false, false), + } + } + + /// Creates a `least` that coerces its arguments as Spark does with + /// `datafusion.execution.enable_ansi_mode` set to `enable_ansi_mode`. + fn new_with_ansi_mode(enable_ansi_mode: bool) -> Self { + Self { + inner: GreatestLeast::new(false, enable_ansi_mode), + } + } +} + +macro_rules! impl_greatest_least_udf { + ($udf:ident) => { + impl ScalarUDFImpl for $udf { + fn name(&self) -> &str { + self.inner.name() + } + + fn signature(&self) -> &Signature { + &self.inner.signature + } + + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { + self.inner.coerce_types(arg_types) + } + + fn return_type(&self, arg_types: &[DataType]) -> Result { + self.inner.check_arg_count(arg_types.len(), true)?; + Ok(arg_types[0].clone()) + } + + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + self.inner.check_arg_count(args.arg_fields.len(), true)?; + // As in Spark, the result is nullable only if every argument is. + let nullable = args.arg_fields.iter().all(|f| f.is_nullable()); + Ok(Arc::new(Field::new( + self.name(), + args.arg_fields[0].data_type().clone(), + nullable, + ))) + } + + fn with_updated_config(&self, config: &ConfigOptions) -> Option { + Some(ScalarUDF::from(Self::new_with_ansi_mode( + config.execution.enable_ansi_mode, + ))) + } + + fn invoke_with_args( + &self, + args: ScalarFunctionArgs, + ) -> Result { + self.inner.invoke(args) + } + + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + } + }; +} + +impl_greatest_least_udf!(SparkGreatest); +impl_greatest_least_udf!(SparkLeast); + +/// The implementation shared by [`SparkGreatest`] and [`SparkLeast`]. +#[derive(Debug, PartialEq, Eq, Hash)] +struct GreatestLeast { + signature: Signature, + greatest: bool, + /// Whether to coerce as Spark's `AnsiTypeCoercion` does, rather than `TypeCoercion`. + ansi_mode: bool, +} + +/// Spark orders a NULL inside an array or struct before every other value. +const SORT_OPTIONS: SortOptions = SortOptions { + descending: false, + nulls_first: true, +}; + +impl GreatestLeast { + fn new(greatest: bool, ansi_mode: bool) -> Self { + Self { + signature: Signature::user_defined(Volatility::Immutable), + greatest, + ansi_mode, + } + } + + fn name(&self) -> &'static str { + if self.greatest { "greatest" } else { "least" } + } + + fn check_arg_count(&self, count: usize, planning: bool) -> Result<()> { + if count > 1 { + return Ok(()); + } + let message = format!( + "[WRONG_NUM_ARGS.WITHOUT_SUGGESTION] The `{}` requires > 1 parameters but the \ + actual number is {count}. Please, refer to \ + 'https://spark.apache.org/docs/latest/sql-ref-functions.html' for a fix. \ + SQLSTATE: 42605", + self.name() + ); + Err(if planning { + DataFusionError::Plan(message) + } else { + DataFusionError::Execution(message) + }) + } + + /// Spark's `TypeCoercion` rule for `Greatest` and `Least`: every argument is cast to + /// `findWiderTypeWithoutStringPromotion` of all the argument types, and the result must be + /// orderable. + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { + self.check_arg_count(arg_types.len(), true)?; + let common = arg_types + .iter() + .skip(1) + .try_fold(arg_types[0].clone(), |common, data_type| { + wider_type(&common, data_type, self.ansi_mode) + }); + let Some(common) = common else { + return plan_err!( + "[DATATYPE_MISMATCH.DATA_DIFF_TYPES] Input to `{}` should all be the same \ + type, but it's [{}]. SQLSTATE: 42K09", + self.name(), + arg_types + .iter() + .map(|t| format!("\"{}\"", spark_sql_type(t))) + .collect::>() + .join(", ") + ); + }; + if !is_orderable(&common) { + return plan_err!( + "[DATATYPE_MISMATCH.INVALID_ORDERING_TYPE] The `{}` does not support ordering \ + on type \"{}\". SQLSTATE: 42K09", + self.name(), + spark_sql_type(&common) + ); + } + Ok(vec![common; arg_types.len()]) + } + + fn invoke(&self, args: ScalarFunctionArgs) -> Result { + self.check_arg_count(args.args.len(), false)?; + let data_type = args.return_field.data_type(); + let all_scalars = args + .args + .iter() + .all(|arg| matches!(arg, ColumnarValue::Scalar(_))); + let rows = if all_scalars { 1 } else { args.number_rows }; + + if data_type.is_null() { + return Ok(if all_scalars { + ColumnarValue::Scalar(ScalarValue::Null) + } else { + ColumnarValue::Array(new_null_array(&DataType::Null, rows)) + }); + } + + // The planner casts every argument to the common type, but a field name or the + // nullability of a nested field can still differ when the function is called directly, + // and `zip` needs identical types. + let to_array = |arg: &ColumnarValue, rows: usize| -> Result { + let array = arg.to_array(rows)?; + Ok(if array.data_type() == data_type { + array + } else { + cast(&array, data_type)? + }) + }; + // Nested floats are compared in a copy where `-0.0` is `0.0` and every NaN is the + // canonical NaN, where Arrow's total order agrees with Spark's. The result keeps the + // original. Flat floats are compared in place by `spark_float_ordered`. + let normalize = data_type.is_nested() && contains_float(data_type); + let comparable = |array: &ArrayRef| -> Result { + if normalize { + normalize_array(array) + } else { + Ok(Arc::clone(array)) + } + }; + + let mut result = to_array(&args.args[0], rows)?; + let mut result_cmp = comparable(&result)?; + for arg in &args.args[1..] { + let candidate = match arg { + // A NULL never replaces the result. + ColumnarValue::Scalar(scalar) if scalar.is_null() => continue, + ColumnarValue::Scalar(_) => { + let value = to_array(arg, 1)?; + let cmp = comparable(&value)?; + Candidate { + value, + cmp, + scalar: true, + } + } + ColumnarValue::Array(_) => { + let value = to_array(arg, rows)?; + let cmp = comparable(&value)?; + Candidate { + value, + cmp, + scalar: false, + } + } + }; + let take = self.replaces(&candidate, &result, &result_cmp)?; + let take = BooleanArray::new(take, None); + let new_result = zip(&take, candidate.value_datum().as_ref(), &result)?; + result_cmp = if normalize { + zip(&take, candidate.cmp_datum().as_ref(), &result_cmp)? + } else { + Arc::clone(&new_result) + }; + result = new_result; + } + + if all_scalars { + Ok(ColumnarValue::Scalar(ScalarValue::try_from_array( + &result, 0, + )?)) + } else { + Ok(ColumnarValue::Array(result)) + } + } + + /// For each row, whether `candidate` replaces the current result: it is not NULL, and + /// either the result is NULL or the candidate ranks strictly before it. + fn replaces( + &self, + candidate: &Candidate, + current: &ArrayRef, + current_cmp: &ArrayRef, + ) -> Result { + let len = current.len(); + let current_nulls = current.logical_nulls(); + let candidate_nulls = if candidate.scalar { + // A NULL constant is skipped before this is called. + None + } else { + candidate.value.logical_nulls() + }; + + if !current.data_type().is_nested() { + // Where either side is NULL the comparison's value is arbitrary, but such rows are + // decided by the null masks below. + let mut take = match current.data_type() { + DataType::Float32 => spark_float_ordered::( + &candidate.cmp, + candidate.scalar, + current_cmp, + self.greatest, + ), + DataType::Float64 => spark_float_ordered::( + &candidate.cmp, + candidate.scalar, + current_cmp, + self.greatest, + ), + _ => { + let candidate_cmp = candidate.cmp_datum(); + let ordered = if self.greatest { + gt(candidate_cmp.as_ref(), current_cmp)? + } else { + lt(candidate_cmp.as_ref(), current_cmp)? + }; + ordered.values().clone() + } + }; + if let Some(nulls) = ¤t_nulls { + take = &take | &!nulls.inner(); + } + if let Some(nulls) = &candidate_nulls { + take = &take & nulls.inner(); + } + return Ok(take); + } + + let compare = + make_comparator(candidate.cmp.as_ref(), current_cmp.as_ref(), SORT_OPTIONS)?; + let scalar = candidate.scalar; + Ok(BooleanBuffer::collect_bool(len, |row| { + let candidate_row = if scalar { 0 } else { row }; + if candidate_nulls + .as_ref() + .is_some_and(|nulls| nulls.is_null(candidate_row)) + { + return false; + } + if current_nulls + .as_ref() + .is_some_and(|nulls| nulls.is_null(row)) + { + return true; + } + let ordering = compare(candidate_row, row); + if self.greatest { + ordering.is_gt() + } else { + ordering.is_lt() + } + })) + } +} + +/// For each row, whether `candidate` ranks strictly before `current` in Spark's float order, +/// where NaN is greater than every other value and `-0.0` equals `0.0`. A scalar candidate is +/// a one-row array. +fn spark_float_ordered( + candidate: &ArrayRef, + scalar: bool, + current: &ArrayRef, + greatest: bool, +) -> BooleanBuffer +where + T: ArrowPrimitiveType, + T::Native: Float, +{ + let candidate: &[T::Native] = candidate.as_primitive::().values(); + let current: &[T::Native] = current.as_primitive::().values(); + // IEEE 754 `>` already treats `-0.0` as equal to `0.0` and is false when either side is + // NaN, so only a NaN `a` over a non-NaN `b` needs adding. `&` and `|` instead of `&&` and + // `||` keep the loop free of branches so it vectorizes. + let gt = |a: T::Native, b: T::Native| (a > b) | (a.is_nan() & !b.is_nan()); + let len = current.len(); + if scalar { + let c = candidate[0]; + if greatest { + BooleanBuffer::collect_bool(len, |row| gt(c, current[row])) + } else { + BooleanBuffer::collect_bool(len, |row| gt(current[row], c)) + } + } else if greatest { + BooleanBuffer::collect_bool(len, |row| gt(candidate[row], current[row])) + } else { + BooleanBuffer::collect_bool(len, |row| gt(current[row], candidate[row])) + } +} + +/// An argument after the first, with its values as they are and as they are compared. A +/// constant is a one-row array. +struct Candidate { + value: ArrayRef, + cmp: ArrayRef, + scalar: bool, +} + +impl Candidate { + fn datum(&self, array: &ArrayRef) -> Box { + if self.scalar { + Box::new(Scalar::new(Arc::clone(array))) + } else { + Box::new(Arc::clone(array)) + } + } + + fn value_datum(&self) -> Box { + self.datum(&self.value) + } + + fn cmp_datum(&self) -> Box { + self.datum(&self.cmp) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{ + AsArray, Float32Array, Float64Array, Int32Array, ListArray, StringArray, + StructArray, + }; + use arrow::buffer::OffsetBuffer; + use arrow::datatypes::{Fields, TimeUnit}; + use arrow::datatypes::{Float64Type, Int32Type}; + + /// Values on which Spark's ordering and IEEE 754 total order disagree, with neighbors and a + /// NULL. `NEGATIVE_NAN` is what arithmetic produces on x86-64. + const NEGATIVE_NAN: f64 = f64::from_bits(0xfff8_0000_0000_0000); + const PAYLOAD_NAN: f64 = f64::from_bits(0x7ff0_0000_0000_0001); + const EDGE_VALUES: [Option; 10] = [ + Some(f64::NEG_INFINITY), + Some(-1.0), + Some(-0.0), + Some(0.0), + Some(1.0), + Some(f64::INFINITY), + Some(f64::NAN), + Some(NEGATIVE_NAN), + Some(PAYLOAD_NAN), + None, + ]; + + /// `SQLOrderingUtil.compareDoubles`. + fn compare_doubles(left: f64, right: f64) -> std::cmp::Ordering { + use std::cmp::Ordering::*; + match (left.is_nan(), right.is_nan()) { + (true, true) => Equal, + (true, false) => Greater, + (false, true) => Less, + _ => left.partial_cmp(&right).unwrap(), + } + } + + /// Spark's `Greatest.eval` and `Least.eval`, over doubles. + fn spark_extreme(values: &[Option], greatest: bool) -> Option { + values + .iter() + .flatten() + .fold(None, |best, &value| match best { + None => Some(value), + Some(best) => { + let ordering = compare_doubles(value, best); + let replace = if greatest { + ordering.is_gt() + } else { + ordering.is_lt() + }; + Some(if replace { value } else { best }) + } + }) + } + + fn udf(greatest: bool) -> GreatestLeast { + GreatestLeast::new(greatest, true) + } + + fn invoke( + greatest: bool, + args: Vec, + rows: usize, + return_type: DataType, + ) -> Result { + let return_field: FieldRef = Arc::new(Field::new("result", return_type, true)); + udf(greatest).invoke(ScalarFunctionArgs { + args, + arg_fields: vec![], + number_rows: rows, + return_field, + config_options: Arc::new(ConfigOptions::default()), + }) + } + + fn bits(array: &ArrayRef) -> Vec> { + let array = array.as_primitive::(); + (0..array.len()) + .map(|i| array.is_valid(i).then(|| array.value(i).to_bits())) + .collect() + } + + /// Every combination of three edge values, as columns and with the middle argument also as + /// a constant. The combinations without a NULL run again on their own. + #[test] + fn floats_follow_spark_order() -> Result<()> { + let all_rows: Vec<[Option; 3]> = EDGE_VALUES + .iter() + .flat_map(|&a| { + EDGE_VALUES + .iter() + .flat_map(move |&b| EDGE_VALUES.iter().map(move |&c| [a, b, c])) + }) + .collect(); + let without_nulls: Vec<_> = all_rows + .iter() + .copied() + .filter(|row| row.iter().all(Option::is_some)) + .collect(); + for rows in [all_rows, without_nulls] { + let column = |i: usize| -> ColumnarValue { + ColumnarValue::Array(Arc::new(Float64Array::from( + rows.iter().map(|row| row[i]).collect::>(), + ))) + }; + for greatest in [true, false] { + let args = vec![column(0), column(1), column(2)]; + let result = invoke(greatest, args, rows.len(), DataType::Float64)? + .into_array(rows.len())?; + let expected: Vec> = rows + .iter() + .map(|row| spark_extreme(row, greatest).map(f64::to_bits)) + .collect(); + assert_eq!(bits(&result), expected, "greatest={greatest}"); + + for middle in EDGE_VALUES { + let args = vec![ + column(0), + ColumnarValue::Scalar(ScalarValue::Float64(middle)), + column(2), + ]; + let result = invoke(greatest, args, rows.len(), DataType::Float64)? + .into_array(rows.len())?; + let expected: Vec> = rows + .iter() + .map(|row| { + spark_extreme(&[row[0], middle, row[2]], greatest) + .map(f64::to_bits) + }) + .collect(); + assert_eq!( + bits(&result), + expected, + "middle={middle:?} greatest={greatest}" + ); + } + } + } + Ok(()) + } + + #[test] + fn float32_ties_keep_the_first_argument() -> Result<()> { + let zero = ColumnarValue::Array(Arc::new(Float32Array::from(vec![0.0f32]))); + let negative_zero = + ColumnarValue::Array(Arc::new(Float32Array::from(vec![-0.0f32]))); + for greatest in [true, false] { + for (args, expected) in [ + (vec![negative_zero.clone(), zero.clone()], -0.0f32), + (vec![zero.clone(), negative_zero.clone()], 0.0f32), + ] { + let result = + invoke(greatest, args, 1, DataType::Float32)?.into_array(1)?; + let value = result.as_primitive::().value(0); + assert_eq!(value.to_bits(), expected.to_bits(), "greatest={greatest}"); + } + } + Ok(()) + } + + #[test] + fn constants_only_return_a_constant() -> Result<()> { + let scalar = |v: Option| ColumnarValue::Scalar(ScalarValue::Float64(v)); + for (greatest, args, expected) in [ + ( + true, + vec![scalar(Some(-0.0)), scalar(Some(0.0))], + Some(-0.0), + ), + ( + false, + vec![scalar(Some(0.0)), scalar(Some(-0.0))], + Some(0.0), + ), + ( + true, + vec![ + scalar(None), + scalar(Some(f64::NAN)), + scalar(Some(f64::INFINITY)), + ], + Some(f64::NAN), + ), + (false, vec![scalar(None), scalar(None)], None), + ] { + match invoke(greatest, args, 5, DataType::Float64)? { + ColumnarValue::Scalar(ScalarValue::Float64(actual)) => { + assert_eq!(actual.map(f64::to_bits), expected.map(f64::to_bits)) + } + other => panic!("expected a double scalar, got {other:?}"), + } + } + Ok(()) + } + + #[test] + fn integers_and_strings_with_nulls() -> Result<()> { + let ints = + |v: Vec>| ColumnarValue::Array(Arc::new(Int32Array::from(v))); + let args = vec![ + ints(vec![Some(1), None, None, Some(5)]), + ColumnarValue::Scalar(ScalarValue::Int32(Some(3))), + ints(vec![Some(2), Some(7), None, None]), + ]; + let result = invoke(true, args.clone(), 4, DataType::Int32)?.into_array(4)?; + assert_eq!( + result.as_ref(), + &Int32Array::from(vec![Some(3), Some(7), Some(3), Some(5)]) + ); + let result = invoke(false, args, 4, DataType::Int32)?.into_array(4)?; + assert_eq!( + result.as_ref(), + &Int32Array::from(vec![Some(1), Some(3), Some(3), Some(3)]) + ); + + let strings = + |v: Vec>| ColumnarValue::Array(Arc::new(StringArray::from(v))); + let args = vec![ + strings(vec![Some("abc"), None, Some("")]), + strings(vec![Some("abd"), None, Some("a")]), + ]; + let result = invoke(true, args.clone(), 3, DataType::Utf8)?.into_array(3)?; + assert_eq!( + result.as_ref(), + &StringArray::from(vec![Some("abd"), None, Some("a")]) + ); + let result = invoke(false, args, 3, DataType::Utf8)?.into_array(3)?; + assert_eq!( + result.as_ref(), + &StringArray::from(vec![Some("abc"), None, Some("")]) + ); + Ok(()) + } + + /// A NULL inside a list ranks first, lists compare element by element and then by length, + /// and floats inside lists follow Spark's order. The second list's element field has a + /// different name and nullability, which the result type decides. + #[test] + fn nested_arguments() -> Result<()> { + let list = |values: Vec>>>| -> ColumnarValue { + ColumnarValue::Array(Arc::new(ListArray::from_iter_primitive::< + Int32Type, + _, + _, + >(values))) + }; + let args = vec![ + list(vec![ + Some(vec![Some(1), None]), + Some(vec![Some(1), Some(2)]), + None, + ]), + list(vec![ + Some(vec![Some(1), Some(0)]), + Some(vec![Some(1), Some(2), Some(3)]), + None, + ]), + ]; + let list_type = args[0].data_type(); + for (greatest, expected) in [ + ( + true, + list(vec![ + Some(vec![Some(1), Some(0)]), + Some(vec![Some(1), Some(2), Some(3)]), + None, + ]), + ), + ( + false, + list(vec![ + Some(vec![Some(1), None]), + Some(vec![Some(1), Some(2)]), + None, + ]), + ), + ] { + let result = + invoke(greatest, args.clone(), 3, list_type.clone())?.into_array(3)?; + assert_eq!(result.as_ref(), expected.into_array(3)?.as_ref()); + } + + let float_list = + |name: &str, nullable: bool, values: Vec| -> ColumnarValue { + let field = Arc::new(Field::new(name, DataType::Float64, nullable)); + ColumnarValue::Array(Arc::new(ListArray::new( + field, + OffsetBuffer::from_lengths(vec![1; values.len()]), + Arc::new(Float64Array::from(values)), + None, + ))) + }; + let return_type = + DataType::List(Arc::new(Field::new("item", DataType::Float64, true))); + for (greatest, expected) in + [(true, [-0.0, f64::NAN]), (false, [-0.0, f64::INFINITY])] + { + let args = vec![ + float_list("item", true, vec![-0.0, f64::INFINITY]), + float_list("element", false, vec![0.0, f64::NAN]), + ]; + let result = invoke(greatest, args, 2, return_type.clone())?.into_array(2)?; + assert_eq!(result.data_type(), &return_type); + let values: ArrayRef = Arc::clone(result.as_list::().values()); + let expected: Vec> = + expected.iter().map(|v| Some(v.to_bits())).collect(); + assert_eq!(bits(&values), expected, "greatest={greatest}"); + } + + // A constant struct against a column of structs. + let fields = Fields::from(vec![Field::new("a", DataType::Float64, true)]); + let structs = |values: Vec>| -> ArrayRef { + Arc::new(StructArray::new( + fields.clone(), + vec![Arc::new(Float64Array::from(values))], + None, + )) + }; + let constant = ScalarValue::try_from_array(&structs(vec![Some(f64::NAN)]), 0)?; + let args = vec![ + ColumnarValue::Array(structs(vec![Some(1.0), None])), + ColumnarValue::Scalar(constant), + ]; + let struct_type = DataType::Struct(fields.clone()); + let result = invoke(true, args.clone(), 2, struct_type.clone())?.into_array(2)?; + let a: ArrayRef = Arc::clone(result.as_struct().column(0)); + assert_eq!( + bits(&a), + vec![Some(f64::NAN.to_bits()), Some(f64::NAN.to_bits())] + ); + let result = invoke(false, args, 2, struct_type)?.into_array(2)?; + let a: ArrayRef = Arc::clone(result.as_struct().column(0)); + // A NULL field ranks first. + assert_eq!(bits(&a), vec![Some(1.0f64.to_bits()), None]); + Ok(()) + } + + #[test] + fn null_type() -> Result<()> { + let args = vec![ + ColumnarValue::Scalar(ScalarValue::Null), + ColumnarValue::Scalar(ScalarValue::Null), + ]; + assert!(matches!( + invoke(true, args, 3, DataType::Null)?, + ColumnarValue::Scalar(ScalarValue::Null) + )); + Ok(()) + } + + #[test] + fn rejects_fewer_than_two_arguments() { + let scalar = ColumnarValue::Scalar(ScalarValue::Float64(Some(1.0))); + for greatest in [true, false] { + for arg_types in [vec![], vec![DataType::Float64]] { + let error = udf(greatest).coerce_types(&arg_types).unwrap_err(); + assert!( + error + .to_string() + .contains("[WRONG_NUM_ARGS.WITHOUT_SUGGESTION]"), + "unexpected error: {error}" + ); + } + let error = + invoke(greatest, vec![scalar.clone()], 1, DataType::Float64).unwrap_err(); + assert!( + error.to_string().contains("requires > 1 parameters"), + "unexpected error: {error}" + ); + } + } + + #[test] + fn coercion() { + use DataType::*; + let coerce = |types: Vec, ansi: bool| { + GreatestLeast::new(true, ansi) + .coerce_types(&types) + .map(|t| t[0].clone()) + }; + let list = |t: DataType| List(Arc::new(Field::new("item", t, true))); + let ok = [ + (vec![Int32, Float64], true, Float64), + (vec![Int32, Float32], true, Float64), + (vec![Int64, Float32], false, Float32), + (vec![Int8, Int16], true, Int16), + (vec![Null, Int32, Null], true, Int32), + (vec![Null, Null], true, Null), + (vec![Decimal128(11, 1), Int32], true, Decimal128(11, 1)), + (vec![Decimal128(2, 1), Int32], true, Decimal128(11, 1)), + (vec![Decimal128(5, 2), Int64], true, Decimal128(22, 2)), + ( + vec![Decimal128(38, 10), Decimal128(38, 0)], + true, + Decimal128(38, 0), + ), + (vec![Decimal128(10, 2), Float32], true, Float64), + (vec![Utf8, Utf8View], true, Utf8View), + ( + vec![Date32, Timestamp(TimeUnit::Microsecond, Some("UTC".into()))], + true, + Timestamp(TimeUnit::Microsecond, Some("UTC".into())), + ), + (vec![list(Int32), list(Float64)], true, list(Float64)), + ]; + for (types, ansi, expected) in ok { + assert_eq!(coerce(types.clone(), ansi).unwrap(), expected, "{types:?}"); + } + let diff_types = [ + vec![Int32, Utf8], + vec![Date32, Utf8], + vec![Boolean, Int32], + vec![list(Int32), Int32], + ]; + for types in diff_types { + let error = coerce(types.clone(), true).unwrap_err().to_string(); + assert!( + error.contains("[DATATYPE_MISMATCH.DATA_DIFF_TYPES]"), + "{types:?}: {error}" + ); + } + let error = coerce(vec![Int32, Utf8], true).unwrap_err().to_string(); + assert!(error.contains(r#"but it's ["INT", "STRING"]"#), "{error}"); + + let map = Map( + Arc::new(Field::new( + "entries", + Struct(Fields::from(vec![ + Field::new("key", Int32, false), + Field::new("value", Int32, true), + ])), + false, + )), + false, + ); + let error = coerce(vec![map.clone(), map], true) + .unwrap_err() + .to_string(); + assert!( + error.contains(r#"[DATATYPE_MISMATCH.INVALID_ORDERING_TYPE] The `greatest` does not support ordering on type "MAP""#), + "{error}" + ); + } +} diff --git a/crates/spark/src/function/math/mod.rs b/crates/spark/src/function/math/mod.rs index 1367a84f..129d40d4 100644 --- a/crates/spark/src/function/math/mod.rs +++ b/crates/spark/src/function/math/mod.rs @@ -13,6 +13,7 @@ pub mod divide; pub mod expm1; pub mod factorial; pub mod floor; +pub mod greatest_least; pub mod hex; pub mod hypot; pub mod modulus; @@ -35,6 +36,8 @@ make_udf_function!(ceil::SparkCeil, ceil); make_udf_function!(expm1::SparkExpm1, expm1); make_udf_function!(factorial::SparkFactorial, factorial); make_udf_function!(floor::SparkFloor, floor); +make_udf_function!(greatest_least::SparkGreatest, greatest); +make_udf_function!(greatest_least::SparkLeast, least); make_udf_function!(hex::SparkHex, hex); make_udf_function!(hypot::SparkHypot, hypot); make_udf_function!(modulus::SparkMod, modulus); @@ -65,6 +68,16 @@ pub mod expr_fn { arg1 )); export_functions!((floor, "Returns floor of expr.", arg1)); + export_functions!(( + greatest, + "Returns the greatest value of all arguments, skipping nulls.", + args, + )); + export_functions!(( + least, + "Returns the least value of all arguments, skipping nulls.", + args, + )); export_functions!((hex, "Computes hex value of the given column.", arg1)); export_functions!((hypot, "Returns sqrt(a^2 + b^2) without intermediate overflow or underflow.", arg1 arg2)); export_functions!((modulus, "Returns the remainder of division of the first argument by the second argument.", arg1 arg2)); @@ -114,6 +127,8 @@ pub fn functions() -> Vec> { expm1(), factorial(), floor(), + greatest(), + least(), hex(), hypot(), modulus(), diff --git a/crates/spark/src/function/type_coercion.rs b/crates/spark/src/function/type_coercion.rs index 98fbdfa7..0bfd63e1 100644 --- a/crates/spark/src/function/type_coercion.rs +++ b/crates/spark/src/function/type_coercion.rs @@ -68,6 +68,50 @@ pub(crate) fn tightest_common_type( } } +/// Spark's `findWiderTypeWithoutStringPromotionForTwo`: the tightest common type, then a +/// decimal wide enough for both, then the same for the elements of arrays and the fields of +/// structs. Strings are never promoted. `ansi_mode` picks between `AnsiTypeCoercion` and +/// `TypeCoercion`, which differ only in that ANSI widens an integer and a FLOAT to DOUBLE. +pub(crate) fn wider_type( + left: &DataType, + right: &DataType, + ansi_mode: bool, +) -> Option { + tightest_common_type(left, right, ansi_mode) + .or_else(|| wider_decimal_type(left, right)) + .or_else(|| complex_type(left, right, &|l, r| wider_type(l, r, ansi_mode))) +} + +/// Spark's `findWiderTypeForDecimal`, with `DecimalType.boundedPreferIntegralDigits`, which +/// gives up fractional digits rather than integral ones when the precision exceeds 38. +fn wider_decimal_type(left: &DataType, right: &DataType) -> Option { + use DataType::*; + let (p1, s1, p2, s2) = match (left, right) { + (Decimal128(p1, s1), Decimal128(p2, s2)) => { + (*p1 as i16, *s1 as i16, *p2 as i16, *s2 as i16) + } + (int, Decimal128(p, s)) | (Decimal128(p, s), int) + if integral_digits(int).is_some() => + { + (integral_digits(int)?, 0, *p as i16, *s as i16) + } + (Float32 | Float64, Decimal128(_, _)) | (Decimal128(_, _), Float32 | Float64) => { + return Some(Float64); + } + _ => return None, + }; + let scale = s1.max(s2); + let range = (p1 - s1).max(p2 - s2); + let precision = scale + range; + const MAX_PRECISION: i16 = 38; + if precision <= MAX_PRECISION { + Some(Decimal128(precision as u8, scale as i8)) + } else { + let scale = (scale - (precision - MAX_PRECISION)).max(0); + Some(Decimal128(MAX_PRECISION as u8, scale as i8)) + } +} + /// Spark's `findTypeForComplex`: widens the elements of arrays, the keys and values of maps, /// and the fields of structs with `find_type`. A nested field is nullable if it is on either /// side. Struct fields must have the same names in the same order. Spark compares them diff --git a/crates/sqllogictest/test_files/math/greatest.slt b/crates/sqllogictest/test_files/math/greatest.slt index 48b11a8d..e4410a48 100644 --- a/crates/sqllogictest/test_files/math/greatest.slt +++ b/crates/sqllogictest/test_files/math/greatest.slt @@ -10,5 +10,249 @@ ## Original Query: SELECT greatest(10, 9, 2, 4, 3); ## PySpark 3.5.5 Result: {'greatest(10, 9, 2, 4, 3)': 10, 'typeof(greatest(10, 9, 2, 4, 3))': 'int', 'typeof(10)': 'int', 'typeof(9)': 'int', 'typeof(2)': 'int', 'typeof(4)': 'int', 'typeof(3)': 'int'} -#query -#SELECT greatest(10::int, 9::int, 2::int, 4::int, 3::int); +query I +SELECT greatest(10::int, 9::int, 2::int, 4::int, 3::int); +---- +10 + +# NULLs are skipped; the result is NULL only if every argument is NULL +query III +SELECT greatest(1::int, NULL::int, 3::int), greatest(NULL::int, 2::int), greatest(NULL::int, NULL::int); +---- +3 2 NULL + +query ? +SELECT greatest(NULL, NULL); +---- +NULL + +# Integers widen to the widest integer type +query I +SELECT greatest(1::tinyint, 2::smallint); +---- +2 + +query T +SELECT arrow_typeof(greatest(1::tinyint, 2::smallint)); +---- +Int16 + +query I +SELECT greatest(1::int, 3000000000::bigint); +---- +3000000000 + +# An integer and a DOUBLE widen to DOUBLE +query R +SELECT greatest(1::int, 2.5::double); +---- +2.5 + +query T +SELECT arrow_typeof(greatest(1::int, 2.5::double)); +---- +Float64 + +# NaN is greater than every other value, including Infinity +query RRR +SELECT greatest('NaN'::double, 'Infinity'::double), greatest(1::double, 'NaN'::double), greatest('-Infinity'::double, NULL::double); +---- +NaN NaN -Infinity + +query R +SELECT greatest('NaN'::float, 1::float); +---- +NaN + +# -0.0 equals 0.0 and the first of equal values wins. The sign of a zero shows in its string +# form, which starts with '-' only for -0.0. +query BB +SELECT CAST(greatest('-0.0'::double, 0.0::double) AS STRING) LIKE '-%', CAST(greatest(0.0::double, '-0.0'::double) AS STRING) LIKE '-%'; +---- +true false + +# Decimals +query R +SELECT greatest(1.5::decimal(2,1), 2::int); +---- +2.0 + +query T +SELECT arrow_typeof(greatest(1.5::decimal(2,1), 2::int)); +---- +Decimal128(11, 1) + +query T +SELECT arrow_typeof(greatest(1::decimal(5,2), 1::bigint)); +---- +Decimal128(22, 2) + +query T +SELECT arrow_typeof(greatest(1::decimal(38,10), 1::decimal(38,0))); +---- +Decimal128(38, 0) + +# The fractional digits that do not fit are rounded away +query R +SELECT greatest(1.5::decimal(38,10), 1::decimal(38,0)); +---- +2 + +query R +SELECT greatest(1::decimal(38,10), '12345678901234567890123456789'::decimal(38,0)); +---- +12345678901234567890123456789 + +# A decimal and a DOUBLE widen to DOUBLE +query R +SELECT greatest(1.5::decimal(2,1), 2.5::double); +---- +2.5 + +query T +SELECT arrow_typeof(greatest(1.5::decimal(2,1), 2.5::double)); +---- +Float64 + +# Strings +query TTT +SELECT greatest('abc', 'abd', 'ab'), greatest('a', 'b', NULL), greatest('', 'a'); +---- +abd b a + +# Booleans +query B +SELECT greatest(true, false); +---- +true + +# Binary +query ? +SELECT greatest(X'01', X'0102'); +---- +0102 + +# Dates and timestamps; a date and a timestamp widen to the timestamp +query D +SELECT greatest(DATE '2024-01-01', DATE '2023-12-31', NULL::date); +---- +2024-01-01 + +query P +SELECT greatest(DATE '2024-01-01', TIMESTAMP '2023-01-01 10:00:00'); +---- +2024-01-01T00:00:00 + +query P +SELECT greatest(TIMESTAMP '2024-01-01 10:00:00', TIMESTAMP '2024-01-01 10:00:01'); +---- +2024-01-01T10:00:01 + +# Arrays compare element by element and then by length, a NULL element is smaller than any +# value, and elements widen +query ?? +SELECT greatest(array(1, 2), array(1, 2, 3)), greatest(array(1, NULL), array(1, 0)); +---- +[1, 2, 3] [1, 0] + +query ? +SELECT greatest(array(1), array(1.5::double)); +---- +[1.5] + +query ? +SELECT greatest(array('NaN'::double), array(1::double)); +---- +[NaN] + +# Structs compare field by field +query ? +SELECT greatest(named_struct('a', 1, 'b', 2), named_struct('a', 1, 'b', 3)); +---- +{a: 1, b: 3} + +# Columns +statement ok +CREATE TABLE t AS VALUES + (1::int, 2::int, 'NaN'::double, 1::double, 'b'), + (5::int, NULL::int, -1::double, NULL::double, 'a'), + (NULL::int, NULL::int, NULL::double, NULL::double, NULL), + (NULL::int, 7::int, 'Infinity'::double, 'NaN'::double, 'c'); + +query IIRRT +SELECT greatest(column1, column2), greatest(column1, 3::int, column2), greatest(column3, column4), greatest(column3, 0::double), greatest(column5, 'b') FROM t; +---- +2 3 NaN NaN b +5 5 -1 0 b +NULL 3 NULL 0 b +7 7 NaN Infinity c + +# A constant first argument, followed by columns. A NULL constant is skipped. +query IIIR +SELECT greatest(3::int, column1), greatest(NULL::int, column1), greatest(NULL::int, column1, column2), greatest(0::double, column3) FROM t; +---- +3 1 2 NaN +5 5 5 0 +3 NULL NULL 0 +3 NULL 7 Infinity + +statement ok +DROP TABLE t; + +# Errors + +# At least two arguments +query error \[WRONG_NUM_ARGS\.WITHOUT_SUGGESTION\] The `greatest` requires > 1 parameters but the actual number is 1\. Please, refer to 'https://spark\.apache\.org/docs/latest/sql-ref-functions\.html' for a fix\. SQLSTATE: 42605 +SELECT greatest(1::int); + +query error +SELECT greatest(); + +# Strings are not promoted +query error \[DATATYPE_MISMATCH\.DATA_DIFF_TYPES\] Input to `greatest` should all be the same type, but it's \["INT", "STRING"\]\. SQLSTATE: 42K09 +SELECT greatest(1::int, 'a'); + +query error \[DATATYPE_MISMATCH\.DATA_DIFF_TYPES\] Input to `greatest` should all be the same type, but it's \["DATE", "STRING"\] +SELECT greatest(DATE '2024-01-01', '2025-01-01'); + +query error \[DATATYPE_MISMATCH\.DATA_DIFF_TYPES\] Input to `greatest` should all be the same type, but it's \["BOOLEAN", "INT"\] +SELECT greatest(true, 1::int); + +query error \[DATATYPE_MISMATCH\.DATA_DIFF_TYPES\] +SELECT greatest(array(1), 1::int); + +query error \[DATATYPE_MISMATCH\.DATA_DIFF_TYPES\] +SELECT greatest(named_struct('a', 1), named_struct('b', 1)); + +# Maps cannot be ordered +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `greatest` does not support ordering on type "MAP"\. SQLSTATE: 42K09 +SELECT greatest(map(1::int, 2::int), map(1::int, 3::int)); + +# Intervals cannot be ordered +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `greatest` does not support ordering on type "INTERVAL"\. SQLSTATE: 42K09 +SELECT greatest(make_interval(1), make_interval(2)); + +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `greatest` does not support ordering on type "ARRAY"\. SQLSTATE: 42K09 +SELECT greatest(array(make_interval(1)), array(make_interval(2))); + +# An integer and a FLOAT widen to FLOAT with ANSI mode off, and to DOUBLE with it on +query T +SELECT arrow_typeof(greatest(1::int, 2.5::float)); +---- +Float32 + +statement ok +set datafusion.execution.enable_ansi_mode = true; + +query T +SELECT arrow_typeof(greatest(1::int, 2.5::float)); +---- +Float64 + +query R +SELECT greatest(1::bigint, 2.5::float); +---- +2.5 + +statement ok +set datafusion.execution.enable_ansi_mode = false; diff --git a/crates/sqllogictest/test_files/math/least.slt b/crates/sqllogictest/test_files/math/least.slt index db0a890a..dbc8d8fb 100644 --- a/crates/sqllogictest/test_files/math/least.slt +++ b/crates/sqllogictest/test_files/math/least.slt @@ -10,5 +10,203 @@ ## Original Query: SELECT least(10, 9, 2, 4, 3); ## PySpark 3.5.5 Result: {'least(10, 9, 2, 4, 3)': 2, 'typeof(least(10, 9, 2, 4, 3))': 'int', 'typeof(10)': 'int', 'typeof(9)': 'int', 'typeof(2)': 'int', 'typeof(4)': 'int', 'typeof(3)': 'int'} -#query -#SELECT least(10::int, 9::int, 2::int, 4::int, 3::int); +query I +SELECT least(10::int, 9::int, 2::int, 4::int, 3::int); +---- +2 + +# NULLs are skipped; the result is NULL only if every argument is NULL +query III +SELECT least(NULL::int, 2::int, NULL::int), least(3::int, NULL::int, 1::int), least(NULL::int, NULL::int); +---- +2 1 NULL + +query ? +SELECT least(NULL, NULL); +---- +NULL + +# Integers widen to the widest integer type +query I +SELECT least(-1::tinyint, 2::bigint); +---- +-1 + +query T +SELECT arrow_typeof(least(-1::tinyint, 2::bigint)); +---- +Int64 + +# An integer and a DOUBLE widen to DOUBLE +query R +SELECT least(3::int, 2.5::double); +---- +2.5 + +# NaN is greater than every other value, so least returns it only if every argument is NaN or NULL +query RRRR +SELECT least('NaN'::double, 'Infinity'::double), least('NaN'::double, 1::double), least('NaN'::double, NULL::double), least('-Infinity'::double, 'NaN'::double); +---- +Infinity 1 NaN -Infinity + +query R +SELECT least('NaN'::float, 1::float); +---- +1 + +# -0.0 equals 0.0 and the first of equal values wins. The sign of a zero shows in its string +# form, which starts with '-' only for -0.0. +query BB +SELECT CAST(least('-0.0'::double, 0.0::double) AS STRING) LIKE '-%', CAST(least(0.0::double, '-0.0'::double) AS STRING) LIKE '-%'; +---- +true false + +# Decimals +query R +SELECT least(1.5::decimal(2,1), 2::int); +---- +1.5 + +query T +SELECT arrow_typeof(least(1.5::decimal(2,1), 2::int)); +---- +Decimal128(11, 1) + +query R +SELECT least(1.25::decimal(3,2), 1.3::decimal(2,1), NULL::decimal(10,0)); +---- +1.25 + +query R +SELECT least(1.5::decimal(2,1), 0.5::double); +---- +0.5 + +# Strings +query TTT +SELECT least('abc', 'abd', 'ab'), least('b', NULL, 'a'), least('', 'a'); +---- +ab a (empty) + +# Booleans +query B +SELECT least(true, false); +---- +false + +# Binary +query ? +SELECT least(X'01', X'0102'); +---- +01 + +# Dates and timestamps; a date and a timestamp widen to the timestamp +query D +SELECT least(DATE '2024-01-01', DATE '2023-12-31', NULL::date); +---- +2023-12-31 + +query P +SELECT least(DATE '2024-01-01', TIMESTAMP '2023-01-01 10:00:00'); +---- +2023-01-01T10:00:00 + +# Arrays compare element by element and then by length, and a NULL element is smaller than any +# value +query ?? +SELECT least(array(1, 2), array(1, 2, 3)), least(array(1, NULL), array(1, 0)); +---- +[1, 2] [1, NULL] + +query ? +SELECT least(array('NaN'::double), array(1::double)); +---- +[1.0] + +# Structs compare field by field, and a NULL field is smaller than any value +query ? +SELECT least(named_struct('a', 1, 'b', 2), named_struct('a', 1, 'b', 3)); +---- +{a: 1, b: 2} + +query ? +SELECT least(named_struct('a', 1, 'b', 2), named_struct('a', 1, 'b', NULL::int)); +---- +{a: 1, b: NULL} + +# Columns +statement ok +CREATE TABLE t AS VALUES + (1::int, 2::int, 'NaN'::double, 1::double, 'b'), + (5::int, NULL::int, -1::double, NULL::double, 'a'), + (NULL::int, NULL::int, NULL::double, NULL::double, NULL), + (NULL::int, 7::int, 'Infinity'::double, 'NaN'::double, 'c'); + +query IIRRT +SELECT least(column1, column2), least(column1, 3::int, column2), least(column3, column4), least(column3, 0::double), least(column5, 'b') FROM t; +---- +1 1 1 0 b +5 3 -1 -1 a +NULL 3 NULL 0 b +7 3 Infinity 0 b + +# A constant first argument, followed by columns. A NULL constant is skipped. +query IIIR +SELECT least(3::int, column1), least(NULL::int, column1), least(NULL::int, column1, column2), least(0::double, column3) FROM t; +---- +1 1 1 0 +3 5 5 -1 +3 NULL NULL 0 +3 NULL 7 0 + +statement ok +DROP TABLE t; + +# Errors + +# At least two arguments +query error \[WRONG_NUM_ARGS\.WITHOUT_SUGGESTION\] The `least` requires > 1 parameters but the actual number is 1\. Please, refer to 'https://spark\.apache\.org/docs/latest/sql-ref-functions\.html' for a fix\. SQLSTATE: 42605 +SELECT least(1::int); + +query error +SELECT least(); + +# Strings are not promoted +query error \[DATATYPE_MISMATCH\.DATA_DIFF_TYPES\] Input to `least` should all be the same type, but it's \["INT", "STRING"\]\. SQLSTATE: 42K09 +SELECT least(1::int, '2'); + +query error \[DATATYPE_MISMATCH\.DATA_DIFF_TYPES\] +SELECT least(1::int, true); + +# Maps cannot be ordered +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `least` does not support ordering on type "MAP"\. SQLSTATE: 42K09 +SELECT least(map(1::int, 2::int), map(1::int, 3::int)); + +# Intervals cannot be ordered +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `least` does not support ordering on type "INTERVAL"\. SQLSTATE: 42K09 +SELECT least(make_interval(1), make_interval(2)); + +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `least` does not support ordering on type "ARRAY"\. SQLSTATE: 42K09 +SELECT least(array(make_interval(1)), array(make_interval(2))); + +# An integer and a FLOAT widen to FLOAT with ANSI mode off, and to DOUBLE with it on +query T +SELECT arrow_typeof(least(1::int, 2.5::float)); +---- +Float32 + +statement ok +set datafusion.execution.enable_ansi_mode = true; + +query T +SELECT arrow_typeof(least(1::int, 2.5::float)); +---- +Float64 + +query R +SELECT least(3::bigint, 2.5::float); +---- +2.5 + +statement ok +set datafusion.execution.enable_ansi_mode = false; diff --git a/docs/functions/README.md b/docs/functions/README.md index ba6a387d..efb17586 100644 --- a/docs/functions/README.md +++ b/docs/functions/README.md @@ -17,7 +17,7 @@ function differs from the DataFusion built-in of the same name, if there is one. ## Scalar Functions -- [Math Functions](scalar/math.md) (25) +- [Math Functions](scalar/math.md) (27) - [Conditional Functions](scalar/conditional.md) (2) - [String Functions](scalar/string.md) (17) - [Binary String Functions](scalar/binary_string.md) (5) @@ -86,6 +86,7 @@ function differs from the DataFusion built-in of the same name, if there is one. - [format_string](scalar/string.md#format_string) - [from_utc_timestamp](scalar/time_and_date.md#from_utc_timestamp) - [getbit](scalar/math.md#getbit) +- [greatest](scalar/math.md#greatest) - [hash](scalar/hashing.md#hash) - [hex](scalar/math.md#hex) - [hour](scalar/time_and_date.md#hour) @@ -96,6 +97,7 @@ function differs from the DataFusion built-in of the same name, if there is one. - [json_array_length](scalar/json.md#json_array_length) - [json_tuple](scalar/json.md#json_tuple) - [last_day](scalar/time_and_date.md#last_day) +- [least](scalar/math.md#least) - [len](scalar/string.md#len) - [length](scalar/string.md#length) - [like](scalar/string.md#like) diff --git a/docs/functions/scalar/math.md b/docs/functions/scalar/math.md index 0a0a673d..ce76c2f9 100644 --- a/docs/functions/scalar/math.md +++ b/docs/functions/scalar/math.md @@ -28,8 +28,10 @@ Part of the [datafusion-spark function reference](../README.md). - [factorial](#factorial) - [floor](#floor) - [getbit](#getbit) +- [greatest](#greatest) - [hex](#hex) - [hypot](#hypot) +- [least](#least) - [negative](#negative) - [positive](#positive) - [pow](#pow) @@ -344,6 +346,29 @@ floor(expr) _Alias of [bit_get](#bit_get)._ +## `greatest` + +Returns the greatest value of all arguments, skipping NULLs. The result is NULL only if every argument is NULL. At least two arguments are required. The arguments are cast to a common type as in Spark: numbers widen (an integer and a FLOAT widen to DOUBLE with `datafusion.execution.enable_ansi_mode` on and to FLOAT with it off), integers and decimals widen to a decimal that holds both, a decimal and a float widen to DOUBLE, a date and a timestamp widen to the timestamp, and arrays and structs widen element by element. Strings are never promoted: a string and a number, a string and a date, or a boolean and a number are an error, as are maps, which cannot be ordered. NaN is greater than every other value and `-0.0` equals `0.0`, also inside arrays and structs, and of equal values the first argument wins, so `greatest(-0.0, 0.0)` is `-0.0`. A NULL inside an array or struct is smaller than every other value. DataFusion's built-in `greatest` orders floats by IEEE 754 total order, lets a later argument win a tie, accepts a single argument, and compares numbers with strings. + +```sql +greatest(expr1, expr2[, ..., expr_n]) +``` + +### Arguments + +- **expr1, expr2, ..., expr_n**: Expressions of an orderable type that share a common type. NULLs are skipped. + +### Example + +```sql +> SELECT greatest(10, 9, 2, 4, 3); +10 +> SELECT greatest(1, NULL, 3); +3 +> SELECT greatest(1.0, 'NaN'::double); +NaN +``` + ## `hex` Returns the hexadecimal representation of `expr` as an upper-case string. Numbers are cast to BIGINT and rendered with no leading zeros; negative values are rendered as their 64-bit two's complement (so -1 is `FFFFFFFFFFFFFFFF`). Strings are encoded as the hex of their UTF-8 bytes, and binary values as the hex of their bytes. DataFusion's `to_hex` only accepts integers and returns lower case. Dictionary-encoded input returns a dictionary-encoded string. Returns NULL if the input is NULL. @@ -389,6 +414,29 @@ Infinity NULL ``` +## `least` + +Returns the least value of all arguments, skipping NULLs. The result is NULL only if every argument is NULL. At least two arguments are required. The arguments are cast to a common type as in `greatest`. NaN is greater than every other value and `-0.0` equals `0.0`, also inside arrays and structs, and of equal values the first argument wins, so `least(0.0, -0.0)` is `0.0`. A NULL inside an array or struct is smaller than every other value, so `least(array(1, NULL), array(1, 0))` is `[1, NULL]`. DataFusion's built-in `least` orders floats by IEEE 754 total order, lets a later argument win a tie, orders a NULL inside an array or struct after every other value, accepts a single argument, and compares numbers with strings. + +```sql +least(expr1, expr2[, ..., expr_n]) +``` + +### Arguments + +- **expr1, expr2, ..., expr_n**: Expressions of an orderable type that share a common type. NULLs are skipped. + +### Example + +```sql +> SELECT least(10, 9, 2, 4, 3); +2 +> SELECT least(NULL, 2, NULL); +2 +> SELECT least(1.0, 'NaN'::double); +1.0 +``` + ## `negative` Returns the negation of `expr` (the same as unary minus), with the same type as the input. DataFusion has no built-in `negative`. Accepts integer, floating-point, decimal and interval values. Negating the minimum value of a signed integer type (for example `-2147483648` for INT) depends on ANSI mode: with `datafusion.execution.enable_ansi_mode` off it wraps around and returns the same value, and with it on the function fails with an overflow error. Returns NULL if the input is NULL and NaN if it is NaN.