From bce98954fb97d103cacffa3bd69c842b0023c760 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 16 Nov 2026 06:32:19 -0700 Subject: [PATCH 1/8] feat(spark): add Spark-compatible array_remove Port array_remove from DataFusion Comet. Elements equal to the element are removed using Spark's equality (NaN == NaN, -0.0 == 0.0 at any depth, nested NULLs equal), NULL elements are kept, and a NULL array or element returns NULL. The array element and the element are coerced to Spark's tightest common type, reusing the rules from array_position. Closes #142. --- .../src/function/array/array_position.rs | 7 +- .../spark/src/function/array/array_remove.rs | 502 ++++++++++++++++++ crates/spark/src/function/array/mod.rs | 10 +- .../test_files/array/array_remove.slt | 453 ++++++++++++++++ 4 files changed, 969 insertions(+), 3 deletions(-) create mode 100644 crates/spark/src/function/array/array_remove.rs create mode 100644 crates/sqllogictest/test_files/array/array_remove.slt diff --git a/crates/spark/src/function/array/array_position.rs b/crates/spark/src/function/array/array_position.rs index 4ca52da9..71aa2aae 100644 --- a/crates/spark/src/function/array/array_position.rs +++ b/crates/spark/src/function/array/array_position.rs @@ -202,7 +202,10 @@ fn null_type_err(name: &str) -> Result { /// Spark's `TypeCoercion.findTightestCommonType`, on the arrow types that Spark's types /// map to. -fn tightest_common_type(left: &DataType, right: &DataType) -> Option { +pub(super) fn tightest_common_type( + left: &DataType, + right: &DataType, +) -> Option { use DataType::*; if left == right { return Some(left.clone()); @@ -349,7 +352,7 @@ fn finer_unit(left: TimeUnit, right: TimeUnit) -> TimeUnit { } /// Spark cannot order maps, so it rejects them at any depth of the element type. -fn contains_map(data_type: &DataType) -> bool { +pub(super) fn contains_map(data_type: &DataType) -> bool { match data_type { DataType::Map(_, _) => true, DataType::List(field) diff --git a/crates/spark/src/function/array/array_remove.rs b/crates/spark/src/function/array/array_remove.rs new file mode 100644 index 00000000..ed0bb707 --- /dev/null +++ b/crates/spark/src/function/array/array_remove.rs @@ -0,0 +1,502 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use super::array_position::{contains_map, tightest_common_type}; +use super::float_normalize::normalize_array; +use arrow::array::{ + Array, ArrayRef, AsArray, BooleanArray, BooleanBufferBuilder, GenericListArray, + OffsetSizeTrait, Scalar, make_comparator, +}; +use arrow::buffer::{BooleanBuffer, NullBuffer, OffsetBuffer}; +use arrow::compute::kernels::cmp::eq; +use arrow::compute::{SortOptions, filter}; +use arrow::datatypes::{DataType, Field}; +use datafusion_common::utils::take_function_args; +use datafusion_common::{Result, ScalarValue, exec_err, plan_err}; +use datafusion_expr::{ + ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, + Volatility, +}; +use datafusion_macros::user_doc; +use std::sync::Arc; + +/// Spark-compatible `array_remove` expression. +/// +/// +#[user_doc( + doc_section(label = "Array Functions"), + description = "Removes all elements that are equal to the element from the array. The \ + result is NULL when the array or the element is NULL, while the DataFusion built-in \ + removes the NULL elements when the element is NULL. NULL elements in the array are \ + always kept. Corresponding NULLs nested inside compared arrays or structs are equal. \ + For floating point values, also inside nested arrays and structs, NaN is equal to NaN \ + and -0.0 is equal to 0.0; the elements that are kept keep their original values. As \ + in Spark, the array element and the element are cast to their tightest common type, \ + and the result is an array of that type: numbers widen, an integer widens to a \ + decimal only if the decimal can hold all its values, and a date widens to a \ + timestamp. Other combinations are an error, for example a string and a number, two \ + different decimals, or structs whose field names or field types differ, as in Spark \ + with ANSI mode off (with ANSI mode on, Spark widens struct field types). Map \ + elements are also an error.", + syntax_example = "array_remove(array, element)", + sql_example = r#"```sql +> SELECT array_remove(array(1, 2, 3, NULL, 3), 3); +[1, 2, NULL] +```"#, + argument(name = "array", description = "Array expression. NULL returns NULL."), + argument( + name = "element", + description = "Element to remove. NULL returns NULL." + ) +)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkArrayRemove { + signature: Signature, +} + +impl Default for SparkArrayRemove { + fn default() -> Self { + Self::new() + } +} + +impl SparkArrayRemove { + pub fn new() -> Self { + Self { + signature: Signature::user_defined(Volatility::Immutable), + } + } +} + +impl ScalarUDFImpl for SparkArrayRemove { + fn name(&self) -> &str { + "array_remove" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, arg_types: &[DataType]) -> Result { + let [array, _] = take_function_args(self.name(), arg_types)?; + Ok(array.clone()) + } + + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { + let [array, element] = take_function_args(self.name(), arg_types)?; + coerce_array_and_element(self.name(), array, element) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + let [array, element] = take_function_args(self.name(), &args.args)?; + let is_scalar = matches!( + (array, element), + (ColumnarValue::Scalar(_), ColumnarValue::Scalar(_)) + ); + let rows = if is_scalar { 1 } else { args.number_rows }; + let array = array.to_array(rows)?; + // A constant element is compared as a single value against every row. + let (element, constant) = match element { + ColumnarValue::Scalar(scalar) => (scalar.to_array()?, true), + ColumnarValue::Array(element) => (Arc::clone(element), false), + }; + + let result = match array.data_type() { + DataType::List(_) => remove::(&array, &element, constant)?, + DataType::LargeList(_) => remove::(&array, &element, constant)?, + other => return exec_err!("array_remove does not support type {other}"), + }; + + if is_scalar { + Ok(ColumnarValue::Scalar(ScalarValue::try_from_array( + &result, 0, + )?)) + } else { + Ok(ColumnarValue::Array(result)) + } + } + + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } +} + +/// Removes from each row of `array` the non-null elements equal to the row's element, which +/// is `element[0]` for every row if `constant`, and `element[row]` otherwise. A row is NULL +/// when the array or the element is NULL. +fn remove( + array: &ArrayRef, + element: &ArrayRef, + constant: bool, +) -> Result { + let list = array.as_list::(); + let offsets = list.offsets(); + // Only the values that this (possibly sliced) list refers to are looked at. + let base = offsets[0].as_usize(); + let len = offsets[offsets.len() - 1].as_usize() - base; + let values = list.values().slice(base, len); + + // Logical nulls, so that a NULL-typed element, which has no null buffer, is NULL. + let element_nulls = element.logical_nulls(); + let row_nulls = if constant { + if element_nulls.as_ref().is_some_and(|n| n.is_null(0)) { + Some(NullBuffer::new_null(list.len())) + } else { + list.nulls().cloned() + } + } else { + NullBuffer::union(list.nulls(), element_nulls.as_ref()) + }; + let row_is_valid = |row: usize| row_nulls.as_ref().is_none_or(|n| n.is_valid(row)); + + // Equality is computed on copies with canonical NaN and 0.0 at any depth, so that it + // follows Spark's float equality, while the kept elements are taken from the original + // values. The comparator treats NULLs nested inside the compared values as equal. + let mut is_equal: Box bool> = Box::new(|_, _| false); + // When every element is NULL there is nothing to compare: NULL elements are kept and a + // NULL element makes every row NULL. This also covers the NULL type. + let any_comparison = + values.logical_null_count() < values.len() && (0..list.len()).any(row_is_valid); + if any_comparison { + let normalized = normalize_array(&values)?; + let needle = normalize_array(element)?; + is_equal = if constant && !normalized.data_type().is_nested() { + // Compare every value with the constant at once. + let equal = eq(&normalized, &Scalar::new(Arc::clone(&needle)))?; + let equal = equal.values().clone(); + Box::new(move |i, _| equal.value(i)) + } else { + let cmp = make_comparator( + normalized.as_ref(), + needle.as_ref(), + SortOptions::default(), + )?; + Box::new(move |i, row| cmp(i, if constant { 0 } else { row }).is_eq()) + }; + } + + let value_nulls = values.logical_nulls(); + let mut keep = BooleanBufferBuilder::new(len); + let mut new_offsets = Vec::with_capacity(list.len() + 1); + new_offsets.push(O::usize_as(0)); + let mut kept = 0usize; + for (row, w) in offsets.windows(2).enumerate() { + let (start, end) = (w[0].as_usize() - base, w[1].as_usize() - base); + if row_is_valid(row) { + for i in start..end { + let keep_it = value_nulls.as_ref().is_some_and(|n| n.is_null(i)) + || !is_equal(i, row); + keep.append(keep_it); + kept += usize::from(keep_it); + } + } else { + keep.append_n(end - start, false); + } + new_offsets.push(O::usize_as(kept)); + } + let keep: BooleanBuffer = keep.finish(); + let kept_values = if kept == len { + values + } else { + filter(&values, &BooleanArray::new(keep, None))? + }; + + let (DataType::List(field) | DataType::LargeList(field)) = list.data_type() else { + unreachable!("remove is only called with lists"); + }; + Ok(Arc::new(GenericListArray::::try_new( + Arc::clone(field), + OffsetBuffer::new(new_offsets.into()), + kept_values, + row_nulls, + )?)) +} + +/// Casts the array element and the element to their tightest common type, and rejects the +/// argument types that Spark's `ArrayRemove.checkInputDataTypes` rejects. +/// +/// Unlike `array_position`, Spark accepts a NULL-typed common element type here (for +/// example `array_remove(array(), NULL)`, which returns NULL), and reports an argument that +/// is not an array, including an untyped NULL, as `ARRAY_FUNCTION_DIFF_TYPES`. +fn coerce_array_and_element( + name: &str, + array: &DataType, + element: &DataType, +) -> Result> { + let diff_types_err = || { + plan_err!( + "[DATATYPE_MISMATCH.ARRAY_FUNCTION_DIFF_TYPES] Input to `{name}` should have \ + been \"ARRAY\" followed by a value with same element type, but it's \ + [{array}, {element}]" + ) + }; + let (item, large) = match array { + DataType::List(field) | DataType::FixedSizeList(field, _) => (field, false), + DataType::LargeList(field) => (field, true), + _ => return diff_types_err(), + }; + let Some(common) = tightest_common_type(item.data_type(), element) else { + return diff_types_err(); + }; + if contains_map(&common) { + return plan_err!( + "[DATATYPE_MISMATCH.INVALID_ORDERING_TYPE] The `{name}` does not support \ + ordering on type {common}" + ); + } + let item = Arc::new(Field::new(item.name(), common.clone(), true)); + let array = if large { + DataType::LargeList(item) + } else { + DataType::List(item) + }; + Ok(vec![array, common]) +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{Float32Array, Float64Array, Int32Array, ListArray, StructArray}; + use arrow::datatypes::{Fields, Float32Type, Float64Type, Int32Type}; + + fn negative_nan() -> f64 { + f64::from_bits(f64::NAN.to_bits() | (1 << 63)) + } + + fn invoke( + array: ColumnarValue, + element: ColumnarValue, + rows: usize, + ) -> ColumnarValue { + let return_field = Arc::new(Field::new("result", array.data_type(), true)); + SparkArrayRemove::new() + .invoke_with_args(ScalarFunctionArgs { + args: vec![array, element], + arg_fields: vec![], + number_rows: rows, + return_field, + config_options: Default::default(), + }) + .unwrap() + } + + fn float_list(rows: &[Option>>]) -> ArrayRef { + Arc::new(ListArray::from_iter_primitive::( + rows.to_vec(), + )) + } + + /// The rows of a list of doubles, as bits so that the zeros and NaNs can be told apart. + fn bits(array: &ArrayRef) -> Vec>>> { + array + .as_list::() + .iter() + .map(|row| { + row.map(|row| { + row.as_primitive::() + .iter() + .map(|v| v.map(f64::to_bits)) + .collect() + }) + }) + .collect() + } + + /// Spark's result for one row: drop the non-null elements equal to the element. + fn spark_remove(row: &[Option], element: f64) -> Vec> { + let equal = |a: f64| (a.is_nan() && element.is_nan()) || a == element; + row.iter() + .filter(|v| v.is_none_or(|v| !equal(v))) + .map(|v| v.map(f64::to_bits)) + .collect() + } + + const EDGE_VALUES: [Option; 7] = [ + Some(0.0), + Some(-0.0), + Some(f64::NAN), + None, + Some(1.0), + Some(f64::INFINITY), + Some(f64::NEG_INFINITY), + ]; + + /// Every edge value removed from an array of all edge values, both as a column and as a + /// constant: each zero and each NaN meets every other, and the kept elements keep their + /// bits. + #[test] + fn floats_follow_spark_equality() { + let mut elements = EDGE_VALUES.to_vec(); + elements.push(Some(negative_nan())); + let removable: Vec = elements.iter().flatten().copied().collect(); + let rows: Vec<_> = removable.iter().map(|_| Some(elements.clone())).collect(); + let expected: Vec<_> = removable + .iter() + .map(|&v| Some(spark_remove(&elements, v))) + .collect(); + + let result = invoke( + ColumnarValue::Array(float_list(&rows)), + ColumnarValue::Array(Arc::new(Float64Array::from(removable.clone()))), + rows.len(), + ) + .into_array(rows.len()) + .unwrap(); + assert_eq!(bits(&result), expected); + + for (value, expected) in removable.iter().zip(expected) { + let result = invoke( + ColumnarValue::Array(float_list(&rows[..1])), + ColumnarValue::Scalar(ScalarValue::Float64(Some(*value))), + 1, + ) + .into_array(1) + .unwrap(); + assert_eq!(bits(&result), vec![expected], "remove {value}"); + } + } + + #[test] + fn nulls() { + let rows = vec![ + Some(vec![Some(0.0), None, Some(-0.0)]), + None, + Some(vec![Some(1.0)]), + ]; + let element = Float64Array::from(vec![Some(0.0), Some(0.0), None]); + let result = invoke( + ColumnarValue::Array(float_list(&rows)), + ColumnarValue::Array(Arc::new(element)), + 3, + ) + .into_array(3) + .unwrap(); + // The NULL element stays; a NULL array or a NULL element gives a NULL row. + assert_eq!(bits(&result), vec![Some(vec![None]), None, None]); + + let result = invoke( + ColumnarValue::Array(float_list(&rows)), + ColumnarValue::Scalar(ScalarValue::Float64(None)), + 3, + ) + .into_array(3) + .unwrap(); + assert_eq!(bits(&result), vec![None, None, None]); + } + + /// The offsets of a sliced list do not start at 0, and the elements of NULL rows are not + /// carried into the result. + #[test] + fn sliced_list() { + let list = ListArray::from_iter_primitive::(vec![ + Some(vec![Some(9), Some(9)]), + Some(vec![Some(1), Some(2), Some(1)]), + None, + Some(vec![Some(3), None, Some(4)]), + ]); + let array: ArrayRef = Arc::new(list.slice(1, 3)); + for element in [ + ColumnarValue::Array(Arc::new(Int32Array::from(vec![1, 1, 1]))), + ColumnarValue::Scalar(ScalarValue::Int32(Some(1))), + ] { + let result = invoke(ColumnarValue::Array(Arc::clone(&array)), element, 3) + .into_array(3) + .unwrap(); + let expected = ListArray::from_iter_primitive::(vec![ + Some(vec![Some(2)]), + None, + Some(vec![Some(3), None, Some(4)]), + ]); + assert_eq!(result.as_list::(), &expected); + assert_eq!(result.as_list::().values().len(), 4); + } + } + + #[test] + fn large_list_and_scalars() { + let array = ScalarValue::LargeList(Arc::new( + arrow::array::LargeListArray::from_iter_primitive::(vec![ + Some(vec![Some(-0.0f32), Some(f32::NAN), Some(1.0)]), + ]), + )); + let result = invoke( + ColumnarValue::Scalar(array), + ColumnarValue::Scalar(ScalarValue::Float32(Some(0.0))), + 4, + ); + let ColumnarValue::Scalar(ScalarValue::LargeList(result)) = result else { + panic!("expected a large list scalar"); + }; + let values = result.value(0); + let values: &Float32Array = values.as_primitive(); + assert_eq!(values.len(), 2); + assert!(values.value(0).is_nan()); + assert_eq!(values.value(1), 1.0); + } + + /// Struct elements compare their float fields with Spark's equality, and keep their bits. + #[test] + fn nested_elements() { + let fields = Fields::from(vec![Field::new("x", DataType::Float64, true)]); + let structs = |values: Vec| -> ArrayRef { + Arc::new(StructArray::new( + fields.clone(), + vec![Arc::new(Float64Array::from(values))], + None, + )) + }; + let item = Field::new("item", DataType::Struct(fields.clone()), true); + let array: ArrayRef = Arc::new(ListArray::new( + Arc::new(item), + OffsetBuffer::from_lengths([3]), + structs(vec![-0.0, negative_nan(), 1.0]), + None, + )); + for (value, expected) in [ + (0.0, vec![negative_nan(), 1.0]), + (f64::NAN, vec![-0.0, 1.0]), + ] { + for element in [ + ColumnarValue::Array(structs(vec![value])), + ColumnarValue::Scalar( + ScalarValue::try_from_array(&structs(vec![value]), 0).unwrap(), + ), + ] { + let result = invoke(ColumnarValue::Array(Arc::clone(&array)), element, 1) + .into_array(1) + .unwrap(); + let row = result.as_list::().value(0); + let x = row.as_struct().column(0).as_primitive::(); + let x: Vec = x.values().iter().map(|v| v.to_bits()).collect(); + let expected: Vec = expected.iter().map(|v| v.to_bits()).collect(); + assert_eq!(x, expected, "remove {value}"); + } + } + } + + #[test] + fn coerce_types_rules() { + let udf = SparkArrayRemove::new(); + let list = |t: DataType| DataType::new_list(t, true); + assert_eq!( + udf.coerce_types(&[list(DataType::Int32), DataType::Int64]) + .unwrap(), + vec![list(DataType::Int64), DataType::Int64] + ); + assert_eq!( + udf.coerce_types(&[list(DataType::Null), DataType::Null]) + .unwrap(), + vec![list(DataType::Null), DataType::Null] + ); + for (array, element) in [ + (DataType::Null, DataType::Int32), + (DataType::Int32, DataType::Int32), + (list(DataType::Int32), DataType::Utf8), + (list(DataType::Int32), DataType::Decimal128(2, 1)), + ] { + let err = udf.coerce_types(&[array, element]).unwrap_err().to_string(); + assert!(err.contains("ARRAY_FUNCTION_DIFF_TYPES"), "{err}"); + } + } +} diff --git a/crates/spark/src/function/array/mod.rs b/crates/spark/src/function/array/mod.rs index 7ff0f2aa..8b74c33e 100644 --- a/crates/spark/src/function/array/mod.rs +++ b/crates/spark/src/function/array/mod.rs @@ -2,10 +2,11 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -//! Spark array functions, such as `array`, `array_contains`, `array_repeat`, `shuffle` and `slice`. +//! Spark array functions, such as `array`, `array_contains`, `array_remove`, `array_repeat`, `shuffle` and `slice`. pub mod array_contains; pub mod array_position; +pub mod array_remove; mod float_normalize; pub mod repeat; pub mod shuffle; @@ -18,6 +19,7 @@ use std::sync::Arc; make_udf_function!(array_contains::SparkArrayContains, spark_array_contains); make_udf_function!(array_position::SparkArrayPosition, array_position); +make_udf_function!(array_remove::SparkArrayRemove, array_remove); make_udf_function!(spark_array::SparkArray, array); make_udf_function!(shuffle::SparkShuffle, shuffle); make_udf_function!(repeat::SparkArrayRepeat, array_repeat); @@ -36,6 +38,11 @@ pub mod expr_fn { "Returns the 1-based position of the element in the array, or 0 if not found (Spark semantics).", array element )); + export_functions!(( + array_remove, + "Removes all elements equal to the element from the array (Spark semantics).", + array element + )); export_functions!((array, "Returns an array with the given elements.", args)); export_functions!(( shuffle, @@ -58,6 +65,7 @@ pub fn functions() -> Vec> { vec![ spark_array_contains(), array_position(), + array_remove(), array(), shuffle(), array_repeat(), diff --git a/crates/sqllogictest/test_files/array/array_remove.slt b/crates/sqllogictest/test_files/array/array_remove.slt new file mode 100644 index 00000000..c37e5181 --- /dev/null +++ b/crates/sqllogictest/test_files/array/array_remove.slt @@ -0,0 +1,453 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +# Tests for Spark-compatible array_remove. +# Removes every element equal to the element; NULL elements are kept; the result is NULL if the +# array or the element is NULL. +# Expected values verified with pyspark==4.2.0. + +query ? +SELECT array_remove(array(1, 2, 3, 2), 2); +---- +[1, 3] + +query ? +SELECT array_remove(array(1, 2, 3), 4); +---- +[1, 2, 3] + +query ? +SELECT array_remove(array('a', 'b', 'a'), 'a'); +---- +[b] + +# NULL elements in the array are kept +query ? +SELECT array_remove(array(1, 2, NULL, 2), 2); +---- +[1, NULL] + +query ? +SELECT array_remove(array(NULL), 1); +---- +[NULL] + +query ? +SELECT array_remove(array(), 1); +---- +[] + +query ? +SELECT array_remove(array(1, 1), 1); +---- +[] + +# A NULL element returns NULL. The DataFusion built-in removes the NULL elements instead. +query ? +SELECT array_remove(array(1, 2, 3), CAST(NULL AS INT)); +---- +NULL + +query ? +SELECT array_remove(array(1, NULL, 3), CAST(NULL AS INT)); +---- +NULL + +# The tightest common type of the array element and an untyped NULL is the array element type +query ? +SELECT array_remove(array(1, 2, 3), NULL); +---- +NULL + +query ? +SELECT array_remove(CAST(NULL AS ARRAY), 1); +---- +NULL + +query ? +SELECT array_remove(CAST(NULL AS ARRAY), CAST(NULL AS INT)); +---- +NULL + +# Unlike array_position, a NULL common element type is accepted, and the result is NULL +query ? +SELECT array_remove(array(), NULL); +---- +NULL + +query ? +SELECT array_remove(array(NULL, NULL), NULL); +---- +NULL + +# An untyped NULL array is not an array +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] Input to `array_remove` should have been "ARRAY" followed by a value with same element type +SELECT array_remove(NULL, 1); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_remove(NULL, NULL); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_remove(1, 1); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_remove(array(1, 2, 3, 2), array(2)); + +query ? +SELECT array_remove(array(true, false, true), true); +---- +[false] + +query ? +SELECT array_remove(array(X'01', X'02'), X'01'); +---- +[02] + +query ? +SELECT array_remove(array(CAST(1.5 AS DECIMAL(3,1)), CAST(2.5 AS DECIMAL(3,1))), CAST(2.5 AS DECIMAL(3,1))); +---- +[1.5] + +query ? +SELECT array_remove(array(DATE '2020-01-01', DATE '2020-01-02'), DATE '2020-01-01'); +---- +[2020-01-02] + +### +### Floating point equality +### + +# -0.0 equals 0.0, in either direction +query ? +SELECT array_remove(array(CAST('-0.0' AS DOUBLE), CAST(0.0 AS DOUBLE), CAST(1.0 AS DOUBLE)), CAST(0.0 AS DOUBLE)); +---- +[1.0] + +query ? +SELECT array_remove(array(CAST('-0.0' AS DOUBLE), CAST(0.0 AS DOUBLE), CAST(1.0 AS DOUBLE)), CAST('-0.0' AS DOUBLE)); +---- +[1.0] + +query ? +SELECT array_remove(array(CAST('-0.0' AS FLOAT), CAST(1.0 AS FLOAT)), CAST(0.0 AS FLOAT)); +---- +[1.0] + +# NaN equals NaN +query ? +SELECT array_remove(array(CAST('NaN' AS DOUBLE), CAST(1.0 AS DOUBLE), CAST('NaN' AS DOUBLE)), CAST('NaN' AS DOUBLE)); +---- +[1.0] + +query ? +SELECT array_remove(array(CAST('NaN' AS FLOAT), CAST(1.0 AS FLOAT)), CAST('NaN' AS FLOAT)); +---- +[1.0] + +# Kept elements keep their sign +query ? +SELECT array_remove(array(CAST('-0.0' AS DOUBLE), CAST(1.0 AS DOUBLE)), CAST(1.0 AS DOUBLE)); +---- +[-0.0] + +query ? +SELECT array_remove(array(CAST('-0.0' AS DOUBLE), CAST('NaN' AS DOUBLE)), CAST('NaN' AS DOUBLE)); +---- +[-0.0] + +### +### Nested elements +### + +query ? +SELECT array_remove(array(array(1, 2), array(3), NULL, array(1, 2)), array(1, 2)); +---- +[[3], NULL] + +# NULLs nested inside compared arrays and structs are equal +query ? +SELECT array_remove(array(array(1, CAST(NULL AS INT)), array(3)), array(1, CAST(NULL AS INT))); +---- +[[3]] + +query ? +SELECT array_remove(array(named_struct('a', 1, 'b', CAST(NULL AS DOUBLE)), named_struct('a', 2, 'b', CAST(1.0 AS DOUBLE))), named_struct('a', 1, 'b', CAST(NULL AS DOUBLE))); +---- +[{a: 2, b: 1.0}] + +# Nested floats follow Spark's equality, and kept elements keep their values +query ? +SELECT array_remove(array(array(CAST('-0.0' AS DOUBLE)), array(CAST('NaN' AS DOUBLE)), array(CAST(1.0 AS DOUBLE))), array(CAST(0.0 AS DOUBLE))); +---- +[[NaN], [1.0]] + +query ? +SELECT array_remove(array(array(CAST('-0.0' AS DOUBLE)), array(CAST('NaN' AS DOUBLE)), array(CAST(1.0 AS DOUBLE))), array(CAST('NaN' AS DOUBLE))); +---- +[[-0.0], [1.0]] + +query ? +SELECT array_remove(array(named_struct('a', 1, 'b', CAST('-0.0' AS DOUBLE)), named_struct('a', 2, 'b', CAST(1.0 AS DOUBLE))), named_struct('a', 1, 'b', CAST(0.0 AS DOUBLE))); +---- +[{a: 2, b: 1.0}] + +# Maps cannot be ordered, so map elements are rejected at any depth +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_remove` does not support ordering on type +SELECT array_remove(array(map(1, 2)), map(1, 2)); + +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] +SELECT array_remove(array(named_struct('m', map(1, 2))), named_struct('m', map(1, 2))); + +### +### Type coercion +### + +# The array element and the element are cast to their tightest common type, which is also the +# result's element type +query ?T +SELECT array_remove(array(CAST(1 AS INT), CAST(2 AS INT)), 2L), arrow_typeof(array_remove(array(CAST(1 AS INT), CAST(2 AS INT)), 2L)); +---- +[1] List(Int64, field: 'element') + +query ?T +SELECT array_remove(array(1L, 2L), CAST(2 AS INT)), arrow_typeof(array_remove(array(1L, 2L), CAST(2 AS INT))); +---- +[1] List(Int64, field: 'element') + +query ?T +SELECT array_remove(array(CAST(1 AS INT), CAST(2 AS INT)), CAST(2 AS TINYINT)), arrow_typeof(array_remove(array(CAST(1 AS INT), CAST(2 AS INT)), CAST(2 AS TINYINT))); +---- +[1] List(Int32, field: 'element') + +query ?T +SELECT array_remove(array(CAST(1 AS INT), CAST(2 AS INT)), CAST(1.5 AS DOUBLE)), arrow_typeof(array_remove(array(CAST(1 AS INT), CAST(2 AS INT)), CAST(1.5 AS DOUBLE))); +---- +[1.0, 2.0] List(Float64, field: 'element') + +query ? +SELECT array_remove(array(CAST(1 AS FLOAT), CAST(2 AS FLOAT)), CAST(1.0 AS DOUBLE)); +---- +[2.0] + +query ? +SELECT array_remove(array(array(CAST(1 AS INT))), array(1L)); +---- +[] + +# An integral type widens to a decimal only if the decimal holds all its values: +# INT is DECIMAL(10,0), so DECIMAL(12,2) is wide enough and DECIMAL(2,1) is not. +query ?T +SELECT array_remove(array(CAST(1 AS INT), CAST(2 AS INT)), CAST(1 AS DECIMAL(12,2))), arrow_typeof(array_remove(array(CAST(1 AS INT), CAST(2 AS INT)), CAST(1 AS DECIMAL(12,2)))); +---- +[2.00] List(Decimal128(12, 2), field: 'element') + +query ? +SELECT array_remove(array(CAST(1 AS DECIMAL(12,2))), CAST(1 AS INT)); +---- +[] + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_remove(array(CAST(1 AS INT), CAST(2 AS INT)), CAST(1 AS DECIMAL(2,1))); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_remove(array(CAST(1.5 AS DECIMAL(2,1))), CAST(2 AS INT)); + +# Decimals and floating point numbers have no tightest common type +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_remove(array(CAST(1 AS DECIMAL(5,2))), CAST(1.0 AS DOUBLE)); + +# Strings and numbers are not cast to each other +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_remove(array(1, 2), '1'); + +# DATE and TIMESTAMP become TIMESTAMP +query ? +SELECT array_remove(array(DATE '2020-01-01', DATE '2020-01-02'), TIMESTAMP '2020-01-01 00:00:00'); +---- +[2020-01-02T00:00:00] + +# Struct fields must have the same names +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_remove(array(named_struct('a', 1)), named_struct('b', 1)); + +# Struct field types are not widened, as in Spark with ANSI mode off. With ANSI mode on, Spark +# widens them and returns [] here. +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_remove(array(named_struct('a', CAST(1 AS INT))), named_struct('a', 1L)); + +### +### Column inputs +### + +statement ok +CREATE TABLE t AS VALUES + (array(1, 2, 3, 2), 2), + (array(1, NULL, 3), 3), + (array(1, 2), 5), + (NULL, 1), + (array(1, 2), NULL); + +query ? +SELECT array_remove(column1, column2) FROM t; +---- +[1, 3] +[1, NULL] +[1, 2] +NULL +NULL + +# A literal array with a column element, including a NULL element +query ? +SELECT array_remove(array(1, 2, 3), column2) FROM t; +---- +[1, 3] +[1, 2] +[1, 2, 3] +[2, 3] +NULL + +# A column array with a literal element +query ? +SELECT array_remove(column1, 1) FROM t; +---- +[2, 3, 2] +[NULL, 3] +[2] +NULL +[2] + +query ? +SELECT array_remove(column1, CAST(NULL AS INT)) FROM t; +---- +NULL +NULL +NULL +NULL +NULL + +# A batch sliced by OFFSET: the list offsets do not start at 0 +query ? +SELECT array_remove(column1, column2) FROM (SELECT * FROM t LIMIT 3 OFFSET 1); +---- +[1, NULL] +[1, 2] +NULL + +statement ok +DROP TABLE t; + +statement ok +CREATE TABLE floats AS VALUES + (array(CAST(1.0 AS DOUBLE), CAST('NaN' AS DOUBLE)), CAST('NaN' AS DOUBLE)), + (array(CAST('-0.0' AS DOUBLE), CAST(1.0 AS DOUBLE)), CAST(0.0 AS DOUBLE)), + (array(CAST(0.0 AS DOUBLE), CAST(1.0 AS DOUBLE)), CAST('-0.0' AS DOUBLE)), + (array(CAST(1.0 AS DOUBLE), CAST(NULL AS DOUBLE)), CAST(1.0 AS DOUBLE)), + (array(CAST(1.0 AS DOUBLE), CAST(2.0 AS DOUBLE)), CAST(NULL AS DOUBLE)), + (CAST(NULL AS ARRAY), CAST(1.0 AS DOUBLE)); + +query ? +SELECT array_remove(column1, column2) FROM floats; +---- +[1.0] +[1.0] +[1.0] +[NULL] +NULL +NULL + +query ? +SELECT array_remove(column1, CAST(0.0 AS DOUBLE)) FROM floats; +---- +[1.0, NaN] +[1.0] +[1.0] +[1.0, NULL] +[1.0, 2.0] +NULL + +statement ok +DROP TABLE floats; + +statement ok +CREATE TABLE strings AS VALUES + (array('a', 'b', 'a'), 'a'), + (array('a', NULL, 'b'), 'b'), + (array('', 'a'), ''), + (array('a', 'b'), CAST(NULL AS STRING)), + (CAST(NULL AS ARRAY), 'a'); + +query ? +SELECT array_remove(column1, column2) FROM strings; +---- +[b] +[a, NULL] +[a] +NULL +NULL + +statement ok +DROP TABLE strings; + +statement ok +CREATE TABLE nested AS VALUES + (array(array(CAST(1.0 AS DOUBLE)), array(CAST('NaN' AS DOUBLE), CAST('-0.0' AS DOUBLE))), array(CAST('NaN' AS DOUBLE), CAST(0.0 AS DOUBLE))), + (array(array(CAST(0.0 AS DOUBLE)), array(CAST(1.0 AS DOUBLE))), array(CAST('-0.0' AS DOUBLE))), + (array(array(CAST(1.0 AS DOUBLE), CAST(NULL AS DOUBLE))), array(CAST(1.0 AS DOUBLE), CAST(NULL AS DOUBLE))), + (array(array(CAST(1.0 AS DOUBLE))), array(CAST(2.0 AS DOUBLE))); + +query ? +SELECT array_remove(column1, column2) FROM nested; +---- +[[1.0]] +[[1.0]] +[] +[[1.0]] + +statement ok +DROP TABLE nested; + +statement ok +CREATE TABLE structs AS VALUES + (array(named_struct('a', CAST(1.0 AS DOUBLE), 'b', 1), named_struct('a', CAST('-0.0' AS DOUBLE), 'b', 2)), named_struct('a', CAST(0.0 AS DOUBLE), 'b', 2)), + (array(named_struct('a', CAST('NaN' AS DOUBLE), 'b', 1)), named_struct('a', CAST('NaN' AS DOUBLE), 'b', 1)), + (array(named_struct('a', CAST(0.0 AS DOUBLE), 'b', 1)), named_struct('a', CAST(0.0 AS DOUBLE), 'b', 2)); + +query ? +SELECT array_remove(column1, column2) FROM structs; +---- +[{a: 1.0, b: 1}] +[] +[{a: 0.0, b: 1}] + +statement ok +DROP TABLE structs; + +### +### LargeList +### + +query ?T +SELECT array_remove(arrow_cast(array(1, 2, 3, 2), 'LargeList(Int64)'), 2), arrow_typeof(array_remove(arrow_cast(array(1, 2, 3, 2), 'LargeList(Int64)'), 2)); +---- +[1, 3] LargeList(Int64) + +query ? +SELECT array_remove(arrow_cast(array(1, NULL, 3), 'LargeList(Int64)'), 3); +---- +[1, NULL] + +query ? +SELECT array_remove(arrow_cast(array(1, 2, 3), 'LargeList(Int64)'), CAST(NULL AS BIGINT)); +---- +NULL + +query ? +SELECT array_remove(arrow_cast(array(CAST('-0.0' AS DOUBLE), CAST('NaN' AS DOUBLE), CAST(1.0 AS DOUBLE)), 'LargeList(Float64)'), CAST(0.0 AS DOUBLE)); +---- +[NaN, 1.0] + +query ? +SELECT array_remove(arrow_cast(array(CAST('-0.0' AS DOUBLE), CAST('NaN' AS DOUBLE), CAST(1.0 AS DOUBLE)), 'LargeList(Float64)'), CAST('NaN' AS DOUBLE)); +---- +[-0.0, 1.0] From 01d8874a489db5b48b98f422b0c002c6825e9054 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 10:17:23 -0700 Subject: [PATCH 2/8] refactor(spark): share Spark's common-type rules between functions Move tightest_common_type from array_position into function/type_coercion, with an ANSI flag, a findTypeForComplex that widens map keys and values, Spark's isOrderable (which also rejects INTERVAL), and Spark type names for error messages. Move float_normalize up to function/ so functions outside array can use it. array_position now reports a map with different key or value types as INVALID_ORDERING_TYPE, rejects INTERVAL elements, and names Spark types and the SQLSTATE in its errors, as Spark does. --- .../src/function/array/array_contains.rs | 2 +- .../src/function/array/array_position.rs | 178 ++-------- crates/spark/src/function/array/mod.rs | 1 - .../function/{array => }/float_normalize.rs | 8 +- crates/spark/src/function/mod.rs | 2 + crates/spark/src/function/type_coercion.rs | 327 ++++++++++++++++++ .../test_files/array/array_position.slt | 15 +- 7 files changed, 382 insertions(+), 151 deletions(-) rename crates/spark/src/function/{array => }/float_normalize.rs (97%) create mode 100644 crates/spark/src/function/type_coercion.rs diff --git a/crates/spark/src/function/array/array_contains.rs b/crates/spark/src/function/array/array_contains.rs index 3802667e..029e6c09 100644 --- a/crates/spark/src/function/array/array_contains.rs +++ b/crates/spark/src/function/array/array_contains.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use super::float_normalize::normalize_floats; +use crate::function::float_normalize::normalize_floats; use arrow::array::{ Array, AsArray, BooleanArray, BooleanBufferBuilder, GenericListArray, OffsetSizeTrait, }; diff --git a/crates/spark/src/function/array/array_position.rs b/crates/spark/src/function/array/array_position.rs index 4ca52da9..d1a64701 100644 --- a/crates/spark/src/function/array/array_position.rs +++ b/crates/spark/src/function/array/array_position.rs @@ -2,13 +2,16 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use super::float_normalize::normalize_array; +use crate::function::float_normalize::normalize_array; +use crate::function::type_coercion::{ + is_orderable, spark_sql_type, tightest_common_type, +}; use arrow::array::{ Array, ArrayRef, AsArray, Int64Array, OffsetSizeTrait, make_comparator, }; use arrow::buffer::{NullBuffer, ScalarBuffer}; use arrow::compute::SortOptions; -use arrow::datatypes::{DataType, Field, Fields, TimeUnit}; +use arrow::datatypes::{DataType, Field}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, exec_err, plan_err}; use datafusion_expr::{ @@ -36,7 +39,7 @@ use std::sync::Arc; a date widens to a timestamp. Other combinations are an error, for example a string \ and a number, two different decimals, or structs whose field names or field types \ differ, as in Spark with ANSI mode off (with ANSI mode on, Spark widens struct field \ - types). A NULL-typed array and map elements are also an error.", + types). A NULL-typed array, and map or INTERVAL elements at any depth, are also an error.", syntax_example = "array_position(array, element)", sql_example = r#"```sql > SELECT array_position(array(3, 2, 1), 1); @@ -164,24 +167,31 @@ fn coerce_array_and_element( other => { return plan_err!( "[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] The first parameter of `{name}` \ - requires the \"ARRAY\" type, however it has the type {other}" + requires the \"ARRAY\" type, however it has the type \"{}\". SQLSTATE: 42K09", + spark_sql_type(other) ); } }; - let Some(common) = tightest_common_type(item.data_type(), element) else { + // Spark's `ArrayPosition` calls the non-ANSI `TypeCoercion.findTightestCommonType`. + let common = tightest_common_type(item.data_type(), element, false) + .filter(|_| struct_field_types_match(item.data_type(), element)); + let Some(common) = common else { return plan_err!( "[DATATYPE_MISMATCH.ARRAY_FUNCTION_DIFF_TYPES] Input to `{name}` should have \ been \"ARRAY\" followed by a value with same element type, but it's \ - [{array}, {element}]" + [\"{}\", \"{}\"]. SQLSTATE: 42K09", + spark_sql_type(array), + spark_sql_type(element) ); }; if common.is_null() { return null_type_err(name); } - if contains_map(&common) { + if !is_orderable(&common) { return plan_err!( "[DATATYPE_MISMATCH.INVALID_ORDERING_TYPE] The `{name}` does not support \ - ordering on type {common}" + ordering on type \"{}\". SQLSTATE: 42K09", + spark_sql_type(&common) ); } let item = Arc::new(Field::new(item.name(), common.clone(), true)); @@ -196,82 +206,30 @@ fn coerce_array_and_element( fn null_type_err(name: &str) -> Result { plan_err!( "[DATATYPE_MISMATCH.NULL_TYPE] Null typed values cannot be used as arguments of \ - `{name}`" + `{name}`. SQLSTATE: 42K09" ) } -/// Spark's `TypeCoercion.findTightestCommonType`, on the arrow types that Spark's types -/// map to. -fn tightest_common_type(left: &DataType, right: &DataType) -> Option { +/// With ANSI mode off, Spark cannot implicitly cast between structs whose field types +/// differ, so `array_position` rejects them although their tightest common type exists. +/// With ANSI mode on, Spark widens the field types. +fn struct_field_types_match(left: &DataType, right: &DataType) -> bool { use DataType::*; - if left == right { - return Some(left.clone()); - } - if let (Some(l), Some(r)) = (numeric_precedence(left), numeric_precedence(right)) { - return Some(if l >= r { left.clone() } else { right.clone() }); - } match (left, right) { - (Null, other) | (other, Null) => Some(other.clone()), - // Spark has a single string type and a single binary type. - (l, r) if is_string(l) && is_string(r) => Some(l.clone()), - (l, r) if is_binary(l) && is_binary(r) => Some(l.clone()), - // The decimal must be at least as wide as the integral type's DECIMAL(n, 0). - (int, Decimal128(p, s)) | (Decimal128(p, s), int) => { - let digits = integral_digits(int)?; - (*p as i16 - *s as i16 >= digits).then_some(Decimal128(*p, *s)) - } - (Date32, ts @ Timestamp(_, _)) | (ts @ Timestamp(_, _), Date32) => { - Some(ts.clone()) - } - // TIMESTAMP_NTZ and TIMESTAMP widen to TIMESTAMP. - (Timestamp(lu, ltz), Timestamp(ru, rtz)) => Some(Timestamp( - finer_unit(*lu, *ru), - ltz.clone().or_else(|| rtz.clone()), - )), ( - List(lf) | LargeList(lf) | FixedSizeList(lf, _), - List(rf) | LargeList(rf) | FixedSizeList(rf, _), - ) => { - let item = tightest_common_type(lf.data_type(), rf.data_type())?; - let field = Arc::new(Field::new(lf.name(), item, true)); - Some( - if matches!(left, LargeList(_)) || matches!(right, LargeList(_)) { - LargeList(field) - } else { - List(field) - }, - ) - } - // The fields must have the same names and types, ignoring nullability, in the same - // order. This follows Spark with ANSI mode off; with ANSI mode on, Spark also widens - // the field types. Spark compares the names ignoring case, but DataFusion cannot cast - // between structs whose field names differ only in case, so they must match exactly. - (Struct(lf), Struct(rf)) if lf.len() == rf.len() => { - let fields = lf + List(l) | LargeList(l) | FixedSizeList(l, _), + List(r) | LargeList(r) | FixedSizeList(r, _), + ) => struct_field_types_match(l.data_type(), r.data_type()), + // The entries are a struct of the key and the value, which are checked on their own. + (Map(l, _), Map(r, _)) => match (l.data_type(), r.data_type()) { + (Struct(l), Struct(r)) => l .iter() - .zip(rf.iter()) - .map(|(a, b)| { - if a.name() != b.name() || !same_type(a.data_type(), b.data_type()) { - return None; - } - let data_type = tightest_common_type(a.data_type(), b.data_type())?; - Some(Field::new( - a.name(), - data_type, - a.is_nullable() || b.is_nullable(), - )) - }) - .collect::>>()?; - Some(Struct(Fields::from(fields))) - } - (Map(lf, sorted), Map(rf, _)) => { - let entries = tightest_common_type(lf.data_type(), rf.data_type())?; - Some(Map( - Arc::new(Field::new(lf.name(), entries, false)), - *sorted, - )) - } - _ => None, + .zip(r.iter()) + .all(|(a, b)| struct_field_types_match(a.data_type(), b.data_type())), + _ => true, + }, + (Struct(_), Struct(_)) => same_type(left, right), + _ => true, } } @@ -296,72 +254,6 @@ fn same_type(left: &DataType, right: &DataType) -> bool { } } -fn is_string(data_type: &DataType) -> bool { - matches!( - data_type, - DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View - ) -} - -fn is_binary(data_type: &DataType) -> bool { - matches!( - data_type, - DataType::Binary | DataType::LargeBinary | DataType::BinaryView - ) -} - -/// The precision of Spark's `DecimalType.forType` for an integral type. -fn integral_digits(data_type: &DataType) -> Option { - match data_type { - DataType::Int8 => Some(3), - DataType::Int16 => Some(5), - DataType::Int32 => Some(10), - DataType::Int64 => Some(20), - _ => None, - } -} - -/// Spark's `numericPrecedence`, which leaves out decimals. -fn numeric_precedence(data_type: &DataType) -> Option { - match data_type { - DataType::Int8 => Some(0), - DataType::Int16 => Some(1), - DataType::Int32 => Some(2), - DataType::Int64 => Some(3), - DataType::Float32 => Some(4), - DataType::Float64 => Some(5), - _ => None, - } -} - -fn finer_unit(left: TimeUnit, right: TimeUnit) -> TimeUnit { - let rank = |unit: TimeUnit| match unit { - TimeUnit::Second => 0, - TimeUnit::Millisecond => 1, - TimeUnit::Microsecond => 2, - TimeUnit::Nanosecond => 3, - }; - if rank(left) >= rank(right) { - left - } else { - right - } -} - -/// Spark cannot order maps, so it rejects them at any depth of the element type. -fn contains_map(data_type: &DataType) -> bool { - match data_type { - DataType::Map(_, _) => true, - DataType::List(field) - | DataType::LargeList(field) - | DataType::FixedSizeList(field, _) => contains_map(field.data_type()), - DataType::Struct(fields) => { - fields.iter().any(|field| contains_map(field.data_type())) - } - _ => false, - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/spark/src/function/array/mod.rs b/crates/spark/src/function/array/mod.rs index 7ff0f2aa..36dcccc9 100644 --- a/crates/spark/src/function/array/mod.rs +++ b/crates/spark/src/function/array/mod.rs @@ -6,7 +6,6 @@ pub mod array_contains; pub mod array_position; -mod float_normalize; pub mod repeat; pub mod shuffle; pub mod slice; diff --git a/crates/spark/src/function/array/float_normalize.rs b/crates/spark/src/function/float_normalize.rs similarity index 97% rename from crates/spark/src/function/array/float_normalize.rs rename to crates/spark/src/function/float_normalize.rs index 07bbb28a..1a76b60d 100644 --- a/crates/spark/src/function/array/float_normalize.rs +++ b/crates/spark/src/function/float_normalize.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -//! Float normalization shared by the array functions that compare elements. +//! Float normalization shared by the functions that compare values. //! //! Spark compares floating point values with `-0.0 == 0.0` and `NaN == NaN`, while arrow //! compares them by their bits or by the IEEE total order. Rewriting `-0.0` to `0.0` and @@ -19,7 +19,7 @@ use datafusion_expr::ColumnarValue; use std::sync::Arc; /// Normalizes the floats in a [`ColumnarValue`], see [`normalize_array`]. -pub(super) fn normalize_floats(value: &ColumnarValue) -> Result { +pub(crate) fn normalize_floats(value: &ColumnarValue) -> Result { if !contains_float(&value.data_type()) { return Ok(value.clone()); } @@ -35,7 +35,7 @@ pub(super) fn normalize_floats(value: &ColumnarValue) -> Result { } /// Returns whether values of this type are, or hold at any depth, floating point numbers. -pub(super) fn contains_float(data_type: &DataType) -> bool { +pub(crate) fn contains_float(data_type: &DataType) -> bool { match data_type { DataType::Float32 | DataType::Float64 => true, DataType::List(field) @@ -51,7 +51,7 @@ pub(super) fn contains_float(data_type: &DataType) -> bool { /// Rewrites `-0.0` to `0.0` and every NaN to the canonical NaN at any depth of list, large /// list, fixed-size list, struct and map. Children without floats are reused as they are. -pub(super) fn normalize_array(array: &ArrayRef) -> Result { +pub(crate) fn normalize_array(array: &ArrayRef) -> Result { if !contains_float(array.data_type()) { return Ok(Arc::clone(array)); } diff --git a/crates/spark/src/function/mod.rs b/crates/spark/src/function/mod.rs index 483f9b64..4098d8e0 100644 --- a/crates/spark/src/function/mod.rs +++ b/crates/spark/src/function/mod.rs @@ -12,6 +12,7 @@ pub mod conversion; pub mod csv; pub mod datetime; pub mod error_utils; +pub(crate) mod float_normalize; pub mod functions_nested_utils; pub mod generator; pub mod hash; @@ -25,6 +26,7 @@ pub mod predicate; pub mod string; pub mod r#struct; pub mod table; +pub(crate) mod type_coercion; pub mod url; pub mod utils; pub mod window; diff --git a/crates/spark/src/function/type_coercion.rs b/crates/spark/src/function/type_coercion.rs new file mode 100644 index 00000000..035acbf6 --- /dev/null +++ b/crates/spark/src/function/type_coercion.rs @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Spark's rules for finding a common type of several arguments, from its `TypeCoercion` +//! and `AnsiTypeCoercion`, on the Arrow types that Spark's types map to. Also the checks +//! and type names that the errors for those arguments use. +//! +//! These are for functions that need one type for several arguments, such as +//! `array_position`. They are not Spark's implicit casts to a function's declared input +//! types, which the functions do through their [`datafusion_expr::Signature`]. + +use std::sync::Arc; + +use arrow::datatypes::{DataType, Field, Fields, IntervalUnit, TimeUnit}; + +/// Spark's `findTightestCommonType`. `ansi_mode` picks between `AnsiTypeCoercion` and +/// `TypeCoercion`, which differ only in that ANSI widens an integer and a FLOAT to DOUBLE. +/// Arrays, maps and structs are widened element by element with this function. +pub(crate) fn tightest_common_type( + left: &DataType, + right: &DataType, + ansi_mode: bool, +) -> Option { + use DataType::*; + if left == right { + return Some(left.clone()); + } + if let (Some(l), Some(r)) = (numeric_precedence(left), numeric_precedence(right)) { + let wider = if l >= r { left } else { right }; + // With ANSI mode on, an integer and a FLOAT widen to DOUBLE, which holds every INT + // exactly. + return Some(if ansi_mode && *wider == Float32 { + Float64 + } else { + wider.clone() + }); + } + match (left, right) { + (Null, other) | (other, Null) => Some(other.clone()), + // Spark has one string type and one binary type. + (l, r) if is_string(l) && is_string(r) => Some(wider_physical_type(l, r)), + (l, r) if is_binary(l) && is_binary(r) => Some(wider_physical_type(l, r)), + // The decimal must hold every value of the integral type. + (int, Decimal128(p, s)) | (Decimal128(p, s), int) => { + let digits = integral_digits(int)?; + (*p as i16 - *s as i16 >= digits).then_some(Decimal128(*p, *s)) + } + (Date32, ts @ Timestamp(_, _)) | (ts @ Timestamp(_, _), Date32) => { + Some(ts.clone()) + } + // TIMESTAMP_NTZ and TIMESTAMP widen to TIMESTAMP. + (Timestamp(lu, ltz), Timestamp(ru, rtz)) => Some(Timestamp( + finer_unit(*lu, *ru), + ltz.clone().or_else(|| rtz.clone()), + )), + _ => complex_type(left, right, &|l, r| tightest_common_type(l, r, ansi_mode)), + } +} + +/// Spark's `findTypeForComplex`: widens the elements of arrays, the keys and values of maps, +/// and the fields of structs with `find_type`. A nested field is nullable if it is on either +/// side. Struct fields must have the same names in the same order. Spark compares them +/// ignoring case, but DataFusion cannot cast between structs whose field names differ only +/// in case, so this requires an exact match. +pub(crate) fn complex_type( + left: &DataType, + right: &DataType, + find_type: &dyn Fn(&DataType, &DataType) -> Option, +) -> Option { + use DataType::*; + match (left, right) { + ( + List(lf) | LargeList(lf) | FixedSizeList(lf, _), + List(rf) | LargeList(rf) | FixedSizeList(rf, _), + ) => { + let item = find_type(lf.data_type(), rf.data_type())?; + let field = Arc::new(Field::new( + lf.name(), + item, + lf.is_nullable() || rf.is_nullable(), + )); + Some(match (left, right) { + (FixedSizeList(_, ln), FixedSizeList(_, rn)) if ln == rn => { + FixedSizeList(field, *ln) + } + (LargeList(_), _) | (_, LargeList(_)) => LargeList(field), + _ => List(field), + }) + } + (Struct(lf), Struct(rf)) if lf.len() == rf.len() => { + let fields = lf + .iter() + .zip(rf.iter()) + .map(|(a, b)| { + if a.name() != b.name() { + return None; + } + let data_type = find_type(a.data_type(), b.data_type())?; + Some(Field::new( + a.name(), + data_type, + a.is_nullable() || b.is_nullable(), + )) + }) + .collect::>>()?; + Some(Struct(Fields::from(fields))) + } + // The entries are a struct of the key and the value, so the struct arm widens both. + (Map(lf, sorted), Map(rf, _)) => { + let entries = complex_type(lf.data_type(), rf.data_type(), find_type)?; + Some(Map( + Arc::new(Field::new(lf.name(), entries, false)), + *sorted, + )) + } + _ => None, + } +} + +/// Spark's `OrderUtils.isOrderable`. Spark cannot order maps or `CalendarIntervalType`, which +/// is `Interval(MonthDayNano)` in Arrow, at any depth. +pub(crate) fn is_orderable(data_type: &DataType) -> bool { + match data_type { + DataType::Map(_, _) + | DataType::Union(_, _) + | DataType::Interval(IntervalUnit::MonthDayNano) => false, + DataType::List(field) + | DataType::LargeList(field) + | DataType::ListView(field) + | DataType::LargeListView(field) + | DataType::FixedSizeList(field, _) => is_orderable(field.data_type()), + DataType::Struct(fields) => fields.iter().all(|f| is_orderable(f.data_type())), + DataType::Dictionary(_, value) => is_orderable(value), + _ => true, + } +} + +/// The name Spark's `toSQLType` gives the Spark type that an Arrow type maps to, for error +/// messages. Arrow types without a Spark counterpart use their Arrow name. +pub(crate) fn spark_sql_type(data_type: &DataType) -> String { + use DataType::*; + match data_type { + Null => "VOID".to_string(), + Boolean => "BOOLEAN".to_string(), + Int8 => "TINYINT".to_string(), + Int16 => "SMALLINT".to_string(), + Int32 => "INT".to_string(), + Int64 => "BIGINT".to_string(), + Float32 => "FLOAT".to_string(), + Float64 => "DOUBLE".to_string(), + Decimal128(p, s) => format!("DECIMAL({p},{s})"), + Utf8 | LargeUtf8 | Utf8View => "STRING".to_string(), + Binary | LargeBinary | BinaryView => "BINARY".to_string(), + Date32 => "DATE".to_string(), + Timestamp(_, Some(_)) => "TIMESTAMP".to_string(), + Timestamp(_, None) => "TIMESTAMP_NTZ".to_string(), + Interval(IntervalUnit::MonthDayNano) => "INTERVAL".to_string(), + List(field) | LargeList(field) | FixedSizeList(field, _) => { + format!("ARRAY<{}>", spark_sql_type(field.data_type())) + } + Struct(fields) => format!( + "STRUCT<{}>", + fields + .iter() + .map(|f| format!( + "{}: {}{}", + f.name(), + spark_sql_type(f.data_type()), + if f.is_nullable() { "" } else { " NOT NULL" } + )) + .collect::>() + .join(", ") + ), + Map(entries, _) => match entries.data_type() { + Struct(kv) if kv.len() == 2 => format!( + "MAP<{}, {}>", + spark_sql_type(kv[0].data_type()), + spark_sql_type(kv[1].data_type()) + ), + other => format!("MAP<{other}>"), + }, + other => other.to_string(), + } +} + +fn is_string(data_type: &DataType) -> bool { + matches!( + data_type, + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View + ) +} + +fn is_binary(data_type: &DataType) -> bool { + matches!( + data_type, + DataType::Binary | DataType::LargeBinary | DataType::BinaryView + ) +} + +/// Of two physical layouts of the same Spark string or binary type, the one that can hold the +/// values of both. +fn wider_physical_type(left: &DataType, right: &DataType) -> DataType { + use DataType::*; + match (left, right) { + (LargeUtf8, _) | (_, LargeUtf8) => LargeUtf8, + (LargeBinary, _) | (_, LargeBinary) => LargeBinary, + (Utf8View, _) | (_, Utf8View) => Utf8View, + (BinaryView, _) | (_, BinaryView) => BinaryView, + _ => left.clone(), + } +} + +/// The precision of Spark's `DecimalType.forType` for an integral type. +fn integral_digits(data_type: &DataType) -> Option { + match data_type { + DataType::Int8 => Some(3), + DataType::Int16 => Some(5), + DataType::Int32 => Some(10), + DataType::Int64 => Some(20), + _ => None, + } +} + +/// Spark's `numericPrecedence`, which leaves out decimals. +fn numeric_precedence(data_type: &DataType) -> Option { + match data_type { + DataType::Int8 => Some(0), + DataType::Int16 => Some(1), + DataType::Int32 => Some(2), + DataType::Int64 => Some(3), + DataType::Float32 => Some(4), + DataType::Float64 => Some(5), + _ => None, + } +} + +fn finer_unit(left: TimeUnit, right: TimeUnit) -> TimeUnit { + let rank = |unit: TimeUnit| match unit { + TimeUnit::Second => 0, + TimeUnit::Millisecond => 1, + TimeUnit::Microsecond => 2, + TimeUnit::Nanosecond => 3, + }; + if rank(left) >= rank(right) { + left + } else { + right + } +} + +#[cfg(test)] +mod tests { + use super::*; + use DataType::*; + + fn list(item: DataType) -> DataType { + List(Arc::new(Field::new("item", item, true))) + } + + fn map(key: DataType, value: DataType) -> DataType { + let entries = Struct(Fields::from(vec![ + Field::new("key", key, false), + Field::new("value", value, true), + ])); + Map(Arc::new(Field::new("entries", entries, false)), false) + } + + #[test] + fn numbers_widen_and_ansi_widens_int_and_float_to_double() { + assert_eq!(tightest_common_type(&Int32, &Int64, false), Some(Int64)); + assert_eq!(tightest_common_type(&Int32, &Float32, false), Some(Float32)); + assert_eq!(tightest_common_type(&Int32, &Float32, true), Some(Float64)); + assert_eq!( + tightest_common_type(&Int32, &Decimal128(10, 0), false), + Some(Decimal128(10, 0)) + ); + assert_eq!( + tightest_common_type(&Int32, &Decimal128(10, 1), false), + None + ); + assert_eq!(tightest_common_type(&Int32, &Utf8, false), None); + } + + #[test] + fn string_layouts_widen_to_the_one_that_holds_both() { + assert_eq!( + tightest_common_type(&Utf8, &LargeUtf8, false), + Some(LargeUtf8) + ); + assert_eq!( + tightest_common_type(&LargeUtf8, &Utf8, false), + Some(LargeUtf8) + ); + assert_eq!( + tightest_common_type(&Utf8, &Utf8View, false), + Some(Utf8View) + ); + } + + #[test] + fn complex_types_widen_element_by_element() { + assert_eq!( + tightest_common_type(&list(Int32), &list(Int64), false), + Some(list(Int64)) + ); + assert_eq!( + tightest_common_type(&map(Int32, Int32), &map(Int64, Int16), false), + Some(map(Int64, Int32)) + ); + let a = Struct(Fields::from(vec![Field::new("a", Int32, false)])); + let b = Struct(Fields::from(vec![Field::new("b", Int32, false)])); + let wide = Struct(Fields::from(vec![Field::new("a", Int64, true)])); + assert_eq!(tightest_common_type(&a, &wide, false), Some(wide)); + assert_eq!(tightest_common_type(&a, &b, false), None); + } + + #[test] + fn maps_and_calendar_intervals_are_not_orderable() { + let interval = Interval(IntervalUnit::MonthDayNano); + assert!(!is_orderable(&map(Int32, Int32))); + assert!(!is_orderable(&interval)); + assert!(!is_orderable(&list(interval.clone()))); + assert!(is_orderable(&list(Int32))); + assert_eq!(spark_sql_type(&list(interval)), "ARRAY"); + } +} diff --git a/crates/sqllogictest/test_files/array/array_position.slt b/crates/sqllogictest/test_files/array/array_position.slt index cf9ef3e8..25ebfd45 100644 --- a/crates/sqllogictest/test_files/array/array_position.slt +++ b/crates/sqllogictest/test_files/array/array_position.slt @@ -274,8 +274,8 @@ SELECT array_position(array(CAST(1.0 AS DOUBLE)), CAST(1 AS DECIMAL(5,1))); query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] SELECT array_position(array(1), '1'); -query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] -SELECT array_position(array('1'), 1); +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] Input to `array_position` should have been "ARRAY" followed by a value with same element type, but it's \["ARRAY", "INT"\]\. SQLSTATE: 42K09 +SELECT array_position(array('1'), CAST(1 AS INT)); query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] SELECT array_position(array(1), CAST(NULL AS STRING)); @@ -332,6 +332,17 @@ SELECT array_position(array(named_struct('m', map(1, 2))), named_struct('m', map query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] SELECT array_position(array(array(map(1, 2))), array(map(1, 2))); +# Map keys and values are widened, and the common map type is then rejected +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_position` does not support ordering on type "MAP"\. SQLSTATE: 42K09 +SELECT array_position(array(map(CAST(1 AS INT), CAST(1 AS INT))), map(1L, 1L)); + +# Spark cannot order intervals +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_position` does not support ordering on type "INTERVAL"\. SQLSTATE: 42K09 +SELECT array_position(array(make_interval(1)), make_interval(2)); + +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_position` does not support ordering on type "ARRAY"\. SQLSTATE: 42K09 +SELECT array_position(array(array(make_interval(1))), array(make_interval(2))); + # Signed zero: -0.0 equals 0.0 query I SELECT array_position(array(CAST('-0.0' AS DOUBLE), CAST(1.0 AS DOUBLE)), CAST(0.0 AS DOUBLE)); From 3e9008cff980d3f1546656cbd8a71737c6bc22ac Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 10:19:52 -0700 Subject: [PATCH 3/8] refactor(spark): match map keys and values by position in complex_type Spark's MapType has no field names, so map entries named differently still have a common type. Keep contains_float private. --- crates/spark/src/function/float_normalize.rs | 2 +- crates/spark/src/function/type_coercion.rs | 29 ++++++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/crates/spark/src/function/float_normalize.rs b/crates/spark/src/function/float_normalize.rs index 1a76b60d..e166b362 100644 --- a/crates/spark/src/function/float_normalize.rs +++ b/crates/spark/src/function/float_normalize.rs @@ -35,7 +35,7 @@ pub(crate) fn normalize_floats(value: &ColumnarValue) -> Result { } /// Returns whether values of this type are, or hold at any depth, floating point numbers. -pub(crate) fn contains_float(data_type: &DataType) -> bool { +fn contains_float(data_type: &DataType) -> bool { match data_type { DataType::Float32 | DataType::Float64 => true, DataType::List(field) diff --git a/crates/spark/src/function/type_coercion.rs b/crates/spark/src/function/type_coercion.rs index 035acbf6..d4a8bbb5 100644 --- a/crates/spark/src/function/type_coercion.rs +++ b/crates/spark/src/function/type_coercion.rs @@ -106,9 +106,24 @@ pub(crate) fn complex_type( .collect::>>()?; Some(Struct(Fields::from(fields))) } - // The entries are a struct of the key and the value, so the struct arm widens both. + // The entries are a struct of the key and the value. Spark's `MapType` has no field + // names, so the key and the value are matched by position. (Map(lf, sorted), Map(rf, _)) => { - let entries = complex_type(lf.data_type(), rf.data_type(), find_type)?; + let (Struct(lkv), Struct(rkv)) = (lf.data_type(), rf.data_type()) else { + return None; + }; + let [lk, lv] = lkv.iter().collect::>()[..] else { + return None; + }; + let [rk, rv] = rkv.iter().collect::>()[..] else { + return None; + }; + let key = find_type(lk.data_type(), rk.data_type())?; + let value = find_type(lv.data_type(), rv.data_type())?; + let entries = Struct(Fields::from(vec![ + Field::new(lk.name(), key, false), + Field::new(lv.name(), value, lv.is_nullable() || rv.is_nullable()), + ])); Some(Map( Arc::new(Field::new(lf.name(), entries, false)), *sorted, @@ -310,6 +325,16 @@ mod tests { ); let a = Struct(Fields::from(vec![Field::new("a", Int32, false)])); let b = Struct(Fields::from(vec![Field::new("b", Int32, false)])); + // Map entries named differently are matched by position. + let renamed = Struct(Fields::from(vec![ + Field::new("keys", Int64, false), + Field::new("values", Int16, true), + ])); + let renamed = Map(Arc::new(Field::new("entries", renamed, false)), false); + assert_eq!( + tightest_common_type(&map(Int32, Int32), &renamed, false), + Some(map(Int64, Int32)) + ); let wide = Struct(Fields::from(vec![Field::new("a", Int64, true)])); assert_eq!(tightest_common_type(&a, &wide, false), Some(wide)); assert_eq!(tightest_common_type(&a, &b, false), None); From f5571e9d044f852c04e7c07035c16d30e19e33dd Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 10:22:21 -0700 Subject: [PATCH 4/8] refactor(spark): simplify the shared common-type rules Use DataType::is_string and TimeUnit's ordering, match map entries without collecting, keep complex_type private, and move the non-ANSI struct field check and same_type into type_coercion beside the rules they qualify. --- .../src/function/array/array_position.rs | 46 +------- crates/spark/src/function/type_coercion.rs | 106 ++++++++++++------ 2 files changed, 75 insertions(+), 77 deletions(-) diff --git a/crates/spark/src/function/array/array_position.rs b/crates/spark/src/function/array/array_position.rs index d1a64701..445a8b48 100644 --- a/crates/spark/src/function/array/array_position.rs +++ b/crates/spark/src/function/array/array_position.rs @@ -4,7 +4,7 @@ use crate::function::float_normalize::normalize_array; use crate::function::type_coercion::{ - is_orderable, spark_sql_type, tightest_common_type, + is_orderable, spark_sql_type, struct_field_types_match, tightest_common_type, }; use arrow::array::{ Array, ArrayRef, AsArray, Int64Array, OffsetSizeTrait, make_comparator, @@ -210,50 +210,6 @@ fn null_type_err(name: &str) -> Result { ) } -/// With ANSI mode off, Spark cannot implicitly cast between structs whose field types -/// differ, so `array_position` rejects them although their tightest common type exists. -/// With ANSI mode on, Spark widens the field types. -fn struct_field_types_match(left: &DataType, right: &DataType) -> bool { - use DataType::*; - match (left, right) { - ( - List(l) | LargeList(l) | FixedSizeList(l, _), - List(r) | LargeList(r) | FixedSizeList(r, _), - ) => struct_field_types_match(l.data_type(), r.data_type()), - // The entries are a struct of the key and the value, which are checked on their own. - (Map(l, _), Map(r, _)) => match (l.data_type(), r.data_type()) { - (Struct(l), Struct(r)) => l - .iter() - .zip(r.iter()) - .all(|(a, b)| struct_field_types_match(a.data_type(), b.data_type())), - _ => true, - }, - (Struct(_), Struct(_)) => same_type(left, right), - _ => true, - } -} - -/// Whether two types are equal when the nullability of nested fields is ignored, like -/// Spark's `DataTypeUtils.sameType`. -fn same_type(left: &DataType, right: &DataType) -> bool { - use DataType::*; - match (left, right) { - (List(l), List(r)) | (LargeList(l), LargeList(r)) | (Map(l, _), Map(r, _)) => { - same_type(l.data_type(), r.data_type()) - } - (FixedSizeList(l, ln), FixedSizeList(r, rn)) => { - ln == rn && same_type(l.data_type(), r.data_type()) - } - (Struct(l), Struct(r)) => { - l.len() == r.len() - && l.iter().zip(r.iter()).all(|(a, b)| { - a.name() == b.name() && same_type(a.data_type(), b.data_type()) - }) - } - _ => left == right, - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/spark/src/function/type_coercion.rs b/crates/spark/src/function/type_coercion.rs index d4a8bbb5..9c23939a 100644 --- a/crates/spark/src/function/type_coercion.rs +++ b/crates/spark/src/function/type_coercion.rs @@ -12,7 +12,7 @@ use std::sync::Arc; -use arrow::datatypes::{DataType, Field, Fields, IntervalUnit, TimeUnit}; +use arrow::datatypes::{DataType, Field, Fields, IntervalUnit}; /// Spark's `findTightestCommonType`. `ansi_mode` picks between `AnsiTypeCoercion` and /// `TypeCoercion`, which differ only in that ANSI widens an integer and a FLOAT to DOUBLE. @@ -39,7 +39,7 @@ pub(crate) fn tightest_common_type( match (left, right) { (Null, other) | (other, Null) => Some(other.clone()), // Spark has one string type and one binary type. - (l, r) if is_string(l) && is_string(r) => Some(wider_physical_type(l, r)), + (l, r) if l.is_string() && r.is_string() => Some(wider_physical_type(l, r)), (l, r) if is_binary(l) && is_binary(r) => Some(wider_physical_type(l, r)), // The decimal must hold every value of the integral type. (int, Decimal128(p, s)) | (Decimal128(p, s), int) => { @@ -51,7 +51,8 @@ pub(crate) fn tightest_common_type( } // TIMESTAMP_NTZ and TIMESTAMP widen to TIMESTAMP. (Timestamp(lu, ltz), Timestamp(ru, rtz)) => Some(Timestamp( - finer_unit(*lu, *ru), + // Arrow orders the units from Second to Nanosecond. + (*lu).max(*ru), ltz.clone().or_else(|| rtz.clone()), )), _ => complex_type(left, right, &|l, r| tightest_common_type(l, r, ansi_mode)), @@ -63,7 +64,7 @@ pub(crate) fn tightest_common_type( /// side. Struct fields must have the same names in the same order. Spark compares them /// ignoring case, but DataFusion cannot cast between structs whose field names differ only /// in case, so this requires an exact match. -pub(crate) fn complex_type( +fn complex_type( left: &DataType, right: &DataType, find_type: &dyn Fn(&DataType, &DataType) -> Option, @@ -112,10 +113,7 @@ pub(crate) fn complex_type( let (Struct(lkv), Struct(rkv)) = (lf.data_type(), rf.data_type()) else { return None; }; - let [lk, lv] = lkv.iter().collect::>()[..] else { - return None; - }; - let [rk, rv] = rkv.iter().collect::>()[..] else { + let ([lk, lv], [rk, rv]) = (&lkv[..], &rkv[..]) else { return None; }; let key = find_type(lk.data_type(), rk.data_type())?; @@ -151,6 +149,51 @@ pub(crate) fn is_orderable(data_type: &DataType) -> bool { } } +/// Whether the structs in two types, at any depth, have the same field types. With ANSI mode +/// off, Spark's implicit cast cannot change the field types of a struct, so a function that +/// casts its arguments to their tightest common type rejects such structs, as +/// `array_position` does. With ANSI mode on, Spark widens the field types. +pub(crate) fn struct_field_types_match(left: &DataType, right: &DataType) -> bool { + use DataType::*; + match (left, right) { + ( + List(l) | LargeList(l) | FixedSizeList(l, _), + List(r) | LargeList(r) | FixedSizeList(r, _), + ) => struct_field_types_match(l.data_type(), r.data_type()), + // The entries are a struct of the key and the value, which are checked on their own. + (Map(l, _), Map(r, _)) => match (l.data_type(), r.data_type()) { + (Struct(l), Struct(r)) => l + .iter() + .zip(r.iter()) + .all(|(a, b)| struct_field_types_match(a.data_type(), b.data_type())), + _ => true, + }, + (Struct(_), Struct(_)) => same_type(left, right), + _ => true, + } +} + +/// Whether two types are equal when the nullability of nested fields is ignored, like +/// Spark's `DataTypeUtils.sameType`. +fn same_type(left: &DataType, right: &DataType) -> bool { + use DataType::*; + match (left, right) { + (List(l), List(r)) | (LargeList(l), LargeList(r)) | (Map(l, _), Map(r, _)) => { + same_type(l.data_type(), r.data_type()) + } + (FixedSizeList(l, ln), FixedSizeList(r, rn)) => { + ln == rn && same_type(l.data_type(), r.data_type()) + } + (Struct(l), Struct(r)) => { + l.len() == r.len() + && l.iter().zip(r.iter()).all(|(a, b)| { + a.name() == b.name() && same_type(a.data_type(), b.data_type()) + }) + } + _ => left == right, + } +} + /// The name Spark's `toSQLType` gives the Spark type that an Arrow type maps to, for error /// messages. Arrow types without a Spark counterpart use their Arrow name. pub(crate) fn spark_sql_type(data_type: &DataType) -> String { @@ -188,10 +231,10 @@ pub(crate) fn spark_sql_type(data_type: &DataType) -> String { .join(", ") ), Map(entries, _) => match entries.data_type() { - Struct(kv) if kv.len() == 2 => format!( + Struct(kv) if let [k, v] = &kv[..] => format!( "MAP<{}, {}>", - spark_sql_type(kv[0].data_type()), - spark_sql_type(kv[1].data_type()) + spark_sql_type(k.data_type()), + spark_sql_type(v.data_type()) ), other => format!("MAP<{other}>"), }, @@ -199,13 +242,8 @@ pub(crate) fn spark_sql_type(data_type: &DataType) -> String { } } -fn is_string(data_type: &DataType) -> bool { - matches!( - data_type, - DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View - ) -} - +/// Unlike [`DataType::is_binary`], this leaves out `FixedSizeBinary`, which +/// [`wider_physical_type`] cannot widen. fn is_binary(data_type: &DataType) -> bool { matches!( data_type, @@ -250,20 +288,6 @@ fn numeric_precedence(data_type: &DataType) -> Option { } } -fn finer_unit(left: TimeUnit, right: TimeUnit) -> TimeUnit { - let rank = |unit: TimeUnit| match unit { - TimeUnit::Second => 0, - TimeUnit::Millisecond => 1, - TimeUnit::Microsecond => 2, - TimeUnit::Nanosecond => 3, - }; - if rank(left) >= rank(right) { - left - } else { - right - } -} - #[cfg(test)] mod tests { use super::*; @@ -340,6 +364,24 @@ mod tests { assert_eq!(tightest_common_type(&a, &b, false), None); } + #[test] + fn struct_field_types_must_match_at_any_depth() { + let int = Struct(Fields::from(vec![Field::new("a", Int32, false)])); + let nullable_int = Struct(Fields::from(vec![Field::new("a", Int32, true)])); + let long = Struct(Fields::from(vec![Field::new("a", Int64, false)])); + assert!(struct_field_types_match(&int, &nullable_int)); + assert!(!struct_field_types_match(&int, &long)); + assert!(!struct_field_types_match( + &list(int.clone()), + &list(long.clone()) + )); + assert!(!struct_field_types_match( + &map(Int32, int), + &map(Int64, long) + )); + assert!(struct_field_types_match(&list(Int32), &list(Int64))); + } + #[test] fn maps_and_calendar_intervals_are_not_orderable() { let interval = Interval(IntervalUnit::MonthDayNano); From 552d8d721677914511a61d8a165e1ecbdf6169e7 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 10:36:50 -0700 Subject: [PATCH 5/8] fix(spark): reject INTERVAL and name Spark types in array_remove errors Use the shared isOrderable, which also rejects INTERVAL, and render the error types the way Spark does, with its SQLSTATE. The description now says the DataFusion built-in removes only the first matching element. --- .../spark/src/function/array/array_remove.rs | 21 +++++++++++-------- .../test_files/array/array_remove.slt | 17 +++++++++++---- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/crates/spark/src/function/array/array_remove.rs b/crates/spark/src/function/array/array_remove.rs index 88653b69..0a5296c6 100644 --- a/crates/spark/src/function/array/array_remove.rs +++ b/crates/spark/src/function/array/array_remove.rs @@ -4,7 +4,7 @@ use crate::function::float_normalize::normalize_array; use crate::function::type_coercion::{ - is_orderable, struct_field_types_match, tightest_common_type, + is_orderable, spark_sql_type, struct_field_types_match, tightest_common_type, }; use arrow::array::{ Array, ArrayRef, AsArray, BooleanArray, BooleanBufferBuilder, GenericListArray, @@ -28,10 +28,9 @@ use std::sync::Arc; /// #[user_doc( doc_section(label = "Array Functions"), - description = "Removes all elements that are equal to the element from the array. The \ - result is NULL when the array or the element is NULL, while the DataFusion built-in \ - removes the NULL elements when the element is NULL. NULL elements in the array are \ - always kept. Corresponding NULLs nested inside compared arrays or structs are equal. \ + description = "Removes all elements that are equal to the element from the array, while \ + the DataFusion built-in removes only the first one. The result is NULL when the array \ + or the element is NULL. NULL elements in the array are always kept. Corresponding NULLs nested inside compared arrays or structs are equal. \ For floating point values, also inside nested arrays and structs, NaN is equal to NaN \ and -0.0 is equal to 0.0; the elements that are kept keep their original values. As \ in Spark, the array element and the element are cast to their tightest common type, \ @@ -39,8 +38,8 @@ use std::sync::Arc; decimal only if the decimal can hold all its values, and a date widens to a \ timestamp. Other combinations are an error, for example a string and a number, two \ different decimals, or structs whose field names or field types differ, as in Spark \ - with ANSI mode off (with ANSI mode on, Spark widens struct field types). Map \ - elements are also an error.", + with ANSI mode off (with ANSI mode on, Spark widens struct field types). Map and \ + INTERVAL elements, at any depth, are also an error.", syntax_example = "array_remove(array, element)", sql_example = r#"```sql > SELECT array_remove(array(1, 2, 3, NULL, 3), 3); @@ -230,7 +229,9 @@ fn coerce_array_and_element( plan_err!( "[DATATYPE_MISMATCH.ARRAY_FUNCTION_DIFF_TYPES] Input to `{name}` should have \ been \"ARRAY\" followed by a value with same element type, but it's \ - [{array}, {element}]" + [\"{}\", \"{}\"]. SQLSTATE: 42K09", + spark_sql_type(array), + spark_sql_type(element) ) }; let (item, large) = match array { @@ -238,6 +239,7 @@ fn coerce_array_and_element( DataType::LargeList(field) => (field, true), _ => return diff_types_err(), }; + // Spark's `ArrayRemove` calls the non-ANSI `TypeCoercion.findTightestCommonType`. let common = tightest_common_type(item.data_type(), element, false) .filter(|_| struct_field_types_match(item.data_type(), element)); let Some(common) = common else { @@ -246,7 +248,8 @@ fn coerce_array_and_element( if !is_orderable(&common) { return plan_err!( "[DATATYPE_MISMATCH.INVALID_ORDERING_TYPE] The `{name}` does not support \ - ordering on type {common}" + ordering on type \"{}\". SQLSTATE: 42K09", + spark_sql_type(&common) ); } let item = Arc::new(Field::new(item.name(), common.clone(), true)); diff --git a/crates/sqllogictest/test_files/array/array_remove.slt b/crates/sqllogictest/test_files/array/array_remove.slt index c37e5181..a07f34bd 100644 --- a/crates/sqllogictest/test_files/array/array_remove.slt +++ b/crates/sqllogictest/test_files/array/array_remove.slt @@ -7,6 +7,8 @@ # array or the element is NULL. # Expected values verified with pyspark==4.2.0. +# Every matching element is removed. The DataFusion built-in removes only the first, and +# returns [1, 3, 2] here. query ? SELECT array_remove(array(1, 2, 3, 2), 2); ---- @@ -43,7 +45,7 @@ SELECT array_remove(array(1, 1), 1); ---- [] -# A NULL element returns NULL. The DataFusion built-in removes the NULL elements instead. +# A NULL element returns NULL query ? SELECT array_remove(array(1, 2, 3), CAST(NULL AS INT)); ---- @@ -82,7 +84,7 @@ SELECT array_remove(array(NULL, NULL), NULL); NULL # An untyped NULL array is not an array -query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] Input to `array_remove` should have been "ARRAY" followed by a value with same element type +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] Input to `array_remove` should have been "ARRAY" followed by a value with same element type, but it's \["VOID", "BIGINT"\]\. SQLSTATE: 42K09 SELECT array_remove(NULL, 1); query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] @@ -91,7 +93,7 @@ SELECT array_remove(NULL, NULL); query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] SELECT array_remove(1, 1); -query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] Input to `array_remove` should have been "ARRAY" followed by a value with same element type, but it's \["ARRAY", "ARRAY"\]\. SQLSTATE: 42K09 SELECT array_remove(array(1, 2, 3, 2), array(2)); query ? @@ -193,12 +195,19 @@ SELECT array_remove(array(named_struct('a', 1, 'b', CAST('-0.0' AS DOUBLE)), nam [{a: 2, b: 1.0}] # Maps cannot be ordered, so map elements are rejected at any depth -query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_remove` does not support ordering on type +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_remove` does not support ordering on type "MAP"\. SQLSTATE: 42K09 SELECT array_remove(array(map(1, 2)), map(1, 2)); query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] SELECT array_remove(array(named_struct('m', map(1, 2))), named_struct('m', map(1, 2))); +# Intervals cannot be ordered either +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_remove` does not support ordering on type "INTERVAL"\. SQLSTATE: 42K09 +SELECT array_remove(array(make_interval(1)), make_interval(1)); + +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_remove` does not support ordering on type "ARRAY"\. SQLSTATE: 42K09 +SELECT array_remove(array(array(make_interval(1))), array(make_interval(1))); + ### ### Type coercion ### From 7af017e4f6bac29bf9b83571b20c8ea3ff5e9fa9 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 10:41:06 -0700 Subject: [PATCH 6/8] perf(spark): bitmap path for a constant element in array_remove For a constant element that is not nested, compare every value with it at once (in place for floats, with Arrow's eq kernel otherwise) and find the elements to keep with bitmap operations, instead of normalizing a copy and calling a comparator per element. About 3.5x faster at 8192 rows. Add a criterion bench and a test that the two paths agree. --- crates/spark/Cargo.toml | 4 + crates/spark/benches/array_remove.rs | 112 +++++++++++ .../spark/src/function/array/array_remove.rs | 180 ++++++++++++++---- 3 files changed, 257 insertions(+), 39 deletions(-) create mode 100644 crates/spark/benches/array_remove.rs diff --git a/crates/spark/Cargo.toml b/crates/spark/Cargo.toml index 829ce8a1..e916aa8c 100644 --- a/crates/spark/Cargo.toml +++ b/crates/spark/Cargo.toml @@ -110,3 +110,7 @@ name = "from_utc_timestamp" [[bench]] harness = false name = "stack" + +[[bench]] +harness = false +name = "array_remove" diff --git a/crates/spark/benches/array_remove.rs b/crates/spark/benches/array_remove.rs new file mode 100644 index 00000000..5c6b05f2 --- /dev/null +++ b/crates/spark/benches/array_remove.rs @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use arrow::array::{ArrayRef, ListArray, PrimitiveArray}; +use arrow::buffer::OffsetBuffer; +use arrow::datatypes::{Field, Float64Type, Int64Type}; +use criterion::{Criterion, criterion_group, criterion_main}; +use datafusion_common::ScalarValue; +use datafusion_common::config::ConfigOptions; +use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDF}; +use datafusion_spark::function::array::array_remove; +use rand::rngs::StdRng; +use rand::{Rng, SeedableRng}; +use std::hint::black_box; +use std::sync::Arc; + +const ROWS: usize = 8192; +const MAX_LIST_LEN: usize = 32; +const NULL_DENSITY: f32 = 0.1; + +fn list_of(values: ArrayRef, rng: &mut StdRng) -> ColumnarValue { + let lengths: Vec = (0..ROWS) + .map(|_| rng.random_range(0..MAX_LIST_LEN)) + .collect(); + let values = values.slice(0, lengths.iter().sum()); + let field = Arc::new(Field::new_list_field(values.data_type().clone(), true)); + ColumnarValue::Array(Arc::new(ListArray::new( + field, + OffsetBuffer::from_lengths(lengths), + values, + None, + ))) +} + +/// Small values, so that the element matches often. Includes NaN and -0.0. +fn float64_values(rng: &mut StdRng) -> ArrayRef { + let values: PrimitiveArray = (0..ROWS * MAX_LIST_LEN) + .map(|_| { + (rng.random::() >= NULL_DENSITY).then(|| match rng.random_range(0..20) { + 0 => f64::NAN, + 1 => -0.0, + n => f64::from(n % 8), + }) + }) + .collect(); + Arc::new(values) +} + +fn int64_values(rng: &mut StdRng) -> ArrayRef { + let values: PrimitiveArray = (0..ROWS * MAX_LIST_LEN) + .map(|_| (rng.random::() >= NULL_DENSITY).then(|| rng.random_range(0..8))) + .collect(); + Arc::new(values) +} + +fn bench(c: &mut Criterion, udf: &ScalarUDF, label: &str, args: [ColumnarValue; 2]) { + let arg_fields: Vec<_> = args + .iter() + .map(|arg| Arc::new(Field::new("a", arg.data_type(), true))) + .collect(); + let return_field = Arc::new(Field::new("r", args[0].data_type(), true)); + let config = Arc::new(ConfigOptions::default()); + c.bench_function(&format!("array_remove {label}"), |b| { + b.iter(|| { + black_box( + udf.invoke_with_args(ScalarFunctionArgs { + args: args.to_vec(), + arg_fields: arg_fields.clone(), + number_rows: ROWS, + return_field: Arc::clone(&return_field), + config_options: Arc::clone(&config), + }) + .unwrap(), + ) + }) + }); +} + +fn criterion_benchmark(c: &mut Criterion) { + let rng = &mut StdRng::seed_from_u64(42); + let udf = array_remove(); + let floats = list_of(float64_values(rng), rng); + let ints = list_of(int64_values(rng), rng); + let float_column = + ColumnarValue::Array(Arc::new(PrimitiveArray::::from( + (0..ROWS).map(|i| Some((i % 8) as f64)).collect::>(), + ))); + let scalar = |v: ScalarValue| ColumnarValue::Scalar(v); + bench( + c, + &udf, + "float64 constant", + [floats.clone(), scalar(ScalarValue::Float64(Some(3.0)))], + ); + bench( + c, + &udf, + "float64 constant NaN", + [floats.clone(), scalar(ScalarValue::Float64(Some(f64::NAN)))], + ); + bench(c, &udf, "float64 column", [floats, float_column]); + bench( + c, + &udf, + "int64 constant", + [ints, scalar(ScalarValue::Int64(Some(3)))], + ); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/crates/spark/src/function/array/array_remove.rs b/crates/spark/src/function/array/array_remove.rs index 0a5296c6..076da6f5 100644 --- a/crates/spark/src/function/array/array_remove.rs +++ b/crates/spark/src/function/array/array_remove.rs @@ -7,13 +7,13 @@ use crate::function::type_coercion::{ is_orderable, spark_sql_type, struct_field_types_match, tightest_common_type, }; use arrow::array::{ - Array, ArrayRef, AsArray, BooleanArray, BooleanBufferBuilder, GenericListArray, - OffsetSizeTrait, Scalar, make_comparator, + Array, ArrayRef, ArrowPrimitiveType, AsArray, BooleanArray, BooleanBufferBuilder, + GenericListArray, OffsetSizeTrait, Scalar, make_comparator, }; use arrow::buffer::{BooleanBuffer, NullBuffer, OffsetBuffer}; use arrow::compute::kernels::cmp::eq; use arrow::compute::{SortOptions, filter}; -use arrow::datatypes::{DataType, Field}; +use arrow::datatypes::{DataType, Field, Float32Type, Float64Type}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, exec_err, plan_err}; use datafusion_expr::{ @@ -21,6 +21,7 @@ use datafusion_expr::{ Volatility, }; use datafusion_macros::user_doc; +use num_traits::Float; use std::sync::Arc; /// Spark-compatible `array_remove` expression. @@ -151,52 +152,77 @@ fn remove( }; let row_is_valid = |row: usize| row_nulls.as_ref().is_none_or(|n| n.is_valid(row)); - // Equality is computed on copies with canonical NaN and 0.0 at any depth, so that it - // follows Spark's float equality, while the kept elements are taken from the original - // values. The comparator treats NULLs nested inside the compared values as equal. - let mut is_equal: Box bool> = Box::new(|_, _| false); // When every element is NULL there is nothing to compare: NULL elements are kept and a // NULL element makes every row NULL. This also covers the NULL type. let any_comparison = values.logical_null_count() < values.len() && (0..list.len()).any(row_is_valid); - if any_comparison { - let normalized = normalize_array(&values)?; - let needle = normalize_array(element)?; - is_equal = if constant && !normalized.data_type().is_nested() { - // Compare every value with the constant at once. - let equal = eq(&normalized, &Scalar::new(Arc::clone(&needle)))?; - let equal = equal.values().clone(); - Box::new(move |i, _| equal.value(i)) - } else { + let value_nulls = values.logical_nulls(); + let mut new_offsets = Vec::with_capacity(list.len() + 1); + new_offsets.push(O::usize_as(0)); + let mut kept = 0usize; + + let keep = if any_comparison && constant && !values.data_type().is_nested() { + // A constant element that is not nested is compared with every value at once, and + // the elements to keep are found with bitmap operations. + let equal = match values.data_type() { + DataType::Float32 => float_equal::(&values, element), + DataType::Float64 => float_equal::(&values, element), + _ => eq(&values, &Scalar::new(Arc::clone(element)))? + .values() + .clone(), + }; + let mut keep = !&equal; + if let Some(nulls) = &value_nulls { + keep = &keep | &!nulls.inner(); + } + if row_nulls.as_ref().is_some_and(|n| n.null_count() > 0) { + // The elements of a NULL row are dropped. + let mut in_valid_row = BooleanBufferBuilder::new(len); + for (row, w) in offsets.windows(2).enumerate() { + in_valid_row.append_n((w[1] - w[0]).as_usize(), row_is_valid(row)); + } + keep = &keep & &in_valid_row.finish(); + } + for w in offsets.windows(2) { + let (start, end) = (w[0].as_usize() - base, w[1].as_usize() - base); + kept += keep.slice(start, end - start).count_set_bits(); + new_offsets.push(O::usize_as(kept)); + } + keep + } else { + // Equality is computed on copies with canonical NaN and 0.0 at any depth, so that it + // follows Spark's float equality, while the kept elements are taken from the + // original values. The comparator treats NULLs nested inside the compared values as + // equal. + let mut is_equal: Box bool> = Box::new(|_, _| false); + if any_comparison { + let normalized = normalize_array(&values)?; + let needle = normalize_array(element)?; let cmp = make_comparator( normalized.as_ref(), needle.as_ref(), SortOptions::default(), )?; - Box::new(move |i, row| cmp(i, if constant { 0 } else { row }).is_eq()) - }; - } - - let value_nulls = values.logical_nulls(); - let mut keep = BooleanBufferBuilder::new(len); - let mut new_offsets = Vec::with_capacity(list.len() + 1); - new_offsets.push(O::usize_as(0)); - let mut kept = 0usize; - for (row, w) in offsets.windows(2).enumerate() { - let (start, end) = (w[0].as_usize() - base, w[1].as_usize() - base); - if row_is_valid(row) { - for i in start..end { - let keep_it = value_nulls.as_ref().is_some_and(|n| n.is_null(i)) - || !is_equal(i, row); - keep.append(keep_it); - kept += usize::from(keep_it); + is_equal = + Box::new(move |i, row| cmp(i, if constant { 0 } else { row }).is_eq()); + } + let mut keep = BooleanBufferBuilder::new(len); + for (row, w) in offsets.windows(2).enumerate() { + let (start, end) = (w[0].as_usize() - base, w[1].as_usize() - base); + if row_is_valid(row) { + for i in start..end { + let keep_it = value_nulls.as_ref().is_some_and(|n| n.is_null(i)) + || !is_equal(i, row); + keep.append(keep_it); + kept += usize::from(keep_it); + } + } else { + keep.append_n(end - start, false); } - } else { - keep.append_n(end - start, false); + new_offsets.push(O::usize_as(kept)); } - new_offsets.push(O::usize_as(kept)); - } - let keep: BooleanBuffer = keep.finish(); + keep.finish() + }; let kept_values = if kept == len { values } else { @@ -214,6 +240,23 @@ fn remove( )?)) } +/// For each value, whether it equals the one-row `element` under Spark's float equality, +/// where NaN equals NaN and `-0.0` equals `0.0`. The bits of NULL values are arbitrary. +fn float_equal(values: &ArrayRef, element: &ArrayRef) -> BooleanBuffer +where + T: ArrowPrimitiveType, + T::Native: Float, +{ + let values = values.as_primitive::().values(); + let needle = element.as_primitive::().value(0); + if needle.is_nan() { + BooleanBuffer::collect_bool(values.len(), |i| values[i].is_nan()) + } else { + // IEEE 754 `==` already treats `-0.0` as equal to `0.0`. + BooleanBuffer::collect_bool(values.len(), |i| values[i] == needle) + } +} + /// Casts the array element and the element to their tightest common type, and rejects the /// argument types that Spark's `ArrayRemove.checkInputDataTypes` rejects. /// @@ -265,7 +308,7 @@ fn coerce_array_and_element( mod tests { use super::*; use arrow::array::{Float32Array, Float64Array, Int32Array, ListArray, StructArray}; - use arrow::datatypes::{Fields, Float32Type, Float64Type, Int32Type}; + use arrow::datatypes::{Fields, Int32Type}; fn negative_nan() -> f64 { f64::from_bits(f64::NAN.to_bits() | (1 << 63)) @@ -506,4 +549,63 @@ mod tests { assert!(err.contains("ARRAY_FUNCTION_DIFF_TYPES"), "{err}"); } } + + /// The bitmap path for a constant element that is not nested must agree with the + /// comparator path, which a column holding the same element takes. + #[test] + fn constant_fast_path_matches_comparator_path() { + let edge = [ + Some(1.0), + Some(-0.0), + Some(0.0), + Some(f64::NAN), + Some(negative_nan()), + Some(f64::INFINITY), + None, + ]; + // Every edge value in several orders, with NULL rows and empty rows. + let rows: Vec>>> = (0..edge.len()) + .map(|i| Some(edge.iter().cycle().skip(i).take(i + 2).copied().collect())) + .chain([None, Some(vec![]), Some(vec![None])]) + .collect(); + let list = ListArray::from_iter_primitive::(rows.clone()); + let n = rows.len(); + for array in [ + Arc::new(list.clone()) as ArrayRef, + Arc::new(list.slice(1, n - 2)), + ] { + let len = array.len(); + for needle in edge { + let constant: ArrayRef = Arc::new(Float64Array::from(vec![needle])); + let column: ArrayRef = Arc::new(Float64Array::from(vec![needle; len])); + let fast = remove::(&array, &constant, true).unwrap(); + let generic = remove::(&array, &column, false).unwrap(); + assert_eq!( + format!("{fast:?}"), + format!("{generic:?}"), + "needle {needle:?}" + ); + } + } + + let ints = ListArray::from_iter_primitive::(vec![ + Some(vec![Some(1), None, Some(2), Some(1)]), + None, + Some(vec![Some(2)]), + ]); + let ints: ArrayRef = Arc::new(ints); + let fast = remove::( + &ints, + &(Arc::new(Int32Array::from(vec![1])) as ArrayRef), + true, + ) + .unwrap(); + let generic = remove::( + &ints, + &(Arc::new(Int32Array::from(vec![1; 3])) as ArrayRef), + false, + ) + .unwrap(); + assert_eq!(format!("{fast:?}"), format!("{generic:?}")); + } } From 72b5ce0e84149abca6dd499dc375d1ae1540a698 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 16:54:22 -0700 Subject: [PATCH 7/8] fix(spark): return an error, not a panic, for a mismatched constant element in array_remove --- .../spark/src/function/array/array_remove.rs | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/crates/spark/src/function/array/array_remove.rs b/crates/spark/src/function/array/array_remove.rs index 076da6f5..e7f7b175 100644 --- a/crates/spark/src/function/array/array_remove.rs +++ b/crates/spark/src/function/array/array_remove.rs @@ -161,7 +161,14 @@ fn remove( new_offsets.push(O::usize_as(0)); let mut kept = 0usize; - let keep = if any_comparison && constant && !values.data_type().is_nested() { + // A caller that skips coercion, such as Comet, can pass an element of another type. The + // bitmap path would panic on its downcast, so such an element takes the comparator path, + // which returns an error. + let keep = if any_comparison + && constant + && !values.data_type().is_nested() + && element.data_type() == values.data_type() + { // A constant element that is not nested is compared with every value at once, and // the elements to keep are found with bitmap operations. let equal = match values.data_type() { @@ -525,6 +532,33 @@ mod tests { } } + /// Without coercion, an element whose type differs from the array's element type is an + /// error, whether it is a constant or a column. + #[test] + fn mismatched_element_type_is_an_error() { + let array = float_list(&[Some(vec![Some(1.0), Some(2.0)])]); + let ints: ArrayRef = + Arc::new(ListArray::from_iter_primitive::(vec![ + Some(vec![Some(1), Some(2)]), + ])); + for (array, element) in [ + (&array, ScalarValue::Float32(Some(1.0))), + (&array, ScalarValue::Int64(Some(1))), + (&ints, ScalarValue::Int64(Some(1))), + ] { + for constant in [true, false] { + let element = element.to_array().unwrap(); + let err = remove::(array, &element, constant) + .unwrap_err() + .to_string(); + assert!( + err.contains("Can't compare arrays of different types"), + "{err}" + ); + } + } + } + #[test] fn coerce_types_rules() { let udf = SparkArrayRemove::new(); From 1ce5ba6062d5d1725305a0c6ea8d1f696c7ae0ca Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 16:54:22 -0700 Subject: [PATCH 8/8] perf(spark): count kept elements without slicing the bitmap in array_remove --- crates/spark/src/function/array/array_remove.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/spark/src/function/array/array_remove.rs b/crates/spark/src/function/array/array_remove.rs index e7f7b175..77b1da94 100644 --- a/crates/spark/src/function/array/array_remove.rs +++ b/crates/spark/src/function/array/array_remove.rs @@ -192,7 +192,10 @@ fn remove( } for w in offsets.windows(2) { let (start, end) = (w[0].as_usize() - base, w[1].as_usize() - base); - kept += keep.slice(start, end - start).count_set_bits(); + // Counted in place: `BooleanBuffer::slice` would clone the buffer for every row. + kept += keep + .inner() + .count_set_bits_offset(keep.offset() + start, end - start); new_offsets.push(O::usize_as(kept)); } keep