diff --git a/crates/spark/Cargo.toml b/crates/spark/Cargo.toml index 829ce8a1..e916aa8c 100644 --- a/crates/spark/Cargo.toml +++ b/crates/spark/Cargo.toml @@ -110,3 +110,7 @@ name = "from_utc_timestamp" [[bench]] harness = false name = "stack" + +[[bench]] +harness = false +name = "array_remove" diff --git a/crates/spark/benches/array_remove.rs b/crates/spark/benches/array_remove.rs new file mode 100644 index 00000000..5c6b05f2 --- /dev/null +++ b/crates/spark/benches/array_remove.rs @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use arrow::array::{ArrayRef, ListArray, PrimitiveArray}; +use arrow::buffer::OffsetBuffer; +use arrow::datatypes::{Field, Float64Type, Int64Type}; +use criterion::{Criterion, criterion_group, criterion_main}; +use datafusion_common::ScalarValue; +use datafusion_common::config::ConfigOptions; +use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDF}; +use datafusion_spark::function::array::array_remove; +use rand::rngs::StdRng; +use rand::{Rng, SeedableRng}; +use std::hint::black_box; +use std::sync::Arc; + +const ROWS: usize = 8192; +const MAX_LIST_LEN: usize = 32; +const NULL_DENSITY: f32 = 0.1; + +fn list_of(values: ArrayRef, rng: &mut StdRng) -> ColumnarValue { + let lengths: Vec = (0..ROWS) + .map(|_| rng.random_range(0..MAX_LIST_LEN)) + .collect(); + let values = values.slice(0, lengths.iter().sum()); + let field = Arc::new(Field::new_list_field(values.data_type().clone(), true)); + ColumnarValue::Array(Arc::new(ListArray::new( + field, + OffsetBuffer::from_lengths(lengths), + values, + None, + ))) +} + +/// Small values, so that the element matches often. Includes NaN and -0.0. +fn float64_values(rng: &mut StdRng) -> ArrayRef { + let values: PrimitiveArray = (0..ROWS * MAX_LIST_LEN) + .map(|_| { + (rng.random::() >= NULL_DENSITY).then(|| match rng.random_range(0..20) { + 0 => f64::NAN, + 1 => -0.0, + n => f64::from(n % 8), + }) + }) + .collect(); + Arc::new(values) +} + +fn int64_values(rng: &mut StdRng) -> ArrayRef { + let values: PrimitiveArray = (0..ROWS * MAX_LIST_LEN) + .map(|_| (rng.random::() >= NULL_DENSITY).then(|| rng.random_range(0..8))) + .collect(); + Arc::new(values) +} + +fn bench(c: &mut Criterion, udf: &ScalarUDF, label: &str, args: [ColumnarValue; 2]) { + let arg_fields: Vec<_> = args + .iter() + .map(|arg| Arc::new(Field::new("a", arg.data_type(), true))) + .collect(); + let return_field = Arc::new(Field::new("r", args[0].data_type(), true)); + let config = Arc::new(ConfigOptions::default()); + c.bench_function(&format!("array_remove {label}"), |b| { + b.iter(|| { + black_box( + udf.invoke_with_args(ScalarFunctionArgs { + args: args.to_vec(), + arg_fields: arg_fields.clone(), + number_rows: ROWS, + return_field: Arc::clone(&return_field), + config_options: Arc::clone(&config), + }) + .unwrap(), + ) + }) + }); +} + +fn criterion_benchmark(c: &mut Criterion) { + let rng = &mut StdRng::seed_from_u64(42); + let udf = array_remove(); + let floats = list_of(float64_values(rng), rng); + let ints = list_of(int64_values(rng), rng); + let float_column = + ColumnarValue::Array(Arc::new(PrimitiveArray::::from( + (0..ROWS).map(|i| Some((i % 8) as f64)).collect::>(), + ))); + let scalar = |v: ScalarValue| ColumnarValue::Scalar(v); + bench( + c, + &udf, + "float64 constant", + [floats.clone(), scalar(ScalarValue::Float64(Some(3.0)))], + ); + bench( + c, + &udf, + "float64 constant NaN", + [floats.clone(), scalar(ScalarValue::Float64(Some(f64::NAN)))], + ); + bench(c, &udf, "float64 column", [floats, float_column]); + bench( + c, + &udf, + "int64 constant", + [ints, scalar(ScalarValue::Int64(Some(3)))], + ); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/crates/spark/src/function/array/array_remove.rs b/crates/spark/src/function/array/array_remove.rs new file mode 100644 index 00000000..77b1da94 --- /dev/null +++ b/crates/spark/src/function/array/array_remove.rs @@ -0,0 +1,648 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use crate::function::float_normalize::normalize_array; +use crate::function::type_coercion::{ + is_orderable, spark_sql_type, struct_field_types_match, tightest_common_type, +}; +use arrow::array::{ + Array, ArrayRef, ArrowPrimitiveType, AsArray, BooleanArray, BooleanBufferBuilder, + GenericListArray, OffsetSizeTrait, Scalar, make_comparator, +}; +use arrow::buffer::{BooleanBuffer, NullBuffer, OffsetBuffer}; +use arrow::compute::kernels::cmp::eq; +use arrow::compute::{SortOptions, filter}; +use arrow::datatypes::{DataType, Field, Float32Type, Float64Type}; +use datafusion_common::utils::take_function_args; +use datafusion_common::{Result, ScalarValue, exec_err, plan_err}; +use datafusion_expr::{ + ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, + Volatility, +}; +use datafusion_macros::user_doc; +use num_traits::Float; +use std::sync::Arc; + +/// Spark-compatible `array_remove` expression. +/// +/// +#[user_doc( + doc_section(label = "Array Functions"), + description = "Removes all elements that are equal to the element from the array, while \ + the DataFusion built-in removes only the first one. The result is NULL when the array \ + or the element is NULL. NULL elements in the array are always kept. Corresponding NULLs nested inside compared arrays or structs are equal. \ + For floating point values, also inside nested arrays and structs, NaN is equal to NaN \ + and -0.0 is equal to 0.0; the elements that are kept keep their original values. As \ + in Spark, the array element and the element are cast to their tightest common type, \ + and the result is an array of that type: numbers widen, an integer widens to a \ + decimal only if the decimal can hold all its values, and a date widens to a \ + timestamp. Other combinations are an error, for example a string and a number, two \ + different decimals, or structs whose field names or field types differ, as in Spark \ + with ANSI mode off (with ANSI mode on, Spark widens struct field types). Map and \ + INTERVAL elements, at any depth, are also an error.", + syntax_example = "array_remove(array, element)", + sql_example = r#"```sql +> SELECT array_remove(array(1, 2, 3, NULL, 3), 3); +[1, 2, NULL] +```"#, + argument(name = "array", description = "Array expression. NULL returns NULL."), + argument( + name = "element", + description = "Element to remove. NULL returns NULL." + ) +)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkArrayRemove { + signature: Signature, +} + +impl Default for SparkArrayRemove { + fn default() -> Self { + Self::new() + } +} + +impl SparkArrayRemove { + pub fn new() -> Self { + Self { + signature: Signature::user_defined(Volatility::Immutable), + } + } +} + +impl ScalarUDFImpl for SparkArrayRemove { + fn name(&self) -> &str { + "array_remove" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, arg_types: &[DataType]) -> Result { + let [array, _] = take_function_args(self.name(), arg_types)?; + Ok(array.clone()) + } + + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { + let [array, element] = take_function_args(self.name(), arg_types)?; + coerce_array_and_element(self.name(), array, element) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + let [array, element] = take_function_args(self.name(), &args.args)?; + let is_scalar = matches!( + (array, element), + (ColumnarValue::Scalar(_), ColumnarValue::Scalar(_)) + ); + let rows = if is_scalar { 1 } else { args.number_rows }; + let array = array.to_array(rows)?; + // A constant element is compared as a single value against every row. + let (element, constant) = match element { + ColumnarValue::Scalar(scalar) => (scalar.to_array()?, true), + ColumnarValue::Array(element) => (Arc::clone(element), false), + }; + + let result = match array.data_type() { + DataType::List(_) => remove::(&array, &element, constant)?, + DataType::LargeList(_) => remove::(&array, &element, constant)?, + other => return exec_err!("array_remove does not support type {other}"), + }; + + if is_scalar { + Ok(ColumnarValue::Scalar(ScalarValue::try_from_array( + &result, 0, + )?)) + } else { + Ok(ColumnarValue::Array(result)) + } + } + + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } +} + +/// Removes from each row of `array` the non-null elements equal to the row's element, which +/// is `element[0]` for every row if `constant`, and `element[row]` otherwise. A row is NULL +/// when the array or the element is NULL. +fn remove( + array: &ArrayRef, + element: &ArrayRef, + constant: bool, +) -> Result { + let list = array.as_list::(); + let offsets = list.offsets(); + // Only the values that this (possibly sliced) list refers to are looked at. + let base = offsets[0].as_usize(); + let len = offsets[offsets.len() - 1].as_usize() - base; + let values = list.values().slice(base, len); + + // Logical nulls, so that a NULL-typed element, which has no null buffer, is NULL. + let element_nulls = element.logical_nulls(); + let row_nulls = if constant { + if element_nulls.as_ref().is_some_and(|n| n.is_null(0)) { + Some(NullBuffer::new_null(list.len())) + } else { + list.nulls().cloned() + } + } else { + NullBuffer::union(list.nulls(), element_nulls.as_ref()) + }; + let row_is_valid = |row: usize| row_nulls.as_ref().is_none_or(|n| n.is_valid(row)); + + // When every element is NULL there is nothing to compare: NULL elements are kept and a + // NULL element makes every row NULL. This also covers the NULL type. + let any_comparison = + values.logical_null_count() < values.len() && (0..list.len()).any(row_is_valid); + let value_nulls = values.logical_nulls(); + let mut new_offsets = Vec::with_capacity(list.len() + 1); + new_offsets.push(O::usize_as(0)); + let mut kept = 0usize; + + // A caller that skips coercion, such as Comet, can pass an element of another type. The + // bitmap path would panic on its downcast, so such an element takes the comparator path, + // which returns an error. + let keep = if any_comparison + && constant + && !values.data_type().is_nested() + && element.data_type() == values.data_type() + { + // A constant element that is not nested is compared with every value at once, and + // the elements to keep are found with bitmap operations. + let equal = match values.data_type() { + DataType::Float32 => float_equal::(&values, element), + DataType::Float64 => float_equal::(&values, element), + _ => eq(&values, &Scalar::new(Arc::clone(element)))? + .values() + .clone(), + }; + let mut keep = !&equal; + if let Some(nulls) = &value_nulls { + keep = &keep | &!nulls.inner(); + } + if row_nulls.as_ref().is_some_and(|n| n.null_count() > 0) { + // The elements of a NULL row are dropped. + let mut in_valid_row = BooleanBufferBuilder::new(len); + for (row, w) in offsets.windows(2).enumerate() { + in_valid_row.append_n((w[1] - w[0]).as_usize(), row_is_valid(row)); + } + keep = &keep & &in_valid_row.finish(); + } + for w in offsets.windows(2) { + let (start, end) = (w[0].as_usize() - base, w[1].as_usize() - base); + // Counted in place: `BooleanBuffer::slice` would clone the buffer for every row. + kept += keep + .inner() + .count_set_bits_offset(keep.offset() + start, end - start); + new_offsets.push(O::usize_as(kept)); + } + keep + } else { + // Equality is computed on copies with canonical NaN and 0.0 at any depth, so that it + // follows Spark's float equality, while the kept elements are taken from the + // original values. The comparator treats NULLs nested inside the compared values as + // equal. + let mut is_equal: Box bool> = Box::new(|_, _| false); + if any_comparison { + let normalized = normalize_array(&values)?; + let needle = normalize_array(element)?; + let cmp = make_comparator( + normalized.as_ref(), + needle.as_ref(), + SortOptions::default(), + )?; + is_equal = + Box::new(move |i, row| cmp(i, if constant { 0 } else { row }).is_eq()); + } + let mut keep = BooleanBufferBuilder::new(len); + for (row, w) in offsets.windows(2).enumerate() { + let (start, end) = (w[0].as_usize() - base, w[1].as_usize() - base); + if row_is_valid(row) { + for i in start..end { + let keep_it = value_nulls.as_ref().is_some_and(|n| n.is_null(i)) + || !is_equal(i, row); + keep.append(keep_it); + kept += usize::from(keep_it); + } + } else { + keep.append_n(end - start, false); + } + new_offsets.push(O::usize_as(kept)); + } + keep.finish() + }; + let kept_values = if kept == len { + values + } else { + filter(&values, &BooleanArray::new(keep, None))? + }; + + let (DataType::List(field) | DataType::LargeList(field)) = list.data_type() else { + unreachable!("remove is only called with lists"); + }; + Ok(Arc::new(GenericListArray::::try_new( + Arc::clone(field), + OffsetBuffer::new(new_offsets.into()), + kept_values, + row_nulls, + )?)) +} + +/// For each value, whether it equals the one-row `element` under Spark's float equality, +/// where NaN equals NaN and `-0.0` equals `0.0`. The bits of NULL values are arbitrary. +fn float_equal(values: &ArrayRef, element: &ArrayRef) -> BooleanBuffer +where + T: ArrowPrimitiveType, + T::Native: Float, +{ + let values = values.as_primitive::().values(); + let needle = element.as_primitive::().value(0); + if needle.is_nan() { + BooleanBuffer::collect_bool(values.len(), |i| values[i].is_nan()) + } else { + // IEEE 754 `==` already treats `-0.0` as equal to `0.0`. + BooleanBuffer::collect_bool(values.len(), |i| values[i] == needle) + } +} + +/// Casts the array element and the element to their tightest common type, and rejects the +/// argument types that Spark's `ArrayRemove.checkInputDataTypes` rejects. +/// +/// Unlike `array_position`, Spark accepts a NULL-typed common element type here (for +/// example `array_remove(array(), NULL)`, which returns NULL), and reports an argument that +/// is not an array, including an untyped NULL, as `ARRAY_FUNCTION_DIFF_TYPES`. +fn coerce_array_and_element( + name: &str, + array: &DataType, + element: &DataType, +) -> Result> { + let diff_types_err = || { + plan_err!( + "[DATATYPE_MISMATCH.ARRAY_FUNCTION_DIFF_TYPES] Input to `{name}` should have \ + been \"ARRAY\" followed by a value with same element type, but it's \ + [\"{}\", \"{}\"]. SQLSTATE: 42K09", + spark_sql_type(array), + spark_sql_type(element) + ) + }; + let (item, large) = match array { + DataType::List(field) | DataType::FixedSizeList(field, _) => (field, false), + DataType::LargeList(field) => (field, true), + _ => return diff_types_err(), + }; + // Spark's `ArrayRemove` calls the non-ANSI `TypeCoercion.findTightestCommonType`. + let common = tightest_common_type(item.data_type(), element, false) + .filter(|_| struct_field_types_match(item.data_type(), element)); + let Some(common) = common else { + return diff_types_err(); + }; + if !is_orderable(&common) { + return plan_err!( + "[DATATYPE_MISMATCH.INVALID_ORDERING_TYPE] The `{name}` does not support \ + ordering on type \"{}\". SQLSTATE: 42K09", + spark_sql_type(&common) + ); + } + let item = Arc::new(Field::new(item.name(), common.clone(), true)); + let array = if large { + DataType::LargeList(item) + } else { + DataType::List(item) + }; + Ok(vec![array, common]) +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{Float32Array, Float64Array, Int32Array, ListArray, StructArray}; + use arrow::datatypes::{Fields, Int32Type}; + + fn negative_nan() -> f64 { + f64::from_bits(f64::NAN.to_bits() | (1 << 63)) + } + + fn invoke( + array: ColumnarValue, + element: ColumnarValue, + rows: usize, + ) -> ColumnarValue { + let return_field = Arc::new(Field::new("result", array.data_type(), true)); + SparkArrayRemove::new() + .invoke_with_args(ScalarFunctionArgs { + args: vec![array, element], + arg_fields: vec![], + number_rows: rows, + return_field, + config_options: Default::default(), + }) + .unwrap() + } + + fn float_list(rows: &[Option>>]) -> ArrayRef { + Arc::new(ListArray::from_iter_primitive::( + rows.to_vec(), + )) + } + + /// The rows of a list of doubles, as bits so that the zeros and NaNs can be told apart. + fn bits(array: &ArrayRef) -> Vec>>> { + array + .as_list::() + .iter() + .map(|row| { + row.map(|row| { + row.as_primitive::() + .iter() + .map(|v| v.map(f64::to_bits)) + .collect() + }) + }) + .collect() + } + + /// Spark's result for one row: drop the non-null elements equal to the element. + fn spark_remove(row: &[Option], element: f64) -> Vec> { + let equal = |a: f64| (a.is_nan() && element.is_nan()) || a == element; + row.iter() + .filter(|v| v.is_none_or(|v| !equal(v))) + .map(|v| v.map(f64::to_bits)) + .collect() + } + + const EDGE_VALUES: [Option; 7] = [ + Some(0.0), + Some(-0.0), + Some(f64::NAN), + None, + Some(1.0), + Some(f64::INFINITY), + Some(f64::NEG_INFINITY), + ]; + + /// Every edge value removed from an array of all edge values, both as a column and as a + /// constant: each zero and each NaN meets every other, and the kept elements keep their + /// bits. + #[test] + fn floats_follow_spark_equality() { + let mut elements = EDGE_VALUES.to_vec(); + elements.push(Some(negative_nan())); + let removable: Vec = elements.iter().flatten().copied().collect(); + let rows: Vec<_> = removable.iter().map(|_| Some(elements.clone())).collect(); + let expected: Vec<_> = removable + .iter() + .map(|&v| Some(spark_remove(&elements, v))) + .collect(); + + let result = invoke( + ColumnarValue::Array(float_list(&rows)), + ColumnarValue::Array(Arc::new(Float64Array::from(removable.clone()))), + rows.len(), + ) + .into_array(rows.len()) + .unwrap(); + assert_eq!(bits(&result), expected); + + for (value, expected) in removable.iter().zip(expected) { + let result = invoke( + ColumnarValue::Array(float_list(&rows[..1])), + ColumnarValue::Scalar(ScalarValue::Float64(Some(*value))), + 1, + ) + .into_array(1) + .unwrap(); + assert_eq!(bits(&result), vec![expected], "remove {value}"); + } + } + + #[test] + fn nulls() { + let rows = vec![ + Some(vec![Some(0.0), None, Some(-0.0)]), + None, + Some(vec![Some(1.0)]), + ]; + let element = Float64Array::from(vec![Some(0.0), Some(0.0), None]); + let result = invoke( + ColumnarValue::Array(float_list(&rows)), + ColumnarValue::Array(Arc::new(element)), + 3, + ) + .into_array(3) + .unwrap(); + // The NULL element stays; a NULL array or a NULL element gives a NULL row. + assert_eq!(bits(&result), vec![Some(vec![None]), None, None]); + + let result = invoke( + ColumnarValue::Array(float_list(&rows)), + ColumnarValue::Scalar(ScalarValue::Float64(None)), + 3, + ) + .into_array(3) + .unwrap(); + assert_eq!(bits(&result), vec![None, None, None]); + } + + /// The offsets of a sliced list do not start at 0, and the elements of NULL rows are not + /// carried into the result. + #[test] + fn sliced_list() { + let list = ListArray::from_iter_primitive::(vec![ + Some(vec![Some(9), Some(9)]), + Some(vec![Some(1), Some(2), Some(1)]), + None, + Some(vec![Some(3), None, Some(4)]), + ]); + let array: ArrayRef = Arc::new(list.slice(1, 3)); + for element in [ + ColumnarValue::Array(Arc::new(Int32Array::from(vec![1, 1, 1]))), + ColumnarValue::Scalar(ScalarValue::Int32(Some(1))), + ] { + let result = invoke(ColumnarValue::Array(Arc::clone(&array)), element, 3) + .into_array(3) + .unwrap(); + let expected = ListArray::from_iter_primitive::(vec![ + Some(vec![Some(2)]), + None, + Some(vec![Some(3), None, Some(4)]), + ]); + assert_eq!(result.as_list::(), &expected); + assert_eq!(result.as_list::().values().len(), 4); + } + } + + #[test] + fn large_list_and_scalars() { + let array = ScalarValue::LargeList(Arc::new( + arrow::array::LargeListArray::from_iter_primitive::(vec![ + Some(vec![Some(-0.0f32), Some(f32::NAN), Some(1.0)]), + ]), + )); + let result = invoke( + ColumnarValue::Scalar(array), + ColumnarValue::Scalar(ScalarValue::Float32(Some(0.0))), + 4, + ); + let ColumnarValue::Scalar(ScalarValue::LargeList(result)) = result else { + panic!("expected a large list scalar"); + }; + let values = result.value(0); + let values: &Float32Array = values.as_primitive(); + assert_eq!(values.len(), 2); + assert!(values.value(0).is_nan()); + assert_eq!(values.value(1), 1.0); + } + + /// Struct elements compare their float fields with Spark's equality, and keep their bits. + #[test] + fn nested_elements() { + let fields = Fields::from(vec![Field::new("x", DataType::Float64, true)]); + let structs = |values: Vec| -> ArrayRef { + Arc::new(StructArray::new( + fields.clone(), + vec![Arc::new(Float64Array::from(values))], + None, + )) + }; + let item = Field::new("item", DataType::Struct(fields.clone()), true); + let array: ArrayRef = Arc::new(ListArray::new( + Arc::new(item), + OffsetBuffer::from_lengths([3]), + structs(vec![-0.0, negative_nan(), 1.0]), + None, + )); + for (value, expected) in [ + (0.0, vec![negative_nan(), 1.0]), + (f64::NAN, vec![-0.0, 1.0]), + ] { + for element in [ + ColumnarValue::Array(structs(vec![value])), + ColumnarValue::Scalar( + ScalarValue::try_from_array(&structs(vec![value]), 0).unwrap(), + ), + ] { + let result = invoke(ColumnarValue::Array(Arc::clone(&array)), element, 1) + .into_array(1) + .unwrap(); + let row = result.as_list::().value(0); + let x = row.as_struct().column(0).as_primitive::(); + let x: Vec = x.values().iter().map(|v| v.to_bits()).collect(); + let expected: Vec = expected.iter().map(|v| v.to_bits()).collect(); + assert_eq!(x, expected, "remove {value}"); + } + } + } + + /// Without coercion, an element whose type differs from the array's element type is an + /// error, whether it is a constant or a column. + #[test] + fn mismatched_element_type_is_an_error() { + let array = float_list(&[Some(vec![Some(1.0), Some(2.0)])]); + let ints: ArrayRef = + Arc::new(ListArray::from_iter_primitive::(vec![ + Some(vec![Some(1), Some(2)]), + ])); + for (array, element) in [ + (&array, ScalarValue::Float32(Some(1.0))), + (&array, ScalarValue::Int64(Some(1))), + (&ints, ScalarValue::Int64(Some(1))), + ] { + for constant in [true, false] { + let element = element.to_array().unwrap(); + let err = remove::(array, &element, constant) + .unwrap_err() + .to_string(); + assert!( + err.contains("Can't compare arrays of different types"), + "{err}" + ); + } + } + } + + #[test] + fn coerce_types_rules() { + let udf = SparkArrayRemove::new(); + let list = |t: DataType| DataType::new_list(t, true); + assert_eq!( + udf.coerce_types(&[list(DataType::Int32), DataType::Int64]) + .unwrap(), + vec![list(DataType::Int64), DataType::Int64] + ); + assert_eq!( + udf.coerce_types(&[list(DataType::Null), DataType::Null]) + .unwrap(), + vec![list(DataType::Null), DataType::Null] + ); + for (array, element) in [ + (DataType::Null, DataType::Int32), + (DataType::Int32, DataType::Int32), + (list(DataType::Int32), DataType::Utf8), + (list(DataType::Int32), DataType::Decimal128(2, 1)), + ] { + let err = udf.coerce_types(&[array, element]).unwrap_err().to_string(); + assert!(err.contains("ARRAY_FUNCTION_DIFF_TYPES"), "{err}"); + } + } + + /// The bitmap path for a constant element that is not nested must agree with the + /// comparator path, which a column holding the same element takes. + #[test] + fn constant_fast_path_matches_comparator_path() { + let edge = [ + Some(1.0), + Some(-0.0), + Some(0.0), + Some(f64::NAN), + Some(negative_nan()), + Some(f64::INFINITY), + None, + ]; + // Every edge value in several orders, with NULL rows and empty rows. + let rows: Vec>>> = (0..edge.len()) + .map(|i| Some(edge.iter().cycle().skip(i).take(i + 2).copied().collect())) + .chain([None, Some(vec![]), Some(vec![None])]) + .collect(); + let list = ListArray::from_iter_primitive::(rows.clone()); + let n = rows.len(); + for array in [ + Arc::new(list.clone()) as ArrayRef, + Arc::new(list.slice(1, n - 2)), + ] { + let len = array.len(); + for needle in edge { + let constant: ArrayRef = Arc::new(Float64Array::from(vec![needle])); + let column: ArrayRef = Arc::new(Float64Array::from(vec![needle; len])); + let fast = remove::(&array, &constant, true).unwrap(); + let generic = remove::(&array, &column, false).unwrap(); + assert_eq!( + format!("{fast:?}"), + format!("{generic:?}"), + "needle {needle:?}" + ); + } + } + + let ints = ListArray::from_iter_primitive::(vec![ + Some(vec![Some(1), None, Some(2), Some(1)]), + None, + Some(vec![Some(2)]), + ]); + let ints: ArrayRef = Arc::new(ints); + let fast = remove::( + &ints, + &(Arc::new(Int32Array::from(vec![1])) as ArrayRef), + true, + ) + .unwrap(); + let generic = remove::( + &ints, + &(Arc::new(Int32Array::from(vec![1; 3])) as ArrayRef), + false, + ) + .unwrap(); + assert_eq!(format!("{fast:?}"), format!("{generic:?}")); + } +} diff --git a/crates/spark/src/function/array/mod.rs b/crates/spark/src/function/array/mod.rs index 36dcccc9..72e750f1 100644 --- a/crates/spark/src/function/array/mod.rs +++ b/crates/spark/src/function/array/mod.rs @@ -2,10 +2,11 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -//! Spark array functions, such as `array`, `array_contains`, `array_repeat`, `shuffle` and `slice`. +//! Spark array functions, such as `array`, `array_contains`, `array_remove`, `array_repeat`, `shuffle` and `slice`. pub mod array_contains; pub mod array_position; +pub mod array_remove; pub mod repeat; pub mod shuffle; pub mod slice; @@ -17,6 +18,7 @@ use std::sync::Arc; make_udf_function!(array_contains::SparkArrayContains, spark_array_contains); make_udf_function!(array_position::SparkArrayPosition, array_position); +make_udf_function!(array_remove::SparkArrayRemove, array_remove); make_udf_function!(spark_array::SparkArray, array); make_udf_function!(shuffle::SparkShuffle, shuffle); make_udf_function!(repeat::SparkArrayRepeat, array_repeat); @@ -35,6 +37,11 @@ pub mod expr_fn { "Returns the 1-based position of the element in the array, or 0 if not found (Spark semantics).", array element )); + export_functions!(( + array_remove, + "Removes all elements equal to the element from the array (Spark semantics).", + array element + )); export_functions!((array, "Returns an array with the given elements.", args)); export_functions!(( shuffle, @@ -57,6 +64,7 @@ pub fn functions() -> Vec> { vec![ spark_array_contains(), array_position(), + array_remove(), array(), shuffle(), array_repeat(), diff --git a/crates/sqllogictest/test_files/array/array_remove.slt b/crates/sqllogictest/test_files/array/array_remove.slt new file mode 100644 index 00000000..a07f34bd --- /dev/null +++ b/crates/sqllogictest/test_files/array/array_remove.slt @@ -0,0 +1,462 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +# Tests for Spark-compatible array_remove. +# Removes every element equal to the element; NULL elements are kept; the result is NULL if the +# array or the element is NULL. +# Expected values verified with pyspark==4.2.0. + +# Every matching element is removed. The DataFusion built-in removes only the first, and +# returns [1, 3, 2] here. +query ? +SELECT array_remove(array(1, 2, 3, 2), 2); +---- +[1, 3] + +query ? +SELECT array_remove(array(1, 2, 3), 4); +---- +[1, 2, 3] + +query ? +SELECT array_remove(array('a', 'b', 'a'), 'a'); +---- +[b] + +# NULL elements in the array are kept +query ? +SELECT array_remove(array(1, 2, NULL, 2), 2); +---- +[1, NULL] + +query ? +SELECT array_remove(array(NULL), 1); +---- +[NULL] + +query ? +SELECT array_remove(array(), 1); +---- +[] + +query ? +SELECT array_remove(array(1, 1), 1); +---- +[] + +# A NULL element returns NULL +query ? +SELECT array_remove(array(1, 2, 3), CAST(NULL AS INT)); +---- +NULL + +query ? +SELECT array_remove(array(1, NULL, 3), CAST(NULL AS INT)); +---- +NULL + +# The tightest common type of the array element and an untyped NULL is the array element type +query ? +SELECT array_remove(array(1, 2, 3), NULL); +---- +NULL + +query ? +SELECT array_remove(CAST(NULL AS ARRAY), 1); +---- +NULL + +query ? +SELECT array_remove(CAST(NULL AS ARRAY), CAST(NULL AS INT)); +---- +NULL + +# Unlike array_position, a NULL common element type is accepted, and the result is NULL +query ? +SELECT array_remove(array(), NULL); +---- +NULL + +query ? +SELECT array_remove(array(NULL, NULL), NULL); +---- +NULL + +# An untyped NULL array is not an array +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] Input to `array_remove` should have been "ARRAY" followed by a value with same element type, but it's \["VOID", "BIGINT"\]\. SQLSTATE: 42K09 +SELECT array_remove(NULL, 1); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_remove(NULL, NULL); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_remove(1, 1); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] Input to `array_remove` should have been "ARRAY" followed by a value with same element type, but it's \["ARRAY", "ARRAY"\]\. SQLSTATE: 42K09 +SELECT array_remove(array(1, 2, 3, 2), array(2)); + +query ? +SELECT array_remove(array(true, false, true), true); +---- +[false] + +query ? +SELECT array_remove(array(X'01', X'02'), X'01'); +---- +[02] + +query ? +SELECT array_remove(array(CAST(1.5 AS DECIMAL(3,1)), CAST(2.5 AS DECIMAL(3,1))), CAST(2.5 AS DECIMAL(3,1))); +---- +[1.5] + +query ? +SELECT array_remove(array(DATE '2020-01-01', DATE '2020-01-02'), DATE '2020-01-01'); +---- +[2020-01-02] + +### +### Floating point equality +### + +# -0.0 equals 0.0, in either direction +query ? +SELECT array_remove(array(CAST('-0.0' AS DOUBLE), CAST(0.0 AS DOUBLE), CAST(1.0 AS DOUBLE)), CAST(0.0 AS DOUBLE)); +---- +[1.0] + +query ? +SELECT array_remove(array(CAST('-0.0' AS DOUBLE), CAST(0.0 AS DOUBLE), CAST(1.0 AS DOUBLE)), CAST('-0.0' AS DOUBLE)); +---- +[1.0] + +query ? +SELECT array_remove(array(CAST('-0.0' AS FLOAT), CAST(1.0 AS FLOAT)), CAST(0.0 AS FLOAT)); +---- +[1.0] + +# NaN equals NaN +query ? +SELECT array_remove(array(CAST('NaN' AS DOUBLE), CAST(1.0 AS DOUBLE), CAST('NaN' AS DOUBLE)), CAST('NaN' AS DOUBLE)); +---- +[1.0] + +query ? +SELECT array_remove(array(CAST('NaN' AS FLOAT), CAST(1.0 AS FLOAT)), CAST('NaN' AS FLOAT)); +---- +[1.0] + +# Kept elements keep their sign +query ? +SELECT array_remove(array(CAST('-0.0' AS DOUBLE), CAST(1.0 AS DOUBLE)), CAST(1.0 AS DOUBLE)); +---- +[-0.0] + +query ? +SELECT array_remove(array(CAST('-0.0' AS DOUBLE), CAST('NaN' AS DOUBLE)), CAST('NaN' AS DOUBLE)); +---- +[-0.0] + +### +### Nested elements +### + +query ? +SELECT array_remove(array(array(1, 2), array(3), NULL, array(1, 2)), array(1, 2)); +---- +[[3], NULL] + +# NULLs nested inside compared arrays and structs are equal +query ? +SELECT array_remove(array(array(1, CAST(NULL AS INT)), array(3)), array(1, CAST(NULL AS INT))); +---- +[[3]] + +query ? +SELECT array_remove(array(named_struct('a', 1, 'b', CAST(NULL AS DOUBLE)), named_struct('a', 2, 'b', CAST(1.0 AS DOUBLE))), named_struct('a', 1, 'b', CAST(NULL AS DOUBLE))); +---- +[{a: 2, b: 1.0}] + +# Nested floats follow Spark's equality, and kept elements keep their values +query ? +SELECT array_remove(array(array(CAST('-0.0' AS DOUBLE)), array(CAST('NaN' AS DOUBLE)), array(CAST(1.0 AS DOUBLE))), array(CAST(0.0 AS DOUBLE))); +---- +[[NaN], [1.0]] + +query ? +SELECT array_remove(array(array(CAST('-0.0' AS DOUBLE)), array(CAST('NaN' AS DOUBLE)), array(CAST(1.0 AS DOUBLE))), array(CAST('NaN' AS DOUBLE))); +---- +[[-0.0], [1.0]] + +query ? +SELECT array_remove(array(named_struct('a', 1, 'b', CAST('-0.0' AS DOUBLE)), named_struct('a', 2, 'b', CAST(1.0 AS DOUBLE))), named_struct('a', 1, 'b', CAST(0.0 AS DOUBLE))); +---- +[{a: 2, b: 1.0}] + +# Maps cannot be ordered, so map elements are rejected at any depth +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_remove` does not support ordering on type "MAP"\. SQLSTATE: 42K09 +SELECT array_remove(array(map(1, 2)), map(1, 2)); + +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] +SELECT array_remove(array(named_struct('m', map(1, 2))), named_struct('m', map(1, 2))); + +# Intervals cannot be ordered either +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_remove` does not support ordering on type "INTERVAL"\. SQLSTATE: 42K09 +SELECT array_remove(array(make_interval(1)), make_interval(1)); + +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_remove` does not support ordering on type "ARRAY"\. SQLSTATE: 42K09 +SELECT array_remove(array(array(make_interval(1))), array(make_interval(1))); + +### +### Type coercion +### + +# The array element and the element are cast to their tightest common type, which is also the +# result's element type +query ?T +SELECT array_remove(array(CAST(1 AS INT), CAST(2 AS INT)), 2L), arrow_typeof(array_remove(array(CAST(1 AS INT), CAST(2 AS INT)), 2L)); +---- +[1] List(Int64, field: 'element') + +query ?T +SELECT array_remove(array(1L, 2L), CAST(2 AS INT)), arrow_typeof(array_remove(array(1L, 2L), CAST(2 AS INT))); +---- +[1] List(Int64, field: 'element') + +query ?T +SELECT array_remove(array(CAST(1 AS INT), CAST(2 AS INT)), CAST(2 AS TINYINT)), arrow_typeof(array_remove(array(CAST(1 AS INT), CAST(2 AS INT)), CAST(2 AS TINYINT))); +---- +[1] List(Int32, field: 'element') + +query ?T +SELECT array_remove(array(CAST(1 AS INT), CAST(2 AS INT)), CAST(1.5 AS DOUBLE)), arrow_typeof(array_remove(array(CAST(1 AS INT), CAST(2 AS INT)), CAST(1.5 AS DOUBLE))); +---- +[1.0, 2.0] List(Float64, field: 'element') + +query ? +SELECT array_remove(array(CAST(1 AS FLOAT), CAST(2 AS FLOAT)), CAST(1.0 AS DOUBLE)); +---- +[2.0] + +query ? +SELECT array_remove(array(array(CAST(1 AS INT))), array(1L)); +---- +[] + +# An integral type widens to a decimal only if the decimal holds all its values: +# INT is DECIMAL(10,0), so DECIMAL(12,2) is wide enough and DECIMAL(2,1) is not. +query ?T +SELECT array_remove(array(CAST(1 AS INT), CAST(2 AS INT)), CAST(1 AS DECIMAL(12,2))), arrow_typeof(array_remove(array(CAST(1 AS INT), CAST(2 AS INT)), CAST(1 AS DECIMAL(12,2)))); +---- +[2.00] List(Decimal128(12, 2), field: 'element') + +query ? +SELECT array_remove(array(CAST(1 AS DECIMAL(12,2))), CAST(1 AS INT)); +---- +[] + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_remove(array(CAST(1 AS INT), CAST(2 AS INT)), CAST(1 AS DECIMAL(2,1))); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_remove(array(CAST(1.5 AS DECIMAL(2,1))), CAST(2 AS INT)); + +# Decimals and floating point numbers have no tightest common type +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_remove(array(CAST(1 AS DECIMAL(5,2))), CAST(1.0 AS DOUBLE)); + +# Strings and numbers are not cast to each other +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_remove(array(1, 2), '1'); + +# DATE and TIMESTAMP become TIMESTAMP +query ? +SELECT array_remove(array(DATE '2020-01-01', DATE '2020-01-02'), TIMESTAMP '2020-01-01 00:00:00'); +---- +[2020-01-02T00:00:00] + +# Struct fields must have the same names +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_remove(array(named_struct('a', 1)), named_struct('b', 1)); + +# Struct field types are not widened, as in Spark with ANSI mode off. With ANSI mode on, Spark +# widens them and returns [] here. +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_remove(array(named_struct('a', CAST(1 AS INT))), named_struct('a', 1L)); + +### +### Column inputs +### + +statement ok +CREATE TABLE t AS VALUES + (array(1, 2, 3, 2), 2), + (array(1, NULL, 3), 3), + (array(1, 2), 5), + (NULL, 1), + (array(1, 2), NULL); + +query ? +SELECT array_remove(column1, column2) FROM t; +---- +[1, 3] +[1, NULL] +[1, 2] +NULL +NULL + +# A literal array with a column element, including a NULL element +query ? +SELECT array_remove(array(1, 2, 3), column2) FROM t; +---- +[1, 3] +[1, 2] +[1, 2, 3] +[2, 3] +NULL + +# A column array with a literal element +query ? +SELECT array_remove(column1, 1) FROM t; +---- +[2, 3, 2] +[NULL, 3] +[2] +NULL +[2] + +query ? +SELECT array_remove(column1, CAST(NULL AS INT)) FROM t; +---- +NULL +NULL +NULL +NULL +NULL + +# A batch sliced by OFFSET: the list offsets do not start at 0 +query ? +SELECT array_remove(column1, column2) FROM (SELECT * FROM t LIMIT 3 OFFSET 1); +---- +[1, NULL] +[1, 2] +NULL + +statement ok +DROP TABLE t; + +statement ok +CREATE TABLE floats AS VALUES + (array(CAST(1.0 AS DOUBLE), CAST('NaN' AS DOUBLE)), CAST('NaN' AS DOUBLE)), + (array(CAST('-0.0' AS DOUBLE), CAST(1.0 AS DOUBLE)), CAST(0.0 AS DOUBLE)), + (array(CAST(0.0 AS DOUBLE), CAST(1.0 AS DOUBLE)), CAST('-0.0' AS DOUBLE)), + (array(CAST(1.0 AS DOUBLE), CAST(NULL AS DOUBLE)), CAST(1.0 AS DOUBLE)), + (array(CAST(1.0 AS DOUBLE), CAST(2.0 AS DOUBLE)), CAST(NULL AS DOUBLE)), + (CAST(NULL AS ARRAY), CAST(1.0 AS DOUBLE)); + +query ? +SELECT array_remove(column1, column2) FROM floats; +---- +[1.0] +[1.0] +[1.0] +[NULL] +NULL +NULL + +query ? +SELECT array_remove(column1, CAST(0.0 AS DOUBLE)) FROM floats; +---- +[1.0, NaN] +[1.0] +[1.0] +[1.0, NULL] +[1.0, 2.0] +NULL + +statement ok +DROP TABLE floats; + +statement ok +CREATE TABLE strings AS VALUES + (array('a', 'b', 'a'), 'a'), + (array('a', NULL, 'b'), 'b'), + (array('', 'a'), ''), + (array('a', 'b'), CAST(NULL AS STRING)), + (CAST(NULL AS ARRAY), 'a'); + +query ? +SELECT array_remove(column1, column2) FROM strings; +---- +[b] +[a, NULL] +[a] +NULL +NULL + +statement ok +DROP TABLE strings; + +statement ok +CREATE TABLE nested AS VALUES + (array(array(CAST(1.0 AS DOUBLE)), array(CAST('NaN' AS DOUBLE), CAST('-0.0' AS DOUBLE))), array(CAST('NaN' AS DOUBLE), CAST(0.0 AS DOUBLE))), + (array(array(CAST(0.0 AS DOUBLE)), array(CAST(1.0 AS DOUBLE))), array(CAST('-0.0' AS DOUBLE))), + (array(array(CAST(1.0 AS DOUBLE), CAST(NULL AS DOUBLE))), array(CAST(1.0 AS DOUBLE), CAST(NULL AS DOUBLE))), + (array(array(CAST(1.0 AS DOUBLE))), array(CAST(2.0 AS DOUBLE))); + +query ? +SELECT array_remove(column1, column2) FROM nested; +---- +[[1.0]] +[[1.0]] +[] +[[1.0]] + +statement ok +DROP TABLE nested; + +statement ok +CREATE TABLE structs AS VALUES + (array(named_struct('a', CAST(1.0 AS DOUBLE), 'b', 1), named_struct('a', CAST('-0.0' AS DOUBLE), 'b', 2)), named_struct('a', CAST(0.0 AS DOUBLE), 'b', 2)), + (array(named_struct('a', CAST('NaN' AS DOUBLE), 'b', 1)), named_struct('a', CAST('NaN' AS DOUBLE), 'b', 1)), + (array(named_struct('a', CAST(0.0 AS DOUBLE), 'b', 1)), named_struct('a', CAST(0.0 AS DOUBLE), 'b', 2)); + +query ? +SELECT array_remove(column1, column2) FROM structs; +---- +[{a: 1.0, b: 1}] +[] +[{a: 0.0, b: 1}] + +statement ok +DROP TABLE structs; + +### +### LargeList +### + +query ?T +SELECT array_remove(arrow_cast(array(1, 2, 3, 2), 'LargeList(Int64)'), 2), arrow_typeof(array_remove(arrow_cast(array(1, 2, 3, 2), 'LargeList(Int64)'), 2)); +---- +[1, 3] LargeList(Int64) + +query ? +SELECT array_remove(arrow_cast(array(1, NULL, 3), 'LargeList(Int64)'), 3); +---- +[1, NULL] + +query ? +SELECT array_remove(arrow_cast(array(1, 2, 3), 'LargeList(Int64)'), CAST(NULL AS BIGINT)); +---- +NULL + +query ? +SELECT array_remove(arrow_cast(array(CAST('-0.0' AS DOUBLE), CAST('NaN' AS DOUBLE), CAST(1.0 AS DOUBLE)), 'LargeList(Float64)'), CAST(0.0 AS DOUBLE)); +---- +[NaN, 1.0] + +query ? +SELECT array_remove(arrow_cast(array(CAST('-0.0' AS DOUBLE), CAST('NaN' AS DOUBLE), CAST(1.0 AS DOUBLE)), 'LargeList(Float64)'), CAST('NaN' AS DOUBLE)); +---- +[-0.0, 1.0]