diff --git a/crates/spark/src/function/array/array_contains.rs b/crates/spark/src/function/array/array_contains.rs index 0d948fb6..3802667e 100644 --- a/crates/spark/src/function/array/array_contains.rs +++ b/crates/spark/src/function/array/array_contains.rs @@ -2,13 +2,13 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use super::float_normalize::normalize_floats; use arrow::array::{ - Array, ArrayRef, AsArray, BooleanArray, BooleanBufferBuilder, FixedSizeListArray, - GenericListArray, OffsetSizeTrait, StructArray, + Array, AsArray, BooleanArray, BooleanBufferBuilder, GenericListArray, OffsetSizeTrait, }; use arrow::buffer::{BooleanBuffer, NullBuffer}; -use arrow::datatypes::{DataType, FieldRef, Float32Type, Float64Type}; -use datafusion_common::{Result, ScalarValue, exec_err}; +use arrow::datatypes::DataType; +use datafusion_common::{Result, exec_err}; use datafusion_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; @@ -72,110 +72,6 @@ impl ScalarUDFImpl for SparkArrayContains { } } -/// Rewrites `-0.0` to `0.0` and every NaN to the canonical NaN, so that `array_has` -/// compares floating point values the way Spark does. -fn normalize_floats(value: &ColumnarValue) -> Result { - if !contains_float(&value.data_type()) { - return Ok(value.clone()); - } - match value { - ColumnarValue::Array(array) => Ok(ColumnarValue::Array(normalize_array(array)?)), - ColumnarValue::Scalar(scalar) => { - let array = normalize_array(&scalar.to_array()?)?; - Ok(ColumnarValue::Scalar(ScalarValue::try_from_array( - &array, 0, - )?)) - } - } -} - -/// Returns whether values of this type are, or hold at any depth, floating point numbers. -fn contains_float(data_type: &DataType) -> bool { - match data_type { - DataType::Float32 | DataType::Float64 => true, - DataType::List(field) - | DataType::LargeList(field) - | DataType::FixedSizeList(field, _) => contains_float(field.data_type()), - DataType::Struct(fields) => { - fields.iter().any(|field| contains_float(field.data_type())) - } - _ => false, - } -} - -fn normalize_array(array: &ArrayRef) -> Result { - if !contains_float(array.data_type()) { - return Ok(Arc::clone(array)); - } - Ok(match array.data_type() { - DataType::Float32 => Arc::new( - array - .as_primitive::() - .unary::<_, Float32Type>(|v| { - if v.is_nan() { - f32::NAN - } else if v == 0.0 { - 0.0 - } else { - v - } - }), - ), - DataType::Float64 => Arc::new( - array - .as_primitive::() - .unary::<_, Float64Type>(|v| { - if v.is_nan() { - f64::NAN - } else if v == 0.0 { - 0.0 - } else { - v - } - }), - ), - DataType::List(field) => normalize_list(array.as_list::(), field)?, - DataType::LargeList(field) => normalize_list(array.as_list::(), field)?, - DataType::FixedSizeList(field, size) => { - let list = array.as_fixed_size_list(); - // The length is passed explicitly: a zero-width list has no values to infer it from. - Arc::new(FixedSizeListArray::try_new_with_length( - Arc::clone(field), - *size, - normalize_array(list.values())?, - list.nulls().cloned(), - list.len(), - )?) - } - DataType::Struct(fields) => { - let columns = array - .as_struct() - .columns() - .iter() - .map(normalize_array) - .collect::>>()?; - Arc::new(StructArray::try_new( - fields.clone(), - columns, - array.nulls().cloned(), - )?) - } - _ => Arc::clone(array), - }) -} - -fn normalize_list( - list: &GenericListArray, - field: &FieldRef, -) -> Result { - Ok(Arc::new(GenericListArray::::try_new( - Arc::clone(field), - list.offsets().clone(), - normalize_array(list.values())?, - list.nulls().cloned(), - )?)) -} - /// For each row where `array_has` returned `false`, set the output to null /// if that row's input array contains any null elements. fn apply_spark_null_semantics( @@ -264,131 +160,22 @@ fn mask_with_list_nulls( #[cfg(test)] mod tests { use super::*; - use arrow::array::{ - Float32Array, Float64Array, Int32Array, Int32Builder, ListArray, ListBuilder, - }; - use arrow::buffer::OffsetBuffer; - use arrow::datatypes::{Field, Fields, Int32Type}; - - fn negative_nan() -> f64 { - f64::from_bits(f64::NAN.to_bits() | (1 << 63)) - } - - #[test] - fn normalize_array_canonicalizes_floats() { - let array: ArrayRef = Arc::new(Float64Array::from(vec![ - Some(-0.0), - Some(negative_nan()), - Some(1.5), - None, - ])); - let normalized = normalize_array(&array).unwrap(); - let normalized = normalized.as_primitive::(); - assert_eq!(normalized.value(0).to_bits(), 0.0_f64.to_bits()); - assert_eq!(normalized.value(1).to_bits(), f64::NAN.to_bits()); - assert_eq!(normalized.value(2), 1.5); - assert!(normalized.is_null(3)); - - let array: ArrayRef = Arc::new(Float32Array::from(vec![-0.0_f32, f32::NAN])); - let normalized = normalize_array(&array).unwrap(); - let normalized = normalized.as_primitive::(); - assert_eq!(normalized.value(0).to_bits(), 0.0_f32.to_bits()); - assert_eq!(normalized.value(1).to_bits(), f32::NAN.to_bits()); - } - - #[test] - fn normalize_array_keeps_the_shape_of_nested_values() { - let fields = Fields::from(vec![ - Field::new("a", DataType::Float64, true), - Field::new("b", DataType::Int32, true), - ]); - let item = Arc::new(Field::new("item", DataType::Struct(fields.clone()), true)); - let structs = StructArray::try_new( - fields, - vec![ - Arc::new(Float64Array::from(vec![-0.0, 2.0, 3.0])) as ArrayRef, - Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef, - ], - Some(NullBuffer::from(vec![true, false, true])), - ) - .unwrap(); - let list = ListArray::try_new( - item, - OffsetBuffer::new(vec![0, 2, 3].into()), - Arc::new(structs), - Some(NullBuffer::from(vec![true, false])), - ) - .unwrap(); - let array: ArrayRef = Arc::new(list); - assert!(contains_float(array.data_type())); - - let normalized = normalize_array(&array).unwrap(); - assert_eq!(normalized.data_type(), array.data_type()); - assert_eq!(normalized.nulls(), array.nulls()); - let original = array.as_list::(); - let list = normalized.as_list::(); - assert_eq!(list.offsets(), original.offsets()); - let structs = list.values().as_struct(); - assert_eq!(structs.nulls(), original.values().nulls()); - let a = structs.column(0).as_primitive::(); - assert_eq!(a.value(0).to_bits(), 0.0_f64.to_bits()); - assert_eq!(a.value(1), 2.0); - assert_eq!(structs.column(1).as_primitive::().value(2), 3); - } + use arrow::array::{FixedSizeListArray, Float64Array}; + use arrow::datatypes::Field; + use datafusion_common::ScalarValue; #[test] - fn normalize_floats_leaves_other_types_untouched() { - let mut builder = ListBuilder::new(Int32Builder::new()); - builder.values().append_value(1); - builder.append(true); - let array: ArrayRef = Arc::new(builder.finish()); - assert!(!contains_float(array.data_type())); - let value = ColumnarValue::Array(Arc::clone(&array)); - match normalize_floats(&value).unwrap() { - ColumnarValue::Array(result) => assert!(Arc::ptr_eq(&result, &array)), - ColumnarValue::Scalar(_) => panic!("expected an array"), - } - } - - #[test] - fn normalize_array_reuses_children_without_floats() { - let mut builder = ListBuilder::new(Int32Builder::new()); - builder.values().append_value(1); - builder.append(true); - let ints: ArrayRef = Arc::new(builder.finish()); - let structs = StructArray::try_new( - Fields::from(vec![ - Field::new("a", DataType::Float64, true), - Field::new("b", ints.data_type().clone(), true), - ]), - vec![ - Arc::new(Float64Array::from(vec![-0.0])) as ArrayRef, - Arc::clone(&ints), - ], - None, - ) - .unwrap(); - - let normalized = normalize_array(&(Arc::new(structs) as ArrayRef)).unwrap(); - assert!(Arc::ptr_eq(normalized.as_struct().column(1), &ints)); - } - - /// One valid row of a zero-width list, without a null buffer to infer the length from. - fn zero_width_list(field: &FieldRef) -> FixedSizeListArray { - FixedSizeListArray::try_new_with_length( - Arc::clone(field), + fn zero_width_fixed_size_list_keeps_its_row() { + // One valid row of a zero-width list, without a null buffer to infer the length from. + let field = Arc::new(Field::new("item", DataType::Float64, true)); + let haystack = FixedSizeListArray::try_new_with_length( + Arc::clone(&field), 0, Arc::new(Float64Array::from(Vec::::new())), None, 1, ) - .unwrap() - } - - #[test] - fn zero_width_fixed_size_list_keeps_its_row() { - let field = Arc::new(Field::new("item", DataType::Float64, true)); - let haystack = zero_width_list(&field); + .unwrap(); assert_eq!(haystack.len(), 1); let result = SparkArrayContains::new() @@ -407,26 +194,4 @@ mod tests { .unwrap(); assert_eq!(result.as_boolean(), &BooleanArray::from(vec![Some(false)])); } - - #[test] - fn zero_width_fixed_size_list_nested_in_a_list_is_not_rebuilt() { - let item = Arc::new(Field::new("item", DataType::Float64, true)); - let inner = Arc::new(Field::new( - "inner", - DataType::FixedSizeList(Arc::clone(&item), 0), - true, - )); - let list = ListArray::try_new( - inner, - OffsetBuffer::new(vec![0, 1].into()), - Arc::new(zero_width_list(&item)), - None, - ) - .unwrap(); - let array: ArrayRef = Arc::new(list); - - let normalized = normalize_array(&array).unwrap(); - assert_eq!(normalized.len(), 1); - assert_eq!(normalized.as_list::().values().len(), 1); - } } diff --git a/crates/spark/src/function/array/array_position.rs b/crates/spark/src/function/array/array_position.rs new file mode 100644 index 00000000..4ca52da9 --- /dev/null +++ b/crates/spark/src/function/array/array_position.rs @@ -0,0 +1,414 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use super::float_normalize::normalize_array; +use arrow::array::{ + Array, ArrayRef, AsArray, Int64Array, OffsetSizeTrait, make_comparator, +}; +use arrow::buffer::{NullBuffer, ScalarBuffer}; +use arrow::compute::SortOptions; +use arrow::datatypes::{DataType, Field, Fields, TimeUnit}; +use datafusion_common::utils::take_function_args; +use datafusion_common::{Result, ScalarValue, exec_err, plan_err}; +use datafusion_expr::{ + ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, + Volatility, +}; +use datafusion_macros::user_doc; +use std::sync::Arc; + +/// Spark-compatible `array_position` expression. +/// +/// +#[user_doc( + doc_section(label = "Array Functions"), + description = "Returns the 1-based position of the first occurrence of the element in the \ + array, as a `BIGINT`. The DataFusion built-in returns a `UInt64`, and returns NULL when \ + the element is not found, where this returns 0. The result is NULL only when the array \ + or the element is NULL: a NULL element returns NULL even if the array contains NULL, \ + while in the DataFusion built-in a NULL element matches the first NULL in the array. \ + Top-level NULL elements in the array never match, while corresponding NULLs nested \ + inside compared arrays or structs are equal. For floating point values, also inside \ + nested arrays and structs, NaN is equal to NaN and -0.0 is equal to 0.0. As in Spark, \ + the array element and the element are cast to their tightest common type: numbers \ + widen, an integer widens to a decimal only if the decimal can hold all its values, and \ + a date widens to a timestamp. Other combinations are an error, for example a string \ + and a number, two different decimals, or structs whose field names or field types \ + differ, as in Spark with ANSI mode off (with ANSI mode on, Spark widens struct field \ + types). A NULL-typed array and map elements are also an error.", + syntax_example = "array_position(array, element)", + sql_example = r#"```sql +> SELECT array_position(array(3, 2, 1), 1); +3 +> SELECT array_position(array(3, 2, 1), 4); +0 +```"#, + argument(name = "array", description = "Array expression. NULL returns NULL."), + argument( + name = "element", + description = "Element to search for. NULL returns NULL." + ) +)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkArrayPosition { + signature: Signature, +} + +impl Default for SparkArrayPosition { + fn default() -> Self { + Self::new() + } +} + +impl SparkArrayPosition { + pub fn new() -> Self { + Self { + signature: Signature::user_defined(Volatility::Immutable), + } + } +} + +impl ScalarUDFImpl for SparkArrayPosition { + fn name(&self) -> &str { + "array_position" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Int64) + } + + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { + let [array, element] = take_function_args(self.name(), arg_types)?; + coerce_array_and_element(self.name(), array, element) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + let is_scalar = args + .args + .iter() + .all(|a| matches!(a, ColumnarValue::Scalar(_))); + let arrays = ColumnarValue::values_to_arrays(&args.args)?; + let [array, element] = take_function_args(self.name(), &arrays)?; + + let result = match array.data_type() { + DataType::List(_) => position::(array, element)?, + DataType::LargeList(_) => position::(array, element)?, + other => return exec_err!("array_position does not support type {other}"), + }; + + if is_scalar { + Ok(ColumnarValue::Scalar(ScalarValue::try_from_array( + &result, 0, + )?)) + } else { + Ok(ColumnarValue::Array(result)) + } + } + + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } +} + +fn position( + array: &ArrayRef, + element: &ArrayRef, +) -> Result { + let list = array.as_list::(); + let offsets = list.offsets(); + // Only the values that this (possibly sliced) list refers to are normalized. + let base = offsets[0].as_usize(); + let len = offsets[offsets.len() - 1].as_usize() - base; + // Canonicalize NaN and -0.0 at any depth so that the comparator follows Spark's float + // equality. The comparator treats NULLs nested inside the compared values as equal. + let values = normalize_array(&list.values().slice(base, len))?; + let element = normalize_array(element)?; + let nulls = NullBuffer::union(list.nulls(), element.nulls()); + let cmp = make_comparator(values.as_ref(), element.as_ref(), SortOptions::default())?; + + let result: Vec = offsets + .windows(2) + .enumerate() + .map(|(row, w)| { + if nulls.as_ref().is_some_and(|n| n.is_null(row)) { + return 0; + } + let (start, end) = (w[0].as_usize() - base, w[1].as_usize() - base); + (start..end) + .find(|&i| values.is_valid(i) && cmp(i, row).is_eq()) + .map_or(0, |i| (i - start + 1) as i64) + }) + .collect(); + + Ok(Arc::new(Int64Array::new(ScalarBuffer::from(result), nulls))) +} + +/// Casts the array element and the element to their tightest common type, and rejects the +/// argument types that Spark's `ArrayPosition.checkInputDataTypes` rejects. +fn coerce_array_and_element( + name: &str, + array: &DataType, + element: &DataType, +) -> Result> { + if array.is_null() { + return null_type_err(name); + } + let (item, large) = match array { + DataType::List(field) | DataType::FixedSizeList(field, _) => (field, false), + DataType::LargeList(field) => (field, true), + other => { + return plan_err!( + "[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] The first parameter of `{name}` \ + requires the \"ARRAY\" type, however it has the type {other}" + ); + } + }; + let Some(common) = tightest_common_type(item.data_type(), element) else { + return plan_err!( + "[DATATYPE_MISMATCH.ARRAY_FUNCTION_DIFF_TYPES] Input to `{name}` should have \ + been \"ARRAY\" followed by a value with same element type, but it's \ + [{array}, {element}]" + ); + }; + if common.is_null() { + return null_type_err(name); + } + if contains_map(&common) { + return plan_err!( + "[DATATYPE_MISMATCH.INVALID_ORDERING_TYPE] The `{name}` does not support \ + ordering on type {common}" + ); + } + let item = Arc::new(Field::new(item.name(), common.clone(), true)); + let array = if large { + DataType::LargeList(item) + } else { + DataType::List(item) + }; + Ok(vec![array, common]) +} + +fn null_type_err(name: &str) -> Result { + plan_err!( + "[DATATYPE_MISMATCH.NULL_TYPE] Null typed values cannot be used as arguments of \ + `{name}`" + ) +} + +/// Spark's `TypeCoercion.findTightestCommonType`, on the arrow types that Spark's types +/// map to. +fn tightest_common_type(left: &DataType, right: &DataType) -> Option { + use DataType::*; + if left == right { + return Some(left.clone()); + } + if let (Some(l), Some(r)) = (numeric_precedence(left), numeric_precedence(right)) { + return Some(if l >= r { left.clone() } else { right.clone() }); + } + match (left, right) { + (Null, other) | (other, Null) => Some(other.clone()), + // Spark has a single string type and a single binary type. + (l, r) if is_string(l) && is_string(r) => Some(l.clone()), + (l, r) if is_binary(l) && is_binary(r) => Some(l.clone()), + // The decimal must be at least as wide as the integral type's DECIMAL(n, 0). + (int, Decimal128(p, s)) | (Decimal128(p, s), int) => { + let digits = integral_digits(int)?; + (*p as i16 - *s as i16 >= digits).then_some(Decimal128(*p, *s)) + } + (Date32, ts @ Timestamp(_, _)) | (ts @ Timestamp(_, _), Date32) => { + Some(ts.clone()) + } + // TIMESTAMP_NTZ and TIMESTAMP widen to TIMESTAMP. + (Timestamp(lu, ltz), Timestamp(ru, rtz)) => Some(Timestamp( + finer_unit(*lu, *ru), + ltz.clone().or_else(|| rtz.clone()), + )), + ( + List(lf) | LargeList(lf) | FixedSizeList(lf, _), + List(rf) | LargeList(rf) | FixedSizeList(rf, _), + ) => { + let item = tightest_common_type(lf.data_type(), rf.data_type())?; + let field = Arc::new(Field::new(lf.name(), item, true)); + Some( + if matches!(left, LargeList(_)) || matches!(right, LargeList(_)) { + LargeList(field) + } else { + List(field) + }, + ) + } + // The fields must have the same names and types, ignoring nullability, in the same + // order. This follows Spark with ANSI mode off; with ANSI mode on, Spark also widens + // the field types. Spark compares the names ignoring case, but DataFusion cannot cast + // between structs whose field names differ only in case, so they must match exactly. + (Struct(lf), Struct(rf)) if lf.len() == rf.len() => { + let fields = lf + .iter() + .zip(rf.iter()) + .map(|(a, b)| { + if a.name() != b.name() || !same_type(a.data_type(), b.data_type()) { + return None; + } + let data_type = tightest_common_type(a.data_type(), b.data_type())?; + Some(Field::new( + a.name(), + data_type, + a.is_nullable() || b.is_nullable(), + )) + }) + .collect::>>()?; + Some(Struct(Fields::from(fields))) + } + (Map(lf, sorted), Map(rf, _)) => { + let entries = tightest_common_type(lf.data_type(), rf.data_type())?; + Some(Map( + Arc::new(Field::new(lf.name(), entries, false)), + *sorted, + )) + } + _ => None, + } +} + +/// Whether two types are equal when the nullability of nested fields is ignored, like +/// Spark's `DataTypeUtils.sameType`. +fn same_type(left: &DataType, right: &DataType) -> bool { + use DataType::*; + match (left, right) { + (List(l), List(r)) | (LargeList(l), LargeList(r)) | (Map(l, _), Map(r, _)) => { + same_type(l.data_type(), r.data_type()) + } + (FixedSizeList(l, ln), FixedSizeList(r, rn)) => { + ln == rn && same_type(l.data_type(), r.data_type()) + } + (Struct(l), Struct(r)) => { + l.len() == r.len() + && l.iter().zip(r.iter()).all(|(a, b)| { + a.name() == b.name() && same_type(a.data_type(), b.data_type()) + }) + } + _ => left == right, + } +} + +fn is_string(data_type: &DataType) -> bool { + matches!( + data_type, + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View + ) +} + +fn is_binary(data_type: &DataType) -> bool { + matches!( + data_type, + DataType::Binary | DataType::LargeBinary | DataType::BinaryView + ) +} + +/// The precision of Spark's `DecimalType.forType` for an integral type. +fn integral_digits(data_type: &DataType) -> Option { + match data_type { + DataType::Int8 => Some(3), + DataType::Int16 => Some(5), + DataType::Int32 => Some(10), + DataType::Int64 => Some(20), + _ => None, + } +} + +/// Spark's `numericPrecedence`, which leaves out decimals. +fn numeric_precedence(data_type: &DataType) -> Option { + match data_type { + DataType::Int8 => Some(0), + DataType::Int16 => Some(1), + DataType::Int32 => Some(2), + DataType::Int64 => Some(3), + DataType::Float32 => Some(4), + DataType::Float64 => Some(5), + _ => None, + } +} + +fn finer_unit(left: TimeUnit, right: TimeUnit) -> TimeUnit { + let rank = |unit: TimeUnit| match unit { + TimeUnit::Second => 0, + TimeUnit::Millisecond => 1, + TimeUnit::Microsecond => 2, + TimeUnit::Nanosecond => 3, + }; + if rank(left) >= rank(right) { + left + } else { + right + } +} + +/// Spark cannot order maps, so it rejects them at any depth of the element type. +fn contains_map(data_type: &DataType) -> bool { + match data_type { + DataType::Map(_, _) => true, + DataType::List(field) + | DataType::LargeList(field) + | DataType::FixedSizeList(field, _) => contains_map(field.data_type()), + DataType::Struct(fields) => { + fields.iter().any(|field| contains_map(field.data_type())) + } + _ => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{FixedSizeListArray, Float64Array, Int32Array, ListArray}; + use arrow::buffer::OffsetBuffer; + use arrow::datatypes::Int32Type; + + #[test] + fn sliced_list() { + let list = ListArray::from_iter_primitive::(vec![ + Some(vec![Some(9), Some(9)]), + Some(vec![Some(1), Some(2), Some(3)]), + Some(vec![Some(3), Some(4)]), + ]); + let array: ArrayRef = Arc::new(list.slice(1, 2)); + let element: ArrayRef = Arc::new(Int32Array::from(vec![3, 4])); + let result = position::(&array, &element).unwrap(); + assert_eq!(result.as_ref(), &Int64Array::from(vec![3, 2])); + } + + #[test] + fn fixed_size_list_of_floats() { + let item = Arc::new(Field::new("item", DataType::Float64, true)); + let fsl = |values: Vec| -> ArrayRef { + Arc::new( + FixedSizeListArray::try_new( + Arc::clone(&item), + 1, + Arc::new(Float64Array::from(values)), + None, + ) + .unwrap(), + ) + }; + let values = fsl(vec![1.0, -0.0, f64::NAN]); + let field = Arc::new(Field::new("item", values.data_type().clone(), true)); + let array: ArrayRef = Arc::new( + ListArray::try_new( + field, + OffsetBuffer::new(vec![0, 3, 6].into()), + Arc::new(arrow::compute::concat(&[&values, &values]).unwrap()), + None, + ) + .unwrap(), + ); + let element = fsl(vec![0.0, -f64::NAN]); + let result = position::(&array, &element).unwrap(); + assert_eq!(result.as_ref(), &Int64Array::from(vec![2, 3])); + } +} diff --git a/crates/spark/src/function/array/float_normalize.rs b/crates/spark/src/function/array/float_normalize.rs new file mode 100644 index 00000000..07bbb28a --- /dev/null +++ b/crates/spark/src/function/array/float_normalize.rs @@ -0,0 +1,307 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Float normalization shared by the array functions that compare elements. +//! +//! Spark compares floating point values with `-0.0 == 0.0` and `NaN == NaN`, while arrow +//! compares them by their bits or by the IEEE total order. Rewriting `-0.0` to `0.0` and +//! every NaN to the canonical NaN, at any depth of nested types, makes those comparisons +//! follow Spark's float equality. + +use arrow::array::{ + Array, ArrayRef, AsArray, FixedSizeListArray, GenericListArray, MapArray, + OffsetSizeTrait, StructArray, +}; +use arrow::datatypes::{DataType, FieldRef, Float32Type, Float64Type}; +use datafusion_common::{Result, ScalarValue}; +use datafusion_expr::ColumnarValue; +use std::sync::Arc; + +/// Normalizes the floats in a [`ColumnarValue`], see [`normalize_array`]. +pub(super) fn normalize_floats(value: &ColumnarValue) -> Result { + if !contains_float(&value.data_type()) { + return Ok(value.clone()); + } + match value { + ColumnarValue::Array(array) => Ok(ColumnarValue::Array(normalize_array(array)?)), + ColumnarValue::Scalar(scalar) => { + let array = normalize_array(&scalar.to_array()?)?; + Ok(ColumnarValue::Scalar(ScalarValue::try_from_array( + &array, 0, + )?)) + } + } +} + +/// Returns whether values of this type are, or hold at any depth, floating point numbers. +pub(super) fn contains_float(data_type: &DataType) -> bool { + match data_type { + DataType::Float32 | DataType::Float64 => true, + DataType::List(field) + | DataType::LargeList(field) + | DataType::FixedSizeList(field, _) + | DataType::Map(field, _) => contains_float(field.data_type()), + DataType::Struct(fields) => { + fields.iter().any(|field| contains_float(field.data_type())) + } + _ => false, + } +} + +/// Rewrites `-0.0` to `0.0` and every NaN to the canonical NaN at any depth of list, large +/// list, fixed-size list, struct and map. Children without floats are reused as they are. +pub(super) fn normalize_array(array: &ArrayRef) -> Result { + if !contains_float(array.data_type()) { + return Ok(Arc::clone(array)); + } + Ok(match array.data_type() { + DataType::Float32 => Arc::new( + array + .as_primitive::() + .unary::<_, Float32Type>(|v| { + if v.is_nan() { + f32::NAN + } else if v == 0.0 { + 0.0 + } else { + v + } + }), + ), + DataType::Float64 => Arc::new( + array + .as_primitive::() + .unary::<_, Float64Type>(|v| { + if v.is_nan() { + f64::NAN + } else if v == 0.0 { + 0.0 + } else { + v + } + }), + ), + DataType::List(field) => normalize_list(array.as_list::(), field)?, + DataType::LargeList(field) => normalize_list(array.as_list::(), field)?, + DataType::FixedSizeList(field, size) => { + let list = array.as_fixed_size_list(); + // The length is passed explicitly: a zero-width list has no values to infer it from. + Arc::new(FixedSizeListArray::try_new_with_length( + Arc::clone(field), + *size, + normalize_array(list.values())?, + list.nulls().cloned(), + list.len(), + )?) + } + DataType::Struct(fields) => { + let columns = array + .as_struct() + .columns() + .iter() + .map(normalize_array) + .collect::>>()?; + Arc::new(StructArray::try_new( + fields.clone(), + columns, + array.nulls().cloned(), + )?) + } + DataType::Map(field, ordered) => { + let map = array.as_map(); + let entries: ArrayRef = Arc::new(map.entries().clone()); + let entries = normalize_array(&entries)?; + Arc::new(MapArray::try_new( + Arc::clone(field), + map.offsets().clone(), + entries.as_struct().clone(), + map.nulls().cloned(), + *ordered, + )?) + } + _ => Arc::clone(array), + }) +} + +fn normalize_list( + list: &GenericListArray, + field: &FieldRef, +) -> Result { + Ok(Arc::new(GenericListArray::::try_new( + Arc::clone(field), + list.offsets().clone(), + normalize_array(list.values())?, + list.nulls().cloned(), + )?)) +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{ + Float32Array, Float64Array, Int32Array, Int32Builder, ListArray, ListBuilder, + MapBuilder, + }; + use arrow::buffer::{NullBuffer, OffsetBuffer}; + use arrow::datatypes::{Field, Fields, Int32Type}; + + fn negative_nan() -> f64 { + f64::from_bits(f64::NAN.to_bits() | (1 << 63)) + } + + #[test] + fn normalize_array_canonicalizes_floats() { + let array: ArrayRef = Arc::new(Float64Array::from(vec![ + Some(-0.0), + Some(negative_nan()), + Some(1.5), + None, + ])); + let normalized = normalize_array(&array).unwrap(); + let normalized = normalized.as_primitive::(); + assert_eq!(normalized.value(0).to_bits(), 0.0_f64.to_bits()); + assert_eq!(normalized.value(1).to_bits(), f64::NAN.to_bits()); + assert_eq!(normalized.value(2), 1.5); + assert!(normalized.is_null(3)); + + let array: ArrayRef = Arc::new(Float32Array::from(vec![-0.0_f32, f32::NAN])); + let normalized = normalize_array(&array).unwrap(); + let normalized = normalized.as_primitive::(); + assert_eq!(normalized.value(0).to_bits(), 0.0_f32.to_bits()); + assert_eq!(normalized.value(1).to_bits(), f32::NAN.to_bits()); + } + + #[test] + fn normalize_array_keeps_the_shape_of_nested_values() { + let fields = Fields::from(vec![ + Field::new("a", DataType::Float64, true), + Field::new("b", DataType::Int32, true), + ]); + let item = Arc::new(Field::new("item", DataType::Struct(fields.clone()), true)); + let structs = StructArray::try_new( + fields, + vec![ + Arc::new(Float64Array::from(vec![-0.0, 2.0, 3.0])) as ArrayRef, + Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef, + ], + Some(NullBuffer::from(vec![true, false, true])), + ) + .unwrap(); + let list = ListArray::try_new( + item, + OffsetBuffer::new(vec![0, 2, 3].into()), + Arc::new(structs), + Some(NullBuffer::from(vec![true, false])), + ) + .unwrap(); + let array: ArrayRef = Arc::new(list); + assert!(contains_float(array.data_type())); + + let normalized = normalize_array(&array).unwrap(); + assert_eq!(normalized.data_type(), array.data_type()); + assert_eq!(normalized.nulls(), array.nulls()); + let original = array.as_list::(); + let list = normalized.as_list::(); + assert_eq!(list.offsets(), original.offsets()); + let structs = list.values().as_struct(); + assert_eq!(structs.nulls(), original.values().nulls()); + let a = structs.column(0).as_primitive::(); + assert_eq!(a.value(0).to_bits(), 0.0_f64.to_bits()); + assert_eq!(a.value(1), 2.0); + assert_eq!(structs.column(1).as_primitive::().value(2), 3); + } + + #[test] + fn normalize_array_descends_into_maps() { + let mut builder = MapBuilder::new( + None, + Int32Builder::new(), + arrow::array::Float64Builder::new(), + ); + builder.keys().append_value(1); + builder.values().append_value(-0.0); + builder.keys().append_value(2); + builder.values().append_value(negative_nan()); + builder.append(true).unwrap(); + builder.append(false).unwrap(); + let array: ArrayRef = Arc::new(builder.finish()); + assert!(contains_float(array.data_type())); + + let normalized = normalize_array(&array).unwrap(); + assert_eq!(normalized.data_type(), array.data_type()); + assert_eq!(normalized.nulls(), array.nulls()); + let map = normalized.as_map(); + let values = map.values().as_primitive::(); + assert_eq!(values.value(0).to_bits(), 0.0_f64.to_bits()); + assert_eq!(values.value(1).to_bits(), f64::NAN.to_bits()); + assert_eq!(map.keys().as_primitive::().value(1), 2); + } + + #[test] + fn normalize_floats_leaves_other_types_untouched() { + let mut builder = ListBuilder::new(Int32Builder::new()); + builder.values().append_value(1); + builder.append(true); + let array: ArrayRef = Arc::new(builder.finish()); + assert!(!contains_float(array.data_type())); + let value = ColumnarValue::Array(Arc::clone(&array)); + match normalize_floats(&value).unwrap() { + ColumnarValue::Array(result) => assert!(Arc::ptr_eq(&result, &array)), + ColumnarValue::Scalar(_) => panic!("expected an array"), + } + } + + #[test] + fn normalize_array_reuses_children_without_floats() { + let mut builder = ListBuilder::new(Int32Builder::new()); + builder.values().append_value(1); + builder.append(true); + let ints: ArrayRef = Arc::new(builder.finish()); + let structs = StructArray::try_new( + Fields::from(vec![ + Field::new("a", DataType::Float64, true), + Field::new("b", ints.data_type().clone(), true), + ]), + vec![ + Arc::new(Float64Array::from(vec![-0.0])) as ArrayRef, + Arc::clone(&ints), + ], + None, + ) + .unwrap(); + + let normalized = normalize_array(&(Arc::new(structs) as ArrayRef)).unwrap(); + assert!(Arc::ptr_eq(normalized.as_struct().column(1), &ints)); + } + + #[test] + fn zero_width_fixed_size_list_nested_in_a_list_is_not_rebuilt() { + let item = Arc::new(Field::new("item", DataType::Float64, true)); + let inner = Arc::new(Field::new( + "inner", + DataType::FixedSizeList(Arc::clone(&item), 0), + true, + )); + let zero_width = FixedSizeListArray::try_new_with_length( + Arc::clone(&item), + 0, + Arc::new(Float64Array::from(Vec::::new())), + None, + 1, + ) + .unwrap(); + let list = ListArray::try_new( + inner, + OffsetBuffer::new(vec![0, 1].into()), + Arc::new(zero_width), + None, + ) + .unwrap(); + let array: ArrayRef = Arc::new(list); + + let normalized = normalize_array(&array).unwrap(); + assert_eq!(normalized.len(), 1); + assert_eq!(normalized.as_list::().values().len(), 1); + } +} diff --git a/crates/spark/src/function/array/mod.rs b/crates/spark/src/function/array/mod.rs index c4a2c353..f9649d7e 100644 --- a/crates/spark/src/function/array/mod.rs +++ b/crates/spark/src/function/array/mod.rs @@ -3,6 +3,8 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements pub mod array_contains; +pub mod array_position; +mod float_normalize; pub mod repeat; pub mod shuffle; pub mod slice; @@ -13,6 +15,7 @@ use datafusion_functions::make_udf_function; use std::sync::Arc; make_udf_function!(array_contains::SparkArrayContains, spark_array_contains); +make_udf_function!(array_position::SparkArrayPosition, array_position); make_udf_function!(spark_array::SparkArray, array); make_udf_function!(shuffle::SparkShuffle, shuffle); make_udf_function!(repeat::SparkArrayRepeat, array_repeat); @@ -26,6 +29,11 @@ pub mod expr_fn { "Returns true if the array contains the element (Spark semantics).", array element )); + export_functions!(( + array_position, + "Returns the 1-based position of the element in the array, or 0 if not found (Spark semantics).", + array element + )); export_functions!((array, "Returns an array with the given elements.", args)); export_functions!(( shuffle, @@ -47,6 +55,7 @@ pub mod expr_fn { pub fn functions() -> Vec> { vec![ spark_array_contains(), + array_position(), array(), shuffle(), array_repeat(), diff --git a/crates/sqllogictest/test_files/array/array_position.slt b/crates/sqllogictest/test_files/array/array_position.slt new file mode 100644 index 00000000..cf9ef3e8 --- /dev/null +++ b/crates/sqllogictest/test_files/array/array_position.slt @@ -0,0 +1,584 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +# Tests for Spark-compatible array_position. +# Returns the 1-based position of the first match, 0 if not found, NULL if the array or element is NULL. +# Expected values verified with pyspark==4.2.0. + +query I +SELECT array_position(array(1, 2, 3, 4), 3); +---- +3 + +# Not found returns 0 (DataFusion's built-in returns NULL) +query I +SELECT array_position(array(1, 2, 3, 4), 5); +---- +0 + +query I +SELECT array_position(array('a', 'b', 'c'), 'b'); +---- +2 + +# NULL elements in the array do not match and still count towards the position +query I +SELECT array_position(array(1, 2, NULL, 3), 3); +---- +4 + +query I +SELECT array_position(array(1, 2, NULL, 3), 5); +---- +0 + +# First occurrence wins +query I +SELECT array_position(array(1, 2, 1, 3), 1); +---- +1 + +query I +SELECT array_position(array(1, 2, 3), CAST(NULL AS INT)); +---- +NULL + +query I +SELECT array_position(CAST(NULL AS ARRAY), 1); +---- +NULL + +# The tightest common type of the array element and an untyped NULL element is the array +# element type, so the NULL is cast to it and this returns NULL. +query I +SELECT array_position(array(1, 2, 3), NULL); +---- +NULL + +# A NULL element returns NULL even if the array contains NULL. The DataFusion built-in returns +# the position of the first NULL instead. +query I +SELECT array_position(array(1, NULL), CAST(NULL AS BIGINT)); +---- +NULL + +query I +SELECT array_position(CAST(NULL AS ARRAY), CAST(NULL AS INT)); +---- +NULL + +# An untyped NULL array is rejected at planning time, as in Spark. +query error \[DATATYPE_MISMATCH\.NULL_TYPE\] Null typed values cannot be used as arguments of `array_position` +SELECT array_position(NULL, 1); + +query error \[DATATYPE_MISMATCH\.NULL_TYPE\] +SELECT array_position(NULL, NULL); + +# The common type of an empty array and an untyped NULL is the NULL type. +query error \[DATATYPE_MISMATCH\.NULL_TYPE\] +SELECT array_position(array(), NULL); + +query error \[DATATYPE_MISMATCH\.NULL_TYPE\] +SELECT array_position(array(NULL), NULL); + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `array_position` requires the "ARRAY" type +SELECT array_position(1, 1); + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] +SELECT array_position('abc', 'a'); + +query T +SELECT arrow_typeof(array_position(array(1, 2, 3), 2)); +---- +Int64 + +# Several matches return the first position +query I +SELECT array_position(array(312, 773, 708, 708), 708); +---- +3 + +query I +SELECT array_position(array(312, 773, 708, 708), 414); +---- +0 + +query I +SELECT array_position(array(NULL, ''), ''); +---- +2 + +query I +SELECT array_position(array(NULL, ''), 'a'); +---- +0 + +query I +SELECT array_position(array(NULL), 1L); +---- +0 + +query I +SELECT array_position(CAST(NULL AS ARRAY), ''); +---- +NULL + +# Spark's `array(1, NULL)[0]` is its first element; arrays in this crate are 1-based, so the +# same element is `[1]` here. +query I +SELECT array_position(array(1, NULL), array(1, NULL)[1]); +---- +1 + +query I +SELECT array_position(array(), 1); +---- +0 + +# NaN equals NaN +query I +SELECT array_position(array(CAST('NaN' AS DOUBLE), CAST(1.0 AS DOUBLE)), CAST('NaN' AS DOUBLE)); +---- +1 + +query I +SELECT array_position(array(CAST('NaN' AS FLOAT), CAST(1.0 AS FLOAT)), CAST('NaN' AS FLOAT)); +---- +1 + +query I +SELECT array_position(array(true, false), false); +---- +2 + +query I +SELECT array_position(array(CAST(1.5 AS DECIMAL(3,1)), CAST(2.5 AS DECIMAL(3,1))), CAST(2.5 AS DECIMAL(3,1))); +---- +2 + +query I +SELECT array_position(array(DATE '2020-01-01', DATE '2021-01-01'), DATE '2021-01-01'); +---- +2 + +query I +SELECT array_position(array(X'01', X'02'), X'02'); +---- +2 + +# Nested arrays +query I +SELECT array_position(array(array(1, 2), array(3)), array(3)); +---- +2 + +query I +SELECT array_position(array(array(1, 2), array(3, 4)), array(1, 2)); +---- +1 + +query I +SELECT array_position(array(array(5, 6), array(2, 1)), array(1, 2)); +---- +0 + +# NULLs nested inside compared arrays and structs are equal +query I +SELECT array_position(array(array(1, 2), array(1, NULL)), array(1, NULL)); +---- +2 + +query I +SELECT array_position(array(named_struct('a', 1, 'b', CAST(NULL AS INT))), named_struct('a', 1, 'b', CAST(NULL AS INT))); +---- +1 + +### +### Type coercion +### + +# The array element and the element are both cast to their tightest common type +# (Spark's findTightestCommonType): here INT and BIGINT become BIGINT. +query I +SELECT array_position(array(CAST(1 AS INT), CAST(2 AS INT), CAST(3 AS INT)), 2L); +---- +2 + +query I +SELECT array_position(array(1, 2, 3), CAST(2.0 AS DOUBLE)); +---- +2 + +# INT and DOUBLE become DOUBLE, so 1.23 does not match 1 +query I +SELECT array_position(array(CAST(1 AS INT)), CAST(1.23 AS DOUBLE)); +---- +0 + +query I +SELECT array_position(array(CAST(1 AS INT)), CAST(1.0 AS DOUBLE)); +---- +1 + +query I +SELECT array_position(array(CAST(1 AS DOUBLE)), CAST(1 AS INT)); +---- +1 + +query I +SELECT array_position(array(CAST(1 AS FLOAT)), 1L); +---- +1 + +query I +SELECT array_position(array(array(CAST(1 AS INT))), array(CAST(1.0 AS DOUBLE))); +---- +1 + +# An integral type widens to a decimal only if the decimal holds all its values: +# INT is DECIMAL(10,0), so DECIMAL(38,2) and DECIMAL(12,2) are wide enough and DECIMAL(10,2) is not. +query I +SELECT array_position(array(CAST(1 AS INT)), CAST(1 AS DECIMAL(38,2))); +---- +1 + +query I +SELECT array_position(array(CAST(1 AS DECIMAL(12,2))), CAST(1 AS INT)); +---- +1 + +query I +SELECT array_position(array(CAST(1 AS DECIMAL(3,0))), CAST(1 AS TINYINT)); +---- +1 + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] Input to `array_position` should have been "ARRAY" followed by a value with same element type +SELECT array_position(array(CAST(1 AS INT)), CAST(1 AS DECIMAL(10,2))); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_position(array(CAST(1 AS DECIMAL(12,2))), CAST(1 AS BIGINT)); + +# Two different decimals have no tightest common type +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_position(array(CAST(1 AS DECIMAL(3,1))), CAST(1 AS DECIMAL(4,2))); + +# Decimals and floating point numbers have no tightest common type +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_position(array(CAST(1 AS DECIMAL(5,1))), CAST(1.0 AS DOUBLE)); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_position(array(CAST(1.0 AS DOUBLE)), CAST(1 AS DECIMAL(5,1))); + +# Strings and numbers are not cast to each other, in either direction +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_position(array(1), '1'); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_position(array('1'), 1); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_position(array(1), CAST(NULL AS STRING)); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_position(array(array(1)), array('1')); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_position(array(true), 1); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_position(array(X'01'), '1'); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_position(array(DATE '2020-01-01'), '2020-01-01'); + +# DATE and TIMESTAMP become TIMESTAMP +query I +SELECT array_position(array(DATE '2020-01-01'), TIMESTAMP '2020-01-01 00:00:00'); +---- +1 + +query I +SELECT array_position(array(TIMESTAMP '2020-01-01 00:00:00'), DATE '2020-01-01'); +---- +1 + +# Struct fields are matched by name, in order. Spark ignores the case of the names and returns 1 +# here, but DataFusion cannot cast between struct fields whose names differ only in case. +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_position(array(named_struct('a', 1)), named_struct('A', 1)); + +# Struct field types are not widened, as in Spark with ANSI mode off. With ANSI mode on, Spark +# widens them and returns 0 here. +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_position(array(named_struct('a', 1)), named_struct('a', CAST(1.5 AS DOUBLE))); + +# Fields may differ in nullability +query I +SELECT array_position(array(named_struct('a', 1, 'b', CAST(NULL AS BIGINT))), named_struct('a', 1, 'b', 2L)); +---- +0 + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_position(array(named_struct('a', 1, 'b', 'x')), named_struct('b', 'x', 'a', 1)); + +# Maps cannot be ordered, so map elements are rejected at any depth +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_position` does not support ordering on type +SELECT array_position(array(map(1, 2)), map(1, 2)); + +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] +SELECT array_position(array(named_struct('m', map(1, 2))), named_struct('m', map(1, 2))); + +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] +SELECT array_position(array(array(map(1, 2))), array(map(1, 2))); + +# Signed zero: -0.0 equals 0.0 +query I +SELECT array_position(array(CAST('-0.0' AS DOUBLE), CAST(1.0 AS DOUBLE)), CAST(0.0 AS DOUBLE)); +---- +1 + +query I +SELECT array_position(array(CAST(0.0 AS DOUBLE), CAST(1.0 AS DOUBLE)), CAST('-0.0' AS DOUBLE)); +---- +1 + +query I +SELECT array_position(array(CAST(1.0 AS DOUBLE), CAST('-0.0' AS DOUBLE)), CAST(0.0 AS DOUBLE)); +---- +2 + +query I +SELECT array_position(array(CAST('-0.0' AS FLOAT), CAST(1.0 AS FLOAT)), CAST(0.0 AS FLOAT)); +---- +1 + +# Nested floats follow Spark's ordering: NaN equals NaN, -0.0 equals 0.0 +query I +SELECT array_position(array(array(CAST('NaN' AS DOUBLE)), array(CAST(1.0 AS DOUBLE))), array(CAST('NaN' AS DOUBLE))); +---- +1 + +query I +SELECT array_position(array(array(CAST('-0.0' AS DOUBLE)), array(CAST(1.0 AS DOUBLE))), array(CAST(0.0 AS DOUBLE))); +---- +1 + +query I +SELECT array_position(array(array(CAST('-0.0' AS FLOAT)), array(CAST(1.0 AS FLOAT))), array(CAST(0.0 AS FLOAT))); +---- +1 + +query I +SELECT array_position(array(named_struct('a', CAST('NaN' AS DOUBLE)), named_struct('a', CAST(1.0 AS DOUBLE))), named_struct('a', CAST('NaN' AS DOUBLE))); +---- +1 + +query I +SELECT array_position(array(named_struct('a', CAST('-0.0' AS DOUBLE)), named_struct('a', CAST(1.0 AS DOUBLE))), named_struct('a', CAST(0.0 AS DOUBLE))); +---- +1 + +# NULLs nested inside the compared values are equal to each other +query I +SELECT array_position(array(array(1, CAST(NULL AS INT)), array(1, 2)), array(1, CAST(NULL AS INT))); +---- +1 + +query I +SELECT array_position(array(array(CAST(NULL AS INT)), array(1)), array(CAST(NULL AS INT))); +---- +1 + +query I +SELECT array_position(array(named_struct('a', CAST(NULL AS INT)), named_struct('a', CAST(1 AS INT))), named_struct('a', CAST(NULL AS INT))); +---- +1 + +query I +SELECT array_position(array(array(CAST(1.0 AS DOUBLE), CAST(NULL AS DOUBLE))), array(CAST(1.0 AS DOUBLE), CAST(NULL AS DOUBLE))); +---- +1 + +### +### Column inputs +### + +statement ok +CREATE TABLE t AS VALUES + (array(1, 2, 3), 3), + (array(1, NULL, 3), 3), + (array(1, 2), 5), + (NULL, 1), + (array(1, 2), NULL); + +query I +SELECT array_position(column1, column2) FROM t; +---- +3 +3 +0 +NULL +NULL + +# A literal array with a column element, including a NULL element +query I +SELECT array_position(array(1, 2), column2) FROM t; +---- +0 +0 +0 +1 +NULL + +query I +SELECT array_position(array(1, 2, 3), column2) FROM t; +---- +3 +3 +0 +1 +NULL + +# A column array with a literal element +query I +SELECT array_position(column1, 3) FROM t; +---- +3 +3 +0 +NULL +0 + +# A batch sliced by OFFSET: the list offsets do not start at 0 +query I +SELECT array_position(column1, column2) FROM (SELECT * FROM t LIMIT 3 OFFSET 1); +---- +3 +0 +NULL + +statement ok +DROP TABLE t; + +### +### LargeList +### + +query I +SELECT array_position(arrow_cast(array(1, 2, 3), 'LargeList(Int64)'), 3); +---- +3 + +query I +SELECT array_position(arrow_cast(array(1, 2, 3), 'LargeList(Int64)'), 5); +---- +0 + +query I +SELECT array_position(arrow_cast(array(1, NULL, 3), 'LargeList(Int64)'), 3); +---- +3 + +query I +SELECT array_position(arrow_cast(array(1, 2, 3), 'LargeList(Int64)'), CAST(NULL AS BIGINT)); +---- +NULL + +query I +SELECT array_position(arrow_cast(array(CAST('-0.0' AS DOUBLE), CAST('NaN' AS DOUBLE)), 'LargeList(Float64)'), CAST('NaN' AS DOUBLE)); +---- +2 + +query I +SELECT array_position(arrow_cast(array(CAST('-0.0' AS DOUBLE), CAST('NaN' AS DOUBLE)), 'LargeList(Float64)'), CAST(0.0 AS DOUBLE)); +---- +1 + +### +### Column inputs with floats and strings +### + +statement ok +CREATE TABLE floats AS VALUES + (array(CAST(1.0 AS DOUBLE), CAST('NaN' AS DOUBLE)), CAST('NaN' AS DOUBLE)), + (array(CAST('-0.0' AS DOUBLE), CAST(1.0 AS DOUBLE)), CAST(0.0 AS DOUBLE)), + (array(CAST(0.0 AS DOUBLE), CAST(1.0 AS DOUBLE)), CAST('-0.0' AS DOUBLE)), + (array(CAST(1.0 AS DOUBLE), CAST(NULL AS DOUBLE)), CAST(2.0 AS DOUBLE)), + (array(CAST(1.0 AS DOUBLE), CAST(2.0 AS DOUBLE)), CAST(NULL AS DOUBLE)), + (CAST(NULL AS ARRAY), CAST(1.0 AS DOUBLE)); + +query I +SELECT array_position(column1, column2) FROM floats; +---- +2 +1 +1 +0 +NULL +NULL + +statement ok +DROP TABLE floats; + +statement ok +CREATE TABLE strings AS VALUES + (array('a', 'b', 'c'), 'c'), + (array('a', NULL, 'b'), 'b'), + (array('a', 'b'), 'z'), + (array('', 'a'), ''), + (array('a', 'b'), CAST(NULL AS STRING)), + (CAST(NULL AS ARRAY), 'a'); + +query I +SELECT array_position(column1, column2) FROM strings; +---- +3 +3 +0 +1 +NULL +NULL + +statement ok +DROP TABLE strings; + +### +### Nested floats and structs in column inputs +### + +statement ok +CREATE TABLE nested AS VALUES + (array(array(CAST(1.0 AS DOUBLE)), array(CAST('NaN' AS DOUBLE), CAST('-0.0' AS DOUBLE))), array(CAST('NaN' AS DOUBLE), CAST(0.0 AS DOUBLE))), + (array(array(CAST(0.0 AS DOUBLE)), array(CAST(1.0 AS DOUBLE))), array(CAST('-0.0' AS DOUBLE))), + (array(array(CAST(1.0 AS DOUBLE), CAST(NULL AS DOUBLE))), array(CAST(1.0 AS DOUBLE), CAST(NULL AS DOUBLE))), + (array(array(CAST(1.0 AS DOUBLE))), array(CAST(2.0 AS DOUBLE))); + +query I +SELECT array_position(column1, column2) FROM nested; +---- +2 +1 +1 +0 + +statement ok +DROP TABLE nested; + +statement ok +CREATE TABLE structs AS VALUES + (array(named_struct('a', CAST(1.0 AS DOUBLE), 'b', 1), named_struct('a', CAST('-0.0' AS DOUBLE), 'b', 2)), named_struct('a', CAST(0.0 AS DOUBLE), 'b', 2)), + (array(named_struct('a', CAST(0.0 AS DOUBLE), 'b', 1)), named_struct('a', CAST('-0.0' AS DOUBLE), 'b', 1)), + (array(named_struct('a', CAST('NaN' AS DOUBLE), 'b', 1)), named_struct('a', CAST('NaN' AS DOUBLE), 'b', 1)), + (array(named_struct('a', CAST(0.0 AS DOUBLE), 'b', 1)), named_struct('a', CAST(0.0 AS DOUBLE), 'b', 2)); + +query I +SELECT array_position(column1, column2) FROM structs; +---- +2 +1 +1 +0 + +statement ok +DROP TABLE structs;