From 7901e209989c84e202673bb41c7e9a9d5e73fa14 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 11 Nov 2026 20:54:43 -0700 Subject: [PATCH 1/5] feat: add Spark-compatible array_position Port of Comet's array_position. Returns the 1-based position of the first match, 0 when not found, and NULL when the array or element is NULL. Floats compare with NaN == NaN and -0.0 == 0.0 at any depth, and NULLs nested inside compared values are equal, matching Spark 4.2.0. Adds datafusion-macros and datafusion-doc dependencies for #[user_doc]. --- Cargo.lock | 2 + Cargo.toml | 2 + crates/spark/Cargo.toml | 2 + .../src/function/array/array_position.rs | 219 ++++++++++++++++++ crates/spark/src/function/array/mod.rs | 8 + .../test_files/array/array_position.slt | 197 ++++++++++++++++ 6 files changed, 430 insertions(+) create mode 100644 crates/spark/src/function/array/array_position.rs create mode 100644 crates/sqllogictest/test_files/array/array_position.slt diff --git a/Cargo.lock b/Cargo.lock index 5f1a84d1..461a5116 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1601,11 +1601,13 @@ dependencies = [ "criterion", "datafusion", "datafusion-common", + "datafusion-doc", "datafusion-expr", "datafusion-functions", "datafusion-functions-aggregate", "datafusion-functions-aggregate-common", "datafusion-functions-nested", + "datafusion-macros", "datafusion-physical-expr-common", "datafusion-session", "log", diff --git a/Cargo.toml b/Cargo.toml index a4226ef3..4ad5c0c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,11 +23,13 @@ chrono = { version = "0.4.45", default-features = false } criterion = "0.8" datafusion = { git = "https://github.com/apache/datafusion", branch = "branch-56", default-features = false } datafusion-common = { git = "https://github.com/apache/datafusion", branch = "branch-56", default-features = false } +datafusion-doc = { git = "https://github.com/apache/datafusion", branch = "branch-56" } datafusion-expr = { git = "https://github.com/apache/datafusion", branch = "branch-56", default-features = false } datafusion-functions = { git = "https://github.com/apache/datafusion", branch = "branch-56" } datafusion-functions-aggregate = { git = "https://github.com/apache/datafusion", branch = "branch-56" } datafusion-functions-aggregate-common = { git = "https://github.com/apache/datafusion", branch = "branch-56" } datafusion-functions-nested = { git = "https://github.com/apache/datafusion", branch = "branch-56", default-features = false } +datafusion-macros = { git = "https://github.com/apache/datafusion", branch = "branch-56" } datafusion-physical-expr-common = { git = "https://github.com/apache/datafusion", branch = "branch-56", default-features = false } datafusion-session = { git = "https://github.com/apache/datafusion", branch = "branch-56" } log = "^0.4" diff --git a/crates/spark/Cargo.toml b/crates/spark/Cargo.toml index 3c82dbb9..d9b26328 100644 --- a/crates/spark/Cargo.toml +++ b/crates/spark/Cargo.toml @@ -38,11 +38,13 @@ crc32fast = "1.4" # Optional dependency for SessionStateBuilderSpark extension trait datafusion = { workspace = true, optional = true, default-features = false } datafusion-common = { workspace = true } +datafusion-doc = { workspace = true } datafusion-expr = { workspace = true } datafusion-functions = { workspace = true } datafusion-functions-aggregate = { workspace = true } datafusion-functions-aggregate-common = { workspace = true } datafusion-functions-nested = { workspace = true } +datafusion-macros = { workspace = true } datafusion-physical-expr-common = { workspace = true } datafusion-session = { workspace = true } log = { workspace = true } diff --git a/crates/spark/src/function/array/array_position.rs b/crates/spark/src/function/array/array_position.rs new file mode 100644 index 00000000..eb9a9528 --- /dev/null +++ b/crates/spark/src/function/array/array_position.rs @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use arrow::array::{ + Array, ArrayRef, AsArray, GenericListArray, Int64Array, OffsetSizeTrait, + make_comparator, +}; +use arrow::buffer::{NullBuffer, ScalarBuffer}; +use arrow::compute::SortOptions; +use arrow::datatypes::{DataType, Float32Type, Float64Type}; +use datafusion_common::{Result, ScalarValue, exec_err, not_impl_err}; +use datafusion_expr::{ + ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, + Volatility, +}; +use datafusion_macros::user_doc; +use std::sync::Arc; + +/// Spark-compatible `array_position` expression. +/// +/// +#[user_doc( + doc_section(label = "Array Functions"), + description = "Returns the 1-based position of the first occurrence of the element in the \ + array, as a `BIGINT`. Unlike the DataFusion built-in, which returns NULL when the element \ + is not found, this returns 0. The result is NULL only when the array or the element is \ + NULL. NULL elements inside the array never match. For floating point values, NaN is \ + equal to NaN and -0.0 is equal to 0.0.", + syntax_example = "array_position(array, element)", + sql_example = r#"```sql +> SELECT array_position(array(3, 2, 1), 1); +3 +> SELECT array_position(array(3, 2, 1), 4); +0 +```"#, + argument(name = "array", description = "Array expression. NULL returns NULL."), + argument( + name = "element", + description = "Element to search for. NULL returns NULL." + ) +)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkArrayPosition { + signature: Signature, +} + +impl Default for SparkArrayPosition { + fn default() -> Self { + Self::new() + } +} + +impl SparkArrayPosition { + pub fn new() -> Self { + Self { + signature: Signature::array_and_element(Volatility::Immutable), + } + } +} + +impl ScalarUDFImpl for SparkArrayPosition { + fn name(&self) -> &str { + "array_position" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Int64) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + let is_scalar = args + .args + .iter() + .all(|a| matches!(a, ColumnarValue::Scalar(_))); + let arrays = ColumnarValue::values_to_arrays(&args.args)?; + let [array, element] = arrays.as_slice() else { + return exec_err!("array_position expects exactly two arguments"); + }; + + let result = match array.data_type() { + DataType::List(_) => position::(array, element)?, + DataType::LargeList(_) => position::(array, element)?, + other => return exec_err!("array_position does not support type {other}"), + }; + + if is_scalar { + Ok(ColumnarValue::Scalar(ScalarValue::try_from_array( + &result, 0, + )?)) + } else { + Ok(ColumnarValue::Array(result)) + } + } + + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } +} + +type Equality = Box bool + Send + Sync>; + +fn has_float(data_type: &DataType) -> bool { + match data_type { + DataType::Float32 | DataType::Float64 => true, + DataType::List(f) | DataType::LargeList(f) | DataType::FixedSizeList(f, _) => { + has_float(f.data_type()) + } + DataType::Struct(fields) => fields.iter().any(|f| has_float(f.data_type())), + DataType::Map(f, _) => has_float(f.data_type()), + _ => false, + } +} + +/// Wraps a non-null equality so that NULL equals NULL and NULL differs from any value, which +/// is how Spark compares values nested inside arrays and structs. +fn null_aware( + left: &ArrayRef, + right: &ArrayRef, + eq: impl Fn(usize, usize) -> bool + Send + Sync + 'static, +) -> Equality { + let (left, right) = (Arc::clone(left), Arc::clone(right)); + Box::new(move |i, j| match (left.is_null(i), right.is_null(j)) { + (true, true) => true, + (false, false) => eq(i, j), + _ => false, + }) +} + +/// Builds `eq(left_index, right_index)` in Spark's equality, where NaN equals NaN and -0.0 +/// equals 0.0 at any depth. Subtrees without a float use Arrow's comparator. +fn equality(left: &ArrayRef, right: &ArrayRef) -> Result { + macro_rules! float_eq { + ($t:ty) => {{ + let (l, r) = ( + left.as_primitive::<$t>().clone(), + right.as_primitive::<$t>().clone(), + ); + null_aware(left, right, move |i, j| { + let (a, b) = (l.value(i), r.value(j)); + a == b || (a.is_nan() && b.is_nan()) + }) + }}; + } + Ok(match left.data_type() { + DataType::Float32 => float_eq!(Float32Type), + DataType::Float64 => float_eq!(Float64Type), + DataType::List(_) => list_equality::(left, right)?, + DataType::LargeList(_) => list_equality::(left, right)?, + DataType::Struct(_) => { + let (l, r) = (left.as_struct(), right.as_struct()); + let fields = l + .columns() + .iter() + .zip(r.columns()) + .map(|(lc, rc)| equality(lc, rc)) + .collect::>>()?; + null_aware(left, right, move |i, j| fields.iter().all(|eq| eq(i, j))) + } + dt if has_float(dt) => { + return not_impl_err!("array_position does not support {dt} with floats"); + } + _ => { + let cmp = + make_comparator(left.as_ref(), right.as_ref(), SortOptions::default())?; + Box::new(move |i, j| cmp(i, j).is_eq()) + } + }) +} + +fn list_equality( + left: &ArrayRef, + right: &ArrayRef, +) -> Result { + let (l, r) = (left.as_list::().clone(), right.as_list::().clone()); + let inner = equality(l.values(), r.values())?; + Ok(null_aware(left, right, move |i, j| { + let (lo, ro) = (l.value_offsets(), r.value_offsets()); + let (ls, le) = (lo[i].as_usize(), lo[i + 1].as_usize()); + let (rs, re) = (ro[j].as_usize(), ro[j + 1].as_usize()); + le - ls == re - rs && (0..le - ls).all(|k| inner(ls + k, rs + k)) + })) +} + +fn position( + array: &ArrayRef, + element: &ArrayRef, +) -> Result { + let list = array + .as_any() + .downcast_ref::>() + .ok_or_else(|| { + datafusion_common::DataFusionError::Internal("expected list array".into()) + })?; + let values = list.values(); + let nulls = NullBuffer::union(list.nulls(), element.nulls()); + let eq = equality(values, element)?; + + let result: Vec = list + .offsets() + .windows(2) + .enumerate() + .map(|(row, w)| { + if nulls.as_ref().is_some_and(|n| n.is_null(row)) { + return 0; + } + let (start, end) = (w[0].as_usize(), w[1].as_usize()); + (start..end) + .find(|&i| values.is_valid(i) && eq(i, row)) + .map_or(0, |i| (i - start + 1) as i64) + }) + .collect(); + + Ok(Arc::new(Int64Array::new(ScalarBuffer::from(result), nulls))) +} diff --git a/crates/spark/src/function/array/mod.rs b/crates/spark/src/function/array/mod.rs index c4a2c353..70eb556a 100644 --- a/crates/spark/src/function/array/mod.rs +++ b/crates/spark/src/function/array/mod.rs @@ -3,6 +3,7 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements pub mod array_contains; +pub mod array_position; pub mod repeat; pub mod shuffle; pub mod slice; @@ -13,6 +14,7 @@ use datafusion_functions::make_udf_function; use std::sync::Arc; make_udf_function!(array_contains::SparkArrayContains, spark_array_contains); +make_udf_function!(array_position::SparkArrayPosition, array_position); make_udf_function!(spark_array::SparkArray, array); make_udf_function!(shuffle::SparkShuffle, shuffle); make_udf_function!(repeat::SparkArrayRepeat, array_repeat); @@ -26,6 +28,11 @@ pub mod expr_fn { "Returns true if the array contains the element (Spark semantics).", array element )); + export_functions!(( + array_position, + "Returns the 1-based position of the element in the array, or 0 if not found (Spark semantics).", + array element + )); export_functions!((array, "Returns an array with the given elements.", args)); export_functions!(( shuffle, @@ -47,6 +54,7 @@ pub mod expr_fn { pub fn functions() -> Vec> { vec![ spark_array_contains(), + array_position(), array(), shuffle(), array_repeat(), diff --git a/crates/sqllogictest/test_files/array/array_position.slt b/crates/sqllogictest/test_files/array/array_position.slt new file mode 100644 index 00000000..80604e47 --- /dev/null +++ b/crates/sqllogictest/test_files/array/array_position.slt @@ -0,0 +1,197 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +# Tests for Spark-compatible array_position. +# Returns the 1-based position of the first match, 0 if not found, NULL if the array or element is NULL. +# Expected values verified with pyspark==4.2.0. + +query I +SELECT array_position(array(1, 2, 3, 4), 3); +---- +3 + +# Not found returns 0 (DataFusion's built-in returns NULL) +query I +SELECT array_position(array(1, 2, 3, 4), 5); +---- +0 + +query I +SELECT array_position(array('a', 'b', 'c'), 'b'); +---- +2 + +# NULL elements in the array do not match and still count towards the position +query I +SELECT array_position(array(1, 2, NULL, 3), 3); +---- +4 + +query I +SELECT array_position(array(1, 2, NULL, 3), 5); +---- +0 + +# First occurrence wins +query I +SELECT array_position(array(1, 2, 1, 3), 1); +---- +1 + +query I +SELECT array_position(array(1, 2, 3), CAST(NULL AS INT)); +---- +NULL + +query I +SELECT array_position(CAST(NULL AS ARRAY), 1); +---- +NULL + +query I +SELECT array_position(array(1, 2, 3), NULL); +---- +NULL + +query I +SELECT array_position(array(), 1); +---- +0 + +# NaN equals NaN +query I +SELECT array_position(array(CAST('NaN' AS DOUBLE), CAST(1.0 AS DOUBLE)), CAST('NaN' AS DOUBLE)); +---- +1 + +query I +SELECT array_position(array(CAST('NaN' AS FLOAT), CAST(1.0 AS FLOAT)), CAST('NaN' AS FLOAT)); +---- +1 + +query I +SELECT array_position(array(true, false), false); +---- +2 + +query I +SELECT array_position(array(CAST(1.5 AS DECIMAL(3,1)), CAST(2.5 AS DECIMAL(3,1))), CAST(2.5 AS DECIMAL(3,1))); +---- +2 + +query I +SELECT array_position(array(DATE '2020-01-01', DATE '2021-01-01'), DATE '2021-01-01'); +---- +2 + +query I +SELECT array_position(array(X'01', X'02'), X'02'); +---- +2 + +# Nested arrays +query I +SELECT array_position(array(array(1, 2), array(3)), array(3)); +---- +2 + +# Element is coerced to the array's element type +query I +SELECT array_position(array(1, 2, 3), 2L); +---- +2 + +query I +SELECT array_position(array(1, 2, 3), CAST(2.0 AS DOUBLE)); +---- +2 + +# Signed zero: -0.0 equals 0.0 +query I +SELECT array_position(array(CAST('-0.0' AS DOUBLE), CAST(1.0 AS DOUBLE)), CAST(0.0 AS DOUBLE)); +---- +1 + +query I +SELECT array_position(array(CAST(0.0 AS DOUBLE), CAST(1.0 AS DOUBLE)), CAST('-0.0' AS DOUBLE)); +---- +1 + +query I +SELECT array_position(array(CAST(1.0 AS DOUBLE), CAST('-0.0' AS DOUBLE)), CAST(0.0 AS DOUBLE)); +---- +2 + +query I +SELECT array_position(array(CAST('-0.0' AS FLOAT), CAST(1.0 AS FLOAT)), CAST(0.0 AS FLOAT)); +---- +1 + +# Nested floats follow Spark's ordering: NaN equals NaN, -0.0 equals 0.0 +query I +SELECT array_position(array(array(CAST('NaN' AS DOUBLE)), array(CAST(1.0 AS DOUBLE))), array(CAST('NaN' AS DOUBLE))); +---- +1 + +query I +SELECT array_position(array(array(CAST('-0.0' AS DOUBLE)), array(CAST(1.0 AS DOUBLE))), array(CAST(0.0 AS DOUBLE))); +---- +1 + +query I +SELECT array_position(array(array(CAST('-0.0' AS FLOAT)), array(CAST(1.0 AS FLOAT))), array(CAST(0.0 AS FLOAT))); +---- +1 + +query I +SELECT array_position(array(named_struct('a', CAST('NaN' AS DOUBLE)), named_struct('a', CAST(1.0 AS DOUBLE))), named_struct('a', CAST('NaN' AS DOUBLE))); +---- +1 + +query I +SELECT array_position(array(named_struct('a', CAST('-0.0' AS DOUBLE)), named_struct('a', CAST(1.0 AS DOUBLE))), named_struct('a', CAST(0.0 AS DOUBLE))); +---- +1 + +# NULLs nested inside the compared values are equal to each other +query I +SELECT array_position(array(array(1, CAST(NULL AS INT)), array(1, 2)), array(1, CAST(NULL AS INT))); +---- +1 + +query I +SELECT array_position(array(array(CAST(NULL AS INT)), array(1)), array(CAST(NULL AS INT))); +---- +1 + +query I +SELECT array_position(array(named_struct('a', CAST(NULL AS INT)), named_struct('a', CAST(1 AS INT))), named_struct('a', CAST(NULL AS INT))); +---- +1 + +query I +SELECT array_position(array(array(CAST(1.0 AS DOUBLE), CAST(NULL AS DOUBLE))), array(CAST(1.0 AS DOUBLE), CAST(NULL AS DOUBLE))); +---- +1 + +### +### Column inputs +### + +statement ok +CREATE TABLE t AS VALUES + (make_array(1, 2, 3), 3), + (make_array(1, NULL, 3), 3), + (make_array(1, 2), 5), + (NULL, 1), + (make_array(1, 2), NULL); + +query I +SELECT array_position(column1, column2) FROM t; +---- +3 +3 +0 +NULL +NULL From 98787b52f051272ae925f38814dde03159676d8c Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 13 Nov 2026 09:54:32 -0700 Subject: [PATCH 2/5] refactor: share Spark float equality between array_position and array_contains array_position now normalizes NaN and -0.0 with array_contains' normalize_array once per batch and compares every type with make_comparator. This removes the hand-rolled recursive equality and also covers fixed-size lists of floats. Adds tests for LargeList, sliced lists, column inputs with floats, strings, nested floats and structs with signed zero, and the untyped NULL array error. All expected values verified with pyspark 4.2.0. Co-Authored-By: Claude Opus 5.5 --- .../src/function/array/array_contains.rs | 5 +- .../src/function/array/array_position.rs | 152 +++++++----------- .../test_files/array/array_position.slt | 146 ++++++++++++++++- 3 files changed, 207 insertions(+), 96 deletions(-) diff --git a/crates/spark/src/function/array/array_contains.rs b/crates/spark/src/function/array/array_contains.rs index 0d948fb6..fac0fd6b 100644 --- a/crates/spark/src/function/array/array_contains.rs +++ b/crates/spark/src/function/array/array_contains.rs @@ -103,7 +103,10 @@ fn contains_float(data_type: &DataType) -> bool { } } -fn normalize_array(array: &ArrayRef) -> Result { +/// Rewrites `-0.0` to `0.0` and every NaN to the canonical NaN at any depth of list, large +/// list, fixed-size list and struct, so that bitwise or total-order comparisons follow +/// Spark's float equality. +pub(super) fn normalize_array(array: &ArrayRef) -> Result { if !contains_float(array.data_type()) { return Ok(Arc::clone(array)); } diff --git a/crates/spark/src/function/array/array_position.rs b/crates/spark/src/function/array/array_position.rs index eb9a9528..a8c2968e 100644 --- a/crates/spark/src/function/array/array_position.rs +++ b/crates/spark/src/function/array/array_position.rs @@ -2,14 +2,14 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use super::array_contains::normalize_array; use arrow::array::{ - Array, ArrayRef, AsArray, GenericListArray, Int64Array, OffsetSizeTrait, - make_comparator, + Array, ArrayRef, GenericListArray, Int64Array, OffsetSizeTrait, make_comparator, }; use arrow::buffer::{NullBuffer, ScalarBuffer}; use arrow::compute::SortOptions; -use arrow::datatypes::{DataType, Float32Type, Float64Type}; -use datafusion_common::{Result, ScalarValue, exec_err, not_impl_err}; +use arrow::datatypes::DataType; +use datafusion_common::{Result, ScalarValue, exec_err}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -102,90 +102,6 @@ impl ScalarUDFImpl for SparkArrayPosition { } } -type Equality = Box bool + Send + Sync>; - -fn has_float(data_type: &DataType) -> bool { - match data_type { - DataType::Float32 | DataType::Float64 => true, - DataType::List(f) | DataType::LargeList(f) | DataType::FixedSizeList(f, _) => { - has_float(f.data_type()) - } - DataType::Struct(fields) => fields.iter().any(|f| has_float(f.data_type())), - DataType::Map(f, _) => has_float(f.data_type()), - _ => false, - } -} - -/// Wraps a non-null equality so that NULL equals NULL and NULL differs from any value, which -/// is how Spark compares values nested inside arrays and structs. -fn null_aware( - left: &ArrayRef, - right: &ArrayRef, - eq: impl Fn(usize, usize) -> bool + Send + Sync + 'static, -) -> Equality { - let (left, right) = (Arc::clone(left), Arc::clone(right)); - Box::new(move |i, j| match (left.is_null(i), right.is_null(j)) { - (true, true) => true, - (false, false) => eq(i, j), - _ => false, - }) -} - -/// Builds `eq(left_index, right_index)` in Spark's equality, where NaN equals NaN and -0.0 -/// equals 0.0 at any depth. Subtrees without a float use Arrow's comparator. -fn equality(left: &ArrayRef, right: &ArrayRef) -> Result { - macro_rules! float_eq { - ($t:ty) => {{ - let (l, r) = ( - left.as_primitive::<$t>().clone(), - right.as_primitive::<$t>().clone(), - ); - null_aware(left, right, move |i, j| { - let (a, b) = (l.value(i), r.value(j)); - a == b || (a.is_nan() && b.is_nan()) - }) - }}; - } - Ok(match left.data_type() { - DataType::Float32 => float_eq!(Float32Type), - DataType::Float64 => float_eq!(Float64Type), - DataType::List(_) => list_equality::(left, right)?, - DataType::LargeList(_) => list_equality::(left, right)?, - DataType::Struct(_) => { - let (l, r) = (left.as_struct(), right.as_struct()); - let fields = l - .columns() - .iter() - .zip(r.columns()) - .map(|(lc, rc)| equality(lc, rc)) - .collect::>>()?; - null_aware(left, right, move |i, j| fields.iter().all(|eq| eq(i, j))) - } - dt if has_float(dt) => { - return not_impl_err!("array_position does not support {dt} with floats"); - } - _ => { - let cmp = - make_comparator(left.as_ref(), right.as_ref(), SortOptions::default())?; - Box::new(move |i, j| cmp(i, j).is_eq()) - } - }) -} - -fn list_equality( - left: &ArrayRef, - right: &ArrayRef, -) -> Result { - let (l, r) = (left.as_list::().clone(), right.as_list::().clone()); - let inner = equality(l.values(), r.values())?; - Ok(null_aware(left, right, move |i, j| { - let (lo, ro) = (l.value_offsets(), r.value_offsets()); - let (ls, le) = (lo[i].as_usize(), lo[i + 1].as_usize()); - let (rs, re) = (ro[j].as_usize(), ro[j + 1].as_usize()); - le - ls == re - rs && (0..le - ls).all(|k| inner(ls + k, rs + k)) - })) -} - fn position( array: &ArrayRef, element: &ArrayRef, @@ -196,9 +112,12 @@ fn position( .ok_or_else(|| { datafusion_common::DataFusionError::Internal("expected list array".into()) })?; - let values = list.values(); + // Canonicalize NaN and -0.0 at any depth so that the comparator follows Spark's float + // equality. The comparator treats NULLs nested inside the compared values as equal. + let values = normalize_array(list.values())?; + let element = normalize_array(element)?; let nulls = NullBuffer::union(list.nulls(), element.nulls()); - let eq = equality(values, element)?; + let cmp = make_comparator(values.as_ref(), element.as_ref(), SortOptions::default())?; let result: Vec = list .offsets() @@ -210,10 +129,61 @@ fn position( } let (start, end) = (w[0].as_usize(), w[1].as_usize()); (start..end) - .find(|&i| values.is_valid(i) && eq(i, row)) + .find(|&i| values.is_valid(i) && cmp(i, row).is_eq()) .map_or(0, |i| (i - start + 1) as i64) }) .collect(); Ok(Arc::new(Int64Array::new(ScalarBuffer::from(result), nulls))) } + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{FixedSizeListArray, Float64Array, Int32Array, ListArray}; + use arrow::buffer::OffsetBuffer; + use arrow::datatypes::{Field, Int32Type}; + + #[test] + fn sliced_list() { + let list = ListArray::from_iter_primitive::(vec![ + Some(vec![Some(9), Some(9)]), + Some(vec![Some(1), Some(2), Some(3)]), + Some(vec![Some(3), Some(4)]), + ]); + let array: ArrayRef = Arc::new(list.slice(1, 2)); + let element: ArrayRef = Arc::new(Int32Array::from(vec![3, 4])); + let result = position::(&array, &element).unwrap(); + assert_eq!(result.as_ref(), &Int64Array::from(vec![3, 2])); + } + + #[test] + fn fixed_size_list_of_floats() { + let item = Arc::new(Field::new("item", DataType::Float64, true)); + let fsl = |values: Vec| -> ArrayRef { + Arc::new( + FixedSizeListArray::try_new( + Arc::clone(&item), + 1, + Arc::new(Float64Array::from(values)), + None, + ) + .unwrap(), + ) + }; + let values = fsl(vec![1.0, -0.0, f64::NAN]); + let field = Arc::new(Field::new("item", values.data_type().clone(), true)); + let array: ArrayRef = Arc::new( + ListArray::try_new( + field, + OffsetBuffer::new(vec![0, 3, 6].into()), + Arc::new(arrow::compute::concat(&[&values, &values]).unwrap()), + None, + ) + .unwrap(), + ); + let element = fsl(vec![0.0, -f64::NAN]); + let result = position::(&array, &element).unwrap(); + assert_eq!(result.as_ref(), &Int64Array::from(vec![2, 3])); + } +} diff --git a/crates/sqllogictest/test_files/array/array_position.slt b/crates/sqllogictest/test_files/array/array_position.slt index 80604e47..3f6d6474 100644 --- a/crates/sqllogictest/test_files/array/array_position.slt +++ b/crates/sqllogictest/test_files/array/array_position.slt @@ -49,11 +49,16 @@ SELECT array_position(CAST(NULL AS ARRAY), 1); ---- NULL +# Spark coerces an untyped NULL element to the array's element type, so this returns NULL. query I SELECT array_position(array(1, 2, 3), NULL); ---- NULL +# Spark rejects an untyped NULL array with DATATYPE_MISMATCH.NULL_TYPE +query error +SELECT array_position(NULL, 1); + query I SELECT array_position(array(), 1); ---- @@ -181,11 +186,11 @@ SELECT array_position(array(array(CAST(1.0 AS DOUBLE), CAST(NULL AS DOUBLE))), a statement ok CREATE TABLE t AS VALUES - (make_array(1, 2, 3), 3), - (make_array(1, NULL, 3), 3), - (make_array(1, 2), 5), + (array(1, 2, 3), 3), + (array(1, NULL, 3), 3), + (array(1, 2), 5), (NULL, 1), - (make_array(1, 2), NULL); + (array(1, 2), NULL); query I SELECT array_position(column1, column2) FROM t; @@ -195,3 +200,136 @@ SELECT array_position(column1, column2) FROM t; 0 NULL NULL + +# A batch sliced by OFFSET: the list offsets do not start at 0 +query I +SELECT array_position(column1, column2) FROM (SELECT * FROM t LIMIT 3 OFFSET 1); +---- +3 +0 +NULL + +statement ok +DROP TABLE t; + +### +### LargeList +### + +query I +SELECT array_position(arrow_cast(array(1, 2, 3), 'LargeList(Int64)'), 3); +---- +3 + +query I +SELECT array_position(arrow_cast(array(1, 2, 3), 'LargeList(Int64)'), 5); +---- +0 + +query I +SELECT array_position(arrow_cast(array(1, NULL, 3), 'LargeList(Int64)'), 3); +---- +3 + +query I +SELECT array_position(arrow_cast(array(1, 2, 3), 'LargeList(Int64)'), CAST(NULL AS BIGINT)); +---- +NULL + +query I +SELECT array_position(arrow_cast(array(CAST('-0.0' AS DOUBLE), CAST('NaN' AS DOUBLE)), 'LargeList(Float64)'), CAST('NaN' AS DOUBLE)); +---- +2 + +query I +SELECT array_position(arrow_cast(array(CAST('-0.0' AS DOUBLE), CAST('NaN' AS DOUBLE)), 'LargeList(Float64)'), CAST(0.0 AS DOUBLE)); +---- +1 + +### +### Column inputs with floats and strings +### + +statement ok +CREATE TABLE floats AS VALUES + (array(CAST(1.0 AS DOUBLE), CAST('NaN' AS DOUBLE)), CAST('NaN' AS DOUBLE)), + (array(CAST('-0.0' AS DOUBLE), CAST(1.0 AS DOUBLE)), CAST(0.0 AS DOUBLE)), + (array(CAST(0.0 AS DOUBLE), CAST(1.0 AS DOUBLE)), CAST('-0.0' AS DOUBLE)), + (array(CAST(1.0 AS DOUBLE), CAST(NULL AS DOUBLE)), CAST(2.0 AS DOUBLE)), + (array(CAST(1.0 AS DOUBLE), CAST(2.0 AS DOUBLE)), CAST(NULL AS DOUBLE)), + (CAST(NULL AS ARRAY), CAST(1.0 AS DOUBLE)); + +query I +SELECT array_position(column1, column2) FROM floats; +---- +2 +1 +1 +0 +NULL +NULL + +statement ok +DROP TABLE floats; + +statement ok +CREATE TABLE strings AS VALUES + (array('a', 'b', 'c'), 'c'), + (array('a', NULL, 'b'), 'b'), + (array('a', 'b'), 'z'), + (array('', 'a'), ''), + (array('a', 'b'), CAST(NULL AS STRING)), + (CAST(NULL AS ARRAY), 'a'); + +query I +SELECT array_position(column1, column2) FROM strings; +---- +3 +3 +0 +1 +NULL +NULL + +statement ok +DROP TABLE strings; + +### +### Nested floats and structs in column inputs +### + +statement ok +CREATE TABLE nested AS VALUES + (array(array(CAST(1.0 AS DOUBLE)), array(CAST('NaN' AS DOUBLE), CAST('-0.0' AS DOUBLE))), array(CAST('NaN' AS DOUBLE), CAST(0.0 AS DOUBLE))), + (array(array(CAST(0.0 AS DOUBLE)), array(CAST(1.0 AS DOUBLE))), array(CAST('-0.0' AS DOUBLE))), + (array(array(CAST(1.0 AS DOUBLE), CAST(NULL AS DOUBLE))), array(CAST(1.0 AS DOUBLE), CAST(NULL AS DOUBLE))), + (array(array(CAST(1.0 AS DOUBLE))), array(CAST(2.0 AS DOUBLE))); + +query I +SELECT array_position(column1, column2) FROM nested; +---- +2 +1 +1 +0 + +statement ok +DROP TABLE nested; + +statement ok +CREATE TABLE structs AS VALUES + (array(named_struct('a', CAST(1.0 AS DOUBLE), 'b', 1), named_struct('a', CAST('-0.0' AS DOUBLE), 'b', 2)), named_struct('a', CAST(0.0 AS DOUBLE), 'b', 2)), + (array(named_struct('a', CAST(0.0 AS DOUBLE), 'b', 1)), named_struct('a', CAST('-0.0' AS DOUBLE), 'b', 1)), + (array(named_struct('a', CAST('NaN' AS DOUBLE), 'b', 1)), named_struct('a', CAST('NaN' AS DOUBLE), 'b', 1)), + (array(named_struct('a', CAST(0.0 AS DOUBLE), 'b', 1)), named_struct('a', CAST(0.0 AS DOUBLE), 'b', 2)); + +query I +SELECT array_position(column1, column2) FROM structs; +---- +2 +1 +1 +0 + +statement ok +DROP TABLE structs; From 280dd058d684824a8251b59a158b059c07c745d9 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 13 Nov 2026 14:23:12 -0700 Subject: [PATCH 3/5] fix: remove duplicate datafusion-doc workspace dependency after merging main Co-Authored-By: Claude Opus 5.5 --- Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 438c6c6f..8428bc24 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,6 @@ chrono = { version = "0.4.45", default-features = false } criterion = "0.8" datafusion = { git = "https://github.com/apache/datafusion", branch = "branch-56", default-features = false } datafusion-common = { git = "https://github.com/apache/datafusion", branch = "branch-56", default-features = false } -datafusion-doc = { git = "https://github.com/apache/datafusion", branch = "branch-56" } datafusion-expr = { git = "https://github.com/apache/datafusion", branch = "branch-56", default-features = false } datafusion-doc = { git = "https://github.com/apache/datafusion", branch = "branch-56" } datafusion-functions = { git = "https://github.com/apache/datafusion", branch = "branch-56" } From e69c8feacb89147c2e4653099a08700c5247b108 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 15 Nov 2026 11:35:04 -0700 Subject: [PATCH 4/5] fix(spark): follow Spark's type coercion in array_position Replace the array_and_element signature with Spark's findTightestCommonType rules: numbers widen, integers widen to a decimal only when it holds them, dates widen to timestamps, and other mixes (string and number, two decimals, struct field types) are rejected with DATATYPE_MISMATCH.ARRAY_FUNCTION_DIFF_TYPES. A NULL-typed array, a non-array first argument and map elements are rejected at planning time. Move the float normalization helpers into a shared float_normalize module, now also descending into maps, and normalize only the sliced range of list values. Add the missing tests from #220. Co-Authored-By: Claude Opus 5.5 --- .../src/function/array/array_contains.rs | 264 +-------------- .../src/function/array/array_position.rs | 270 +++++++++++++-- .../src/function/array/float_normalize.rs | 307 ++++++++++++++++++ crates/spark/src/function/array/mod.rs | 1 + .../test_files/array/array_position.slt | 239 +++++++++++++- 5 files changed, 802 insertions(+), 279 deletions(-) create mode 100644 crates/spark/src/function/array/float_normalize.rs diff --git a/crates/spark/src/function/array/array_contains.rs b/crates/spark/src/function/array/array_contains.rs index fac0fd6b..3802667e 100644 --- a/crates/spark/src/function/array/array_contains.rs +++ b/crates/spark/src/function/array/array_contains.rs @@ -2,13 +2,13 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use super::float_normalize::normalize_floats; use arrow::array::{ - Array, ArrayRef, AsArray, BooleanArray, BooleanBufferBuilder, FixedSizeListArray, - GenericListArray, OffsetSizeTrait, StructArray, + Array, AsArray, BooleanArray, BooleanBufferBuilder, GenericListArray, OffsetSizeTrait, }; use arrow::buffer::{BooleanBuffer, NullBuffer}; -use arrow::datatypes::{DataType, FieldRef, Float32Type, Float64Type}; -use datafusion_common::{Result, ScalarValue, exec_err}; +use arrow::datatypes::DataType; +use datafusion_common::{Result, exec_err}; use datafusion_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; @@ -72,113 +72,6 @@ impl ScalarUDFImpl for SparkArrayContains { } } -/// Rewrites `-0.0` to `0.0` and every NaN to the canonical NaN, so that `array_has` -/// compares floating point values the way Spark does. -fn normalize_floats(value: &ColumnarValue) -> Result { - if !contains_float(&value.data_type()) { - return Ok(value.clone()); - } - match value { - ColumnarValue::Array(array) => Ok(ColumnarValue::Array(normalize_array(array)?)), - ColumnarValue::Scalar(scalar) => { - let array = normalize_array(&scalar.to_array()?)?; - Ok(ColumnarValue::Scalar(ScalarValue::try_from_array( - &array, 0, - )?)) - } - } -} - -/// Returns whether values of this type are, or hold at any depth, floating point numbers. -fn contains_float(data_type: &DataType) -> bool { - match data_type { - DataType::Float32 | DataType::Float64 => true, - DataType::List(field) - | DataType::LargeList(field) - | DataType::FixedSizeList(field, _) => contains_float(field.data_type()), - DataType::Struct(fields) => { - fields.iter().any(|field| contains_float(field.data_type())) - } - _ => false, - } -} - -/// Rewrites `-0.0` to `0.0` and every NaN to the canonical NaN at any depth of list, large -/// list, fixed-size list and struct, so that bitwise or total-order comparisons follow -/// Spark's float equality. -pub(super) fn normalize_array(array: &ArrayRef) -> Result { - if !contains_float(array.data_type()) { - return Ok(Arc::clone(array)); - } - Ok(match array.data_type() { - DataType::Float32 => Arc::new( - array - .as_primitive::() - .unary::<_, Float32Type>(|v| { - if v.is_nan() { - f32::NAN - } else if v == 0.0 { - 0.0 - } else { - v - } - }), - ), - DataType::Float64 => Arc::new( - array - .as_primitive::() - .unary::<_, Float64Type>(|v| { - if v.is_nan() { - f64::NAN - } else if v == 0.0 { - 0.0 - } else { - v - } - }), - ), - DataType::List(field) => normalize_list(array.as_list::(), field)?, - DataType::LargeList(field) => normalize_list(array.as_list::(), field)?, - DataType::FixedSizeList(field, size) => { - let list = array.as_fixed_size_list(); - // The length is passed explicitly: a zero-width list has no values to infer it from. - Arc::new(FixedSizeListArray::try_new_with_length( - Arc::clone(field), - *size, - normalize_array(list.values())?, - list.nulls().cloned(), - list.len(), - )?) - } - DataType::Struct(fields) => { - let columns = array - .as_struct() - .columns() - .iter() - .map(normalize_array) - .collect::>>()?; - Arc::new(StructArray::try_new( - fields.clone(), - columns, - array.nulls().cloned(), - )?) - } - _ => Arc::clone(array), - }) -} - -fn normalize_list( - list: &GenericListArray, - field: &FieldRef, -) -> Result { - Ok(Arc::new(GenericListArray::::try_new( - Arc::clone(field), - list.offsets().clone(), - normalize_array(list.values())?, - list.nulls().cloned(), - )?)) -} - /// For each row where `array_has` returned `false`, set the output to null /// if that row's input array contains any null elements. fn apply_spark_null_semantics( @@ -267,131 +160,22 @@ fn mask_with_list_nulls( #[cfg(test)] mod tests { use super::*; - use arrow::array::{ - Float32Array, Float64Array, Int32Array, Int32Builder, ListArray, ListBuilder, - }; - use arrow::buffer::OffsetBuffer; - use arrow::datatypes::{Field, Fields, Int32Type}; - - fn negative_nan() -> f64 { - f64::from_bits(f64::NAN.to_bits() | (1 << 63)) - } - - #[test] - fn normalize_array_canonicalizes_floats() { - let array: ArrayRef = Arc::new(Float64Array::from(vec![ - Some(-0.0), - Some(negative_nan()), - Some(1.5), - None, - ])); - let normalized = normalize_array(&array).unwrap(); - let normalized = normalized.as_primitive::(); - assert_eq!(normalized.value(0).to_bits(), 0.0_f64.to_bits()); - assert_eq!(normalized.value(1).to_bits(), f64::NAN.to_bits()); - assert_eq!(normalized.value(2), 1.5); - assert!(normalized.is_null(3)); - - let array: ArrayRef = Arc::new(Float32Array::from(vec![-0.0_f32, f32::NAN])); - let normalized = normalize_array(&array).unwrap(); - let normalized = normalized.as_primitive::(); - assert_eq!(normalized.value(0).to_bits(), 0.0_f32.to_bits()); - assert_eq!(normalized.value(1).to_bits(), f32::NAN.to_bits()); - } - - #[test] - fn normalize_array_keeps_the_shape_of_nested_values() { - let fields = Fields::from(vec![ - Field::new("a", DataType::Float64, true), - Field::new("b", DataType::Int32, true), - ]); - let item = Arc::new(Field::new("item", DataType::Struct(fields.clone()), true)); - let structs = StructArray::try_new( - fields, - vec![ - Arc::new(Float64Array::from(vec![-0.0, 2.0, 3.0])) as ArrayRef, - Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef, - ], - Some(NullBuffer::from(vec![true, false, true])), - ) - .unwrap(); - let list = ListArray::try_new( - item, - OffsetBuffer::new(vec![0, 2, 3].into()), - Arc::new(structs), - Some(NullBuffer::from(vec![true, false])), - ) - .unwrap(); - let array: ArrayRef = Arc::new(list); - assert!(contains_float(array.data_type())); - - let normalized = normalize_array(&array).unwrap(); - assert_eq!(normalized.data_type(), array.data_type()); - assert_eq!(normalized.nulls(), array.nulls()); - let original = array.as_list::(); - let list = normalized.as_list::(); - assert_eq!(list.offsets(), original.offsets()); - let structs = list.values().as_struct(); - assert_eq!(structs.nulls(), original.values().nulls()); - let a = structs.column(0).as_primitive::(); - assert_eq!(a.value(0).to_bits(), 0.0_f64.to_bits()); - assert_eq!(a.value(1), 2.0); - assert_eq!(structs.column(1).as_primitive::().value(2), 3); - } + use arrow::array::{FixedSizeListArray, Float64Array}; + use arrow::datatypes::Field; + use datafusion_common::ScalarValue; #[test] - fn normalize_floats_leaves_other_types_untouched() { - let mut builder = ListBuilder::new(Int32Builder::new()); - builder.values().append_value(1); - builder.append(true); - let array: ArrayRef = Arc::new(builder.finish()); - assert!(!contains_float(array.data_type())); - let value = ColumnarValue::Array(Arc::clone(&array)); - match normalize_floats(&value).unwrap() { - ColumnarValue::Array(result) => assert!(Arc::ptr_eq(&result, &array)), - ColumnarValue::Scalar(_) => panic!("expected an array"), - } - } - - #[test] - fn normalize_array_reuses_children_without_floats() { - let mut builder = ListBuilder::new(Int32Builder::new()); - builder.values().append_value(1); - builder.append(true); - let ints: ArrayRef = Arc::new(builder.finish()); - let structs = StructArray::try_new( - Fields::from(vec![ - Field::new("a", DataType::Float64, true), - Field::new("b", ints.data_type().clone(), true), - ]), - vec![ - Arc::new(Float64Array::from(vec![-0.0])) as ArrayRef, - Arc::clone(&ints), - ], - None, - ) - .unwrap(); - - let normalized = normalize_array(&(Arc::new(structs) as ArrayRef)).unwrap(); - assert!(Arc::ptr_eq(normalized.as_struct().column(1), &ints)); - } - - /// One valid row of a zero-width list, without a null buffer to infer the length from. - fn zero_width_list(field: &FieldRef) -> FixedSizeListArray { - FixedSizeListArray::try_new_with_length( - Arc::clone(field), + fn zero_width_fixed_size_list_keeps_its_row() { + // One valid row of a zero-width list, without a null buffer to infer the length from. + let field = Arc::new(Field::new("item", DataType::Float64, true)); + let haystack = FixedSizeListArray::try_new_with_length( + Arc::clone(&field), 0, Arc::new(Float64Array::from(Vec::::new())), None, 1, ) - .unwrap() - } - - #[test] - fn zero_width_fixed_size_list_keeps_its_row() { - let field = Arc::new(Field::new("item", DataType::Float64, true)); - let haystack = zero_width_list(&field); + .unwrap(); assert_eq!(haystack.len(), 1); let result = SparkArrayContains::new() @@ -410,26 +194,4 @@ mod tests { .unwrap(); assert_eq!(result.as_boolean(), &BooleanArray::from(vec![Some(false)])); } - - #[test] - fn zero_width_fixed_size_list_nested_in_a_list_is_not_rebuilt() { - let item = Arc::new(Field::new("item", DataType::Float64, true)); - let inner = Arc::new(Field::new( - "inner", - DataType::FixedSizeList(Arc::clone(&item), 0), - true, - )); - let list = ListArray::try_new( - inner, - OffsetBuffer::new(vec![0, 1].into()), - Arc::new(zero_width_list(&item)), - None, - ) - .unwrap(); - let array: ArrayRef = Arc::new(list); - - let normalized = normalize_array(&array).unwrap(); - assert_eq!(normalized.len(), 1); - assert_eq!(normalized.as_list::().values().len(), 1); - } } diff --git a/crates/spark/src/function/array/array_position.rs b/crates/spark/src/function/array/array_position.rs index a8c2968e..37ae53ce 100644 --- a/crates/spark/src/function/array/array_position.rs +++ b/crates/spark/src/function/array/array_position.rs @@ -2,14 +2,15 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use super::array_contains::normalize_array; +use super::float_normalize::normalize_array; use arrow::array::{ - Array, ArrayRef, GenericListArray, Int64Array, OffsetSizeTrait, make_comparator, + Array, ArrayRef, AsArray, Int64Array, OffsetSizeTrait, make_comparator, }; use arrow::buffer::{NullBuffer, ScalarBuffer}; use arrow::compute::SortOptions; -use arrow::datatypes::DataType; -use datafusion_common::{Result, ScalarValue, exec_err}; +use arrow::datatypes::{DataType, Field, Fields, TimeUnit}; +use datafusion_common::utils::take_function_args; +use datafusion_common::{Result, ScalarValue, exec_err, plan_err}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -23,10 +24,18 @@ use std::sync::Arc; #[user_doc( doc_section(label = "Array Functions"), description = "Returns the 1-based position of the first occurrence of the element in the \ - array, as a `BIGINT`. Unlike the DataFusion built-in, which returns NULL when the element \ - is not found, this returns 0. The result is NULL only when the array or the element is \ - NULL. NULL elements inside the array never match. For floating point values, NaN is \ - equal to NaN and -0.0 is equal to 0.0.", + array, as a `BIGINT`. The DataFusion built-in returns a `UInt64`, and returns NULL when \ + the element is not found, where this returns 0. The result is NULL only when the array \ + or the element is NULL: a NULL element returns NULL even if the array contains NULL, \ + while in the DataFusion built-in a NULL element matches the first NULL in the array. \ + NULL elements inside the array never match. For floating point values, also inside \ + nested arrays and structs, NaN is equal to NaN and -0.0 is equal to 0.0. As in Spark, \ + the array element and the element are cast to their tightest common type: numbers \ + widen, an integer widens to a decimal only if the decimal can hold all its values, and \ + a date widens to a timestamp. Other combinations are an error, for example a string \ + and a number, two different decimals, or structs whose field names or field types \ + differ, as in Spark with ANSI mode off (with ANSI mode on, Spark widens struct field \ + types). A NULL-typed array and map elements are also an error.", syntax_example = "array_position(array, element)", sql_example = r#"```sql > SELECT array_position(array(3, 2, 1), 1); @@ -54,7 +63,7 @@ impl Default for SparkArrayPosition { impl SparkArrayPosition { pub fn new() -> Self { Self { - signature: Signature::array_and_element(Volatility::Immutable), + signature: Signature::user_defined(Volatility::Immutable), } } } @@ -72,15 +81,18 @@ impl ScalarUDFImpl for SparkArrayPosition { Ok(DataType::Int64) } + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { + let [array, element] = take_function_args(self.name(), arg_types)?; + coerce_array_and_element(self.name(), array, element) + } + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { let is_scalar = args .args .iter() .all(|a| matches!(a, ColumnarValue::Scalar(_))); let arrays = ColumnarValue::values_to_arrays(&args.args)?; - let [array, element] = arrays.as_slice() else { - return exec_err!("array_position expects exactly two arguments"); - }; + let [array, element] = take_function_args(self.name(), &arrays)?; let result = match array.data_type() { DataType::List(_) => position::(array, element)?, @@ -106,28 +118,26 @@ fn position( array: &ArrayRef, element: &ArrayRef, ) -> Result { - let list = array - .as_any() - .downcast_ref::>() - .ok_or_else(|| { - datafusion_common::DataFusionError::Internal("expected list array".into()) - })?; + let list = array.as_list::(); + let offsets = list.offsets(); + // Only the values that this (possibly sliced) list refers to are normalized. + let base = offsets[0].as_usize(); + let len = offsets[offsets.len() - 1].as_usize() - base; // Canonicalize NaN and -0.0 at any depth so that the comparator follows Spark's float // equality. The comparator treats NULLs nested inside the compared values as equal. - let values = normalize_array(list.values())?; + let values = normalize_array(&list.values().slice(base, len))?; let element = normalize_array(element)?; let nulls = NullBuffer::union(list.nulls(), element.nulls()); let cmp = make_comparator(values.as_ref(), element.as_ref(), SortOptions::default())?; - let result: Vec = list - .offsets() + let result: Vec = offsets .windows(2) .enumerate() .map(|(row, w)| { if nulls.as_ref().is_some_and(|n| n.is_null(row)) { return 0; } - let (start, end) = (w[0].as_usize(), w[1].as_usize()); + let (start, end) = (w[0].as_usize() - base, w[1].as_usize() - base); (start..end) .find(|&i| values.is_valid(i) && cmp(i, row).is_eq()) .map_or(0, |i| (i - start + 1) as i64) @@ -137,12 +147,226 @@ fn position( Ok(Arc::new(Int64Array::new(ScalarBuffer::from(result), nulls))) } +/// Casts the array element and the element to their tightest common type, and rejects the +/// argument types that Spark's `ArrayPosition.checkInputDataTypes` rejects. +fn coerce_array_and_element( + name: &str, + array: &DataType, + element: &DataType, +) -> Result> { + if array.is_null() { + return null_type_err(name); + } + let (item, large) = match array { + DataType::List(field) | DataType::FixedSizeList(field, _) => (field, false), + DataType::LargeList(field) => (field, true), + other => { + return plan_err!( + "[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] The first parameter of `{name}` \ + requires the \"ARRAY\" type, however it has the type {other}" + ); + } + }; + let Some(common) = tightest_common_type(item.data_type(), element) else { + return plan_err!( + "[DATATYPE_MISMATCH.ARRAY_FUNCTION_DIFF_TYPES] Input to `{name}` should have \ + been \"ARRAY\" followed by a value with same element type, but it's \ + [{array}, {element}]" + ); + }; + if common.is_null() { + return null_type_err(name); + } + if contains_map(&common) { + return plan_err!( + "[DATATYPE_MISMATCH.INVALID_ORDERING_TYPE] The `{name}` does not support \ + ordering on type {common}" + ); + } + let item = Arc::new(Field::new(item.name(), common.clone(), true)); + let array = if large { + DataType::LargeList(item) + } else { + DataType::List(item) + }; + Ok(vec![array, common]) +} + +fn null_type_err(name: &str) -> Result { + plan_err!( + "[DATATYPE_MISMATCH.NULL_TYPE] Null typed values cannot be used as arguments of \ + `{name}`" + ) +} + +/// Spark's `TypeCoercion.findTightestCommonType`, on the arrow types that Spark's types +/// map to. +fn tightest_common_type(left: &DataType, right: &DataType) -> Option { + use DataType::*; + if left == right { + return Some(left.clone()); + } + if let (Some(l), Some(r)) = (numeric_precedence(left), numeric_precedence(right)) { + return Some(if l >= r { left.clone() } else { right.clone() }); + } + match (left, right) { + (Null, other) | (other, Null) => Some(other.clone()), + // Spark has a single string type and a single binary type. + (l, r) if is_string(l) && is_string(r) => Some(l.clone()), + (l, r) if is_binary(l) && is_binary(r) => Some(l.clone()), + // The decimal must be at least as wide as the integral type's DECIMAL(n, 0). + (int, Decimal128(p, s)) | (Decimal128(p, s), int) => { + let digits = integral_digits(int)?; + (*p as i16 - *s as i16 >= digits).then_some(Decimal128(*p, *s)) + } + (Date32, ts @ Timestamp(_, _)) | (ts @ Timestamp(_, _), Date32) => { + Some(ts.clone()) + } + // TIMESTAMP_NTZ and TIMESTAMP widen to TIMESTAMP. + (Timestamp(lu, ltz), Timestamp(ru, rtz)) => Some(Timestamp( + finer_unit(*lu, *ru), + ltz.clone().or_else(|| rtz.clone()), + )), + ( + List(lf) | LargeList(lf) | FixedSizeList(lf, _), + List(rf) | LargeList(rf) | FixedSizeList(rf, _), + ) => { + let item = tightest_common_type(lf.data_type(), rf.data_type())?; + let field = Arc::new(Field::new(lf.name(), item, true)); + Some( + if matches!(left, LargeList(_)) || matches!(right, LargeList(_)) { + LargeList(field) + } else { + List(field) + }, + ) + } + // The fields must have the same names and types, ignoring nullability, in the same + // order. This follows Spark with ANSI mode off; with ANSI mode on, Spark also widens + // the field types. Spark compares the names ignoring case, but DataFusion cannot cast + // between structs whose field names differ only in case, so they must match exactly. + (Struct(lf), Struct(rf)) if lf.len() == rf.len() => { + let fields = lf + .iter() + .zip(rf.iter()) + .map(|(a, b)| { + if a.name() != b.name() || !same_type(a.data_type(), b.data_type()) { + return None; + } + let data_type = tightest_common_type(a.data_type(), b.data_type())?; + Some(Field::new( + a.name(), + data_type, + a.is_nullable() || b.is_nullable(), + )) + }) + .collect::>>()?; + Some(Struct(Fields::from(fields))) + } + (Map(lf, sorted), Map(rf, _)) => { + let entries = tightest_common_type(lf.data_type(), rf.data_type())?; + Some(Map( + Arc::new(Field::new(lf.name(), entries, false)), + *sorted, + )) + } + _ => None, + } +} + +/// Whether two types are equal when the nullability of nested fields is ignored, like +/// Spark's `DataTypeUtils.sameType`. +fn same_type(left: &DataType, right: &DataType) -> bool { + use DataType::*; + match (left, right) { + (List(l), List(r)) | (LargeList(l), LargeList(r)) | (Map(l, _), Map(r, _)) => { + same_type(l.data_type(), r.data_type()) + } + (FixedSizeList(l, ln), FixedSizeList(r, rn)) => { + ln == rn && same_type(l.data_type(), r.data_type()) + } + (Struct(l), Struct(r)) => { + l.len() == r.len() + && l.iter().zip(r.iter()).all(|(a, b)| { + a.name() == b.name() && same_type(a.data_type(), b.data_type()) + }) + } + _ => left == right, + } +} + +fn is_string(data_type: &DataType) -> bool { + matches!( + data_type, + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View + ) +} + +fn is_binary(data_type: &DataType) -> bool { + matches!( + data_type, + DataType::Binary | DataType::LargeBinary | DataType::BinaryView + ) +} + +/// The precision of Spark's `DecimalType.forType` for an integral type. +fn integral_digits(data_type: &DataType) -> Option { + match data_type { + DataType::Int8 => Some(3), + DataType::Int16 => Some(5), + DataType::Int32 => Some(10), + DataType::Int64 => Some(20), + _ => None, + } +} + +/// Spark's `numericPrecedence`, which leaves out decimals. +fn numeric_precedence(data_type: &DataType) -> Option { + match data_type { + DataType::Int8 => Some(0), + DataType::Int16 => Some(1), + DataType::Int32 => Some(2), + DataType::Int64 => Some(3), + DataType::Float32 => Some(4), + DataType::Float64 => Some(5), + _ => None, + } +} + +fn finer_unit(left: TimeUnit, right: TimeUnit) -> TimeUnit { + let rank = |unit: TimeUnit| match unit { + TimeUnit::Second => 0, + TimeUnit::Millisecond => 1, + TimeUnit::Microsecond => 2, + TimeUnit::Nanosecond => 3, + }; + if rank(left) >= rank(right) { + left + } else { + right + } +} + +/// Spark cannot order maps, so it rejects them at any depth of the element type. +fn contains_map(data_type: &DataType) -> bool { + match data_type { + DataType::Map(_, _) => true, + DataType::List(field) + | DataType::LargeList(field) + | DataType::FixedSizeList(field, _) => contains_map(field.data_type()), + DataType::Struct(fields) => { + fields.iter().any(|field| contains_map(field.data_type())) + } + _ => false, + } +} + #[cfg(test)] mod tests { use super::*; use arrow::array::{FixedSizeListArray, Float64Array, Int32Array, ListArray}; use arrow::buffer::OffsetBuffer; - use arrow::datatypes::{Field, Int32Type}; + use arrow::datatypes::Int32Type; #[test] fn sliced_list() { diff --git a/crates/spark/src/function/array/float_normalize.rs b/crates/spark/src/function/array/float_normalize.rs new file mode 100644 index 00000000..07bbb28a --- /dev/null +++ b/crates/spark/src/function/array/float_normalize.rs @@ -0,0 +1,307 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Float normalization shared by the array functions that compare elements. +//! +//! Spark compares floating point values with `-0.0 == 0.0` and `NaN == NaN`, while arrow +//! compares them by their bits or by the IEEE total order. Rewriting `-0.0` to `0.0` and +//! every NaN to the canonical NaN, at any depth of nested types, makes those comparisons +//! follow Spark's float equality. + +use arrow::array::{ + Array, ArrayRef, AsArray, FixedSizeListArray, GenericListArray, MapArray, + OffsetSizeTrait, StructArray, +}; +use arrow::datatypes::{DataType, FieldRef, Float32Type, Float64Type}; +use datafusion_common::{Result, ScalarValue}; +use datafusion_expr::ColumnarValue; +use std::sync::Arc; + +/// Normalizes the floats in a [`ColumnarValue`], see [`normalize_array`]. +pub(super) fn normalize_floats(value: &ColumnarValue) -> Result { + if !contains_float(&value.data_type()) { + return Ok(value.clone()); + } + match value { + ColumnarValue::Array(array) => Ok(ColumnarValue::Array(normalize_array(array)?)), + ColumnarValue::Scalar(scalar) => { + let array = normalize_array(&scalar.to_array()?)?; + Ok(ColumnarValue::Scalar(ScalarValue::try_from_array( + &array, 0, + )?)) + } + } +} + +/// Returns whether values of this type are, or hold at any depth, floating point numbers. +pub(super) fn contains_float(data_type: &DataType) -> bool { + match data_type { + DataType::Float32 | DataType::Float64 => true, + DataType::List(field) + | DataType::LargeList(field) + | DataType::FixedSizeList(field, _) + | DataType::Map(field, _) => contains_float(field.data_type()), + DataType::Struct(fields) => { + fields.iter().any(|field| contains_float(field.data_type())) + } + _ => false, + } +} + +/// Rewrites `-0.0` to `0.0` and every NaN to the canonical NaN at any depth of list, large +/// list, fixed-size list, struct and map. Children without floats are reused as they are. +pub(super) fn normalize_array(array: &ArrayRef) -> Result { + if !contains_float(array.data_type()) { + return Ok(Arc::clone(array)); + } + Ok(match array.data_type() { + DataType::Float32 => Arc::new( + array + .as_primitive::() + .unary::<_, Float32Type>(|v| { + if v.is_nan() { + f32::NAN + } else if v == 0.0 { + 0.0 + } else { + v + } + }), + ), + DataType::Float64 => Arc::new( + array + .as_primitive::() + .unary::<_, Float64Type>(|v| { + if v.is_nan() { + f64::NAN + } else if v == 0.0 { + 0.0 + } else { + v + } + }), + ), + DataType::List(field) => normalize_list(array.as_list::(), field)?, + DataType::LargeList(field) => normalize_list(array.as_list::(), field)?, + DataType::FixedSizeList(field, size) => { + let list = array.as_fixed_size_list(); + // The length is passed explicitly: a zero-width list has no values to infer it from. + Arc::new(FixedSizeListArray::try_new_with_length( + Arc::clone(field), + *size, + normalize_array(list.values())?, + list.nulls().cloned(), + list.len(), + )?) + } + DataType::Struct(fields) => { + let columns = array + .as_struct() + .columns() + .iter() + .map(normalize_array) + .collect::>>()?; + Arc::new(StructArray::try_new( + fields.clone(), + columns, + array.nulls().cloned(), + )?) + } + DataType::Map(field, ordered) => { + let map = array.as_map(); + let entries: ArrayRef = Arc::new(map.entries().clone()); + let entries = normalize_array(&entries)?; + Arc::new(MapArray::try_new( + Arc::clone(field), + map.offsets().clone(), + entries.as_struct().clone(), + map.nulls().cloned(), + *ordered, + )?) + } + _ => Arc::clone(array), + }) +} + +fn normalize_list( + list: &GenericListArray, + field: &FieldRef, +) -> Result { + Ok(Arc::new(GenericListArray::::try_new( + Arc::clone(field), + list.offsets().clone(), + normalize_array(list.values())?, + list.nulls().cloned(), + )?)) +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{ + Float32Array, Float64Array, Int32Array, Int32Builder, ListArray, ListBuilder, + MapBuilder, + }; + use arrow::buffer::{NullBuffer, OffsetBuffer}; + use arrow::datatypes::{Field, Fields, Int32Type}; + + fn negative_nan() -> f64 { + f64::from_bits(f64::NAN.to_bits() | (1 << 63)) + } + + #[test] + fn normalize_array_canonicalizes_floats() { + let array: ArrayRef = Arc::new(Float64Array::from(vec![ + Some(-0.0), + Some(negative_nan()), + Some(1.5), + None, + ])); + let normalized = normalize_array(&array).unwrap(); + let normalized = normalized.as_primitive::(); + assert_eq!(normalized.value(0).to_bits(), 0.0_f64.to_bits()); + assert_eq!(normalized.value(1).to_bits(), f64::NAN.to_bits()); + assert_eq!(normalized.value(2), 1.5); + assert!(normalized.is_null(3)); + + let array: ArrayRef = Arc::new(Float32Array::from(vec![-0.0_f32, f32::NAN])); + let normalized = normalize_array(&array).unwrap(); + let normalized = normalized.as_primitive::(); + assert_eq!(normalized.value(0).to_bits(), 0.0_f32.to_bits()); + assert_eq!(normalized.value(1).to_bits(), f32::NAN.to_bits()); + } + + #[test] + fn normalize_array_keeps_the_shape_of_nested_values() { + let fields = Fields::from(vec![ + Field::new("a", DataType::Float64, true), + Field::new("b", DataType::Int32, true), + ]); + let item = Arc::new(Field::new("item", DataType::Struct(fields.clone()), true)); + let structs = StructArray::try_new( + fields, + vec![ + Arc::new(Float64Array::from(vec![-0.0, 2.0, 3.0])) as ArrayRef, + Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef, + ], + Some(NullBuffer::from(vec![true, false, true])), + ) + .unwrap(); + let list = ListArray::try_new( + item, + OffsetBuffer::new(vec![0, 2, 3].into()), + Arc::new(structs), + Some(NullBuffer::from(vec![true, false])), + ) + .unwrap(); + let array: ArrayRef = Arc::new(list); + assert!(contains_float(array.data_type())); + + let normalized = normalize_array(&array).unwrap(); + assert_eq!(normalized.data_type(), array.data_type()); + assert_eq!(normalized.nulls(), array.nulls()); + let original = array.as_list::(); + let list = normalized.as_list::(); + assert_eq!(list.offsets(), original.offsets()); + let structs = list.values().as_struct(); + assert_eq!(structs.nulls(), original.values().nulls()); + let a = structs.column(0).as_primitive::(); + assert_eq!(a.value(0).to_bits(), 0.0_f64.to_bits()); + assert_eq!(a.value(1), 2.0); + assert_eq!(structs.column(1).as_primitive::().value(2), 3); + } + + #[test] + fn normalize_array_descends_into_maps() { + let mut builder = MapBuilder::new( + None, + Int32Builder::new(), + arrow::array::Float64Builder::new(), + ); + builder.keys().append_value(1); + builder.values().append_value(-0.0); + builder.keys().append_value(2); + builder.values().append_value(negative_nan()); + builder.append(true).unwrap(); + builder.append(false).unwrap(); + let array: ArrayRef = Arc::new(builder.finish()); + assert!(contains_float(array.data_type())); + + let normalized = normalize_array(&array).unwrap(); + assert_eq!(normalized.data_type(), array.data_type()); + assert_eq!(normalized.nulls(), array.nulls()); + let map = normalized.as_map(); + let values = map.values().as_primitive::(); + assert_eq!(values.value(0).to_bits(), 0.0_f64.to_bits()); + assert_eq!(values.value(1).to_bits(), f64::NAN.to_bits()); + assert_eq!(map.keys().as_primitive::().value(1), 2); + } + + #[test] + fn normalize_floats_leaves_other_types_untouched() { + let mut builder = ListBuilder::new(Int32Builder::new()); + builder.values().append_value(1); + builder.append(true); + let array: ArrayRef = Arc::new(builder.finish()); + assert!(!contains_float(array.data_type())); + let value = ColumnarValue::Array(Arc::clone(&array)); + match normalize_floats(&value).unwrap() { + ColumnarValue::Array(result) => assert!(Arc::ptr_eq(&result, &array)), + ColumnarValue::Scalar(_) => panic!("expected an array"), + } + } + + #[test] + fn normalize_array_reuses_children_without_floats() { + let mut builder = ListBuilder::new(Int32Builder::new()); + builder.values().append_value(1); + builder.append(true); + let ints: ArrayRef = Arc::new(builder.finish()); + let structs = StructArray::try_new( + Fields::from(vec![ + Field::new("a", DataType::Float64, true), + Field::new("b", ints.data_type().clone(), true), + ]), + vec![ + Arc::new(Float64Array::from(vec![-0.0])) as ArrayRef, + Arc::clone(&ints), + ], + None, + ) + .unwrap(); + + let normalized = normalize_array(&(Arc::new(structs) as ArrayRef)).unwrap(); + assert!(Arc::ptr_eq(normalized.as_struct().column(1), &ints)); + } + + #[test] + fn zero_width_fixed_size_list_nested_in_a_list_is_not_rebuilt() { + let item = Arc::new(Field::new("item", DataType::Float64, true)); + let inner = Arc::new(Field::new( + "inner", + DataType::FixedSizeList(Arc::clone(&item), 0), + true, + )); + let zero_width = FixedSizeListArray::try_new_with_length( + Arc::clone(&item), + 0, + Arc::new(Float64Array::from(Vec::::new())), + None, + 1, + ) + .unwrap(); + let list = ListArray::try_new( + inner, + OffsetBuffer::new(vec![0, 1].into()), + Arc::new(zero_width), + None, + ) + .unwrap(); + let array: ArrayRef = Arc::new(list); + + let normalized = normalize_array(&array).unwrap(); + assert_eq!(normalized.len(), 1); + assert_eq!(normalized.as_list::().values().len(), 1); + } +} diff --git a/crates/spark/src/function/array/mod.rs b/crates/spark/src/function/array/mod.rs index 70eb556a..f9649d7e 100644 --- a/crates/spark/src/function/array/mod.rs +++ b/crates/spark/src/function/array/mod.rs @@ -4,6 +4,7 @@ pub mod array_contains; pub mod array_position; +mod float_normalize; pub mod repeat; pub mod shuffle; pub mod slice; diff --git a/crates/sqllogictest/test_files/array/array_position.slt b/crates/sqllogictest/test_files/array/array_position.slt index 3f6d6474..117c5464 100644 --- a/crates/sqllogictest/test_files/array/array_position.slt +++ b/crates/sqllogictest/test_files/array/array_position.slt @@ -49,16 +49,88 @@ SELECT array_position(CAST(NULL AS ARRAY), 1); ---- NULL -# Spark coerces an untyped NULL element to the array's element type, so this returns NULL. +# The tightest common type of the array element and an untyped NULL element is the array +# element type, so the NULL is cast to it and this returns NULL. query I SELECT array_position(array(1, 2, 3), NULL); ---- NULL -# Spark rejects an untyped NULL array with DATATYPE_MISMATCH.NULL_TYPE -query error +# A NULL element returns NULL even if the array contains NULL. The DataFusion built-in returns +# the position of the first NULL instead. +query I +SELECT array_position(array(1, NULL), CAST(NULL AS BIGINT)); +---- +NULL + +query I +SELECT array_position(CAST(NULL AS ARRAY), CAST(NULL AS INT)); +---- +NULL + +# An untyped NULL array is rejected at planning time, as in Spark. +query error \[DATATYPE_MISMATCH\.NULL_TYPE\] Null typed values cannot be used as arguments of `array_position` SELECT array_position(NULL, 1); +query error \[DATATYPE_MISMATCH\.NULL_TYPE\] +SELECT array_position(NULL, NULL); + +# The common type of an empty array and an untyped NULL is the NULL type. +query error \[DATATYPE_MISMATCH\.NULL_TYPE\] +SELECT array_position(array(), NULL); + +query error \[DATATYPE_MISMATCH\.NULL_TYPE\] +SELECT array_position(array(NULL), NULL); + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `array_position` requires the "ARRAY" type +SELECT array_position(1, 1); + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] +SELECT array_position('abc', 'a'); + +query T +SELECT arrow_typeof(array_position(array(1, 2, 3), 2)); +---- +Int64 + +# Several matches return the first position +query I +SELECT array_position(array(312, 773, 708, 708), 708); +---- +3 + +query I +SELECT array_position(array(312, 773, 708, 708), 414); +---- +0 + +query I +SELECT array_position(array(NULL, ''), ''); +---- +2 + +query I +SELECT array_position(array(NULL, ''), 'a'); +---- +0 + +query I +SELECT array_position(array(NULL), 1L); +---- +0 + +query I +SELECT array_position(CAST(NULL AS ARRAY), ''); +---- +NULL + +# Spark's `array(1, NULL)[0]` is its first element; arrays in this crate are 1-based, so the +# same element is `[1]` here. +query I +SELECT array_position(array(1, NULL), array(1, NULL)[1]); +---- +1 + query I SELECT array_position(array(), 1); ---- @@ -101,9 +173,24 @@ SELECT array_position(array(array(1, 2), array(3)), array(3)); ---- 2 -# Element is coerced to the array's element type query I -SELECT array_position(array(1, 2, 3), 2L); +SELECT array_position(array(array(1, 2), array(3, 4)), array(1, 2)); +---- +1 + +query I +SELECT array_position(array(array(5, 6), array(2, 1)), array(1, 2)); +---- +0 + +### +### Type coercion +### + +# The array element and the element are both cast to their tightest common type +# (Spark's findTightestCommonType): here INT and BIGINT become BIGINT. +query I +SELECT array_position(array(CAST(1 AS INT), CAST(2 AS INT), CAST(3 AS INT)), 2L); ---- 2 @@ -112,6 +199,128 @@ SELECT array_position(array(1, 2, 3), CAST(2.0 AS DOUBLE)); ---- 2 +# INT and DOUBLE become DOUBLE, so 1.23 does not match 1 +query I +SELECT array_position(array(CAST(1 AS INT)), CAST(1.23 AS DOUBLE)); +---- +0 + +query I +SELECT array_position(array(CAST(1 AS INT)), CAST(1.0 AS DOUBLE)); +---- +1 + +query I +SELECT array_position(array(CAST(1 AS DOUBLE)), CAST(1 AS INT)); +---- +1 + +query I +SELECT array_position(array(CAST(1 AS FLOAT)), 1L); +---- +1 + +query I +SELECT array_position(array(array(CAST(1 AS INT))), array(CAST(1.0 AS DOUBLE))); +---- +1 + +# An integral type widens to a decimal only if the decimal holds all its values: +# INT is DECIMAL(10,0), so DECIMAL(38,2) and DECIMAL(12,2) are wide enough and DECIMAL(10,2) is not. +query I +SELECT array_position(array(CAST(1 AS INT)), CAST(1 AS DECIMAL(38,2))); +---- +1 + +query I +SELECT array_position(array(CAST(1 AS DECIMAL(12,2))), CAST(1 AS INT)); +---- +1 + +query I +SELECT array_position(array(CAST(1 AS DECIMAL(3,0))), CAST(1 AS TINYINT)); +---- +1 + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] Input to `array_position` should have been "ARRAY" followed by a value with same element type +SELECT array_position(array(CAST(1 AS INT)), CAST(1 AS DECIMAL(10,2))); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_position(array(CAST(1 AS DECIMAL(12,2))), CAST(1 AS BIGINT)); + +# Two different decimals have no tightest common type +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_position(array(CAST(1 AS DECIMAL(3,1))), CAST(1 AS DECIMAL(4,2))); + +# Decimals and floating point numbers have no tightest common type +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_position(array(CAST(1 AS DECIMAL(5,1))), CAST(1.0 AS DOUBLE)); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_position(array(CAST(1.0 AS DOUBLE)), CAST(1 AS DECIMAL(5,1))); + +# Strings and numbers are not cast to each other, in either direction +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_position(array(1), '1'); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_position(array('1'), 1); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_position(array(1), CAST(NULL AS STRING)); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_position(array(array(1)), array('1')); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_position(array(true), 1); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_position(array(X'01'), '1'); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_position(array(DATE '2020-01-01'), '2020-01-01'); + +# DATE and TIMESTAMP become TIMESTAMP +query I +SELECT array_position(array(DATE '2020-01-01'), TIMESTAMP '2020-01-01 00:00:00'); +---- +1 + +query I +SELECT array_position(array(TIMESTAMP '2020-01-01 00:00:00'), DATE '2020-01-01'); +---- +1 + +# Struct fields are matched by name, in order. Spark ignores the case of the names and returns 1 +# here, but DataFusion cannot cast between struct fields whose names differ only in case. +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_position(array(named_struct('a', 1)), named_struct('A', 1)); + +# Struct field types are not widened, as in Spark with ANSI mode off. With ANSI mode on, Spark +# widens them and returns 0 here. +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_position(array(named_struct('a', 1)), named_struct('a', CAST(1.5 AS DOUBLE))); + +# Fields may differ in nullability +query I +SELECT array_position(array(named_struct('a', 1, 'b', CAST(NULL AS BIGINT))), named_struct('a', 1, 'b', 2L)); +---- +0 + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_position(array(named_struct('a', 1, 'b', 'x')), named_struct('b', 'x', 'a', 1)); + +# Maps cannot be ordered, so map elements are rejected at any depth +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_position` does not support ordering on type +SELECT array_position(array(map(1, 2)), map(1, 2)); + +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] +SELECT array_position(array(named_struct('m', map(1, 2))), named_struct('m', map(1, 2))); + +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] +SELECT array_position(array(array(map(1, 2))), array(map(1, 2))); + # Signed zero: -0.0 equals 0.0 query I SELECT array_position(array(CAST('-0.0' AS DOUBLE), CAST(1.0 AS DOUBLE)), CAST(0.0 AS DOUBLE)); @@ -201,6 +410,26 @@ SELECT array_position(column1, column2) FROM t; NULL NULL +# A literal array with a column element, including a NULL element +query I +SELECT array_position(array(1, 2), column2) FROM t; +---- +0 +0 +0 +1 +NULL + +# A column array with a literal element +query I +SELECT array_position(column1, 3) FROM t; +---- +3 +3 +0 +NULL +0 + # A batch sliced by OFFSET: the list offsets do not start at 0 query I SELECT array_position(column1, column2) FROM (SELECT * FROM t LIMIT 3 OFFSET 1); From bed1cf065e8f06d77c2416f429c61043258e0cbc Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 16 Nov 2026 05:02:54 -0700 Subject: [PATCH 5/5] test(spark): cover array_position broadcasting and nested NULL equality Add a literal array with a column element that matches on every non-NULL row, and cases showing NULLs nested inside compared arrays and structs are equal. Clarify the description to distinguish top-level NULL elements, which never match, from nested NULLs. Co-Authored-By: Claude Opus 5.5 --- .../src/function/array/array_position.rs | 3 ++- .../test_files/array/array_position.slt | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/crates/spark/src/function/array/array_position.rs b/crates/spark/src/function/array/array_position.rs index 37ae53ce..4ca52da9 100644 --- a/crates/spark/src/function/array/array_position.rs +++ b/crates/spark/src/function/array/array_position.rs @@ -28,7 +28,8 @@ use std::sync::Arc; the element is not found, where this returns 0. The result is NULL only when the array \ or the element is NULL: a NULL element returns NULL even if the array contains NULL, \ while in the DataFusion built-in a NULL element matches the first NULL in the array. \ - NULL elements inside the array never match. For floating point values, also inside \ + Top-level NULL elements in the array never match, while corresponding NULLs nested \ + inside compared arrays or structs are equal. For floating point values, also inside \ nested arrays and structs, NaN is equal to NaN and -0.0 is equal to 0.0. As in Spark, \ the array element and the element are cast to their tightest common type: numbers \ widen, an integer widens to a decimal only if the decimal can hold all its values, and \ diff --git a/crates/sqllogictest/test_files/array/array_position.slt b/crates/sqllogictest/test_files/array/array_position.slt index 117c5464..cf9ef3e8 100644 --- a/crates/sqllogictest/test_files/array/array_position.slt +++ b/crates/sqllogictest/test_files/array/array_position.slt @@ -183,6 +183,17 @@ SELECT array_position(array(array(5, 6), array(2, 1)), array(1, 2)); ---- 0 +# NULLs nested inside compared arrays and structs are equal +query I +SELECT array_position(array(array(1, 2), array(1, NULL)), array(1, NULL)); +---- +2 + +query I +SELECT array_position(array(named_struct('a', 1, 'b', CAST(NULL AS INT))), named_struct('a', 1, 'b', CAST(NULL AS INT))); +---- +1 + ### ### Type coercion ### @@ -420,6 +431,15 @@ SELECT array_position(array(1, 2), column2) FROM t; 1 NULL +query I +SELECT array_position(array(1, 2, 3), column2) FROM t; +---- +3 +3 +0 +1 +NULL + # A column array with a literal element query I SELECT array_position(column1, 3) FROM t;