From aec8072032f418034fe94ae412832328352e8d1c Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 12 Nov 2026 08:06:35 -0700 Subject: [PATCH 01/10] feat(spark): add Spark-compatible array_insert function Port Comet's array_insert as a scalar UDF. --- .../spark/src/function/array/array_insert.rs | 442 ++++++++++++++++++ crates/spark/src/function/array/mod.rs | 8 + .../test_files/array/array_insert.slt | 102 ++++ 3 files changed, 552 insertions(+) create mode 100644 crates/spark/src/function/array/array_insert.rs create mode 100644 crates/sqllogictest/test_files/array/array_insert.slt diff --git a/crates/spark/src/function/array/array_insert.rs b/crates/spark/src/function/array/array_insert.rs new file mode 100644 index 00000000..c02f6e69 --- /dev/null +++ b/crates/spark/src/function/array/array_insert.rs @@ -0,0 +1,442 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use std::sync::Arc; + +use arrow::array::{ + Array, ArrayRef, AsArray, Capacities, GenericListArray, Int32Array, MutableArrayData, + OffsetSizeTrait, make_array, +}; +use arrow::buffer::{NullBuffer, OffsetBuffer}; +use arrow::datatypes::{DataType, Field, FieldRef}; +use datafusion_common::utils::take_function_args; +use datafusion_common::{Result, exec_err, internal_err}; +use datafusion_expr::type_coercion::binary::binary_numeric_coercion; +use datafusion_expr::{ + ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, + Volatility, +}; + +/// `ByteArrayUtils.MAX_ROUNDED_ARRAY_LENGTH` (`Integer.MAX_VALUE - 15`) in Spark. +const MAX_ROUNDED_ARRAY_LENGTH: i64 = 2_147_483_632; + +/// Spark-compatible `array_insert(array, pos, item)`. +/// +/// +/// - Indexes are 1-based. A negative index counts from the end, where `-1` +/// inserts after the last element. +/// - An index past the end pads with NULLs. A negative index before the start +/// places the item first, then NULL padding, then the original elements. +/// - The result is NULL when the array or the index is NULL. A NULL item is +/// inserted as a NULL element. +/// - Index `0` is an error. +/// - The index may be any integer type and is cast to INT, because DataFusion +/// types bare integer literals as BIGINT. +/// +/// Spark's `spark.sql.legacy.negativeIndexInArrayInsert` (default `false`) makes +/// `-1` insert before the last element. Use [`Self::with_legacy_negative_index`]. +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkArrayInsert { + signature: Signature, + legacy_negative_index: bool, +} + +impl Default for SparkArrayInsert { + fn default() -> Self { + Self::new() + } +} + +impl SparkArrayInsert { + pub fn new() -> Self { + Self::with_legacy_negative_index(false) + } + + pub fn with_legacy_negative_index(legacy_negative_index: bool) -> Self { + Self { + signature: Signature::user_defined(Volatility::Immutable), + legacy_negative_index, + } + } +} + +/// The element field of a list type. +fn list_element_field(data_type: &DataType) -> Option<&FieldRef> { + match data_type { + DataType::List(f) | DataType::LargeList(f) | DataType::FixedSizeList(f, _) => { + Some(f) + } + _ => None, + } +} + +impl ScalarUDFImpl for SparkArrayInsert { + fn name(&self) -> &str { + "array_insert" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + internal_err!("return_field_from_args should be used instead") + } + + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + let [array, pos, _item] = take_function_args(self.name(), args.arg_fields)?; + let Some(elem) = list_element_field(array.data_type()) else { + return internal_err!( + "array_insert expects a list, got {}", + array.data_type() + ); + }; + // Padding can introduce NULLs even when neither input has any, so the + // element field is always nullable. + let elem = Arc::new(Field::new(elem.name(), elem.data_type().clone(), true)); + let data_type = match array.data_type() { + DataType::LargeList(_) => DataType::LargeList(elem), + _ => DataType::List(elem), + }; + Ok(Arc::new(Field::new( + self.name(), + data_type, + array.is_nullable() || pos.is_nullable(), + ))) + } + + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { + let [array, pos, item] = take_function_args(self.name(), arg_types)?; + + let (elem, wrap): (&FieldRef, fn(FieldRef) -> DataType) = match array { + DataType::List(f) | DataType::FixedSizeList(f, _) => (f, DataType::List), + DataType::LargeList(f) => (f, DataType::LargeList), + _ => { + return exec_err!( + "array_insert expects an array as first argument, got {array}" + ); + } + }; + + // Spark takes an INT index. DataFusion types bare integer literals as + // BIGINT, so accept every integer type and cast to INT; a value outside + // the INT range then fails the cast instead of being silently truncated. + if !pos.is_integer() { + return exec_err!( + "array_insert expects an INT index as second argument, got {pos}" + ); + } + + // Spark uses the tightest common type of the element and the item. + let elem_type = elem.data_type(); + let common = if elem_type == item || *item == DataType::Null { + elem_type.clone() + } else if *elem_type == DataType::Null { + item.clone() + } else if let Some(common) = binary_numeric_coercion(elem_type, item) { + common + } else { + return exec_err!( + "array_insert cannot insert a value of type {item} into an array of {elem_type}" + ); + }; + + let new_elem = + Arc::new(Field::new(elem.name(), common.clone(), elem.is_nullable())); + Ok(vec![wrap(new_elem), DataType::Int32, common]) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + let arrays = ColumnarValue::values_to_arrays(&args.args)?; + let [list, pos, item] = take_function_args(self.name(), arrays)?; + let pos = pos.as_primitive::(); + let legacy = self.legacy_negative_index; + let elem_field = match args.return_field.data_type() { + DataType::List(f) | DataType::LargeList(f) => Arc::clone(f), + other => return internal_err!("unexpected array_insert return type {other}"), + }; + let result: ArrayRef = match list.data_type() { + DataType::List(_) => Arc::new(array_insert( + list.as_list::(), + &item, + pos, + legacy, + elem_field, + )?), + DataType::LargeList(_) => Arc::new(array_insert( + list.as_list::(), + &item, + pos, + legacy, + elem_field, + )?), + other => return internal_err!("array_insert expects a list, got {other}"), + }; + Ok(ColumnarValue::Array(result)) + } +} + +/// Where the item goes in a row, and how long the row becomes. +struct Placement { + /// Original elements copied before the item. + before: usize, + /// NULLs between the item and the original elements (negative index + /// before the start of the array). + leading_nulls: usize, + /// NULLs between the original elements and the item (positive index past + /// the end of the array). + trailing_nulls: usize, + /// Length of the resulting row. + new_len: usize, +} + +fn place(pos: i32, len: usize, legacy: bool) -> Result { + if pos == 0 { + return exec_err!( + "[INVALID_INDEX_OF_ZERO] The index 0 is invalid. An index shall be either < 0 or > 0 (the first element has index 1)." + ); + } + let n = len as i64; + let pos = i64::from(pos); + // (original elements before the item, leading NULLs, trailing NULLs, new length) + let (before, leading, trailing, new_len) = if pos > 0 { + let idx = pos - 1; + if idx <= n { + (idx, 0, 0, n + 1) + } else { + (n, 0, idx - n, pos) + } + } else if -pos > n { + let new_len = -pos + i64::from(legacy); + (0, new_len - n - 1, 0, new_len) + } else { + (n + pos + i64::from(!legacy), 0, 0, n + 1) + }; + if new_len > MAX_ROUNDED_ARRAY_LENGTH { + return exec_err!( + "[COLLECTION_SIZE_LIMIT_EXCEEDED.FUNCTION] Can't create array with {new_len} elements which exceeding the array size limit {MAX_ROUNDED_ARRAY_LENGTH}, unsuccessful try to create arrays in the function `array_insert`." + ); + } + Ok(Placement { + before: before as usize, + leading_nulls: leading as usize, + trailing_nulls: trailing as usize, + new_len: new_len as usize, + }) +} + +fn array_insert( + list: &GenericListArray, + item: &ArrayRef, + pos: &Int32Array, + legacy: bool, + elem_field: FieldRef, +) -> Result> { + let values = list.values().to_data(); + let item_data = item.to_data(); + let mut out = MutableArrayData::with_capacities( + vec![&values, &item_data], + true, + Capacities::Array(values.len() + item_data.len()), + ); + + let mut offsets = Vec::with_capacity(list.len() + 1); + offsets.push(O::usize_as(0)); + let mut total = 0usize; + + for row in 0..list.len() { + // A NULL array or NULL index gives a NULL row with no child values. + if !list.is_null(row) && !pos.is_null(row) { + let start = list.value_offsets()[row].as_usize(); + let end = list.value_offsets()[row + 1].as_usize(); + let p = place(pos.value(row), end - start, legacy)?; + + out.try_extend(0, start, start + p.before)?; + if p.leading_nulls == 0 && p.trailing_nulls == 0 { + out.try_extend(1, row, row + 1)?; + out.try_extend(0, start + p.before, end)?; + } else if p.trailing_nulls > 0 { + out.try_extend(0, start + p.before, end)?; + out.try_extend_nulls(p.trailing_nulls)?; + out.try_extend(1, row, row + 1)?; + } else { + out.try_extend(1, row, row + 1)?; + out.try_extend_nulls(p.leading_nulls)?; + out.try_extend(0, start, end)?; + } + total += p.new_len; + } + offsets.push(O::usize_as(total)); + } + + GenericListArray::try_new( + elem_field, + OffsetBuffer::new(offsets.into()), + make_array(out.freeze()), + NullBuffer::union(list.nulls(), pos.nulls()), + ) + .map_err(Into::into) +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::ListArray; + use arrow::datatypes::Int32Type; + + fn run( + rows: Vec>>>, + pos: Vec>, + items: Vec>, + legacy: bool, + ) -> Result { + let list = ListArray::from_iter_primitive::(rows); + let field = Arc::new(Field::new_list_field(DataType::Int32, true)); + let item: ArrayRef = Arc::new(Int32Array::from(items)); + array_insert(&list, &item, &Int32Array::from(pos), legacy, field) + } + + fn list(rows: Vec>>>) -> ListArray { + ListArray::from_iter_primitive::(rows) + } + + // Expected values are taken from Comet's Spark-verified array_insert tests. + #[test] + fn positive_positions() -> Result<()> { + let got = run( + vec![ + Some(vec![Some(1), Some(2), Some(3)]), + Some(vec![Some(4), Some(5)]), + Some(vec![None]), + Some(vec![Some(1), Some(2), Some(3)]), + Some(vec![Some(1), Some(2), Some(3)]), + None, + ], + vec![Some(2), Some(1), Some(1), Some(5), Some(6), Some(1)], + vec![Some(10), Some(20), Some(30), Some(100), Some(100), Some(40)], + false, + )?; + let expected = list(vec![ + Some(vec![Some(1), Some(10), Some(2), Some(3)]), + Some(vec![Some(20), Some(4), Some(5)]), + Some(vec![Some(30), None]), + Some(vec![Some(1), Some(2), Some(3), None, Some(100)]), + Some(vec![Some(1), Some(2), Some(3), None, None, Some(100)]), + None, + ]); + assert_eq!(got.to_data(), expected.to_data()); + Ok(()) + } + + #[test] + fn negative_positions() -> Result<()> { + let got = run( + vec![ + Some(vec![Some(1), Some(2), Some(3)]), + Some(vec![Some(4), Some(5)]), + Some(vec![Some(1)]), + None, + ], + vec![Some(-2), Some(-1), Some(-3), Some(-1)], + vec![Some(10), Some(20), Some(100), Some(30)], + false, + )?; + let expected = list(vec![ + Some(vec![Some(1), Some(2), Some(10), Some(3)]), + Some(vec![Some(4), Some(5), Some(20)]), + Some(vec![Some(100), None, Some(1)]), + None, + ]); + assert_eq!(got.to_data(), expected.to_data()); + Ok(()) + } + + #[test] + fn legacy_negative_index() -> Result<()> { + let got = run( + vec![ + Some(vec![Some(1), Some(2), Some(3)]), + Some(vec![Some(4), Some(5)]), + None, + ], + vec![Some(-1), Some(-1), Some(-1)], + vec![Some(10), Some(20), Some(30)], + true, + )?; + let expected = list(vec![ + Some(vec![Some(1), Some(2), Some(10), Some(3)]), + Some(vec![Some(4), Some(20), Some(5)]), + None, + ]); + assert_eq!(got.to_data(), expected.to_data()); + Ok(()) + } + + #[test] + fn null_item_is_inserted_and_null_pos_gives_null() -> Result<()> { + let got = run( + vec![Some(vec![Some(0), None, Some(0)]), Some(vec![Some(1)])], + vec![Some(1), None], + vec![None, Some(9)], + false, + )?; + let expected = list(vec![Some(vec![None, Some(0), None, Some(0)]), None]); + assert_eq!(got.to_data(), expected.to_data()); + Ok(()) + } + + #[test] + fn zero_index_is_an_error() { + let err = run( + vec![Some(vec![Some(1)])], + vec![Some(0)], + vec![Some(1)], + false, + ) + .unwrap_err(); + assert!(err.to_string().contains("INVALID_INDEX_OF_ZERO")); + } + + #[test] + fn size_limit_is_checked_before_allocating() { + for pos in [i32::MAX, i32::MIN] { + let err = run( + vec![Some(vec![Some(1)])], + vec![Some(pos)], + vec![Some(1)], + false, + ) + .unwrap_err(); + assert!(err.to_string().contains("COLLECTION_SIZE_LIMIT_EXCEEDED")); + } + } + + #[test] + fn coerce_types_rules() -> Result<()> { + let f = SparkArrayInsert::new(); + let list_of = + |t: DataType| DataType::List(Arc::new(Field::new_list_field(t, true))); + // smaller integer index widens to INT, item widens to the common numeric type + let got = + f.coerce_types(&[list_of(DataType::Int32), DataType::Int8, DataType::Int64])?; + assert_eq!( + got, + vec![list_of(DataType::Int64), DataType::Int32, DataType::Int64] + ); + // NULL item takes the element type + let got = + f.coerce_types(&[list_of(DataType::Utf8), DataType::Int32, DataType::Null])?; + assert_eq!(got[2], DataType::Utf8); + // non-integer index and non-array first argument are rejected + assert!( + f.coerce_types(&[list_of(DataType::Int32), DataType::Utf8, DataType::Int32]) + .is_err() + ); + assert!( + f.coerce_types(&[DataType::Int32, DataType::Int32, DataType::Int32]) + .is_err() + ); + Ok(()) + } +} diff --git a/crates/spark/src/function/array/mod.rs b/crates/spark/src/function/array/mod.rs index c4a2c353..e491035a 100644 --- a/crates/spark/src/function/array/mod.rs +++ b/crates/spark/src/function/array/mod.rs @@ -3,6 +3,7 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements pub mod array_contains; +pub mod array_insert; pub mod repeat; pub mod shuffle; pub mod slice; @@ -13,6 +14,7 @@ use datafusion_functions::make_udf_function; use std::sync::Arc; make_udf_function!(array_contains::SparkArrayContains, spark_array_contains); +make_udf_function!(array_insert::SparkArrayInsert, array_insert); make_udf_function!(spark_array::SparkArray, array); make_udf_function!(shuffle::SparkShuffle, shuffle); make_udf_function!(repeat::SparkArrayRepeat, array_repeat); @@ -26,6 +28,11 @@ pub mod expr_fn { "Returns true if the array contains the element (Spark semantics).", array element )); + export_functions!(( + array_insert, + "Places an item into the array at the given 1-based index (negative counts from the end).", + array pos item + )); export_functions!((array, "Returns an array with the given elements.", args)); export_functions!(( shuffle, @@ -47,6 +54,7 @@ pub mod expr_fn { pub fn functions() -> Vec> { vec![ spark_array_contains(), + array_insert(), array(), shuffle(), array_repeat(), diff --git a/crates/sqllogictest/test_files/array/array_insert.slt b/crates/sqllogictest/test_files/array/array_insert.slt new file mode 100644 index 00000000..540c939f --- /dev/null +++ b/crates/sqllogictest/test_files/array/array_insert.slt @@ -0,0 +1,102 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +# Tests for Spark-compatible array_insert +# Reference: https://spark.apache.org/docs/latest/api/sql/index.html#array_insert +# Expected values come from the Spark docs and Comet's Spark-verified tests. + +# Examples from the Spark documentation +query ? +SELECT array_insert(make_array(1, 2, 3, 4), 5, 5); +---- +[1, 2, 3, 4, 5] + +query ? +SELECT array_insert(make_array(5, 4, 3, 2), -1, 1); +---- +[5, 4, 3, 2, 1] + +query ? +SELECT array_insert(make_array(5, 3, 2, 1), -4, 4); +---- +[5, 4, 3, 2, 1] + +# Positive positions +query ? +SELECT array_insert(make_array(1, 2, 3), 2, 10); +---- +[1, 10, 2, 3] + +query ? +SELECT array_insert(make_array(4, 5), 1, 20); +---- +[20, 4, 5] + +# Beyond the end pads with NULLs +query ? +SELECT array_insert(make_array(1, 2, 3), 6, 100); +---- +[1, 2, 3, NULL, NULL, 100] + +# Negative positions +query ? +SELECT array_insert(make_array(1, 2, 3), -2, 10); +---- +[1, 2, 10, 3] + +query ? +SELECT array_insert(make_array(4, 5), -1, 20); +---- +[4, 5, 20] + +# Negative position before the start: item first, then NULL padding +query ? +SELECT array_insert(make_array(1), -3, 100); +---- +[100, NULL, 1] + +# A NULL item is inserted as a NULL element +query ? +SELECT array_insert(make_array(0, NULL, 0), 1, NULL); +---- +[NULL, 0, NULL, 0] + +# NULL array or NULL position gives NULL +query ? +SELECT array_insert(arrow_cast(NULL, 'List(Int32)'), 1, 5); +---- +NULL + +query ? +SELECT array_insert(make_array(1, 2), NULL::INT, 5); +---- +NULL + +# Table data, with per-row positions +statement ok +CREATE TABLE array_insert_t AS VALUES + (make_array(1, 2, 3), 2, 10), + (make_array(4, 5), -1, 20), + (NULL, 1, 30), + (make_array(7), NULL, 40); + +query ? +SELECT array_insert(column1, column2, column3) FROM array_insert_t; +---- +[1, 10, 2, 3] +[4, 5, 20] +NULL +NULL + +# Index 0 is invalid +statement error INVALID_INDEX_OF_ZERO +SELECT array_insert(make_array(1, 2), 0, 5); + +# Exceeding the maximum array length +statement error COLLECTION_SIZE_LIMIT_EXCEEDED +SELECT array_insert(make_array(1, 2), 2147483647, 5); + +# The index must be an integer +statement error +SELECT array_insert(make_array(1, 2), 'a', 5); From d39e1aaaf2b1cf4b34d4e2d39426f2dafa6918cc Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 12 Nov 2026 08:07:26 -0700 Subject: [PATCH 02/10] fix(spark): guard array_insert offset overflow; extend slt coverage --- .../spark/src/function/array/array_insert.rs | 9 ++++- .../test_files/array/array_insert.slt | 35 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/crates/spark/src/function/array/array_insert.rs b/crates/spark/src/function/array/array_insert.rs index c02f6e69..656566b3 100644 --- a/crates/spark/src/function/array/array_insert.rs +++ b/crates/spark/src/function/array/array_insert.rs @@ -267,7 +267,14 @@ fn array_insert( } total += p.new_len; } - offsets.push(O::usize_as(total)); + // A `List` (i32 offsets) can overflow across rows even though each row is + // within Spark's per-array limit. + let Some(offset) = O::from_usize(total) else { + return exec_err!( + "array_insert result is too large for the list's offset type" + ); + }; + offsets.push(offset); } GenericListArray::try_new( diff --git a/crates/sqllogictest/test_files/array/array_insert.slt b/crates/sqllogictest/test_files/array/array_insert.slt index 540c939f..f43c9243 100644 --- a/crates/sqllogictest/test_files/array/array_insert.slt +++ b/crates/sqllogictest/test_files/array/array_insert.slt @@ -100,3 +100,38 @@ SELECT array_insert(make_array(1, 2), 2147483647, 5); # The index must be an integer statement error SELECT array_insert(make_array(1, 2), 'a', 5); + +# String elements +query ? +SELECT array_insert(make_array('a', 'c'), 2, 'b'); +---- +[a, b, c] + +# The item widens to the common numeric type +query ? +SELECT array_insert(make_array(arrow_cast(1, 'Int32')), 1, arrow_cast(3000000000, 'Int64')); +---- +[3000000000, 1] + +# LargeList and FixedSizeList inputs +query ? +SELECT array_insert(arrow_cast(make_array(1, 2), 'LargeList(Int64)'), 2, 9); +---- +[1, 9, 2] + +query ? +SELECT array_insert(arrow_cast(make_array(1, 2), 'FixedSizeList(2, Int64)'), 3, 9); +---- +[1, 2, 9] + +# Empty array +query ? +SELECT array_insert(arrow_cast(make_array(), 'List(Int64)'), 1, 7); +---- +[7] + +# Result type keeps the list type +query T +SELECT arrow_typeof(array_insert(arrow_cast(make_array(1, 2), 'LargeList(Int64)'), 1, 0)); +---- +LargeList(Int64) From e3ee8ae0c1d5abe2dbe06df7ca9d02f3fe027f9b Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 12 Nov 2026 08:08:29 -0700 Subject: [PATCH 03/10] refactor(spark): simplify array_insert row building and error reporting --- .../spark/src/function/array/array_insert.rs | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/crates/spark/src/function/array/array_insert.rs b/crates/spark/src/function/array/array_insert.rs index 656566b3..86f6bbd6 100644 --- a/crates/spark/src/function/array/array_insert.rs +++ b/crates/spark/src/function/array/array_insert.rs @@ -4,6 +4,7 @@ use std::sync::Arc; +use crate::function::error_utils::unsupported_data_type_exec_err; use arrow::array::{ Array, ArrayRef, AsArray, Capacities, GenericListArray, Int32Array, MutableArrayData, OffsetSizeTrait, make_array, @@ -113,9 +114,11 @@ impl ScalarUDFImpl for SparkArrayInsert { DataType::List(f) | DataType::FixedSizeList(f, _) => (f, DataType::List), DataType::LargeList(f) => (f, DataType::LargeList), _ => { - return exec_err!( - "array_insert expects an array as first argument, got {array}" - ); + return Err(unsupported_data_type_exec_err( + self.name(), + "an array as first argument", + array, + )); } }; @@ -123,9 +126,11 @@ impl ScalarUDFImpl for SparkArrayInsert { // BIGINT, so accept every integer type and cast to INT; a value outside // the INT range then fails the cast instead of being silently truncated. if !pos.is_integer() { - return exec_err!( - "array_insert expects an INT index as second argument, got {pos}" - ); + return Err(unsupported_data_type_exec_err( + self.name(), + "an INT index as second argument", + pos, + )); } // Spark uses the tightest common type of the element and the item. @@ -152,10 +157,13 @@ impl ScalarUDFImpl for SparkArrayInsert { let [list, pos, item] = take_function_args(self.name(), arrays)?; let pos = pos.as_primitive::(); let legacy = self.legacy_negative_index; - let elem_field = match args.return_field.data_type() { - DataType::List(f) | DataType::LargeList(f) => Arc::clone(f), - other => return internal_err!("unexpected array_insert return type {other}"), + let Some(elem_field) = list_element_field(args.return_field.data_type()) else { + return internal_err!( + "unexpected array_insert return type {}", + args.return_field.data_type() + ); }; + let elem_field = Arc::clone(elem_field); let result: ArrayRef = match list.data_type() { DataType::List(_) => Arc::new(array_insert( list.as_list::(), @@ -253,17 +261,17 @@ fn array_insert( let p = place(pos.value(row), end - start, legacy)?; out.try_extend(0, start, start + p.before)?; - if p.leading_nulls == 0 && p.trailing_nulls == 0 { - out.try_extend(1, row, row + 1)?; - out.try_extend(0, start + p.before, end)?; - } else if p.trailing_nulls > 0 { + if p.trailing_nulls > 0 { + // Positive index past the end: original elements, NULLs, item. out.try_extend(0, start + p.before, end)?; out.try_extend_nulls(p.trailing_nulls)?; out.try_extend(1, row, row + 1)?; } else { + // Otherwise: item, then (for a negative index before the start) + // NULL padding, then the remaining original elements. out.try_extend(1, row, row + 1)?; out.try_extend_nulls(p.leading_nulls)?; - out.try_extend(0, start, end)?; + out.try_extend(0, start + p.before, end)?; } total += p.new_len; } From 6122a04041e5179050a24696f593863ed934295a Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 13 Nov 2026 09:55:06 -0700 Subject: [PATCH 04/10] fix(spark): unify same-family types in array_insert; move tests to slt Fall back to type_union_resolution when the element and item types are not both numeric, so Utf8View/LargeUtf8 elements accept a Utf8 item, timestamps of different units unify, and nested types differing only in nullability are accepted. Pairs that mix a string or binary type with another kind are still rejected, since type_union_resolution would coerce INT and STRING to STRING while Spark raises ARRAY_FUNCTION_DIFF_TYPES. Move the Rust tests that duplicated array_insert.slt into the .slt, keeping Rust tests only for legacy mode, i32::MIN and a sliced list. Add slt coverage for the -(len + 1) boundary, NULL padding with string and nested elements, Utf8View and LargeUtf8 elements, and a mismatched item type. All new expected values were checked against Spark 4.2.0. Co-Authored-By: Claude Opus 5.5 --- .../spark/src/function/array/array_insert.rs | 230 +++++++++++------- .../test_files/array/array_insert.slt | 102 +++++++- 2 files changed, 243 insertions(+), 89 deletions(-) diff --git a/crates/spark/src/function/array/array_insert.rs b/crates/spark/src/function/array/array_insert.rs index 86f6bbd6..0a01c66d 100644 --- a/crates/spark/src/function/array/array_insert.rs +++ b/crates/spark/src/function/array/array_insert.rs @@ -13,7 +13,9 @@ use arrow::buffer::{NullBuffer, OffsetBuffer}; use arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, exec_err, internal_err}; -use datafusion_expr::type_coercion::binary::binary_numeric_coercion; +use datafusion_expr::type_coercion::binary::{ + binary_numeric_coercion, type_union_resolution, +}; use datafusion_expr::{ ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -72,6 +74,46 @@ fn list_element_field(data_type: &DataType) -> Option<&FieldRef> { } } +/// Whether `a` and `b` pair a string or binary type with a type of another +/// kind, at the top level or inside a list or struct. +/// +/// DataFusion's `type_union_resolution` coerces such pairs (for example INT +/// and STRING to STRING), but Spark's `findTightestCommonType` does not. +fn mixes_string_or_binary(a: &DataType, b: &DataType) -> bool { + #[derive(PartialEq)] + enum Kind { + String, + Binary, + Other, + } + fn kind(t: &DataType) -> Kind { + match t { + DataType::Dictionary(_, v) => kind(v), + DataType::RunEndEncoded(_, v) => kind(v.data_type()), + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => Kind::String, + DataType::Binary + | DataType::LargeBinary + | DataType::BinaryView + | DataType::FixedSizeBinary(_) => Kind::Binary, + _ => Kind::Other, + } + } + if *a == DataType::Null || *b == DataType::Null { + return false; + } + if let (Some(fa), Some(fb)) = (list_element_field(a), list_element_field(b)) { + return mixes_string_or_binary(fa.data_type(), fb.data_type()); + } + if let (DataType::Struct(fa), DataType::Struct(fb)) = (a, b) { + return fa.iter().any(|fa| { + fb.iter() + .find(|fb| fb.name() == fa.name()) + .is_some_and(|fb| mixes_string_or_binary(fa.data_type(), fb.data_type())) + }); + } + kind(a) != kind(b) +} + impl ScalarUDFImpl for SparkArrayInsert { fn name(&self) -> &str { "array_insert" @@ -134,6 +176,9 @@ impl ScalarUDFImpl for SparkArrayInsert { } // Spark uses the tightest common type of the element and the item. + // Besides numeric widening, that unifies types of the same family, such + // as Utf8 and Utf8View (parquet's default for strings), timestamps of + // different units, and nested types that differ only in nullability. let elem_type = elem.data_type(); let common = if elem_type == item || *item == DataType::Null { elem_type.clone() @@ -141,6 +186,11 @@ impl ScalarUDFImpl for SparkArrayInsert { item.clone() } else if let Some(common) = binary_numeric_coercion(elem_type, item) { common + } else if let Some(common) = + type_union_resolution(&[elem_type.clone(), item.clone()]) + .filter(|_| !mixes_string_or_binary(elem_type, item)) + { + common } else { return exec_err!( "array_insert cannot insert a value of type {item} into an array of {elem_type}" @@ -298,7 +348,7 @@ fn array_insert( mod tests { use super::*; use arrow::array::ListArray; - use arrow::datatypes::Int32Type; + use arrow::datatypes::{Int32Type, TimeUnit}; fn run( rows: Vec>>>, @@ -316,117 +366,83 @@ mod tests { ListArray::from_iter_primitive::(rows) } - // Expected values are taken from Comet's Spark-verified array_insert tests. + // Most behavior is covered by array_insert.slt. These cover what SQL + // cannot reach: legacy mode (a session config), `i32::MIN`, and sliced + // input. Expected values were checked against Spark 4.2.0. #[test] - fn positive_positions() -> Result<()> { + fn legacy_negative_index() -> Result<()> { let got = run( vec![ Some(vec![Some(1), Some(2), Some(3)]), Some(vec![Some(4), Some(5)]), - Some(vec![None]), + None, + // before the start: the item, four NULLs, then the elements + Some(vec![Some(1)]), + // -(len + 1) pads one NULL in legacy mode Some(vec![Some(1), Some(2), Some(3)]), + // -len inserts first with no padding in legacy mode Some(vec![Some(1), Some(2), Some(3)]), - None, ], - vec![Some(2), Some(1), Some(1), Some(5), Some(6), Some(1)], - vec![Some(10), Some(20), Some(30), Some(100), Some(100), Some(40)], - false, + vec![Some(-1), Some(-1), Some(-1), Some(-5), Some(-4), Some(-3)], + vec![Some(10), Some(20), Some(30), Some(9), Some(10), Some(10)], + true, )?; let expected = list(vec![ - Some(vec![Some(1), Some(10), Some(2), Some(3)]), - Some(vec![Some(20), Some(4), Some(5)]), - Some(vec![Some(30), None]), - Some(vec![Some(1), Some(2), Some(3), None, Some(100)]), - Some(vec![Some(1), Some(2), Some(3), None, None, Some(100)]), + Some(vec![Some(1), Some(2), Some(10), Some(3)]), + Some(vec![Some(4), Some(20), Some(5)]), None, + Some(vec![Some(9), None, None, None, None, Some(1)]), + Some(vec![Some(10), None, Some(1), Some(2), Some(3)]), + Some(vec![Some(10), Some(1), Some(2), Some(3)]), ]); assert_eq!(got.to_data(), expected.to_data()); Ok(()) } #[test] - fn negative_positions() -> Result<()> { - let got = run( - vec![ - Some(vec![Some(1), Some(2), Some(3)]), - Some(vec![Some(4), Some(5)]), - Some(vec![Some(1)]), - None, - ], - vec![Some(-2), Some(-1), Some(-3), Some(-1)], - vec![Some(10), Some(20), Some(100), Some(30)], - false, - )?; - let expected = list(vec![ - Some(vec![Some(1), Some(2), Some(10), Some(3)]), - Some(vec![Some(4), Some(5), Some(20)]), - Some(vec![Some(100), None, Some(1)]), - None, - ]); - assert_eq!(got.to_data(), expected.to_data()); - Ok(()) + fn min_index_exceeds_size_limit() { + // -i32::MIN overflows i32; Spark reports 2147483648 (2147483649 in legacy mode). + for (legacy, size) in [(false, "2147483648"), (true, "2147483649")] { + let err = run( + vec![Some(vec![Some(1), Some(2), Some(3)])], + vec![Some(i32::MIN)], + vec![Some(10)], + legacy, + ) + .unwrap_err() + .to_string(); + assert!(err.contains("COLLECTION_SIZE_LIMIT_EXCEEDED"), "{err}"); + assert!( + err.contains(&format!("create array with {size} elements")), + "{err}" + ); + } } #[test] - fn legacy_negative_index() -> Result<()> { - let got = run( - vec![ - Some(vec![Some(1), Some(2), Some(3)]), - Some(vec![Some(4), Some(5)]), - None, - ], - vec![Some(-1), Some(-1), Some(-1)], - vec![Some(10), Some(20), Some(30)], - true, - )?; + fn sliced_list() -> Result<()> { + // Row offsets of a sliced list do not start at zero. + let full = list(vec![ + Some(vec![Some(1), Some(2), Some(3)]), + Some(vec![Some(4), Some(5)]), + None, + Some(vec![Some(1)]), + Some(vec![Some(7)]), + ]); + let sliced = full.slice(1, 3); + let field = Arc::new(Field::new_list_field(DataType::Int32, true)); + let item: ArrayRef = Arc::new(Int32Array::from(vec![20, 30, 100])); + let pos = Int32Array::from(vec![-1, 1, -3]); + let got = array_insert(&sliced, &item, &pos, false, field)?; let expected = list(vec![ - Some(vec![Some(1), Some(2), Some(10), Some(3)]), - Some(vec![Some(4), Some(20), Some(5)]), + Some(vec![Some(4), Some(5), Some(20)]), None, + Some(vec![Some(100), None, Some(1)]), ]); assert_eq!(got.to_data(), expected.to_data()); Ok(()) } - #[test] - fn null_item_is_inserted_and_null_pos_gives_null() -> Result<()> { - let got = run( - vec![Some(vec![Some(0), None, Some(0)]), Some(vec![Some(1)])], - vec![Some(1), None], - vec![None, Some(9)], - false, - )?; - let expected = list(vec![Some(vec![None, Some(0), None, Some(0)]), None]); - assert_eq!(got.to_data(), expected.to_data()); - Ok(()) - } - - #[test] - fn zero_index_is_an_error() { - let err = run( - vec![Some(vec![Some(1)])], - vec![Some(0)], - vec![Some(1)], - false, - ) - .unwrap_err(); - assert!(err.to_string().contains("INVALID_INDEX_OF_ZERO")); - } - - #[test] - fn size_limit_is_checked_before_allocating() { - for pos in [i32::MAX, i32::MIN] { - let err = run( - vec![Some(vec![Some(1)])], - vec![Some(pos)], - vec![Some(1)], - false, - ) - .unwrap_err(); - assert!(err.to_string().contains("COLLECTION_SIZE_LIMIT_EXCEEDED")); - } - } - #[test] fn coerce_types_rules() -> Result<()> { let f = SparkArrayInsert::new(); @@ -443,6 +459,48 @@ mod tests { let got = f.coerce_types(&[list_of(DataType::Utf8), DataType::Int32, DataType::Null])?; assert_eq!(got[2], DataType::Utf8); + // Utf8View / LargeUtf8 elements accept a Utf8 item + for elem in [DataType::Utf8View, DataType::LargeUtf8] { + let got = f.coerce_types(&[ + list_of(elem.clone()), + DataType::Int64, + DataType::Utf8, + ])?; + assert_eq!(got, vec![list_of(elem.clone()), DataType::Int32, elem]); + } + // timestamps of different units share a common type + let got = f.coerce_types(&[ + list_of(DataType::Timestamp(TimeUnit::Microsecond, None)), + DataType::Int32, + DataType::Timestamp(TimeUnit::Second, None), + ])?; + assert!(matches!(got[2], DataType::Timestamp(_, None))); + assert_eq!(got[0], list_of(got[2].clone())); + // nested types that differ only in nullability + let inner_nullable = list_of(DataType::Int32); + let inner_non_null = + DataType::List(Arc::new(Field::new_list_field(DataType::Int32, false))); + let got = + f.coerce_types(&[list_of(inner_nullable), DataType::Int32, inner_non_null])?; + assert_eq!(got[0], list_of(got[2].clone())); + // a string item does not widen numeric or nested numeric elements, + // matching Spark's ARRAY_FUNCTION_DIFF_TYPES error + assert!( + f.coerce_types(&[list_of(DataType::Int32), DataType::Int32, DataType::Utf8]) + .is_err() + ); + assert!( + f.coerce_types(&[ + list_of(list_of(DataType::Int32)), + DataType::Int32, + list_of(DataType::Utf8), + ]) + .is_err() + ); + assert!( + f.coerce_types(&[list_of(DataType::Binary), DataType::Int32, DataType::Utf8]) + .is_err() + ); // non-integer index and non-array first argument are rejected assert!( f.coerce_types(&[list_of(DataType::Int32), DataType::Utf8, DataType::Int32]) diff --git a/crates/sqllogictest/test_files/array/array_insert.slt b/crates/sqllogictest/test_files/array/array_insert.slt index f43c9243..98313721 100644 --- a/crates/sqllogictest/test_files/array/array_insert.slt +++ b/crates/sqllogictest/test_files/array/array_insert.slt @@ -4,7 +4,8 @@ # Tests for Spark-compatible array_insert # Reference: https://spark.apache.org/docs/latest/api/sql/index.html#array_insert -# Expected values come from the Spark docs and Comet's Spark-verified tests. +# Expected values come from the Spark docs and Comet's Spark-verified tests, +# and were checked against Spark 4.2.0. # Examples from the Spark documentation query ? @@ -33,7 +34,24 @@ SELECT array_insert(make_array(4, 5), 1, 20); ---- [20, 4, 5] +# Directly after the last element: no padding +query ? +SELECT array_insert(make_array(1, 2, 3), 4, 10); +---- +[1, 2, 3, 10] + +# Insert into an array holding only a NULL +query ? +SELECT array_insert(make_array(NULL::INT), 1, 30); +---- +[30, NULL] + # Beyond the end pads with NULLs +query ? +SELECT array_insert(make_array(1, 2, 3), 5, 100); +---- +[1, 2, 3, NULL, 100] + query ? SELECT array_insert(make_array(1, 2, 3), 6, 100); ---- @@ -56,6 +74,22 @@ SELECT array_insert(make_array(1), -3, 100); ---- [100, NULL, 1] +query ? +SELECT array_insert(make_array(1), -5, 9); +---- +[9, NULL, NULL, NULL, 1] + +# Exactly -(len + 1) inserts first with no padding; one further pads one NULL +query ? +SELECT array_insert(make_array(1, 2, 3), -4, 10); +---- +[10, 1, 2, 3] + +query ? +SELECT array_insert(make_array(1, 2, 3), -5, 10); +---- +[10, NULL, 1, 2, 3] + # A NULL item is inserted as a NULL element query ? SELECT array_insert(make_array(0, NULL, 0), 1, NULL); @@ -73,13 +107,16 @@ SELECT array_insert(make_array(1, 2), NULL::INT, 5); ---- NULL -# Table data, with per-row positions +# Table data, with per-row positions and items statement ok CREATE TABLE array_insert_t AS VALUES (make_array(1, 2, 3), 2, 10), (make_array(4, 5), -1, 20), (NULL, 1, 30), - (make_array(7), NULL, 40); + (make_array(7), NULL, 40), + (make_array(0, NULL, 0), 1, NULL), + (make_array(1), -3, 100), + (make_array(1, 2, 3), 6, 100); query ? SELECT array_insert(column1, column2, column3) FROM array_insert_t; @@ -88,6 +125,13 @@ SELECT array_insert(column1, column2, column3) FROM array_insert_t; [4, 5, 20] NULL NULL +[NULL, 0, NULL, 0] +[100, NULL, 1] +[1, 2, 3, NULL, NULL, 100] + +# Index 0 is invalid for a column position too +statement error INVALID_INDEX_OF_ZERO +SELECT array_insert(column1, 0, column3) FROM array_insert_t WHERE column1 IS NOT NULL; # Index 0 is invalid statement error INVALID_INDEX_OF_ZERO @@ -101,12 +145,64 @@ SELECT array_insert(make_array(1, 2), 2147483647, 5); statement error SELECT array_insert(make_array(1, 2), 'a', 5); +# The item must share a type family with the elements +statement error array_insert cannot insert a value of type Utf8 into an array of Int64 +SELECT array_insert(make_array(1), 1, 'x'); + # String elements query ? SELECT array_insert(make_array('a', 'c'), 2, 'b'); ---- [a, b, c] +# NULL padding with string elements +query ? +SELECT array_insert(make_array('a', 'b'), 5, 'c'); +---- +[a, b, NULL, NULL, c] + +query ? +SELECT array_insert(make_array('a'), -3, 'z'); +---- +[z, NULL, a] + +# Utf8View elements (parquet's default for strings) with a Utf8 item +query ? +SELECT array_insert(arrow_cast(make_array('a', 'b'), 'List(Utf8View)'), 1, 'x'); +---- +[x, a, b] + +query T +SELECT arrow_typeof(array_insert(arrow_cast(make_array('a', 'b'), 'List(Utf8View)'), 1, 'x')); +---- +List(Utf8View) + +statement ok +CREATE TABLE array_insert_view AS + SELECT arrow_cast(make_array('a', 'b'), 'List(Utf8View)') AS arr; + +query ? +SELECT array_insert(arr, -1, 'x') FROM array_insert_view; +---- +[a, b, x] + +# LargeUtf8 elements with a Utf8 item +query ? +SELECT array_insert(arrow_cast(make_array('a', 'b'), 'List(LargeUtf8)'), 2, 'x'); +---- +[a, x, b] + +# NULL padding with nested elements +query ? +SELECT array_insert(make_array(make_array(1)), 3, make_array(2)); +---- +[[1], NULL, [2]] + +query ? +SELECT array_insert(make_array(make_array(1)), -3, make_array(2)); +---- +[[2], NULL, [1]] + # The item widens to the common numeric type query ? SELECT array_insert(make_array(arrow_cast(1, 'Int32')), 1, arrow_cast(3000000000, 'Int64')); From fcf2bf8d0c342ed7c06986be9cb12e13730cd439 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 13 Nov 2026 14:20:54 -0700 Subject: [PATCH 05/10] docs(spark): document array_insert with #[user_doc] Co-Authored-By: Claude Opus 5.5 --- .../spark/src/function/array/array_insert.rs | 61 ++++++++++++++----- 1 file changed, 46 insertions(+), 15 deletions(-) diff --git a/crates/spark/src/function/array/array_insert.rs b/crates/spark/src/function/array/array_insert.rs index 0a01c66d..fb41fd95 100644 --- a/crates/spark/src/function/array/array_insert.rs +++ b/crates/spark/src/function/array/array_insert.rs @@ -17,28 +17,55 @@ use datafusion_expr::type_coercion::binary::{ binary_numeric_coercion, type_union_resolution, }; use datafusion_expr::{ - ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, - Volatility, + ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, + Signature, Volatility, }; +use datafusion_macros::user_doc; /// `ByteArrayUtils.MAX_ROUNDED_ARRAY_LENGTH` (`Integer.MAX_VALUE - 15`) in Spark. const MAX_ROUNDED_ARRAY_LENGTH: i64 = 2_147_483_632; -/// Spark-compatible `array_insert(array, pos, item)`. -/// +/// Spark-compatible `array_insert` expression. /// -/// - Indexes are 1-based. A negative index counts from the end, where `-1` -/// inserts after the last element. -/// - An index past the end pads with NULLs. A negative index before the start -/// places the item first, then NULL padding, then the original elements. -/// - The result is NULL when the array or the index is NULL. A NULL item is -/// inserted as a NULL element. -/// - Index `0` is an error. -/// - The index may be any integer type and is cast to INT, because DataFusion -/// types bare integer literals as BIGINT. +/// /// -/// Spark's `spark.sql.legacy.negativeIndexInArrayInsert` (default `false`) makes -/// `-1` insert before the last element. Use [`Self::with_legacy_negative_index`]. +/// Use [`Self::with_legacy_negative_index`] for Spark's +/// `spark.sql.legacy.negativeIndexInArrayInsert`. +#[user_doc( + doc_section(label = "Array Functions"), + description = "Returns `array` with `item` inserted at index `pos`. DataFusion has no built-in `array_insert`. \ + Indexes are 1-based, and a negative index counts from the end, so -1 inserts after the last element. \ + An index past the end pads with NULLs before the item. A negative index before the start places the item first, \ + then NULL padding, then the original elements. Returns NULL if `array` or `pos` is NULL. \ + An index of 0 is an error (`INVALID_INDEX_OF_ZERO`), as is a result longer than Spark's maximum array length \ + (`COLLECTION_SIZE_LIMIT_EXCEEDED`). With Spark's `spark.sql.legacy.negativeIndexInArrayInsert`, \ + -1 inserts before the last element instead.", + syntax_example = "array_insert(array, pos, item)", + sql_example = r#"```sql +> SELECT array_insert(make_array(1, 2, 3), 2, 10); +[1, 10, 2, 3] +> SELECT array_insert(make_array(5, 4, 3, 2), -1, 1); +[5, 4, 3, 2, 1] +> SELECT array_insert(make_array(1, 2, 3), 5, 100); +[1, 2, 3, NULL, 100] +> SELECT array_insert(make_array(1), -3, 100); +[100, NULL, 1] +> SELECT array_insert(make_array(1, 2), NULL::INT, 5); +NULL +```"#, + argument( + name = "array", + description = "Array expression to insert into. NULL returns NULL." + ), + argument( + name = "pos", + description = "Integer index, 1-based, of any integer type (cast to INT). A negative value counts from the end. 0 is an error. NULL returns NULL." + ), + argument( + name = "item", + description = "Value to insert. Must share a type family with the array's elements. A NULL item is inserted as a NULL element." + ) +)] #[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkArrayInsert { signature: Signature, @@ -123,6 +150,10 @@ impl ScalarUDFImpl for SparkArrayInsert { &self.signature } + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + fn return_type(&self, _arg_types: &[DataType]) -> Result { internal_err!("return_field_from_args should be used instead") } From 02bedc98b85ee5eec90da6689934e2b00998e3fd Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 15 Nov 2026 11:31:24 -0700 Subject: [PATCH 06/10] fix(spark): match Spark's array_insert type rules and error messages - Reject element/item pairs Spark's findTightestCommonType has no type for: decimal with float, decimals of different precision or scale, and an integer with a decimal that cannot hold its range. Recurse into maps too. - Prefix type errors with Spark's DATATYPE_MISMATCH.ARRAY_FUNCTION_DIFF_TYPES and UNEXPECTED_INPUT_TYPE classes and Spark type names. - Make with_legacy_negative_index a chainable builder. - Return a scalar for all-scalar input, size the child buffer exactly, and avoid a panic on an unexpected index type. - Add slt coverage for decimals, maps, DATE with TIMESTAMP, SMALLINT/TINYINT and out-of-range BIGINT positions, and a column position over the size limit; assert full error text. Co-Authored-By: Claude Opus 5.5 --- .../spark/src/function/array/array_insert.rs | 450 ++++++++++++++---- .../test_files/array/array_insert.slt | 104 +++- 2 files changed, 445 insertions(+), 109 deletions(-) diff --git a/crates/spark/src/function/array/array_insert.rs b/crates/spark/src/function/array/array_insert.rs index fb41fd95..90877d73 100644 --- a/crates/spark/src/function/array/array_insert.rs +++ b/crates/spark/src/function/array/array_insert.rs @@ -4,15 +4,17 @@ use std::sync::Arc; -use crate::function::error_utils::unsupported_data_type_exec_err; use arrow::array::{ Array, ArrayRef, AsArray, Capacities, GenericListArray, Int32Array, MutableArrayData, OffsetSizeTrait, make_array, }; use arrow::buffer::{NullBuffer, OffsetBuffer}; -use arrow::datatypes::{DataType, Field, FieldRef}; +use arrow::datatypes::{DataType, Field, FieldRef, Int32Type}; use datafusion_common::utils::take_function_args; -use datafusion_common::{Result, exec_err, internal_err}; +use datafusion_common::{ + DataFusionError, Result, ScalarValue, exec_err, internal_err, plan_datafusion_err, + plan_err, +}; use datafusion_expr::type_coercion::binary::{ binary_numeric_coercion, type_union_resolution, }; @@ -39,7 +41,8 @@ const MAX_ROUNDED_ARRAY_LENGTH: i64 = 2_147_483_632; then NULL padding, then the original elements. Returns NULL if `array` or `pos` is NULL. \ An index of 0 is an error (`INVALID_INDEX_OF_ZERO`), as is a result longer than Spark's maximum array length \ (`COLLECTION_SIZE_LIMIT_EXCEEDED`). With Spark's `spark.sql.legacy.negativeIndexInArrayInsert`, \ - -1 inserts before the last element instead.", + -1 inserts before the last element instead, and a negative index before the start pads one more NULL, \ + so -(n+1) on an array of n elements gives the item, one NULL, then the elements.", syntax_example = "array_insert(array, pos, item)", sql_example = r#"```sql > SELECT array_insert(make_array(1, 2, 3), 2, 10); @@ -80,15 +83,23 @@ impl Default for SparkArrayInsert { impl SparkArrayInsert { pub fn new() -> Self { - Self::with_legacy_negative_index(false) - } - - pub fn with_legacy_negative_index(legacy_negative_index: bool) -> Self { Self { signature: Signature::user_defined(Volatility::Immutable), - legacy_negative_index, + legacy_negative_index: false, } } + + /// Enables Spark's legacy negative index behavior + /// (`spark.sql.legacy.negativeIndexInArrayInsert`): -1 inserts before the + /// last element, and a negative index before the start pads one more NULL. + /// + /// For example `SparkArrayInsert::new().with_legacy_negative_index(true)`. + /// Once `SparkOptions` lands (#92), a DataFusion session would read this + /// from `spark.sql.legacy.negativeIndexInArrayInsert` instead. + pub fn with_legacy_negative_index(mut self, legacy_negative_index: bool) -> Self { + self.legacy_negative_index = legacy_negative_index; + self + } } /// The element field of a list type. @@ -101,12 +112,40 @@ fn list_element_field(data_type: &DataType) -> Option<&FieldRef> { } } -/// Whether `a` and `b` pair a string or binary type with a type of another -/// kind, at the top level or inside a list or struct. +/// The precision and scale of a decimal type. +fn decimal_precision_scale(t: &DataType) -> Option<(u8, i8)> { + match t { + DataType::Decimal32(p, s) + | DataType::Decimal64(p, s) + | DataType::Decimal128(p, s) + | DataType::Decimal256(p, s) => Some((*p, *s)), + _ => None, + } +} + +/// The precision Spark's `DecimalType.forType` gives an integer type. +fn integer_digits(t: &DataType) -> Option { + match t { + DataType::Int8 | DataType::UInt8 => Some(3), + DataType::Int16 | DataType::UInt16 => Some(5), + DataType::Int32 | DataType::UInt32 => Some(10), + DataType::Int64 | DataType::UInt64 => Some(20), + _ => None, + } +} + +/// Whether Spark's `findTightestCommonType` has no common type for `a` and +/// `b` where DataFusion's coercion finds one, at the top level or inside a +/// list, struct or map. /// -/// DataFusion's `type_union_resolution` coerces such pairs (for example INT -/// and STRING to STRING), but Spark's `findTightestCommonType` does not. -fn mixes_string_or_binary(a: &DataType, b: &DataType) -> bool { +/// DataFusion coerces these pairs (for example INT and STRING to STRING, or +/// two decimals to a wider decimal), but Spark rejects them: +/// - a string or binary type with a type of another kind, +/// - a decimal with a floating point type, +/// - two decimals of different precision or scale, +/// - an integer with a decimal that cannot hold the whole integer range +/// (Spark's `DecimalType.isWiderThan`). +fn spark_rejects_pair(a: &DataType, b: &DataType) -> bool { #[derive(PartialEq)] enum Kind { String, @@ -115,8 +154,6 @@ fn mixes_string_or_binary(a: &DataType, b: &DataType) -> bool { } fn kind(t: &DataType) -> Kind { match t { - DataType::Dictionary(_, v) => kind(v), - DataType::RunEndEncoded(_, v) => kind(v.data_type()), DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => Kind::String, DataType::Binary | DataType::LargeBinary @@ -125,20 +162,110 @@ fn mixes_string_or_binary(a: &DataType, b: &DataType) -> bool { _ => Kind::Other, } } - if *a == DataType::Null || *b == DataType::Null { - return false; - } - if let (Some(fa), Some(fb)) = (list_element_field(a), list_element_field(b)) { - return mixes_string_or_binary(fa.data_type(), fb.data_type()); - } - if let (DataType::Struct(fa), DataType::Struct(fb)) = (a, b) { - return fa.iter().any(|fa| { + match (a, b) { + (DataType::Null, _) | (_, DataType::Null) => false, + (DataType::Dictionary(_, v), other) | (other, DataType::Dictionary(_, v)) => { + spark_rejects_pair(v, other) + } + (DataType::RunEndEncoded(_, v), other) + | (other, DataType::RunEndEncoded(_, v)) => { + spark_rejects_pair(v.data_type(), other) + } + (DataType::Struct(fa), DataType::Struct(fb)) => fa.iter().any(|fa| { fb.iter() .find(|fb| fb.name() == fa.name()) - .is_some_and(|fb| mixes_string_or_binary(fa.data_type(), fb.data_type())) - }); + .is_some_and(|fb| spark_rejects_pair(fa.data_type(), fb.data_type())) + }), + // A map's entries are a struct of the key and the value. + (DataType::Map(ea, _), DataType::Map(eb, _)) => { + match (ea.data_type(), eb.data_type()) { + (DataType::Struct(ka), DataType::Struct(kb)) => ka + .iter() + .zip(kb.iter()) + .any(|(fa, fb)| spark_rejects_pair(fa.data_type(), fb.data_type())), + _ => false, + } + } + _ => { + if let (Some(fa), Some(fb)) = (list_element_field(a), list_element_field(b)) { + return spark_rejects_pair(fa.data_type(), fb.data_type()); + } + if kind(a) != kind(b) { + return true; + } + let (decimal, other) = + match (decimal_precision_scale(a), decimal_precision_scale(b)) { + (Some(da), Some(db)) => return da != db, + (Some(d), None) => (d, b), + (None, Some(d)) => (d, a), + (None, None) => return false, + }; + let (precision, scale) = decimal; + if other.is_floating() { + true + } else if let Some(digits) = integer_digits(other) { + scale < 0 || i16::from(precision) - i16::from(scale) < digits + } else { + false + } + } } - kind(a) != kind(b) +} + +/// The Spark SQL name of a type, as Spark prints it in error messages. +fn spark_type_name(t: &DataType) -> String { + match t { + DataType::Null => "VOID".to_string(), + DataType::Boolean => "BOOLEAN".to_string(), + DataType::Int8 => "TINYINT".to_string(), + DataType::Int16 => "SMALLINT".to_string(), + DataType::Int32 => "INT".to_string(), + DataType::Int64 => "BIGINT".to_string(), + DataType::Float32 => "FLOAT".to_string(), + DataType::Float64 => "DOUBLE".to_string(), + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => "STRING".to_string(), + DataType::Binary + | DataType::LargeBinary + | DataType::BinaryView + | DataType::FixedSizeBinary(_) => "BINARY".to_string(), + DataType::Date32 | DataType::Date64 => "DATE".to_string(), + DataType::Timestamp(_, None) => "TIMESTAMP_NTZ".to_string(), + DataType::Timestamp(_, Some(_)) => "TIMESTAMP".to_string(), + DataType::Dictionary(_, v) => spark_type_name(v), + DataType::RunEndEncoded(_, v) => spark_type_name(v.data_type()), + DataType::List(f) | DataType::LargeList(f) | DataType::FixedSizeList(f, _) => { + format!("ARRAY<{}>", spark_type_name(f.data_type())) + } + DataType::Struct(fields) => { + let fields: Vec = fields + .iter() + .map(|f| format!("{}: {}", f.name(), spark_type_name(f.data_type()))) + .collect(); + format!("STRUCT<{}>", fields.join(", ")) + } + DataType::Map(entries, _) => match entries.data_type() { + DataType::Struct(kv) if kv.len() == 2 => format!( + "MAP<{}, {}>", + spark_type_name(kv[0].data_type()), + spark_type_name(kv[1].data_type()) + ), + _ => t.to_string(), + }, + other => match decimal_precision_scale(other) { + Some((p, s)) => format!("DECIMAL({p},{s})"), + None => other.to_string(), + }, + } +} + +/// Spark's error when the first argument is not an array, or the item does +/// not share a type with the array's elements. +fn diff_types_err(array: &DataType, item: &DataType) -> DataFusionError { + plan_datafusion_err!( + "[DATATYPE_MISMATCH.ARRAY_FUNCTION_DIFF_TYPES] Input to `array_insert` should have been \"ARRAY\" followed by a value with same element type, but it's [\"{}\", \"{}\"].", + spark_type_name(array), + spark_type_name(item) + ) } impl ScalarUDFImpl for SparkArrayInsert { @@ -186,24 +313,17 @@ impl ScalarUDFImpl for SparkArrayInsert { let (elem, wrap): (&FieldRef, fn(FieldRef) -> DataType) = match array { DataType::List(f) | DataType::FixedSizeList(f, _) => (f, DataType::List), DataType::LargeList(f) => (f, DataType::LargeList), - _ => { - return Err(unsupported_data_type_exec_err( - self.name(), - "an array as first argument", - array, - )); - } + _ => return Err(diff_types_err(array, item)), }; // Spark takes an INT index. DataFusion types bare integer literals as // BIGINT, so accept every integer type and cast to INT; a value outside // the INT range then fails the cast instead of being silently truncated. if !pos.is_integer() { - return Err(unsupported_data_type_exec_err( - self.name(), - "an INT index as second argument", - pos, - )); + return plan_err!( + "[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] The second parameter of `array_insert` requires the \"INT\" type, however it has the type \"{}\".", + spark_type_name(pos) + ); } // Spark uses the tightest common type of the element and the item. @@ -212,20 +332,18 @@ impl ScalarUDFImpl for SparkArrayInsert { // different units, and nested types that differ only in nullability. let elem_type = elem.data_type(); let common = if elem_type == item || *item == DataType::Null { - elem_type.clone() + Some(elem_type.clone()) } else if *elem_type == DataType::Null { - item.clone() + Some(item.clone()) + } else if spark_rejects_pair(elem_type, item) { + None } else if let Some(common) = binary_numeric_coercion(elem_type, item) { - common - } else if let Some(common) = - type_union_resolution(&[elem_type.clone(), item.clone()]) - .filter(|_| !mixes_string_or_binary(elem_type, item)) - { - common + Some(common) } else { - return exec_err!( - "array_insert cannot insert a value of type {item} into an array of {elem_type}" - ); + type_union_resolution(&[elem_type.clone(), item.clone()]) + }; + let Some(common) = common else { + return Err(diff_types_err(array, item)); }; let new_elem = @@ -234,9 +352,18 @@ impl ScalarUDFImpl for SparkArrayInsert { } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + let all_scalar = args + .args + .iter() + .all(|arg| matches!(arg, ColumnarValue::Scalar(_))); let arrays = ColumnarValue::values_to_arrays(&args.args)?; let [list, pos, item] = take_function_args(self.name(), arrays)?; - let pos = pos.as_primitive::(); + let Some(pos) = pos.as_primitive_opt::() else { + return internal_err!( + "array_insert expects an Int32 index, got {}", + pos.data_type() + ); + }; let legacy = self.legacy_negative_index; let Some(elem_field) = list_element_field(args.return_field.data_type()) else { return internal_err!( @@ -262,7 +389,13 @@ impl ScalarUDFImpl for SparkArrayInsert { )?), other => return internal_err!("array_insert expects a list, got {other}"), }; - Ok(ColumnarValue::Array(result)) + if all_scalar { + Ok(ColumnarValue::Scalar(ScalarValue::try_from_array( + &result, 0, + )?)) + } else { + Ok(ColumnarValue::Array(result)) + } } } @@ -322,40 +455,22 @@ fn array_insert( legacy: bool, elem_field: FieldRef, ) -> Result> { - let values = list.values().to_data(); - let item_data = item.to_data(); - let mut out = MutableArrayData::with_capacities( - vec![&values, &item_data], - true, - Capacities::Array(values.len() + item_data.len()), - ); - + // First pass: validate every row and compute the output offsets, so the + // child data can be allocated once at its final size. + let mut placements = Vec::with_capacity(list.len()); let mut offsets = Vec::with_capacity(list.len() + 1); offsets.push(O::usize_as(0)); let mut total = 0usize; - for row in 0..list.len() { // A NULL array or NULL index gives a NULL row with no child values. - if !list.is_null(row) && !pos.is_null(row) { - let start = list.value_offsets()[row].as_usize(); - let end = list.value_offsets()[row + 1].as_usize(); - let p = place(pos.value(row), end - start, legacy)?; - - out.try_extend(0, start, start + p.before)?; - if p.trailing_nulls > 0 { - // Positive index past the end: original elements, NULLs, item. - out.try_extend(0, start + p.before, end)?; - out.try_extend_nulls(p.trailing_nulls)?; - out.try_extend(1, row, row + 1)?; - } else { - // Otherwise: item, then (for a negative index before the start) - // NULL padding, then the remaining original elements. - out.try_extend(1, row, row + 1)?; - out.try_extend_nulls(p.leading_nulls)?; - out.try_extend(0, start + p.before, end)?; - } + let placement = if list.is_null(row) || pos.is_null(row) { + None + } else { + let p = place(pos.value(row), list.value_length(row).as_usize(), legacy)?; total += p.new_len; - } + Some(p) + }; + placements.push(placement); // A `List` (i32 offsets) can overflow across rows even though each row is // within Spark's per-array limit. let Some(offset) = O::from_usize(total) else { @@ -366,6 +481,32 @@ fn array_insert( offsets.push(offset); } + let values = list.values().to_data(); + let item_data = item.to_data(); + let mut out = MutableArrayData::with_capacities( + vec![&values, &item_data], + true, + Capacities::Array(total), + ); + for (row, p) in placements.iter().enumerate() { + let Some(p) = p else { continue }; + let start = list.value_offsets()[row].as_usize(); + let end = list.value_offsets()[row + 1].as_usize(); + out.try_extend(0, start, start + p.before)?; + if p.trailing_nulls > 0 { + // Positive index past the end: original elements, NULLs, item. + out.try_extend(0, start + p.before, end)?; + out.try_extend_nulls(p.trailing_nulls)?; + out.try_extend(1, row, row + 1)?; + } else { + // Otherwise: item, then (for a negative index before the start) + // NULL padding, then the remaining original elements. + out.try_extend(1, row, row + 1)?; + out.try_extend_nulls(p.leading_nulls)?; + out.try_extend(0, start + p.before, end)?; + } + } + GenericListArray::try_new( elem_field, OffsetBuffer::new(offsets.into()), @@ -399,7 +540,43 @@ mod tests { // Most behavior is covered by array_insert.slt. These cover what SQL // cannot reach: legacy mode (a session config), `i32::MIN`, and sliced - // input. Expected values were checked against Spark 4.2.0. + // input. Expected values were verified with pyspark==4.2.0. + #[test] + fn legacy_builder_through_udf() -> Result<()> { + let list_type = + DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))); + let invoke = |udf: SparkArrayInsert| { + udf.invoke_with_args(ScalarFunctionArgs { + args: vec![ + ColumnarValue::Scalar(ScalarValue::List(Arc::new(list(vec![Some( + vec![Some(1), Some(2), Some(3)], + )])))), + ColumnarValue::Scalar(ScalarValue::Int32(Some(-1))), + ColumnarValue::Scalar(ScalarValue::Int32(Some(10))), + ], + arg_fields: vec![], + number_rows: 1, + return_field: Arc::new(Field::new("r", list_type.clone(), true)), + config_options: Default::default(), + }) + }; + let expect = |values: Vec>| { + ScalarValue::List(Arc::new(list(vec![Some(values)]))) + }; + // All-scalar input gives a scalar result. + let ColumnarValue::Scalar(got) = invoke(SparkArrayInsert::new())? else { + panic!("expected a scalar result"); + }; + assert_eq!(got, expect(vec![Some(1), Some(2), Some(3), Some(10)])); + let ColumnarValue::Scalar(got) = + invoke(SparkArrayInsert::new().with_legacy_negative_index(true))? + else { + panic!("expected a scalar result"); + }; + assert_eq!(got, expect(vec![Some(1), Some(2), Some(10), Some(3)])); + Ok(()) + } + #[test] fn legacy_negative_index() -> Result<()> { let got = run( @@ -432,7 +609,8 @@ mod tests { #[test] fn min_index_exceeds_size_limit() { - // -i32::MIN overflows i32; Spark reports 2147483648 (2147483649 in legacy mode). + // -i32::MIN overflows i32. Spark 4.2.0 reports 2147483648 (2147483649 in + // legacy mode) since SPARK-56567; Spark 4.1.x and older overflow instead. for (legacy, size) in [(false, "2147483648"), (true, "2147483649")] { let err = run( vec![Some(vec![Some(1), Some(2), Some(3)])], @@ -514,33 +692,101 @@ mod tests { let got = f.coerce_types(&[list_of(inner_nullable), DataType::Int32, inner_non_null])?; assert_eq!(got[0], list_of(got[2].clone())); - // a string item does not widen numeric or nested numeric elements, - // matching Spark's ARRAY_FUNCTION_DIFF_TYPES error + // Pairs with no tightest common type in Spark fail with + // ARRAY_FUNCTION_DIFF_TYPES: a string item with numeric or nested + // numeric elements, string with binary, and the decimal rules. + let diff_types = |elem: DataType, item: DataType| { + let err = f + .coerce_types(&[list_of(elem), DataType::Int32, item]) + .unwrap_err() + .to_string(); + assert!( + err.contains("[DATATYPE_MISMATCH.ARRAY_FUNCTION_DIFF_TYPES]"), + "{err}" + ); + err + }; + let err = diff_types(DataType::Int32, DataType::Utf8); assert!( - f.coerce_types(&[list_of(DataType::Int32), DataType::Int32, DataType::Utf8]) - .is_err() + err.contains(r#"but it's ["ARRAY", "STRING"]."#), + "{err}" ); - assert!( - f.coerce_types(&[ - list_of(list_of(DataType::Int32)), - DataType::Int32, - list_of(DataType::Utf8), - ]) - .is_err() + diff_types(list_of(DataType::Int32), list_of(DataType::Utf8)); + diff_types(DataType::Binary, DataType::Utf8); + diff_types(DataType::Decimal128(2, 1), DataType::Float64); + diff_types(DataType::Float32, DataType::Decimal128(5, 1)); + diff_types(DataType::Decimal128(5, 1), DataType::Decimal128(5, 2)); + diff_types(DataType::Decimal128(5, 1), DataType::Decimal128(6, 1)); + diff_types(DataType::Int64, DataType::Decimal128(5, 1)); + diff_types(DataType::Int64, DataType::Decimal128(20, 1)); + diff_types(DataType::Decimal128(9, 0), DataType::Int32); + diff_types( + list_of(DataType::Decimal128(5, 1)), + list_of(DataType::Decimal128(5, 2)), ); - assert!( - f.coerce_types(&[list_of(DataType::Binary), DataType::Int32, DataType::Utf8]) - .is_err() + // ... and inside a map's key or value + let map_of = |k: DataType, v: DataType| { + DataType::Map( + Arc::new(Field::new( + "entries", + DataType::Struct( + vec![Field::new("key", k, false), Field::new("value", v, true)] + .into(), + ), + false, + )), + false, + ) + }; + diff_types( + map_of(DataType::Int32, DataType::Utf8), + map_of(DataType::Int32, DataType::Int32), ); - // non-integer index and non-array first argument are rejected - assert!( - f.coerce_types(&[list_of(DataType::Int32), DataType::Utf8, DataType::Int32]) - .is_err() + diff_types( + map_of(DataType::Utf8, DataType::Int32), + map_of(DataType::Int32, DataType::Int32), ); + // A decimal that holds the whole integer range is the common type. + for (elem, item, common) in [ + ( + DataType::Int64, + DataType::Decimal128(20, 0), + DataType::Decimal128(20, 0), + ), + ( + DataType::Int32, + DataType::Decimal128(11, 1), + DataType::Decimal128(11, 1), + ), + ( + DataType::Decimal128(11, 1), + DataType::Int32, + DataType::Decimal128(11, 1), + ), + ( + DataType::Decimal128(5, 1), + DataType::Decimal128(5, 1), + DataType::Decimal128(5, 1), + ), + ] { + let got = f.coerce_types(&[list_of(elem), DataType::Int32, item])?; + assert_eq!(got[2], common); + } + // A non-integer index fails with UNEXPECTED_INPUT_TYPE; a non-array + // first argument with ARRAY_FUNCTION_DIFF_TYPES. + let err = f + .coerce_types(&[list_of(DataType::Int32), DataType::Utf8, DataType::Int32]) + .unwrap_err() + .to_string(); assert!( - f.coerce_types(&[DataType::Int32, DataType::Int32, DataType::Int32]) - .is_err() + err.contains(r#"[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] The second parameter of `array_insert` requires the "INT" type, however it has the type "STRING"."#), + "{err}" ); + let err = f + .coerce_types(&[DataType::Int32, DataType::Int32, DataType::Int32]) + .unwrap_err() + .to_string(); + assert!(err.contains(r#"but it's ["INT", "INT"]."#), "{err}"); Ok(()) } } diff --git a/crates/sqllogictest/test_files/array/array_insert.slt b/crates/sqllogictest/test_files/array/array_insert.slt index 98313721..8a4ec4df 100644 --- a/crates/sqllogictest/test_files/array/array_insert.slt +++ b/crates/sqllogictest/test_files/array/array_insert.slt @@ -4,8 +4,12 @@ # Tests for Spark-compatible array_insert # Reference: https://spark.apache.org/docs/latest/api/sql/index.html#array_insert -# Expected values come from the Spark docs and Comet's Spark-verified tests, -# and were checked against Spark 4.2.0. +# Expected values come from the Spark docs and Comet's Spark-verified tests. + +# Audited against Spark 4.2.0 on 2026-11-15. +# Expected values verified with pyspark==4.2.0. +# The PySpark runner skips most queries here (Spark has no make_array or +# arrow_typeof), so they were checked with equivalent Spark SQL using array(). # Examples from the Spark documentation query ? @@ -138,16 +142,102 @@ statement error INVALID_INDEX_OF_ZERO SELECT array_insert(make_array(1, 2), 0, 5); # Exceeding the maximum array length -statement error COLLECTION_SIZE_LIMIT_EXCEEDED +query error \[COLLECTION_SIZE_LIMIT_EXCEEDED\.FUNCTION\] Can't create array with 2147483647 elements which exceeding the array size limit 2147483632, unsuccessful try to create arrays in the function `array_insert`\. SELECT array_insert(make_array(1, 2), 2147483647, 5); +# ... also for a position from a column +statement ok +CREATE TABLE array_insert_big AS VALUES (make_array(1, 2), 2147483647, 5); + +query error \[COLLECTION_SIZE_LIMIT_EXCEEDED\.FUNCTION\] Can't create array with 2147483647 elements which exceeding the array size limit 2147483632, unsuccessful try to create arrays in the function `array_insert`\. +SELECT array_insert(column1, column2, column3) FROM array_insert_big; + # The index must be an integer -statement error +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The second parameter of `array_insert` requires the "INT" type, however it has the type "STRING"\. SELECT array_insert(make_array(1, 2), 'a', 5); -# The item must share a type family with the elements -statement error array_insert cannot insert a value of type Utf8 into an array of Int64 -SELECT array_insert(make_array(1), 1, 'x'); +# SMALLINT and TINYINT positions are accepted +query ? +SELECT array_insert(make_array(1::INT), 2::SMALLINT, 5::INT); +---- +[1, 5] + +query ? +SELECT array_insert(make_array(1::INT), (-3)::TINYINT, 5::INT); +---- +[5, NULL, 1] + +# Spark accepts only an INT position. A BIGINT position is accepted here (bare +# integer literals are BIGINT in DataFusion), but a value outside the INT range +# fails the cast instead of wrapping. Spark rejects any BIGINT position with +# DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE. +query error Can't cast value 3000000000 to type Int32 +SELECT array_insert(make_array(1), 3000000000, 5); + +# The first argument must be an array +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] Input to `array_insert` should have been "ARRAY" followed by a value with same element type, but it's \["INT", "INT"\]\. +SELECT array_insert(1::INT, 1, 5::INT); + +# The item must share a type with the elements +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] Input to `array_insert` should have been "ARRAY" followed by a value with same element type, but it's \["ARRAY", "STRING"\]\. +SELECT array_insert(make_array(1::INT), 1, 'x'); + +# Maps whose key or value types differ in kind +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] Input to `array_insert` should have been "ARRAY" followed by a value with same element type, but it's \["ARRAY>", "MAP"\]\. +SELECT array_insert(make_array(map([1::INT], ['a'])), 1, map([1::INT], [2::INT])); + +# Maps whose key and value types widen +query ? +SELECT array_insert(make_array(map([1::INT], [2::INT])), 1, map([1::BIGINT], [3::BIGINT])); +---- +[{1: 3}, {1: 2}] + +# Decimal elements +query ? +SELECT array_insert(make_array(1.5::DECIMAL(5,1)), 3, 2.5::DECIMAL(5,1)); +---- +[1.5, NULL, 2.5] + +query T +SELECT arrow_typeof(array_insert(make_array(1.5::DECIMAL(5,1)), 1, 2.5::DECIMAL(5,1))); +---- +List(Decimal128(5, 1)) + +# An integer item widens to a decimal element type that holds every INT value +query ? +SELECT array_insert(make_array(2.5::DECIMAL(11,1)), 1, 7::INT); +---- +[7.0, 2.5] + +# ... and an integer element type to such a decimal item +query ? +SELECT array_insert(make_array(1::BIGINT), 1, 2.5::DECIMAL(21,1)); +---- +[2.5, 1.0] + +# Spark has no tightest common type for a decimal and a floating point type, +# two different decimal types, or an integer and a narrower decimal +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] Input to `array_insert` should have been "ARRAY" followed by a value with same element type, but it's \["ARRAY", "DOUBLE"\]\. +SELECT array_insert(make_array(1.5::DECIMAL(2,1)), 1, 2.5::DOUBLE); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] Input to `array_insert` should have been "ARRAY" followed by a value with same element type, but it's \["ARRAY", "DECIMAL\(5,1\)"\]\. +SELECT array_insert(make_array(1.5::DOUBLE), 1, 2.5::DECIMAL(5,1)); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] Input to `array_insert` should have been "ARRAY" followed by a value with same element type, but it's \["ARRAY", "DECIMAL\(5,2\)"\]\. +SELECT array_insert(make_array(1.5::DECIMAL(5,1)), 1, 2.25::DECIMAL(5,2)); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] Input to `array_insert` should have been "ARRAY" followed by a value with same element type, but it's \["ARRAY", "DECIMAL\(5,1\)"\]\. +SELECT array_insert(make_array(1::BIGINT), 1, 2.5::DECIMAL(5,1)); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] Input to `array_insert` should have been "ARRAY" followed by a value with same element type, but it's \["ARRAY", "INT"\]\. +SELECT array_insert(make_array(2.5::DECIMAL(5,1)), 1, 7::INT); + +# DATE elements with a TIMESTAMP item widen to TIMESTAMP. DataFusion's +# TIMESTAMP literal has no time zone, like Spark's TIMESTAMP_NTZ. +query ? +SELECT array_insert(make_array(DATE '2024-01-01'), 3, TIMESTAMP '2024-01-02 03:04:05'); +---- +[2024-01-01T00:00:00, NULL, 2024-01-02T03:04:05] # String elements query ? From 01d8874a489db5b48b98f422b0c002c6825e9054 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 10:17:23 -0700 Subject: [PATCH 07/10] refactor(spark): share Spark's common-type rules between functions Move tightest_common_type from array_position into function/type_coercion, with an ANSI flag, a findTypeForComplex that widens map keys and values, Spark's isOrderable (which also rejects INTERVAL), and Spark type names for error messages. Move float_normalize up to function/ so functions outside array can use it. array_position now reports a map with different key or value types as INVALID_ORDERING_TYPE, rejects INTERVAL elements, and names Spark types and the SQLSTATE in its errors, as Spark does. --- .../src/function/array/array_contains.rs | 2 +- .../src/function/array/array_position.rs | 178 ++-------- crates/spark/src/function/array/mod.rs | 1 - .../function/{array => }/float_normalize.rs | 8 +- crates/spark/src/function/mod.rs | 2 + crates/spark/src/function/type_coercion.rs | 327 ++++++++++++++++++ .../test_files/array/array_position.slt | 15 +- 7 files changed, 382 insertions(+), 151 deletions(-) rename crates/spark/src/function/{array => }/float_normalize.rs (97%) create mode 100644 crates/spark/src/function/type_coercion.rs diff --git a/crates/spark/src/function/array/array_contains.rs b/crates/spark/src/function/array/array_contains.rs index 3802667e..029e6c09 100644 --- a/crates/spark/src/function/array/array_contains.rs +++ b/crates/spark/src/function/array/array_contains.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use super::float_normalize::normalize_floats; +use crate::function::float_normalize::normalize_floats; use arrow::array::{ Array, AsArray, BooleanArray, BooleanBufferBuilder, GenericListArray, OffsetSizeTrait, }; diff --git a/crates/spark/src/function/array/array_position.rs b/crates/spark/src/function/array/array_position.rs index 4ca52da9..d1a64701 100644 --- a/crates/spark/src/function/array/array_position.rs +++ b/crates/spark/src/function/array/array_position.rs @@ -2,13 +2,16 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use super::float_normalize::normalize_array; +use crate::function::float_normalize::normalize_array; +use crate::function::type_coercion::{ + is_orderable, spark_sql_type, tightest_common_type, +}; use arrow::array::{ Array, ArrayRef, AsArray, Int64Array, OffsetSizeTrait, make_comparator, }; use arrow::buffer::{NullBuffer, ScalarBuffer}; use arrow::compute::SortOptions; -use arrow::datatypes::{DataType, Field, Fields, TimeUnit}; +use arrow::datatypes::{DataType, Field}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, exec_err, plan_err}; use datafusion_expr::{ @@ -36,7 +39,7 @@ use std::sync::Arc; a date widens to a timestamp. Other combinations are an error, for example a string \ and a number, two different decimals, or structs whose field names or field types \ differ, as in Spark with ANSI mode off (with ANSI mode on, Spark widens struct field \ - types). A NULL-typed array and map elements are also an error.", + types). A NULL-typed array, and map or INTERVAL elements at any depth, are also an error.", syntax_example = "array_position(array, element)", sql_example = r#"```sql > SELECT array_position(array(3, 2, 1), 1); @@ -164,24 +167,31 @@ fn coerce_array_and_element( other => { return plan_err!( "[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] The first parameter of `{name}` \ - requires the \"ARRAY\" type, however it has the type {other}" + requires the \"ARRAY\" type, however it has the type \"{}\". SQLSTATE: 42K09", + spark_sql_type(other) ); } }; - let Some(common) = tightest_common_type(item.data_type(), element) else { + // Spark's `ArrayPosition` calls the non-ANSI `TypeCoercion.findTightestCommonType`. + let common = tightest_common_type(item.data_type(), element, false) + .filter(|_| struct_field_types_match(item.data_type(), element)); + let Some(common) = common else { return plan_err!( "[DATATYPE_MISMATCH.ARRAY_FUNCTION_DIFF_TYPES] Input to `{name}` should have \ been \"ARRAY\" followed by a value with same element type, but it's \ - [{array}, {element}]" + [\"{}\", \"{}\"]. SQLSTATE: 42K09", + spark_sql_type(array), + spark_sql_type(element) ); }; if common.is_null() { return null_type_err(name); } - if contains_map(&common) { + if !is_orderable(&common) { return plan_err!( "[DATATYPE_MISMATCH.INVALID_ORDERING_TYPE] The `{name}` does not support \ - ordering on type {common}" + ordering on type \"{}\". SQLSTATE: 42K09", + spark_sql_type(&common) ); } let item = Arc::new(Field::new(item.name(), common.clone(), true)); @@ -196,82 +206,30 @@ fn coerce_array_and_element( fn null_type_err(name: &str) -> Result { plan_err!( "[DATATYPE_MISMATCH.NULL_TYPE] Null typed values cannot be used as arguments of \ - `{name}`" + `{name}`. SQLSTATE: 42K09" ) } -/// Spark's `TypeCoercion.findTightestCommonType`, on the arrow types that Spark's types -/// map to. -fn tightest_common_type(left: &DataType, right: &DataType) -> Option { +/// With ANSI mode off, Spark cannot implicitly cast between structs whose field types +/// differ, so `array_position` rejects them although their tightest common type exists. +/// With ANSI mode on, Spark widens the field types. +fn struct_field_types_match(left: &DataType, right: &DataType) -> bool { use DataType::*; - if left == right { - return Some(left.clone()); - } - if let (Some(l), Some(r)) = (numeric_precedence(left), numeric_precedence(right)) { - return Some(if l >= r { left.clone() } else { right.clone() }); - } match (left, right) { - (Null, other) | (other, Null) => Some(other.clone()), - // Spark has a single string type and a single binary type. - (l, r) if is_string(l) && is_string(r) => Some(l.clone()), - (l, r) if is_binary(l) && is_binary(r) => Some(l.clone()), - // The decimal must be at least as wide as the integral type's DECIMAL(n, 0). - (int, Decimal128(p, s)) | (Decimal128(p, s), int) => { - let digits = integral_digits(int)?; - (*p as i16 - *s as i16 >= digits).then_some(Decimal128(*p, *s)) - } - (Date32, ts @ Timestamp(_, _)) | (ts @ Timestamp(_, _), Date32) => { - Some(ts.clone()) - } - // TIMESTAMP_NTZ and TIMESTAMP widen to TIMESTAMP. - (Timestamp(lu, ltz), Timestamp(ru, rtz)) => Some(Timestamp( - finer_unit(*lu, *ru), - ltz.clone().or_else(|| rtz.clone()), - )), ( - List(lf) | LargeList(lf) | FixedSizeList(lf, _), - List(rf) | LargeList(rf) | FixedSizeList(rf, _), - ) => { - let item = tightest_common_type(lf.data_type(), rf.data_type())?; - let field = Arc::new(Field::new(lf.name(), item, true)); - Some( - if matches!(left, LargeList(_)) || matches!(right, LargeList(_)) { - LargeList(field) - } else { - List(field) - }, - ) - } - // The fields must have the same names and types, ignoring nullability, in the same - // order. This follows Spark with ANSI mode off; with ANSI mode on, Spark also widens - // the field types. Spark compares the names ignoring case, but DataFusion cannot cast - // between structs whose field names differ only in case, so they must match exactly. - (Struct(lf), Struct(rf)) if lf.len() == rf.len() => { - let fields = lf + List(l) | LargeList(l) | FixedSizeList(l, _), + List(r) | LargeList(r) | FixedSizeList(r, _), + ) => struct_field_types_match(l.data_type(), r.data_type()), + // The entries are a struct of the key and the value, which are checked on their own. + (Map(l, _), Map(r, _)) => match (l.data_type(), r.data_type()) { + (Struct(l), Struct(r)) => l .iter() - .zip(rf.iter()) - .map(|(a, b)| { - if a.name() != b.name() || !same_type(a.data_type(), b.data_type()) { - return None; - } - let data_type = tightest_common_type(a.data_type(), b.data_type())?; - Some(Field::new( - a.name(), - data_type, - a.is_nullable() || b.is_nullable(), - )) - }) - .collect::>>()?; - Some(Struct(Fields::from(fields))) - } - (Map(lf, sorted), Map(rf, _)) => { - let entries = tightest_common_type(lf.data_type(), rf.data_type())?; - Some(Map( - Arc::new(Field::new(lf.name(), entries, false)), - *sorted, - )) - } - _ => None, + .zip(r.iter()) + .all(|(a, b)| struct_field_types_match(a.data_type(), b.data_type())), + _ => true, + }, + (Struct(_), Struct(_)) => same_type(left, right), + _ => true, } } @@ -296,72 +254,6 @@ fn same_type(left: &DataType, right: &DataType) -> bool { } } -fn is_string(data_type: &DataType) -> bool { - matches!( - data_type, - DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View - ) -} - -fn is_binary(data_type: &DataType) -> bool { - matches!( - data_type, - DataType::Binary | DataType::LargeBinary | DataType::BinaryView - ) -} - -/// The precision of Spark's `DecimalType.forType` for an integral type. -fn integral_digits(data_type: &DataType) -> Option { - match data_type { - DataType::Int8 => Some(3), - DataType::Int16 => Some(5), - DataType::Int32 => Some(10), - DataType::Int64 => Some(20), - _ => None, - } -} - -/// Spark's `numericPrecedence`, which leaves out decimals. -fn numeric_precedence(data_type: &DataType) -> Option { - match data_type { - DataType::Int8 => Some(0), - DataType::Int16 => Some(1), - DataType::Int32 => Some(2), - DataType::Int64 => Some(3), - DataType::Float32 => Some(4), - DataType::Float64 => Some(5), - _ => None, - } -} - -fn finer_unit(left: TimeUnit, right: TimeUnit) -> TimeUnit { - let rank = |unit: TimeUnit| match unit { - TimeUnit::Second => 0, - TimeUnit::Millisecond => 1, - TimeUnit::Microsecond => 2, - TimeUnit::Nanosecond => 3, - }; - if rank(left) >= rank(right) { - left - } else { - right - } -} - -/// Spark cannot order maps, so it rejects them at any depth of the element type. -fn contains_map(data_type: &DataType) -> bool { - match data_type { - DataType::Map(_, _) => true, - DataType::List(field) - | DataType::LargeList(field) - | DataType::FixedSizeList(field, _) => contains_map(field.data_type()), - DataType::Struct(fields) => { - fields.iter().any(|field| contains_map(field.data_type())) - } - _ => false, - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/spark/src/function/array/mod.rs b/crates/spark/src/function/array/mod.rs index 7ff0f2aa..36dcccc9 100644 --- a/crates/spark/src/function/array/mod.rs +++ b/crates/spark/src/function/array/mod.rs @@ -6,7 +6,6 @@ pub mod array_contains; pub mod array_position; -mod float_normalize; pub mod repeat; pub mod shuffle; pub mod slice; diff --git a/crates/spark/src/function/array/float_normalize.rs b/crates/spark/src/function/float_normalize.rs similarity index 97% rename from crates/spark/src/function/array/float_normalize.rs rename to crates/spark/src/function/float_normalize.rs index 07bbb28a..1a76b60d 100644 --- a/crates/spark/src/function/array/float_normalize.rs +++ b/crates/spark/src/function/float_normalize.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -//! Float normalization shared by the array functions that compare elements. +//! Float normalization shared by the functions that compare values. //! //! Spark compares floating point values with `-0.0 == 0.0` and `NaN == NaN`, while arrow //! compares them by their bits or by the IEEE total order. Rewriting `-0.0` to `0.0` and @@ -19,7 +19,7 @@ use datafusion_expr::ColumnarValue; use std::sync::Arc; /// Normalizes the floats in a [`ColumnarValue`], see [`normalize_array`]. -pub(super) fn normalize_floats(value: &ColumnarValue) -> Result { +pub(crate) fn normalize_floats(value: &ColumnarValue) -> Result { if !contains_float(&value.data_type()) { return Ok(value.clone()); } @@ -35,7 +35,7 @@ pub(super) fn normalize_floats(value: &ColumnarValue) -> Result { } /// Returns whether values of this type are, or hold at any depth, floating point numbers. -pub(super) fn contains_float(data_type: &DataType) -> bool { +pub(crate) fn contains_float(data_type: &DataType) -> bool { match data_type { DataType::Float32 | DataType::Float64 => true, DataType::List(field) @@ -51,7 +51,7 @@ pub(super) fn contains_float(data_type: &DataType) -> bool { /// Rewrites `-0.0` to `0.0` and every NaN to the canonical NaN at any depth of list, large /// list, fixed-size list, struct and map. Children without floats are reused as they are. -pub(super) fn normalize_array(array: &ArrayRef) -> Result { +pub(crate) fn normalize_array(array: &ArrayRef) -> Result { if !contains_float(array.data_type()) { return Ok(Arc::clone(array)); } diff --git a/crates/spark/src/function/mod.rs b/crates/spark/src/function/mod.rs index 483f9b64..4098d8e0 100644 --- a/crates/spark/src/function/mod.rs +++ b/crates/spark/src/function/mod.rs @@ -12,6 +12,7 @@ pub mod conversion; pub mod csv; pub mod datetime; pub mod error_utils; +pub(crate) mod float_normalize; pub mod functions_nested_utils; pub mod generator; pub mod hash; @@ -25,6 +26,7 @@ pub mod predicate; pub mod string; pub mod r#struct; pub mod table; +pub(crate) mod type_coercion; pub mod url; pub mod utils; pub mod window; diff --git a/crates/spark/src/function/type_coercion.rs b/crates/spark/src/function/type_coercion.rs new file mode 100644 index 00000000..035acbf6 --- /dev/null +++ b/crates/spark/src/function/type_coercion.rs @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Spark's rules for finding a common type of several arguments, from its `TypeCoercion` +//! and `AnsiTypeCoercion`, on the Arrow types that Spark's types map to. Also the checks +//! and type names that the errors for those arguments use. +//! +//! These are for functions that need one type for several arguments, such as +//! `array_position`. They are not Spark's implicit casts to a function's declared input +//! types, which the functions do through their [`datafusion_expr::Signature`]. + +use std::sync::Arc; + +use arrow::datatypes::{DataType, Field, Fields, IntervalUnit, TimeUnit}; + +/// Spark's `findTightestCommonType`. `ansi_mode` picks between `AnsiTypeCoercion` and +/// `TypeCoercion`, which differ only in that ANSI widens an integer and a FLOAT to DOUBLE. +/// Arrays, maps and structs are widened element by element with this function. +pub(crate) fn tightest_common_type( + left: &DataType, + right: &DataType, + ansi_mode: bool, +) -> Option { + use DataType::*; + if left == right { + return Some(left.clone()); + } + if let (Some(l), Some(r)) = (numeric_precedence(left), numeric_precedence(right)) { + let wider = if l >= r { left } else { right }; + // With ANSI mode on, an integer and a FLOAT widen to DOUBLE, which holds every INT + // exactly. + return Some(if ansi_mode && *wider == Float32 { + Float64 + } else { + wider.clone() + }); + } + match (left, right) { + (Null, other) | (other, Null) => Some(other.clone()), + // Spark has one string type and one binary type. + (l, r) if is_string(l) && is_string(r) => Some(wider_physical_type(l, r)), + (l, r) if is_binary(l) && is_binary(r) => Some(wider_physical_type(l, r)), + // The decimal must hold every value of the integral type. + (int, Decimal128(p, s)) | (Decimal128(p, s), int) => { + let digits = integral_digits(int)?; + (*p as i16 - *s as i16 >= digits).then_some(Decimal128(*p, *s)) + } + (Date32, ts @ Timestamp(_, _)) | (ts @ Timestamp(_, _), Date32) => { + Some(ts.clone()) + } + // TIMESTAMP_NTZ and TIMESTAMP widen to TIMESTAMP. + (Timestamp(lu, ltz), Timestamp(ru, rtz)) => Some(Timestamp( + finer_unit(*lu, *ru), + ltz.clone().or_else(|| rtz.clone()), + )), + _ => complex_type(left, right, &|l, r| tightest_common_type(l, r, ansi_mode)), + } +} + +/// Spark's `findTypeForComplex`: widens the elements of arrays, the keys and values of maps, +/// and the fields of structs with `find_type`. A nested field is nullable if it is on either +/// side. Struct fields must have the same names in the same order. Spark compares them +/// ignoring case, but DataFusion cannot cast between structs whose field names differ only +/// in case, so this requires an exact match. +pub(crate) fn complex_type( + left: &DataType, + right: &DataType, + find_type: &dyn Fn(&DataType, &DataType) -> Option, +) -> Option { + use DataType::*; + match (left, right) { + ( + List(lf) | LargeList(lf) | FixedSizeList(lf, _), + List(rf) | LargeList(rf) | FixedSizeList(rf, _), + ) => { + let item = find_type(lf.data_type(), rf.data_type())?; + let field = Arc::new(Field::new( + lf.name(), + item, + lf.is_nullable() || rf.is_nullable(), + )); + Some(match (left, right) { + (FixedSizeList(_, ln), FixedSizeList(_, rn)) if ln == rn => { + FixedSizeList(field, *ln) + } + (LargeList(_), _) | (_, LargeList(_)) => LargeList(field), + _ => List(field), + }) + } + (Struct(lf), Struct(rf)) if lf.len() == rf.len() => { + let fields = lf + .iter() + .zip(rf.iter()) + .map(|(a, b)| { + if a.name() != b.name() { + return None; + } + let data_type = find_type(a.data_type(), b.data_type())?; + Some(Field::new( + a.name(), + data_type, + a.is_nullable() || b.is_nullable(), + )) + }) + .collect::>>()?; + Some(Struct(Fields::from(fields))) + } + // The entries are a struct of the key and the value, so the struct arm widens both. + (Map(lf, sorted), Map(rf, _)) => { + let entries = complex_type(lf.data_type(), rf.data_type(), find_type)?; + Some(Map( + Arc::new(Field::new(lf.name(), entries, false)), + *sorted, + )) + } + _ => None, + } +} + +/// Spark's `OrderUtils.isOrderable`. Spark cannot order maps or `CalendarIntervalType`, which +/// is `Interval(MonthDayNano)` in Arrow, at any depth. +pub(crate) fn is_orderable(data_type: &DataType) -> bool { + match data_type { + DataType::Map(_, _) + | DataType::Union(_, _) + | DataType::Interval(IntervalUnit::MonthDayNano) => false, + DataType::List(field) + | DataType::LargeList(field) + | DataType::ListView(field) + | DataType::LargeListView(field) + | DataType::FixedSizeList(field, _) => is_orderable(field.data_type()), + DataType::Struct(fields) => fields.iter().all(|f| is_orderable(f.data_type())), + DataType::Dictionary(_, value) => is_orderable(value), + _ => true, + } +} + +/// The name Spark's `toSQLType` gives the Spark type that an Arrow type maps to, for error +/// messages. Arrow types without a Spark counterpart use their Arrow name. +pub(crate) fn spark_sql_type(data_type: &DataType) -> String { + use DataType::*; + match data_type { + Null => "VOID".to_string(), + Boolean => "BOOLEAN".to_string(), + Int8 => "TINYINT".to_string(), + Int16 => "SMALLINT".to_string(), + Int32 => "INT".to_string(), + Int64 => "BIGINT".to_string(), + Float32 => "FLOAT".to_string(), + Float64 => "DOUBLE".to_string(), + Decimal128(p, s) => format!("DECIMAL({p},{s})"), + Utf8 | LargeUtf8 | Utf8View => "STRING".to_string(), + Binary | LargeBinary | BinaryView => "BINARY".to_string(), + Date32 => "DATE".to_string(), + Timestamp(_, Some(_)) => "TIMESTAMP".to_string(), + Timestamp(_, None) => "TIMESTAMP_NTZ".to_string(), + Interval(IntervalUnit::MonthDayNano) => "INTERVAL".to_string(), + List(field) | LargeList(field) | FixedSizeList(field, _) => { + format!("ARRAY<{}>", spark_sql_type(field.data_type())) + } + Struct(fields) => format!( + "STRUCT<{}>", + fields + .iter() + .map(|f| format!( + "{}: {}{}", + f.name(), + spark_sql_type(f.data_type()), + if f.is_nullable() { "" } else { " NOT NULL" } + )) + .collect::>() + .join(", ") + ), + Map(entries, _) => match entries.data_type() { + Struct(kv) if kv.len() == 2 => format!( + "MAP<{}, {}>", + spark_sql_type(kv[0].data_type()), + spark_sql_type(kv[1].data_type()) + ), + other => format!("MAP<{other}>"), + }, + other => other.to_string(), + } +} + +fn is_string(data_type: &DataType) -> bool { + matches!( + data_type, + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View + ) +} + +fn is_binary(data_type: &DataType) -> bool { + matches!( + data_type, + DataType::Binary | DataType::LargeBinary | DataType::BinaryView + ) +} + +/// Of two physical layouts of the same Spark string or binary type, the one that can hold the +/// values of both. +fn wider_physical_type(left: &DataType, right: &DataType) -> DataType { + use DataType::*; + match (left, right) { + (LargeUtf8, _) | (_, LargeUtf8) => LargeUtf8, + (LargeBinary, _) | (_, LargeBinary) => LargeBinary, + (Utf8View, _) | (_, Utf8View) => Utf8View, + (BinaryView, _) | (_, BinaryView) => BinaryView, + _ => left.clone(), + } +} + +/// The precision of Spark's `DecimalType.forType` for an integral type. +fn integral_digits(data_type: &DataType) -> Option { + match data_type { + DataType::Int8 => Some(3), + DataType::Int16 => Some(5), + DataType::Int32 => Some(10), + DataType::Int64 => Some(20), + _ => None, + } +} + +/// Spark's `numericPrecedence`, which leaves out decimals. +fn numeric_precedence(data_type: &DataType) -> Option { + match data_type { + DataType::Int8 => Some(0), + DataType::Int16 => Some(1), + DataType::Int32 => Some(2), + DataType::Int64 => Some(3), + DataType::Float32 => Some(4), + DataType::Float64 => Some(5), + _ => None, + } +} + +fn finer_unit(left: TimeUnit, right: TimeUnit) -> TimeUnit { + let rank = |unit: TimeUnit| match unit { + TimeUnit::Second => 0, + TimeUnit::Millisecond => 1, + TimeUnit::Microsecond => 2, + TimeUnit::Nanosecond => 3, + }; + if rank(left) >= rank(right) { + left + } else { + right + } +} + +#[cfg(test)] +mod tests { + use super::*; + use DataType::*; + + fn list(item: DataType) -> DataType { + List(Arc::new(Field::new("item", item, true))) + } + + fn map(key: DataType, value: DataType) -> DataType { + let entries = Struct(Fields::from(vec![ + Field::new("key", key, false), + Field::new("value", value, true), + ])); + Map(Arc::new(Field::new("entries", entries, false)), false) + } + + #[test] + fn numbers_widen_and_ansi_widens_int_and_float_to_double() { + assert_eq!(tightest_common_type(&Int32, &Int64, false), Some(Int64)); + assert_eq!(tightest_common_type(&Int32, &Float32, false), Some(Float32)); + assert_eq!(tightest_common_type(&Int32, &Float32, true), Some(Float64)); + assert_eq!( + tightest_common_type(&Int32, &Decimal128(10, 0), false), + Some(Decimal128(10, 0)) + ); + assert_eq!( + tightest_common_type(&Int32, &Decimal128(10, 1), false), + None + ); + assert_eq!(tightest_common_type(&Int32, &Utf8, false), None); + } + + #[test] + fn string_layouts_widen_to_the_one_that_holds_both() { + assert_eq!( + tightest_common_type(&Utf8, &LargeUtf8, false), + Some(LargeUtf8) + ); + assert_eq!( + tightest_common_type(&LargeUtf8, &Utf8, false), + Some(LargeUtf8) + ); + assert_eq!( + tightest_common_type(&Utf8, &Utf8View, false), + Some(Utf8View) + ); + } + + #[test] + fn complex_types_widen_element_by_element() { + assert_eq!( + tightest_common_type(&list(Int32), &list(Int64), false), + Some(list(Int64)) + ); + assert_eq!( + tightest_common_type(&map(Int32, Int32), &map(Int64, Int16), false), + Some(map(Int64, Int32)) + ); + let a = Struct(Fields::from(vec![Field::new("a", Int32, false)])); + let b = Struct(Fields::from(vec![Field::new("b", Int32, false)])); + let wide = Struct(Fields::from(vec![Field::new("a", Int64, true)])); + assert_eq!(tightest_common_type(&a, &wide, false), Some(wide)); + assert_eq!(tightest_common_type(&a, &b, false), None); + } + + #[test] + fn maps_and_calendar_intervals_are_not_orderable() { + let interval = Interval(IntervalUnit::MonthDayNano); + assert!(!is_orderable(&map(Int32, Int32))); + assert!(!is_orderable(&interval)); + assert!(!is_orderable(&list(interval.clone()))); + assert!(is_orderable(&list(Int32))); + assert_eq!(spark_sql_type(&list(interval)), "ARRAY"); + } +} diff --git a/crates/sqllogictest/test_files/array/array_position.slt b/crates/sqllogictest/test_files/array/array_position.slt index cf9ef3e8..25ebfd45 100644 --- a/crates/sqllogictest/test_files/array/array_position.slt +++ b/crates/sqllogictest/test_files/array/array_position.slt @@ -274,8 +274,8 @@ SELECT array_position(array(CAST(1.0 AS DOUBLE)), CAST(1 AS DECIMAL(5,1))); query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] SELECT array_position(array(1), '1'); -query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] -SELECT array_position(array('1'), 1); +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] Input to `array_position` should have been "ARRAY" followed by a value with same element type, but it's \["ARRAY", "INT"\]\. SQLSTATE: 42K09 +SELECT array_position(array('1'), CAST(1 AS INT)); query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] SELECT array_position(array(1), CAST(NULL AS STRING)); @@ -332,6 +332,17 @@ SELECT array_position(array(named_struct('m', map(1, 2))), named_struct('m', map query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] SELECT array_position(array(array(map(1, 2))), array(map(1, 2))); +# Map keys and values are widened, and the common map type is then rejected +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_position` does not support ordering on type "MAP"\. SQLSTATE: 42K09 +SELECT array_position(array(map(CAST(1 AS INT), CAST(1 AS INT))), map(1L, 1L)); + +# Spark cannot order intervals +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_position` does not support ordering on type "INTERVAL"\. SQLSTATE: 42K09 +SELECT array_position(array(make_interval(1)), make_interval(2)); + +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_position` does not support ordering on type "ARRAY"\. SQLSTATE: 42K09 +SELECT array_position(array(array(make_interval(1))), array(make_interval(2))); + # Signed zero: -0.0 equals 0.0 query I SELECT array_position(array(CAST('-0.0' AS DOUBLE), CAST(1.0 AS DOUBLE)), CAST(0.0 AS DOUBLE)); From 3e9008cff980d3f1546656cbd8a71737c6bc22ac Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 10:19:52 -0700 Subject: [PATCH 08/10] refactor(spark): match map keys and values by position in complex_type Spark's MapType has no field names, so map entries named differently still have a common type. Keep contains_float private. --- crates/spark/src/function/float_normalize.rs | 2 +- crates/spark/src/function/type_coercion.rs | 29 ++++++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/crates/spark/src/function/float_normalize.rs b/crates/spark/src/function/float_normalize.rs index 1a76b60d..e166b362 100644 --- a/crates/spark/src/function/float_normalize.rs +++ b/crates/spark/src/function/float_normalize.rs @@ -35,7 +35,7 @@ pub(crate) fn normalize_floats(value: &ColumnarValue) -> Result { } /// Returns whether values of this type are, or hold at any depth, floating point numbers. -pub(crate) fn contains_float(data_type: &DataType) -> bool { +fn contains_float(data_type: &DataType) -> bool { match data_type { DataType::Float32 | DataType::Float64 => true, DataType::List(field) diff --git a/crates/spark/src/function/type_coercion.rs b/crates/spark/src/function/type_coercion.rs index 035acbf6..d4a8bbb5 100644 --- a/crates/spark/src/function/type_coercion.rs +++ b/crates/spark/src/function/type_coercion.rs @@ -106,9 +106,24 @@ pub(crate) fn complex_type( .collect::>>()?; Some(Struct(Fields::from(fields))) } - // The entries are a struct of the key and the value, so the struct arm widens both. + // The entries are a struct of the key and the value. Spark's `MapType` has no field + // names, so the key and the value are matched by position. (Map(lf, sorted), Map(rf, _)) => { - let entries = complex_type(lf.data_type(), rf.data_type(), find_type)?; + let (Struct(lkv), Struct(rkv)) = (lf.data_type(), rf.data_type()) else { + return None; + }; + let [lk, lv] = lkv.iter().collect::>()[..] else { + return None; + }; + let [rk, rv] = rkv.iter().collect::>()[..] else { + return None; + }; + let key = find_type(lk.data_type(), rk.data_type())?; + let value = find_type(lv.data_type(), rv.data_type())?; + let entries = Struct(Fields::from(vec![ + Field::new(lk.name(), key, false), + Field::new(lv.name(), value, lv.is_nullable() || rv.is_nullable()), + ])); Some(Map( Arc::new(Field::new(lf.name(), entries, false)), *sorted, @@ -310,6 +325,16 @@ mod tests { ); let a = Struct(Fields::from(vec![Field::new("a", Int32, false)])); let b = Struct(Fields::from(vec![Field::new("b", Int32, false)])); + // Map entries named differently are matched by position. + let renamed = Struct(Fields::from(vec![ + Field::new("keys", Int64, false), + Field::new("values", Int16, true), + ])); + let renamed = Map(Arc::new(Field::new("entries", renamed, false)), false); + assert_eq!( + tightest_common_type(&map(Int32, Int32), &renamed, false), + Some(map(Int64, Int32)) + ); let wide = Struct(Fields::from(vec![Field::new("a", Int64, true)])); assert_eq!(tightest_common_type(&a, &wide, false), Some(wide)); assert_eq!(tightest_common_type(&a, &b, false), None); From f5571e9d044f852c04e7c07035c16d30e19e33dd Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 10:22:21 -0700 Subject: [PATCH 09/10] refactor(spark): simplify the shared common-type rules Use DataType::is_string and TimeUnit's ordering, match map entries without collecting, keep complex_type private, and move the non-ANSI struct field check and same_type into type_coercion beside the rules they qualify. --- .../src/function/array/array_position.rs | 46 +------- crates/spark/src/function/type_coercion.rs | 106 ++++++++++++------ 2 files changed, 75 insertions(+), 77 deletions(-) diff --git a/crates/spark/src/function/array/array_position.rs b/crates/spark/src/function/array/array_position.rs index d1a64701..445a8b48 100644 --- a/crates/spark/src/function/array/array_position.rs +++ b/crates/spark/src/function/array/array_position.rs @@ -4,7 +4,7 @@ use crate::function::float_normalize::normalize_array; use crate::function::type_coercion::{ - is_orderable, spark_sql_type, tightest_common_type, + is_orderable, spark_sql_type, struct_field_types_match, tightest_common_type, }; use arrow::array::{ Array, ArrayRef, AsArray, Int64Array, OffsetSizeTrait, make_comparator, @@ -210,50 +210,6 @@ fn null_type_err(name: &str) -> Result { ) } -/// With ANSI mode off, Spark cannot implicitly cast between structs whose field types -/// differ, so `array_position` rejects them although their tightest common type exists. -/// With ANSI mode on, Spark widens the field types. -fn struct_field_types_match(left: &DataType, right: &DataType) -> bool { - use DataType::*; - match (left, right) { - ( - List(l) | LargeList(l) | FixedSizeList(l, _), - List(r) | LargeList(r) | FixedSizeList(r, _), - ) => struct_field_types_match(l.data_type(), r.data_type()), - // The entries are a struct of the key and the value, which are checked on their own. - (Map(l, _), Map(r, _)) => match (l.data_type(), r.data_type()) { - (Struct(l), Struct(r)) => l - .iter() - .zip(r.iter()) - .all(|(a, b)| struct_field_types_match(a.data_type(), b.data_type())), - _ => true, - }, - (Struct(_), Struct(_)) => same_type(left, right), - _ => true, - } -} - -/// Whether two types are equal when the nullability of nested fields is ignored, like -/// Spark's `DataTypeUtils.sameType`. -fn same_type(left: &DataType, right: &DataType) -> bool { - use DataType::*; - match (left, right) { - (List(l), List(r)) | (LargeList(l), LargeList(r)) | (Map(l, _), Map(r, _)) => { - same_type(l.data_type(), r.data_type()) - } - (FixedSizeList(l, ln), FixedSizeList(r, rn)) => { - ln == rn && same_type(l.data_type(), r.data_type()) - } - (Struct(l), Struct(r)) => { - l.len() == r.len() - && l.iter().zip(r.iter()).all(|(a, b)| { - a.name() == b.name() && same_type(a.data_type(), b.data_type()) - }) - } - _ => left == right, - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/spark/src/function/type_coercion.rs b/crates/spark/src/function/type_coercion.rs index d4a8bbb5..9c23939a 100644 --- a/crates/spark/src/function/type_coercion.rs +++ b/crates/spark/src/function/type_coercion.rs @@ -12,7 +12,7 @@ use std::sync::Arc; -use arrow::datatypes::{DataType, Field, Fields, IntervalUnit, TimeUnit}; +use arrow::datatypes::{DataType, Field, Fields, IntervalUnit}; /// Spark's `findTightestCommonType`. `ansi_mode` picks between `AnsiTypeCoercion` and /// `TypeCoercion`, which differ only in that ANSI widens an integer and a FLOAT to DOUBLE. @@ -39,7 +39,7 @@ pub(crate) fn tightest_common_type( match (left, right) { (Null, other) | (other, Null) => Some(other.clone()), // Spark has one string type and one binary type. - (l, r) if is_string(l) && is_string(r) => Some(wider_physical_type(l, r)), + (l, r) if l.is_string() && r.is_string() => Some(wider_physical_type(l, r)), (l, r) if is_binary(l) && is_binary(r) => Some(wider_physical_type(l, r)), // The decimal must hold every value of the integral type. (int, Decimal128(p, s)) | (Decimal128(p, s), int) => { @@ -51,7 +51,8 @@ pub(crate) fn tightest_common_type( } // TIMESTAMP_NTZ and TIMESTAMP widen to TIMESTAMP. (Timestamp(lu, ltz), Timestamp(ru, rtz)) => Some(Timestamp( - finer_unit(*lu, *ru), + // Arrow orders the units from Second to Nanosecond. + (*lu).max(*ru), ltz.clone().or_else(|| rtz.clone()), )), _ => complex_type(left, right, &|l, r| tightest_common_type(l, r, ansi_mode)), @@ -63,7 +64,7 @@ pub(crate) fn tightest_common_type( /// side. Struct fields must have the same names in the same order. Spark compares them /// ignoring case, but DataFusion cannot cast between structs whose field names differ only /// in case, so this requires an exact match. -pub(crate) fn complex_type( +fn complex_type( left: &DataType, right: &DataType, find_type: &dyn Fn(&DataType, &DataType) -> Option, @@ -112,10 +113,7 @@ pub(crate) fn complex_type( let (Struct(lkv), Struct(rkv)) = (lf.data_type(), rf.data_type()) else { return None; }; - let [lk, lv] = lkv.iter().collect::>()[..] else { - return None; - }; - let [rk, rv] = rkv.iter().collect::>()[..] else { + let ([lk, lv], [rk, rv]) = (&lkv[..], &rkv[..]) else { return None; }; let key = find_type(lk.data_type(), rk.data_type())?; @@ -151,6 +149,51 @@ pub(crate) fn is_orderable(data_type: &DataType) -> bool { } } +/// Whether the structs in two types, at any depth, have the same field types. With ANSI mode +/// off, Spark's implicit cast cannot change the field types of a struct, so a function that +/// casts its arguments to their tightest common type rejects such structs, as +/// `array_position` does. With ANSI mode on, Spark widens the field types. +pub(crate) fn struct_field_types_match(left: &DataType, right: &DataType) -> bool { + use DataType::*; + match (left, right) { + ( + List(l) | LargeList(l) | FixedSizeList(l, _), + List(r) | LargeList(r) | FixedSizeList(r, _), + ) => struct_field_types_match(l.data_type(), r.data_type()), + // The entries are a struct of the key and the value, which are checked on their own. + (Map(l, _), Map(r, _)) => match (l.data_type(), r.data_type()) { + (Struct(l), Struct(r)) => l + .iter() + .zip(r.iter()) + .all(|(a, b)| struct_field_types_match(a.data_type(), b.data_type())), + _ => true, + }, + (Struct(_), Struct(_)) => same_type(left, right), + _ => true, + } +} + +/// Whether two types are equal when the nullability of nested fields is ignored, like +/// Spark's `DataTypeUtils.sameType`. +fn same_type(left: &DataType, right: &DataType) -> bool { + use DataType::*; + match (left, right) { + (List(l), List(r)) | (LargeList(l), LargeList(r)) | (Map(l, _), Map(r, _)) => { + same_type(l.data_type(), r.data_type()) + } + (FixedSizeList(l, ln), FixedSizeList(r, rn)) => { + ln == rn && same_type(l.data_type(), r.data_type()) + } + (Struct(l), Struct(r)) => { + l.len() == r.len() + && l.iter().zip(r.iter()).all(|(a, b)| { + a.name() == b.name() && same_type(a.data_type(), b.data_type()) + }) + } + _ => left == right, + } +} + /// The name Spark's `toSQLType` gives the Spark type that an Arrow type maps to, for error /// messages. Arrow types without a Spark counterpart use their Arrow name. pub(crate) fn spark_sql_type(data_type: &DataType) -> String { @@ -188,10 +231,10 @@ pub(crate) fn spark_sql_type(data_type: &DataType) -> String { .join(", ") ), Map(entries, _) => match entries.data_type() { - Struct(kv) if kv.len() == 2 => format!( + Struct(kv) if let [k, v] = &kv[..] => format!( "MAP<{}, {}>", - spark_sql_type(kv[0].data_type()), - spark_sql_type(kv[1].data_type()) + spark_sql_type(k.data_type()), + spark_sql_type(v.data_type()) ), other => format!("MAP<{other}>"), }, @@ -199,13 +242,8 @@ pub(crate) fn spark_sql_type(data_type: &DataType) -> String { } } -fn is_string(data_type: &DataType) -> bool { - matches!( - data_type, - DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View - ) -} - +/// Unlike [`DataType::is_binary`], this leaves out `FixedSizeBinary`, which +/// [`wider_physical_type`] cannot widen. fn is_binary(data_type: &DataType) -> bool { matches!( data_type, @@ -250,20 +288,6 @@ fn numeric_precedence(data_type: &DataType) -> Option { } } -fn finer_unit(left: TimeUnit, right: TimeUnit) -> TimeUnit { - let rank = |unit: TimeUnit| match unit { - TimeUnit::Second => 0, - TimeUnit::Millisecond => 1, - TimeUnit::Microsecond => 2, - TimeUnit::Nanosecond => 3, - }; - if rank(left) >= rank(right) { - left - } else { - right - } -} - #[cfg(test)] mod tests { use super::*; @@ -340,6 +364,24 @@ mod tests { assert_eq!(tightest_common_type(&a, &b, false), None); } + #[test] + fn struct_field_types_must_match_at_any_depth() { + let int = Struct(Fields::from(vec![Field::new("a", Int32, false)])); + let nullable_int = Struct(Fields::from(vec![Field::new("a", Int32, true)])); + let long = Struct(Fields::from(vec![Field::new("a", Int64, false)])); + assert!(struct_field_types_match(&int, &nullable_int)); + assert!(!struct_field_types_match(&int, &long)); + assert!(!struct_field_types_match( + &list(int.clone()), + &list(long.clone()) + )); + assert!(!struct_field_types_match( + &map(Int32, int), + &map(Int64, long) + )); + assert!(struct_field_types_match(&list(Int32), &list(Int64))); + } + #[test] fn maps_and_calendar_intervals_are_not_orderable() { let interval = Interval(IntervalUnit::MonthDayNano); From 918dc421475090e28733c515b7b81044e442bddd Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 11:20:25 -0700 Subject: [PATCH 10/10] fix(spark): use Spark's tightest common type in array_insert Replace the DataFusion coercion fallback, which paired struct fields by name and widened their types, with the shared tightest_common_type and the non-ANSI struct field check, as array_position and array_remove do. End both runtime errors with Spark's SQLSTATE, and return an error rather than panic when the item type differs from the element type. --- .../spark/src/function/array/array_insert.rs | 234 +++++------------- .../test_files/array/array_insert.slt | 16 +- 2 files changed, 76 insertions(+), 174 deletions(-) diff --git a/crates/spark/src/function/array/array_insert.rs b/crates/spark/src/function/array/array_insert.rs index 90877d73..28002d03 100644 --- a/crates/spark/src/function/array/array_insert.rs +++ b/crates/spark/src/function/array/array_insert.rs @@ -15,15 +15,16 @@ use datafusion_common::{ DataFusionError, Result, ScalarValue, exec_err, internal_err, plan_datafusion_err, plan_err, }; -use datafusion_expr::type_coercion::binary::{ - binary_numeric_coercion, type_union_resolution, -}; use datafusion_expr::{ ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; use datafusion_macros::user_doc; +use crate::function::type_coercion::{ + spark_sql_type, struct_field_types_match, tightest_common_type, +}; + /// `ByteArrayUtils.MAX_ROUNDED_ARRAY_LENGTH` (`Integer.MAX_VALUE - 15`) in Spark. const MAX_ROUNDED_ARRAY_LENGTH: i64 = 2_147_483_632; @@ -112,159 +113,13 @@ fn list_element_field(data_type: &DataType) -> Option<&FieldRef> { } } -/// The precision and scale of a decimal type. -fn decimal_precision_scale(t: &DataType) -> Option<(u8, i8)> { - match t { - DataType::Decimal32(p, s) - | DataType::Decimal64(p, s) - | DataType::Decimal128(p, s) - | DataType::Decimal256(p, s) => Some((*p, *s)), - _ => None, - } -} - -/// The precision Spark's `DecimalType.forType` gives an integer type. -fn integer_digits(t: &DataType) -> Option { - match t { - DataType::Int8 | DataType::UInt8 => Some(3), - DataType::Int16 | DataType::UInt16 => Some(5), - DataType::Int32 | DataType::UInt32 => Some(10), - DataType::Int64 | DataType::UInt64 => Some(20), - _ => None, - } -} - -/// Whether Spark's `findTightestCommonType` has no common type for `a` and -/// `b` where DataFusion's coercion finds one, at the top level or inside a -/// list, struct or map. -/// -/// DataFusion coerces these pairs (for example INT and STRING to STRING, or -/// two decimals to a wider decimal), but Spark rejects them: -/// - a string or binary type with a type of another kind, -/// - a decimal with a floating point type, -/// - two decimals of different precision or scale, -/// - an integer with a decimal that cannot hold the whole integer range -/// (Spark's `DecimalType.isWiderThan`). -fn spark_rejects_pair(a: &DataType, b: &DataType) -> bool { - #[derive(PartialEq)] - enum Kind { - String, - Binary, - Other, - } - fn kind(t: &DataType) -> Kind { - match t { - DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => Kind::String, - DataType::Binary - | DataType::LargeBinary - | DataType::BinaryView - | DataType::FixedSizeBinary(_) => Kind::Binary, - _ => Kind::Other, - } - } - match (a, b) { - (DataType::Null, _) | (_, DataType::Null) => false, - (DataType::Dictionary(_, v), other) | (other, DataType::Dictionary(_, v)) => { - spark_rejects_pair(v, other) - } - (DataType::RunEndEncoded(_, v), other) - | (other, DataType::RunEndEncoded(_, v)) => { - spark_rejects_pair(v.data_type(), other) - } - (DataType::Struct(fa), DataType::Struct(fb)) => fa.iter().any(|fa| { - fb.iter() - .find(|fb| fb.name() == fa.name()) - .is_some_and(|fb| spark_rejects_pair(fa.data_type(), fb.data_type())) - }), - // A map's entries are a struct of the key and the value. - (DataType::Map(ea, _), DataType::Map(eb, _)) => { - match (ea.data_type(), eb.data_type()) { - (DataType::Struct(ka), DataType::Struct(kb)) => ka - .iter() - .zip(kb.iter()) - .any(|(fa, fb)| spark_rejects_pair(fa.data_type(), fb.data_type())), - _ => false, - } - } - _ => { - if let (Some(fa), Some(fb)) = (list_element_field(a), list_element_field(b)) { - return spark_rejects_pair(fa.data_type(), fb.data_type()); - } - if kind(a) != kind(b) { - return true; - } - let (decimal, other) = - match (decimal_precision_scale(a), decimal_precision_scale(b)) { - (Some(da), Some(db)) => return da != db, - (Some(d), None) => (d, b), - (None, Some(d)) => (d, a), - (None, None) => return false, - }; - let (precision, scale) = decimal; - if other.is_floating() { - true - } else if let Some(digits) = integer_digits(other) { - scale < 0 || i16::from(precision) - i16::from(scale) < digits - } else { - false - } - } - } -} - -/// The Spark SQL name of a type, as Spark prints it in error messages. -fn spark_type_name(t: &DataType) -> String { - match t { - DataType::Null => "VOID".to_string(), - DataType::Boolean => "BOOLEAN".to_string(), - DataType::Int8 => "TINYINT".to_string(), - DataType::Int16 => "SMALLINT".to_string(), - DataType::Int32 => "INT".to_string(), - DataType::Int64 => "BIGINT".to_string(), - DataType::Float32 => "FLOAT".to_string(), - DataType::Float64 => "DOUBLE".to_string(), - DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => "STRING".to_string(), - DataType::Binary - | DataType::LargeBinary - | DataType::BinaryView - | DataType::FixedSizeBinary(_) => "BINARY".to_string(), - DataType::Date32 | DataType::Date64 => "DATE".to_string(), - DataType::Timestamp(_, None) => "TIMESTAMP_NTZ".to_string(), - DataType::Timestamp(_, Some(_)) => "TIMESTAMP".to_string(), - DataType::Dictionary(_, v) => spark_type_name(v), - DataType::RunEndEncoded(_, v) => spark_type_name(v.data_type()), - DataType::List(f) | DataType::LargeList(f) | DataType::FixedSizeList(f, _) => { - format!("ARRAY<{}>", spark_type_name(f.data_type())) - } - DataType::Struct(fields) => { - let fields: Vec = fields - .iter() - .map(|f| format!("{}: {}", f.name(), spark_type_name(f.data_type()))) - .collect(); - format!("STRUCT<{}>", fields.join(", ")) - } - DataType::Map(entries, _) => match entries.data_type() { - DataType::Struct(kv) if kv.len() == 2 => format!( - "MAP<{}, {}>", - spark_type_name(kv[0].data_type()), - spark_type_name(kv[1].data_type()) - ), - _ => t.to_string(), - }, - other => match decimal_precision_scale(other) { - Some((p, s)) => format!("DECIMAL({p},{s})"), - None => other.to_string(), - }, - } -} - /// Spark's error when the first argument is not an array, or the item does /// not share a type with the array's elements. fn diff_types_err(array: &DataType, item: &DataType) -> DataFusionError { plan_datafusion_err!( - "[DATATYPE_MISMATCH.ARRAY_FUNCTION_DIFF_TYPES] Input to `array_insert` should have been \"ARRAY\" followed by a value with same element type, but it's [\"{}\", \"{}\"].", - spark_type_name(array), - spark_type_name(item) + "[DATATYPE_MISMATCH.ARRAY_FUNCTION_DIFF_TYPES] Input to `array_insert` should have been \"ARRAY\" followed by a value with same element type, but it's [\"{}\", \"{}\"]. SQLSTATE: 42K09", + spark_sql_type(array), + spark_sql_type(item) ) } @@ -321,27 +176,19 @@ impl ScalarUDFImpl for SparkArrayInsert { // the INT range then fails the cast instead of being silently truncated. if !pos.is_integer() { return plan_err!( - "[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] The second parameter of `array_insert` requires the \"INT\" type, however it has the type \"{}\".", - spark_type_name(pos) + "[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] The second parameter of `array_insert` requires the \"INT\" type, however it has the type \"{}\". SQLSTATE: 42K09", + spark_sql_type(pos) ); } - // Spark uses the tightest common type of the element and the item. - // Besides numeric widening, that unifies types of the same family, such - // as Utf8 and Utf8View (parquet's default for strings), timestamps of - // different units, and nested types that differ only in nullability. + // Spark's `ArrayInsert` casts both to the non-ANSI + // `TypeCoercion.findTightestCommonType` of the element and the item. That + // unifies Utf8 and Utf8View (parquet's default for strings), timestamps + // of different units, and nested types that differ only in nullability, + // and pairs struct fields by position. let elem_type = elem.data_type(); - let common = if elem_type == item || *item == DataType::Null { - Some(elem_type.clone()) - } else if *elem_type == DataType::Null { - Some(item.clone()) - } else if spark_rejects_pair(elem_type, item) { - None - } else if let Some(common) = binary_numeric_coercion(elem_type, item) { - Some(common) - } else { - type_union_resolution(&[elem_type.clone(), item.clone()]) - }; + let common = tightest_common_type(elem_type, item, false) + .filter(|_| struct_field_types_match(elem_type, item)); let Some(common) = common else { return Err(diff_types_err(array, item)); }; @@ -416,7 +263,7 @@ struct Placement { fn place(pos: i32, len: usize, legacy: bool) -> Result { if pos == 0 { return exec_err!( - "[INVALID_INDEX_OF_ZERO] The index 0 is invalid. An index shall be either < 0 or > 0 (the first element has index 1)." + "[INVALID_INDEX_OF_ZERO] The index 0 is invalid. An index shall be either < 0 or > 0 (the first element has index 1). SQLSTATE: 22003" ); } let n = len as i64; @@ -437,7 +284,7 @@ fn place(pos: i32, len: usize, legacy: bool) -> Result { }; if new_len > MAX_ROUNDED_ARRAY_LENGTH { return exec_err!( - "[COLLECTION_SIZE_LIMIT_EXCEEDED.FUNCTION] Can't create array with {new_len} elements which exceeding the array size limit {MAX_ROUNDED_ARRAY_LENGTH}, unsuccessful try to create arrays in the function `array_insert`." + "[COLLECTION_SIZE_LIMIT_EXCEEDED.FUNCTION] Can't create array with {new_len} elements which exceeding the array size limit {MAX_ROUNDED_ARRAY_LENGTH}, unsuccessful try to create arrays in the function `array_insert`. SQLSTATE: 54000" ); } Ok(Placement { @@ -483,6 +330,16 @@ fn array_insert( let values = list.values().to_data(); let item_data = item.to_data(); + // `MutableArrayData` panics on mixed types. SQL coerces the item to the + // element type, but a caller that builds the physical expression directly, + // such as Comet, may not. + if values.data_type() != item_data.data_type() { + return internal_err!( + "array_insert expects the item type {} to match the element type {}", + item_data.data_type(), + values.data_type() + ); + } let mut out = MutableArrayData::with_capacities( vec![&values, &item_data], true, @@ -652,6 +509,41 @@ mod tests { Ok(()) } + /// A caller that skips coercion must get an error, not a panic, when the + /// item type differs from the element type. + #[test] + fn mismatched_item_type_is_an_error() { + let list = + ListArray::from_iter_primitive::(vec![Some(vec![Some(1)])]); + let list: ArrayRef = Arc::new(list); + let pos: ArrayRef = Arc::new(Int32Array::from(vec![1])); + let item: ArrayRef = Arc::new(arrow::array::Int64Array::from(vec![2])); + let args = vec![ + ColumnarValue::Array(Arc::clone(&list)), + ColumnarValue::Array(pos), + ColumnarValue::Array(item), + ]; + let arg_fields = args + .iter() + .map(|arg| Arc::new(Field::new("a", arg.data_type(), true))) + .collect(); + let error = SparkArrayInsert::new() + .invoke_with_args(ScalarFunctionArgs { + args, + arg_fields, + number_rows: 1, + return_field: Arc::new(Field::new("r", list.data_type().clone(), true)), + config_options: Arc::new(Default::default()), + }) + .unwrap_err(); + assert!( + error + .to_string() + .contains("expects the item type Int64 to match the element type Int32"), + "{error}" + ); + } + #[test] fn coerce_types_rules() -> Result<()> { let f = SparkArrayInsert::new(); diff --git a/crates/sqllogictest/test_files/array/array_insert.slt b/crates/sqllogictest/test_files/array/array_insert.slt index 8a4ec4df..d7d5f768 100644 --- a/crates/sqllogictest/test_files/array/array_insert.slt +++ b/crates/sqllogictest/test_files/array/array_insert.slt @@ -138,18 +138,28 @@ statement error INVALID_INDEX_OF_ZERO SELECT array_insert(column1, 0, column3) FROM array_insert_t WHERE column1 IS NOT NULL; # Index 0 is invalid -statement error INVALID_INDEX_OF_ZERO +statement error ^.*\[INVALID_INDEX_OF_ZERO\] The index 0 is invalid\. An index shall be either < 0 or > 0 \(the first element has index 1\)\. SQLSTATE: 22003$ SELECT array_insert(make_array(1, 2), 0, 5); +# Struct fields are paired by position, so reordered fields have no common type, +# as in Spark 4.2.0 in both ANSI modes +statement error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_insert(make_array(named_struct('a', 1::INT, 'b', 2::INT)), 1, named_struct('b', 3::INT, 'a', 4::INT)); + +# Struct field types are not widened, as in Spark with ANSI mode off. With ANSI +# mode on, Spark widens them. +statement error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] +SELECT array_insert(make_array(named_struct('a', 1::INT)), 1, named_struct('a', 2::BIGINT)); + # Exceeding the maximum array length -query error \[COLLECTION_SIZE_LIMIT_EXCEEDED\.FUNCTION\] Can't create array with 2147483647 elements which exceeding the array size limit 2147483632, unsuccessful try to create arrays in the function `array_insert`\. +query error \[COLLECTION_SIZE_LIMIT_EXCEEDED\.FUNCTION\] Can't create array with 2147483647 elements which exceeding the array size limit 2147483632, unsuccessful try to create arrays in the function `array_insert`\. SQLSTATE: 54000 SELECT array_insert(make_array(1, 2), 2147483647, 5); # ... also for a position from a column statement ok CREATE TABLE array_insert_big AS VALUES (make_array(1, 2), 2147483647, 5); -query error \[COLLECTION_SIZE_LIMIT_EXCEEDED\.FUNCTION\] Can't create array with 2147483647 elements which exceeding the array size limit 2147483632, unsuccessful try to create arrays in the function `array_insert`\. +query error \[COLLECTION_SIZE_LIMIT_EXCEEDED\.FUNCTION\] Can't create array with 2147483647 elements which exceeding the array size limit 2147483632, unsuccessful try to create arrays in the function `array_insert`\. SQLSTATE: 54000 SELECT array_insert(column1, column2, column3) FROM array_insert_big; # The index must be an integer