From 33ff0578fb122bc1bbc3bd43087981d147f2f3b7 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 18 Nov 2026 09:55:58 -0700 Subject: [PATCH 1/3] fix(spark): cast array_repeat's count to INT as Spark does Spark's ArrayRepeat takes an INT count. Cast any integer count to INT, so an unsigned count no longer fails inside DataFusion's array_repeat and a count outside the INT range fails the cast. Other types fail with Spark's UNEXPECTED_INPUT_TYPE error. Part of #353. --- crates/spark/src/function/array/repeat.rs | 45 ++++++++------ .../test_files/array/array_repeat.slt | 60 +++++++++++++++++++ 2 files changed, 87 insertions(+), 18 deletions(-) diff --git a/crates/spark/src/function/array/repeat.rs b/crates/spark/src/function/array/repeat.rs index e9f6f31b..a9f203b4 100644 --- a/crates/spark/src/function/array/repeat.rs +++ b/crates/spark/src/function/array/repeat.rs @@ -4,7 +4,7 @@ use arrow::datatypes::{DataType, Field}; use datafusion_common::utils::take_function_args; -use datafusion_common::{Result, ScalarValue, exec_err}; +use datafusion_common::{Result, ScalarValue, plan_err}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -16,6 +16,7 @@ use std::sync::Arc; use crate::function::null_utils::{ NullMaskResolution, apply_null_mask, compute_null_mask, }; +use crate::function::type_coercion::spark_sql_type; /// Spark-compatible `array_repeat` expression. /// @@ -40,7 +41,7 @@ NULL ), argument( name = "count", - description = "Integer number of times to repeat `element`. Non-integer types are an error. NULL returns NULL." + description = "Number of times to repeat `element`, cast to INT. Non-integer types are an error. NULL returns NULL." ) )] #[derive(Debug, PartialEq, Eq, Hash)] @@ -89,20 +90,19 @@ impl ScalarUDFImpl for SparkArrayRepeat { fn coerce_types(&self, arg_types: &[DataType]) -> Result> { let [first_type, second_type] = take_function_args(self.name(), arg_types)?; - // Coerce the second argument to Int64/UInt64 if it's a numeric type - let second = match second_type { - DataType::Int8 - | DataType::Int16 - | DataType::Int32 - | DataType::Int64 - | DataType::Null => DataType::Int64, - DataType::UInt8 | DataType::UInt16 | DataType::UInt32 | DataType::UInt64 => { - DataType::UInt64 - } - _ => return exec_err!("count must be an integer type"), - }; - - Ok(vec![first_type.clone(), second]) + // Spark's `ArrayRepeat` takes an INT count. DataFusion types bare integer + // literals as BIGINT, so accept every integer type and cast to INT. A + // value outside the INT range then fails the cast instead of being + // silently truncated. + if !(second_type.is_integer() || second_type.is_null()) { + return plan_err!( + "[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] The second parameter of \ + `array_repeat` requires the \"INT\" type, however it has the type \ + \"{}\". SQLSTATE: 42K09", + spark_sql_type(second_type) + ); + } + Ok(vec![first_type.clone(), DataType::Int32]) } } @@ -110,8 +110,8 @@ impl ScalarUDFImpl for SparkArrayRepeat { /// if the count argument is NULL (Spark behavior), whereas DataFusion's array_repeat ignores NULLs. fn spark_array_repeat(args: ScalarFunctionArgs) -> Result { let ScalarFunctionArgs { - args: arg_values, - arg_fields, + args: mut arg_values, + mut arg_fields, number_rows, return_field, config_options, @@ -126,6 +126,15 @@ fn spark_array_repeat(args: ScalarFunctionArgs) -> Result { return Ok(ColumnarValue::Scalar(ScalarValue::try_from(return_type)?)); } + // DataFusion's `array_repeat` takes a BIGINT count. + arg_values[1] = arg_values[1].cast_to(&DataType::Int64, None)?; + arg_fields[1] = Arc::new( + arg_fields[1] + .as_ref() + .clone() + .with_data_type(DataType::Int64), + ); + let array_repeat_func = ArrayRepeat::new(); let func_args = ScalarFunctionArgs { args: arg_values, diff --git a/crates/sqllogictest/test_files/array/array_repeat.slt b/crates/sqllogictest/test_files/array/array_repeat.slt index 4337ec17..bce725ba 100644 --- a/crates/sqllogictest/test_files/array/array_repeat.slt +++ b/crates/sqllogictest/test_files/array/array_repeat.slt @@ -105,3 +105,63 @@ query ? select array_repeat('a', NULL); ---- NULL + +# Spark's `ArrayRepeat` takes an INT count. An integer count of another type is +# cast to INT, because DataFusion types `2` as BIGINT. +query ? +SELECT array_repeat('a', 2::TINYINT); +---- +[a, a] + +query ? +SELECT array_repeat('a', 2::SMALLINT); +---- +[a, a] + +query ? +SELECT array_repeat('a', 2::BIGINT); +---- +[a, a] + +query ? +SELECT array_repeat('a', arrow_cast(2, 'UInt32')); +---- +[a, a] + +query ? +SELECT array_repeat(1, c) FROM (VALUES (2), (NULL), (0)) AS t(c); +---- +[1, 1] +NULL +[] + +# A BIGINT count outside the INT range fails the cast to INT. +query error Cast error +SELECT array_repeat('a', 3000000000); + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The second parameter of `array_repeat` requires the "INT" type, however it has the type "STRING"\. SQLSTATE: 42K09 +SELECT array_repeat('a', '2'); + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The second parameter of `array_repeat` requires the "INT" type, however it has the type "DOUBLE"\. SQLSTATE: 42K09 +SELECT array_repeat('a', 2.7::DOUBLE); + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The second parameter of `array_repeat` requires the "INT" type, however it has the type "DECIMAL\(2,1\)"\. SQLSTATE: 42K09 +SELECT array_repeat('a', 2.7::DECIMAL(2, 1)); + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The second parameter of `array_repeat` requires the "INT" type, however it has the type "BOOLEAN"\. SQLSTATE: 42K09 +SELECT array_repeat('a', true); + +# The same with ANSI mode on. +statement ok +set datafusion.execution.enable_ansi_mode = true; + +query ? +SELECT array_repeat('a', 2); +---- +[a, a] + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The second parameter of `array_repeat` requires the "INT" type, however it has the type "STRING"\. SQLSTATE: 42K09 +SELECT array_repeat('a', '2'); + +statement ok +set datafusion.execution.enable_ansi_mode = false; From 96666c83ca39d11d53516a7b9803dafa15c8ad0e Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 18 Nov 2026 15:02:02 -0700 Subject: [PATCH 2/3] fix(spark): accept a dictionary-encoded integer count in array_repeat Spark has no dictionary types, so a dictionary count has the type of its values, as in the shared coercion rules. Also note why an out-of-range BIGINT literal fails the cast rather than being rejected at analysis. --- crates/spark/src/function/array/repeat.rs | 9 +++++++-- .../sqllogictest/test_files/array/array_repeat.slt | 14 +++++++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/crates/spark/src/function/array/repeat.rs b/crates/spark/src/function/array/repeat.rs index a9f203b4..1e71b3ca 100644 --- a/crates/spark/src/function/array/repeat.rs +++ b/crates/spark/src/function/array/repeat.rs @@ -93,8 +93,13 @@ impl ScalarUDFImpl for SparkArrayRepeat { // Spark's `ArrayRepeat` takes an INT count. DataFusion types bare integer // literals as BIGINT, so accept every integer type and cast to INT. A // value outside the INT range then fails the cast instead of being - // silently truncated. - if !(second_type.is_integer() || second_type.is_null()) { + // silently truncated. Spark has no dictionary types, so a dictionary-encoded + // count has the type of its values. + let count_type = match second_type { + DataType::Dictionary(_, value) => value.as_ref(), + other => other, + }; + if !(count_type.is_integer() || count_type.is_null()) { return plan_err!( "[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] The second parameter of \ `array_repeat` requires the \"INT\" type, however it has the type \ diff --git a/crates/sqllogictest/test_files/array/array_repeat.slt b/crates/sqllogictest/test_files/array/array_repeat.slt index bce725ba..21e95a1d 100644 --- a/crates/sqllogictest/test_files/array/array_repeat.slt +++ b/crates/sqllogictest/test_files/array/array_repeat.slt @@ -135,7 +135,19 @@ SELECT array_repeat(1, c) FROM (VALUES (2), (NULL), (0)) AS t(c); NULL [] -# A BIGINT count outside the INT range fails the cast to INT. +# A dictionary-encoded integer count has the type of its values. +query ? +SELECT array_repeat('a', arrow_cast(2, 'Dictionary(Int32, Int64)')); +---- +[a, a] + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The second parameter of `array_repeat` requires the "INT" type, however it has the type "STRING"\. SQLSTATE: 42K09 +SELECT array_repeat('a', arrow_cast('2', 'Dictionary(Int32, Utf8)')); + +# A BIGINT count outside the INT range fails the cast to INT. Spark rejects +# this at analysis with UNEXPECTED_INPUT_TYPE, because it types the literal as +# BIGINT. `coerce_types` only sees types, so it accepts every integer type and +# the out-of-range value fails the cast instead. query error Cast error SELECT array_repeat('a', 3000000000); From 8157aefc87bba849c755fcf6f1459f7844e601a9 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 18 Nov 2026 19:01:20 -0700 Subject: [PATCH 3/3] test(spark): mark array_repeat's non-INT integer counts xfail on Spark --- crates/sqllogictest/test_files/array/array_repeat.slt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/sqllogictest/test_files/array/array_repeat.slt b/crates/sqllogictest/test_files/array/array_repeat.slt index 21e95a1d..ddda87f2 100644 --- a/crates/sqllogictest/test_files/array/array_repeat.slt +++ b/crates/sqllogictest/test_files/array/array_repeat.slt @@ -108,16 +108,19 @@ NULL # Spark's `ArrayRepeat` takes an INT count. An integer count of another type is # cast to INT, because DataFusion types `2` as BIGINT. +# xfail spark: intended: Spark rejects a non-INT integer count, it is accepted here because DataFusion types an integer literal as BIGINT (https://github.com/apache/datafusion-spark/issues/97) query ? SELECT array_repeat('a', 2::TINYINT); ---- [a, a] +# xfail spark: intended: Spark rejects a non-INT integer count, it is accepted here because DataFusion types an integer literal as BIGINT (https://github.com/apache/datafusion-spark/issues/97) query ? SELECT array_repeat('a', 2::SMALLINT); ---- [a, a] +# xfail spark: intended: Spark rejects a non-INT integer count, it is accepted here because DataFusion types an integer literal as BIGINT (https://github.com/apache/datafusion-spark/issues/97) query ? SELECT array_repeat('a', 2::BIGINT); ---- @@ -136,6 +139,7 @@ NULL [] # A dictionary-encoded integer count has the type of its values. +# xfail spark: intended: Spark rejects a non-INT integer count, it is accepted here because DataFusion types an integer literal as BIGINT (https://github.com/apache/datafusion-spark/issues/97) query ? SELECT array_repeat('a', arrow_cast(2, 'Dictionary(Int32, Int64)')); ----