Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 31 additions & 17 deletions crates/spark/src/function/array/repeat.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

use arrow::datatypes::{DataType, Field};
use datafusion_common::utils::take_function_args;
use datafusion_common::{Result, ScalarValue, exec_err};
use datafusion_common::{Result, ScalarValue, plan_err};
use datafusion_expr::{
ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
Volatility,
Expand All @@ -16,6 +16,7 @@ use std::sync::Arc;
use crate::function::null_utils::{
NullMaskResolution, apply_null_mask, compute_null_mask,
};
use crate::function::type_coercion::spark_sql_type;

/// Spark-compatible `array_repeat` expression.
///
Expand All @@ -40,7 +41,7 @@ NULL
),
argument(
name = "count",
description = "Integer number of times to repeat `element`. Non-integer types are an error. NULL returns NULL."
description = "Number of times to repeat `element`, cast to INT. Non-integer types are an error. NULL returns NULL."
)
)]
#[derive(Debug, PartialEq, Eq, Hash)]
Expand Down Expand Up @@ -89,29 +90,33 @@ impl ScalarUDFImpl for SparkArrayRepeat {
fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
let [first_type, second_type] = take_function_args(self.name(), arg_types)?;

// Coerce the second argument to Int64/UInt64 if it's a numeric type
let second = match second_type {
DataType::Int8
| DataType::Int16
| DataType::Int32
| DataType::Int64
| DataType::Null => DataType::Int64,
DataType::UInt8 | DataType::UInt16 | DataType::UInt32 | DataType::UInt64 => {
DataType::UInt64
}
_ => return exec_err!("count must be an integer type"),
// Spark's `ArrayRepeat` takes an INT count. DataFusion types bare integer
// literals as BIGINT, so accept every integer type and cast to INT. A
// value outside the INT range then fails the cast instead of being
// silently truncated. Spark has no dictionary types, so a dictionary-encoded
// count has the type of its values.
let count_type = match second_type {
DataType::Dictionary(_, value) => value.as_ref(),
other => other,
};

Ok(vec![first_type.clone(), second])
if !(count_type.is_integer() || count_type.is_null()) {
return plan_err!(
"[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] The second parameter of \
`array_repeat` requires the \"INT\" type, however it has the type \
\"{}\". SQLSTATE: 42K09",
spark_sql_type(second_type)
);
}
Ok(vec![first_type.clone(), DataType::Int32])
}
}

/// This is a Spark-specific wrapper around DataFusion's array_repeat that returns NULL
/// if the count argument is NULL (Spark behavior), whereas DataFusion's array_repeat ignores NULLs.
fn spark_array_repeat(args: ScalarFunctionArgs) -> Result<ColumnarValue> {
let ScalarFunctionArgs {
args: arg_values,
arg_fields,
args: mut arg_values,
mut arg_fields,
number_rows,
return_field,
config_options,
Expand All @@ -126,6 +131,15 @@ fn spark_array_repeat(args: ScalarFunctionArgs) -> Result<ColumnarValue> {
return Ok(ColumnarValue::Scalar(ScalarValue::try_from(return_type)?));
}

// DataFusion's `array_repeat` takes a BIGINT count.
arg_values[1] = arg_values[1].cast_to(&DataType::Int64, None)?;
arg_fields[1] = Arc::new(
arg_fields[1]
.as_ref()
.clone()
.with_data_type(DataType::Int64),
);

let array_repeat_func = ArrayRepeat::new();
let func_args = ScalarFunctionArgs {
args: arg_values,
Expand Down
76 changes: 76 additions & 0 deletions crates/sqllogictest/test_files/array/array_repeat.slt
Original file line number Diff line number Diff line change
Expand Up @@ -105,3 +105,79 @@ query ?
select array_repeat('a', NULL);
----
NULL

# Spark's `ArrayRepeat` takes an INT count. An integer count of another type is
# cast to INT, because DataFusion types `2` as BIGINT.
# xfail spark: intended: Spark rejects a non-INT integer count, it is accepted here because DataFusion types an integer literal as BIGINT (https://github.com/apache/datafusion-spark/issues/97)
query ?
SELECT array_repeat('a', 2::TINYINT);
----
[a, a]

# xfail spark: intended: Spark rejects a non-INT integer count, it is accepted here because DataFusion types an integer literal as BIGINT (https://github.com/apache/datafusion-spark/issues/97)
query ?
SELECT array_repeat('a', 2::SMALLINT);
----
[a, a]

# xfail spark: intended: Spark rejects a non-INT integer count, it is accepted here because DataFusion types an integer literal as BIGINT (https://github.com/apache/datafusion-spark/issues/97)
query ?
SELECT array_repeat('a', 2::BIGINT);
----
[a, a]

query ?
SELECT array_repeat('a', arrow_cast(2, 'UInt32'));
----
[a, a]

query ?
SELECT array_repeat(1, c) FROM (VALUES (2), (NULL), (0)) AS t(c);
----
[1, 1]
NULL
[]

# A dictionary-encoded integer count has the type of its values.
# xfail spark: intended: Spark rejects a non-INT integer count, it is accepted here because DataFusion types an integer literal as BIGINT (https://github.com/apache/datafusion-spark/issues/97)
query ?
SELECT array_repeat('a', arrow_cast(2, 'Dictionary(Int32, Int64)'));
----
[a, a]

query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The second parameter of `array_repeat` requires the "INT" type, however it has the type "STRING"\. SQLSTATE: 42K09
SELECT array_repeat('a', arrow_cast('2', 'Dictionary(Int32, Utf8)'));

# A BIGINT count outside the INT range fails the cast to INT. Spark rejects
# this at analysis with UNEXPECTED_INPUT_TYPE, because it types the literal as
# BIGINT. `coerce_types` only sees types, so it accepts every integer type and
# the out-of-range value fails the cast instead.
query error Cast error

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Optional: Spark rejects this one at analysis with UNEXPECTED_INPUT_TYPE (the literal is BIGINT), rather than a cast error. Since coerce_types only sees types, the cast error makes sense here; a short note in the comment above might help future readers.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added a note above the test saying that Spark rejects the BIGINT literal at analysis, and that we get a cast error because coerce_types only sees types. In 96666c8.

SELECT array_repeat('a', 3000000000);

query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The second parameter of `array_repeat` requires the "INT" type, however it has the type "STRING"\. SQLSTATE: 42K09
SELECT array_repeat('a', '2');

query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The second parameter of `array_repeat` requires the "INT" type, however it has the type "DOUBLE"\. SQLSTATE: 42K09
SELECT array_repeat('a', 2.7::DOUBLE);

query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The second parameter of `array_repeat` requires the "INT" type, however it has the type "DECIMAL\(2,1\)"\. SQLSTATE: 42K09
SELECT array_repeat('a', 2.7::DECIMAL(2, 1));

query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The second parameter of `array_repeat` requires the "INT" type, however it has the type "BOOLEAN"\. SQLSTATE: 42K09
SELECT array_repeat('a', true);

# The same with ANSI mode on.
statement ok
set datafusion.execution.enable_ansi_mode = true;

query ?
SELECT array_repeat('a', 2);
----
[a, a]

query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The second parameter of `array_repeat` requires the "INT" type, however it has the type "STRING"\. SQLSTATE: 42K09
SELECT array_repeat('a', '2');

statement ok
set datafusion.execution.enable_ansi_mode = false;
Loading