Skip to content
2 changes: 1 addition & 1 deletion crates/spark/src/function/array/array_contains.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

use super::float_normalize::normalize_floats;
use crate::function::float_normalize::normalize_floats;
use arrow::array::{
Array, AsArray, BooleanArray, BooleanBufferBuilder, GenericListArray, OffsetSizeTrait,
};
Expand Down
190 changes: 19 additions & 171 deletions crates/spark/src/function/array/array_position.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,16 @@
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

use super::float_normalize::normalize_array;
use crate::function::float_normalize::normalize_array;
use crate::function::type_coercion::{
is_orderable, spark_sql_type, struct_field_types_match, tightest_common_type,
};
use arrow::array::{
Array, ArrayRef, AsArray, Int64Array, OffsetSizeTrait, make_comparator,
};
use arrow::buffer::{NullBuffer, ScalarBuffer};
use arrow::compute::SortOptions;
use arrow::datatypes::{DataType, Field, Fields, TimeUnit};
use arrow::datatypes::{DataType, Field};
use datafusion_common::utils::take_function_args;
use datafusion_common::{Result, ScalarValue, exec_err, plan_err};
use datafusion_expr::{
Expand Down Expand Up @@ -36,7 +39,7 @@ use std::sync::Arc;
a date widens to a timestamp. Other combinations are an error, for example a string \
and a number, two different decimals, or structs whose field names or field types \
differ, as in Spark with ANSI mode off (with ANSI mode on, Spark widens struct field \
types). A NULL-typed array and map elements are also an error.",
types). A NULL-typed array, and map or INTERVAL elements at any depth, are also an error.",
syntax_example = "array_position(array, element)",
sql_example = r#"```sql
> SELECT array_position(array(3, 2, 1), 1);
Expand Down Expand Up @@ -164,24 +167,31 @@ fn coerce_array_and_element(
other => {
return plan_err!(
"[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] The first parameter of `{name}` \
requires the \"ARRAY\" type, however it has the type {other}"
requires the \"ARRAY\" type, however it has the type \"{}\". SQLSTATE: 42K09",
spark_sql_type(other)
);
}
};
let Some(common) = tightest_common_type(item.data_type(), element) else {
// Spark's `ArrayPosition` calls the non-ANSI `TypeCoercion.findTightestCommonType`.
let common = tightest_common_type(item.data_type(), element, false)
.filter(|_| struct_field_types_match(item.data_type(), element));
let Some(common) = common else {
return plan_err!(
"[DATATYPE_MISMATCH.ARRAY_FUNCTION_DIFF_TYPES] Input to `{name}` should have \
been \"ARRAY\" followed by a value with same element type, but it's \
[{array}, {element}]"
[\"{}\", \"{}\"]. SQLSTATE: 42K09",
spark_sql_type(array),
spark_sql_type(element)
);
};
if common.is_null() {
return null_type_err(name);
}
if contains_map(&common) {
if !is_orderable(&common) {
return plan_err!(
"[DATATYPE_MISMATCH.INVALID_ORDERING_TYPE] The `{name}` does not support \
ordering on type {common}"
ordering on type \"{}\". SQLSTATE: 42K09",
spark_sql_type(&common)
);
}
let item = Arc::new(Field::new(item.name(), common.clone(), true));
Expand All @@ -196,172 +206,10 @@ fn coerce_array_and_element(
fn null_type_err<T>(name: &str) -> Result<T> {
plan_err!(
"[DATATYPE_MISMATCH.NULL_TYPE] Null typed values cannot be used as arguments of \
`{name}`"
`{name}`. SQLSTATE: 42K09"
)
}

/// Spark's `TypeCoercion.findTightestCommonType`, on the arrow types that Spark's types
/// map to.
fn tightest_common_type(left: &DataType, right: &DataType) -> Option<DataType> {
use DataType::*;
if left == right {
return Some(left.clone());
}
if let (Some(l), Some(r)) = (numeric_precedence(left), numeric_precedence(right)) {
return Some(if l >= r { left.clone() } else { right.clone() });
}
match (left, right) {
(Null, other) | (other, Null) => Some(other.clone()),
// Spark has a single string type and a single binary type.
(l, r) if is_string(l) && is_string(r) => Some(l.clone()),
(l, r) if is_binary(l) && is_binary(r) => Some(l.clone()),
// The decimal must be at least as wide as the integral type's DECIMAL(n, 0).
(int, Decimal128(p, s)) | (Decimal128(p, s), int) => {
let digits = integral_digits(int)?;
(*p as i16 - *s as i16 >= digits).then_some(Decimal128(*p, *s))
}
(Date32, ts @ Timestamp(_, _)) | (ts @ Timestamp(_, _), Date32) => {
Some(ts.clone())
}
// TIMESTAMP_NTZ and TIMESTAMP widen to TIMESTAMP.
(Timestamp(lu, ltz), Timestamp(ru, rtz)) => Some(Timestamp(
finer_unit(*lu, *ru),
ltz.clone().or_else(|| rtz.clone()),
)),
(
List(lf) | LargeList(lf) | FixedSizeList(lf, _),
List(rf) | LargeList(rf) | FixedSizeList(rf, _),
) => {
let item = tightest_common_type(lf.data_type(), rf.data_type())?;
let field = Arc::new(Field::new(lf.name(), item, true));
Some(
if matches!(left, LargeList(_)) || matches!(right, LargeList(_)) {
LargeList(field)
} else {
List(field)
},
)
}
// The fields must have the same names and types, ignoring nullability, in the same
// order. This follows Spark with ANSI mode off; with ANSI mode on, Spark also widens
// the field types. Spark compares the names ignoring case, but DataFusion cannot cast
// between structs whose field names differ only in case, so they must match exactly.
(Struct(lf), Struct(rf)) if lf.len() == rf.len() => {
let fields = lf
.iter()
.zip(rf.iter())
.map(|(a, b)| {
if a.name() != b.name() || !same_type(a.data_type(), b.data_type()) {
return None;
}
let data_type = tightest_common_type(a.data_type(), b.data_type())?;
Some(Field::new(
a.name(),
data_type,
a.is_nullable() || b.is_nullable(),
))
})
.collect::<Option<Vec<_>>>()?;
Some(Struct(Fields::from(fields)))
}
(Map(lf, sorted), Map(rf, _)) => {
let entries = tightest_common_type(lf.data_type(), rf.data_type())?;
Some(Map(
Arc::new(Field::new(lf.name(), entries, false)),
*sorted,
))
}
_ => None,
}
}

/// Whether two types are equal when the nullability of nested fields is ignored, like
/// Spark's `DataTypeUtils.sameType`.
fn same_type(left: &DataType, right: &DataType) -> bool {
use DataType::*;
match (left, right) {
(List(l), List(r)) | (LargeList(l), LargeList(r)) | (Map(l, _), Map(r, _)) => {
same_type(l.data_type(), r.data_type())
}
(FixedSizeList(l, ln), FixedSizeList(r, rn)) => {
ln == rn && same_type(l.data_type(), r.data_type())
}
(Struct(l), Struct(r)) => {
l.len() == r.len()
&& l.iter().zip(r.iter()).all(|(a, b)| {
a.name() == b.name() && same_type(a.data_type(), b.data_type())
})
}
_ => left == right,
}
}

fn is_string(data_type: &DataType) -> bool {
matches!(
data_type,
DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View
)
}

fn is_binary(data_type: &DataType) -> bool {
matches!(
data_type,
DataType::Binary | DataType::LargeBinary | DataType::BinaryView
)
}

/// The precision of Spark's `DecimalType.forType` for an integral type.
fn integral_digits(data_type: &DataType) -> Option<i16> {
match data_type {
DataType::Int8 => Some(3),
DataType::Int16 => Some(5),
DataType::Int32 => Some(10),
DataType::Int64 => Some(20),
_ => None,
}
}

/// Spark's `numericPrecedence`, which leaves out decimals.
fn numeric_precedence(data_type: &DataType) -> Option<u8> {
match data_type {
DataType::Int8 => Some(0),
DataType::Int16 => Some(1),
DataType::Int32 => Some(2),
DataType::Int64 => Some(3),
DataType::Float32 => Some(4),
DataType::Float64 => Some(5),
_ => None,
}
}

fn finer_unit(left: TimeUnit, right: TimeUnit) -> TimeUnit {
let rank = |unit: TimeUnit| match unit {
TimeUnit::Second => 0,
TimeUnit::Millisecond => 1,
TimeUnit::Microsecond => 2,
TimeUnit::Nanosecond => 3,
};
if rank(left) >= rank(right) {
left
} else {
right
}
}

/// Spark cannot order maps, so it rejects them at any depth of the element type.
fn contains_map(data_type: &DataType) -> bool {
match data_type {
DataType::Map(_, _) => true,
DataType::List(field)
| DataType::LargeList(field)
| DataType::FixedSizeList(field, _) => contains_map(field.data_type()),
DataType::Struct(fields) => {
fields.iter().any(|field| contains_map(field.data_type()))
}
_ => false,
}
}

#[cfg(test)]
mod tests {
use super::*;
Expand Down
1 change: 0 additions & 1 deletion crates/spark/src/function/array/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@

pub mod array_contains;
pub mod array_position;
mod float_normalize;
pub mod repeat;
pub mod shuffle;
pub mod slice;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

//! Float normalization shared by the array functions that compare elements.
//! Float normalization shared by the functions that compare values.
//!
//! Spark compares floating point values with `-0.0 == 0.0` and `NaN == NaN`, while arrow
//! compares them by their bits or by the IEEE total order. Rewriting `-0.0` to `0.0` and
Expand All @@ -19,7 +19,7 @@ use datafusion_expr::ColumnarValue;
use std::sync::Arc;

/// Normalizes the floats in a [`ColumnarValue`], see [`normalize_array`].
pub(super) fn normalize_floats(value: &ColumnarValue) -> Result<ColumnarValue> {
pub(crate) fn normalize_floats(value: &ColumnarValue) -> Result<ColumnarValue> {
if !contains_float(&value.data_type()) {
return Ok(value.clone());
}
Expand All @@ -35,7 +35,7 @@ pub(super) fn normalize_floats(value: &ColumnarValue) -> Result<ColumnarValue> {
}

/// Returns whether values of this type are, or hold at any depth, floating point numbers.
pub(super) fn contains_float(data_type: &DataType) -> bool {
fn contains_float(data_type: &DataType) -> bool {
match data_type {
DataType::Float32 | DataType::Float64 => true,
DataType::List(field)
Expand All @@ -51,7 +51,7 @@ pub(super) fn contains_float(data_type: &DataType) -> bool {

/// Rewrites `-0.0` to `0.0` and every NaN to the canonical NaN at any depth of list, large
/// list, fixed-size list, struct and map. Children without floats are reused as they are.
pub(super) fn normalize_array(array: &ArrayRef) -> Result<ArrayRef> {
pub(crate) fn normalize_array(array: &ArrayRef) -> Result<ArrayRef> {
if !contains_float(array.data_type()) {
return Ok(Arc::clone(array));
}
Expand Down
2 changes: 2 additions & 0 deletions crates/spark/src/function/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ pub mod conversion;
pub mod csv;
pub mod datetime;
pub mod error_utils;
pub(crate) mod float_normalize;
pub mod functions_nested_utils;
pub mod generator;
pub mod hash;
Expand All @@ -26,6 +27,7 @@ pub mod string;
pub mod r#struct;
pub mod table;
pub(crate) mod timezone_utils;
pub(crate) mod type_coercion;
pub mod url;
pub mod utils;
pub mod window;
Expand Down
Loading
Loading