diff --git a/crates/spark/src/function/array/array_contains.rs b/crates/spark/src/function/array/array_contains.rs index 3802667e..029e6c09 100644 --- a/crates/spark/src/function/array/array_contains.rs +++ b/crates/spark/src/function/array/array_contains.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use super::float_normalize::normalize_floats; +use crate::function::float_normalize::normalize_floats; use arrow::array::{ Array, AsArray, BooleanArray, BooleanBufferBuilder, GenericListArray, OffsetSizeTrait, }; diff --git a/crates/spark/src/function/array/array_position.rs b/crates/spark/src/function/array/array_position.rs index 4ca52da9..445a8b48 100644 --- a/crates/spark/src/function/array/array_position.rs +++ b/crates/spark/src/function/array/array_position.rs @@ -2,13 +2,16 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use super::float_normalize::normalize_array; +use crate::function::float_normalize::normalize_array; +use crate::function::type_coercion::{ + is_orderable, spark_sql_type, struct_field_types_match, tightest_common_type, +}; use arrow::array::{ Array, ArrayRef, AsArray, Int64Array, OffsetSizeTrait, make_comparator, }; use arrow::buffer::{NullBuffer, ScalarBuffer}; use arrow::compute::SortOptions; -use arrow::datatypes::{DataType, Field, Fields, TimeUnit}; +use arrow::datatypes::{DataType, Field}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, exec_err, plan_err}; use datafusion_expr::{ @@ -36,7 +39,7 @@ use std::sync::Arc; a date widens to a timestamp. Other combinations are an error, for example a string \ and a number, two different decimals, or structs whose field names or field types \ differ, as in Spark with ANSI mode off (with ANSI mode on, Spark widens struct field \ - types). A NULL-typed array and map elements are also an error.", + types). A NULL-typed array, and map or INTERVAL elements at any depth, are also an error.", syntax_example = "array_position(array, element)", sql_example = r#"```sql > SELECT array_position(array(3, 2, 1), 1); @@ -164,24 +167,31 @@ fn coerce_array_and_element( other => { return plan_err!( "[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] The first parameter of `{name}` \ - requires the \"ARRAY\" type, however it has the type {other}" + requires the \"ARRAY\" type, however it has the type \"{}\". SQLSTATE: 42K09", + spark_sql_type(other) ); } }; - let Some(common) = tightest_common_type(item.data_type(), element) else { + // Spark's `ArrayPosition` calls the non-ANSI `TypeCoercion.findTightestCommonType`. + let common = tightest_common_type(item.data_type(), element, false) + .filter(|_| struct_field_types_match(item.data_type(), element)); + let Some(common) = common else { return plan_err!( "[DATATYPE_MISMATCH.ARRAY_FUNCTION_DIFF_TYPES] Input to `{name}` should have \ been \"ARRAY\" followed by a value with same element type, but it's \ - [{array}, {element}]" + [\"{}\", \"{}\"]. SQLSTATE: 42K09", + spark_sql_type(array), + spark_sql_type(element) ); }; if common.is_null() { return null_type_err(name); } - if contains_map(&common) { + if !is_orderable(&common) { return plan_err!( "[DATATYPE_MISMATCH.INVALID_ORDERING_TYPE] The `{name}` does not support \ - ordering on type {common}" + ordering on type \"{}\". SQLSTATE: 42K09", + spark_sql_type(&common) ); } let item = Arc::new(Field::new(item.name(), common.clone(), true)); @@ -196,172 +206,10 @@ fn coerce_array_and_element( fn null_type_err(name: &str) -> Result { plan_err!( "[DATATYPE_MISMATCH.NULL_TYPE] Null typed values cannot be used as arguments of \ - `{name}`" + `{name}`. SQLSTATE: 42K09" ) } -/// Spark's `TypeCoercion.findTightestCommonType`, on the arrow types that Spark's types -/// map to. -fn tightest_common_type(left: &DataType, right: &DataType) -> Option { - use DataType::*; - if left == right { - return Some(left.clone()); - } - if let (Some(l), Some(r)) = (numeric_precedence(left), numeric_precedence(right)) { - return Some(if l >= r { left.clone() } else { right.clone() }); - } - match (left, right) { - (Null, other) | (other, Null) => Some(other.clone()), - // Spark has a single string type and a single binary type. - (l, r) if is_string(l) && is_string(r) => Some(l.clone()), - (l, r) if is_binary(l) && is_binary(r) => Some(l.clone()), - // The decimal must be at least as wide as the integral type's DECIMAL(n, 0). - (int, Decimal128(p, s)) | (Decimal128(p, s), int) => { - let digits = integral_digits(int)?; - (*p as i16 - *s as i16 >= digits).then_some(Decimal128(*p, *s)) - } - (Date32, ts @ Timestamp(_, _)) | (ts @ Timestamp(_, _), Date32) => { - Some(ts.clone()) - } - // TIMESTAMP_NTZ and TIMESTAMP widen to TIMESTAMP. - (Timestamp(lu, ltz), Timestamp(ru, rtz)) => Some(Timestamp( - finer_unit(*lu, *ru), - ltz.clone().or_else(|| rtz.clone()), - )), - ( - List(lf) | LargeList(lf) | FixedSizeList(lf, _), - List(rf) | LargeList(rf) | FixedSizeList(rf, _), - ) => { - let item = tightest_common_type(lf.data_type(), rf.data_type())?; - let field = Arc::new(Field::new(lf.name(), item, true)); - Some( - if matches!(left, LargeList(_)) || matches!(right, LargeList(_)) { - LargeList(field) - } else { - List(field) - }, - ) - } - // The fields must have the same names and types, ignoring nullability, in the same - // order. This follows Spark with ANSI mode off; with ANSI mode on, Spark also widens - // the field types. Spark compares the names ignoring case, but DataFusion cannot cast - // between structs whose field names differ only in case, so they must match exactly. - (Struct(lf), Struct(rf)) if lf.len() == rf.len() => { - let fields = lf - .iter() - .zip(rf.iter()) - .map(|(a, b)| { - if a.name() != b.name() || !same_type(a.data_type(), b.data_type()) { - return None; - } - let data_type = tightest_common_type(a.data_type(), b.data_type())?; - Some(Field::new( - a.name(), - data_type, - a.is_nullable() || b.is_nullable(), - )) - }) - .collect::>>()?; - Some(Struct(Fields::from(fields))) - } - (Map(lf, sorted), Map(rf, _)) => { - let entries = tightest_common_type(lf.data_type(), rf.data_type())?; - Some(Map( - Arc::new(Field::new(lf.name(), entries, false)), - *sorted, - )) - } - _ => None, - } -} - -/// Whether two types are equal when the nullability of nested fields is ignored, like -/// Spark's `DataTypeUtils.sameType`. -fn same_type(left: &DataType, right: &DataType) -> bool { - use DataType::*; - match (left, right) { - (List(l), List(r)) | (LargeList(l), LargeList(r)) | (Map(l, _), Map(r, _)) => { - same_type(l.data_type(), r.data_type()) - } - (FixedSizeList(l, ln), FixedSizeList(r, rn)) => { - ln == rn && same_type(l.data_type(), r.data_type()) - } - (Struct(l), Struct(r)) => { - l.len() == r.len() - && l.iter().zip(r.iter()).all(|(a, b)| { - a.name() == b.name() && same_type(a.data_type(), b.data_type()) - }) - } - _ => left == right, - } -} - -fn is_string(data_type: &DataType) -> bool { - matches!( - data_type, - DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View - ) -} - -fn is_binary(data_type: &DataType) -> bool { - matches!( - data_type, - DataType::Binary | DataType::LargeBinary | DataType::BinaryView - ) -} - -/// The precision of Spark's `DecimalType.forType` for an integral type. -fn integral_digits(data_type: &DataType) -> Option { - match data_type { - DataType::Int8 => Some(3), - DataType::Int16 => Some(5), - DataType::Int32 => Some(10), - DataType::Int64 => Some(20), - _ => None, - } -} - -/// Spark's `numericPrecedence`, which leaves out decimals. -fn numeric_precedence(data_type: &DataType) -> Option { - match data_type { - DataType::Int8 => Some(0), - DataType::Int16 => Some(1), - DataType::Int32 => Some(2), - DataType::Int64 => Some(3), - DataType::Float32 => Some(4), - DataType::Float64 => Some(5), - _ => None, - } -} - -fn finer_unit(left: TimeUnit, right: TimeUnit) -> TimeUnit { - let rank = |unit: TimeUnit| match unit { - TimeUnit::Second => 0, - TimeUnit::Millisecond => 1, - TimeUnit::Microsecond => 2, - TimeUnit::Nanosecond => 3, - }; - if rank(left) >= rank(right) { - left - } else { - right - } -} - -/// Spark cannot order maps, so it rejects them at any depth of the element type. -fn contains_map(data_type: &DataType) -> bool { - match data_type { - DataType::Map(_, _) => true, - DataType::List(field) - | DataType::LargeList(field) - | DataType::FixedSizeList(field, _) => contains_map(field.data_type()), - DataType::Struct(fields) => { - fields.iter().any(|field| contains_map(field.data_type())) - } - _ => false, - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/spark/src/function/array/mod.rs b/crates/spark/src/function/array/mod.rs index 7ff0f2aa..36dcccc9 100644 --- a/crates/spark/src/function/array/mod.rs +++ b/crates/spark/src/function/array/mod.rs @@ -6,7 +6,6 @@ pub mod array_contains; pub mod array_position; -mod float_normalize; pub mod repeat; pub mod shuffle; pub mod slice; diff --git a/crates/spark/src/function/array/float_normalize.rs b/crates/spark/src/function/float_normalize.rs similarity index 97% rename from crates/spark/src/function/array/float_normalize.rs rename to crates/spark/src/function/float_normalize.rs index 07bbb28a..e166b362 100644 --- a/crates/spark/src/function/array/float_normalize.rs +++ b/crates/spark/src/function/float_normalize.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -//! Float normalization shared by the array functions that compare elements. +//! Float normalization shared by the functions that compare values. //! //! Spark compares floating point values with `-0.0 == 0.0` and `NaN == NaN`, while arrow //! compares them by their bits or by the IEEE total order. Rewriting `-0.0` to `0.0` and @@ -19,7 +19,7 @@ use datafusion_expr::ColumnarValue; use std::sync::Arc; /// Normalizes the floats in a [`ColumnarValue`], see [`normalize_array`]. -pub(super) fn normalize_floats(value: &ColumnarValue) -> Result { +pub(crate) fn normalize_floats(value: &ColumnarValue) -> Result { if !contains_float(&value.data_type()) { return Ok(value.clone()); } @@ -35,7 +35,7 @@ pub(super) fn normalize_floats(value: &ColumnarValue) -> Result { } /// Returns whether values of this type are, or hold at any depth, floating point numbers. -pub(super) fn contains_float(data_type: &DataType) -> bool { +fn contains_float(data_type: &DataType) -> bool { match data_type { DataType::Float32 | DataType::Float64 => true, DataType::List(field) @@ -51,7 +51,7 @@ pub(super) fn contains_float(data_type: &DataType) -> bool { /// Rewrites `-0.0` to `0.0` and every NaN to the canonical NaN at any depth of list, large /// list, fixed-size list, struct and map. Children without floats are reused as they are. -pub(super) fn normalize_array(array: &ArrayRef) -> Result { +pub(crate) fn normalize_array(array: &ArrayRef) -> Result { if !contains_float(array.data_type()) { return Ok(Arc::clone(array)); } diff --git a/crates/spark/src/function/mod.rs b/crates/spark/src/function/mod.rs index f4fd2020..0ec76755 100644 --- a/crates/spark/src/function/mod.rs +++ b/crates/spark/src/function/mod.rs @@ -12,6 +12,7 @@ pub mod conversion; pub mod csv; pub mod datetime; pub mod error_utils; +pub(crate) mod float_normalize; pub mod functions_nested_utils; pub mod generator; pub mod hash; @@ -26,6 +27,7 @@ pub mod string; pub mod r#struct; pub mod table; pub(crate) mod timezone_utils; +pub(crate) mod type_coercion; pub mod url; pub mod utils; pub mod window; diff --git a/crates/spark/src/function/type_coercion.rs b/crates/spark/src/function/type_coercion.rs new file mode 100644 index 00000000..98fbdfa7 --- /dev/null +++ b/crates/spark/src/function/type_coercion.rs @@ -0,0 +1,536 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Spark's rules for finding a common type of several arguments, from its `TypeCoercion` +//! and `AnsiTypeCoercion`, on the Arrow types that Spark's types map to. Also the checks +//! and type names that the errors for those arguments use. +//! +//! These are for functions that need one type for several arguments, such as +//! `array_position`. They are not Spark's implicit casts to a function's declared input +//! types, which the functions do through their [`datafusion_expr::Signature`]. + +use std::sync::Arc; + +use arrow::datatypes::{DataType, Field, Fields, IntervalUnit}; + +/// Spark's `findTightestCommonType`. `ansi_mode` picks between `AnsiTypeCoercion` and +/// `TypeCoercion`, which differ only in that ANSI widens an integer and a FLOAT to DOUBLE. +/// Arrays, maps and structs are widened element by element with this function. +pub(crate) fn tightest_common_type( + left: &DataType, + right: &DataType, + ansi_mode: bool, +) -> Option { + use DataType::*; + // Spark has no dictionary types. A dictionary-encoded argument, such as a + // Parquet string column, has the type of its values. + if let Dictionary(_, value) = left { + return tightest_common_type(value, right, ansi_mode); + } + if let Dictionary(_, value) = right { + return tightest_common_type(left, value, ansi_mode); + } + // Nested types are widened field by field below, which also unwraps nested dictionaries. + if left == right && !left.is_nested() { + return Some(left.clone()); + } + if let (Some(l), Some(r)) = (numeric_precedence(left), numeric_precedence(right)) { + let wider = if l >= r { left } else { right }; + // With ANSI mode on, an integer and a FLOAT widen to DOUBLE, which holds every INT + // exactly. + return Some(if ansi_mode && *wider == Float32 { + Float64 + } else { + wider.clone() + }); + } + match (left, right) { + (Null, other) | (other, Null) => Some(other.clone()), + // Spark has one string type and one binary type. + (l, r) if l.is_string() && r.is_string() => Some(wider_physical_type(l, r)), + (l, r) if is_binary(l) && is_binary(r) => Some(wider_physical_type(l, r)), + // The decimal must hold every value of the integral type. + (int, Decimal128(p, s)) | (Decimal128(p, s), int) => { + let digits = integral_digits(int)?; + (*p as i16 - *s as i16 >= digits).then_some(Decimal128(*p, *s)) + } + (Date32, ts @ Timestamp(_, _)) | (ts @ Timestamp(_, _), Date32) => { + Some(ts.clone()) + } + // TIMESTAMP_NTZ and TIMESTAMP widen to TIMESTAMP. + (Timestamp(lu, ltz), Timestamp(ru, rtz)) => Some(Timestamp( + // Arrow orders the units from Second to Nanosecond. + (*lu).max(*ru), + ltz.clone().or_else(|| rtz.clone()), + )), + _ => complex_type(left, right, &|l, r| tightest_common_type(l, r, ansi_mode)), + } +} + +/// Spark's `findTypeForComplex`: widens the elements of arrays, the keys and values of maps, +/// and the fields of structs with `find_type`. A nested field is nullable if it is on either +/// side. Struct fields must have the same names in the same order. Spark compares them +/// ignoring case, but DataFusion cannot cast between structs whose field names differ only +/// in case, so this requires an exact match. +fn complex_type( + left: &DataType, + right: &DataType, + find_type: &dyn Fn(&DataType, &DataType) -> Option, +) -> Option { + use DataType::*; + match (left, right) { + ( + List(lf) | LargeList(lf) | FixedSizeList(lf, _), + List(rf) | LargeList(rf) | FixedSizeList(rf, _), + ) => { + let item = find_type(lf.data_type(), rf.data_type())?; + let field = Arc::new(Field::new( + lf.name(), + item, + lf.is_nullable() || rf.is_nullable(), + )); + Some(match (left, right) { + (FixedSizeList(_, ln), FixedSizeList(_, rn)) if ln == rn => { + FixedSizeList(field, *ln) + } + (LargeList(_), _) | (_, LargeList(_)) => LargeList(field), + _ => List(field), + }) + } + (Struct(lf), Struct(rf)) if lf.len() == rf.len() => { + let fields = lf + .iter() + .zip(rf.iter()) + .map(|(a, b)| { + if a.name() != b.name() { + return None; + } + let data_type = find_type(a.data_type(), b.data_type())?; + Some(Field::new( + a.name(), + data_type, + a.is_nullable() || b.is_nullable(), + )) + }) + .collect::>>()?; + Some(Struct(Fields::from(fields))) + } + // The entries are a struct of the key and the value. Spark's `MapType` has no field + // names, so the key and the value are matched by position. + (Map(lf, sorted), Map(rf, _)) => { + let (Struct(lkv), Struct(rkv)) = (lf.data_type(), rf.data_type()) else { + return None; + }; + let ([lk, lv], [rk, rv]) = (&lkv[..], &rkv[..]) else { + return None; + }; + // Spark rejects a key type that the cast can make NULL (`Cast.forceNullable`). + let key = find_type(lk.data_type(), rk.data_type()).filter(|key| { + !force_nullable(lk.data_type(), key) + && !force_nullable(rk.data_type(), key) + })?; + let value = find_type(lv.data_type(), rv.data_type())?; + let entries = Struct(Fields::from(vec![ + Field::new(lk.name(), key, false), + Field::new(lv.name(), value, lv.is_nullable() || rv.is_nullable()), + ])); + Some(Map( + Arc::new(Field::new(lf.name(), entries, false)), + *sorted, + )) + } + _ => None, + } +} + +/// Spark's `Cast.forceNullable`, for the casts that these rules produce. Of those, Spark +/// marks only DATE to TIMESTAMP_NTZ as one that can return NULL. +fn force_nullable(from: &DataType, to: &DataType) -> bool { + match from { + DataType::Dictionary(_, value) => force_nullable(value, to), + _ => matches!((from, to), (DataType::Date32, DataType::Timestamp(_, None))), + } +} + +/// Spark's `OrderUtils.isOrderable`. Spark cannot order maps or `CalendarIntervalType`, which +/// is `Interval(MonthDayNano)` in Arrow, at any depth. +pub(crate) fn is_orderable(data_type: &DataType) -> bool { + match data_type { + DataType::Map(_, _) + | DataType::Union(_, _) + | DataType::Interval(IntervalUnit::MonthDayNano) => false, + DataType::List(field) + | DataType::LargeList(field) + | DataType::ListView(field) + | DataType::LargeListView(field) + | DataType::FixedSizeList(field, _) => is_orderable(field.data_type()), + DataType::Struct(fields) => fields.iter().all(|f| is_orderable(f.data_type())), + DataType::Dictionary(_, value) => is_orderable(value), + _ => true, + } +} + +/// Whether the structs in two types, at any depth, have the same field types. With ANSI mode +/// off, Spark's implicit cast cannot change the field types of a struct, so a function that +/// casts its arguments to their tightest common type rejects such structs, as +/// `array_position` does. With ANSI mode on, Spark widens the field types. +pub(crate) fn struct_field_types_match(left: &DataType, right: &DataType) -> bool { + use DataType::*; + match (left, right) { + (Dictionary(_, l), r) | (r, Dictionary(_, l)) => struct_field_types_match(l, r), + ( + List(l) | LargeList(l) | FixedSizeList(l, _), + List(r) | LargeList(r) | FixedSizeList(r, _), + ) => struct_field_types_match(l.data_type(), r.data_type()), + // The entries are a struct of the key and the value, which are checked on their own. + (Map(l, _), Map(r, _)) => match (l.data_type(), r.data_type()) { + (Struct(l), Struct(r)) => l + .iter() + .zip(r.iter()) + .all(|(a, b)| struct_field_types_match(a.data_type(), b.data_type())), + _ => true, + }, + (Struct(_), Struct(_)) => same_type(left, right), + _ => true, + } +} + +/// Whether two types are equal when the nullability of nested fields is ignored, like +/// Spark's `DataTypeUtils.sameType`. +fn same_type(left: &DataType, right: &DataType) -> bool { + use DataType::*; + match (left, right) { + (Dictionary(_, l), r) | (r, Dictionary(_, l)) => same_type(l, r), + (List(l), List(r)) | (LargeList(l), LargeList(r)) | (Map(l, _), Map(r, _)) => { + same_type(l.data_type(), r.data_type()) + } + (FixedSizeList(l, ln), FixedSizeList(r, rn)) => { + ln == rn && same_type(l.data_type(), r.data_type()) + } + (Struct(l), Struct(r)) => { + l.len() == r.len() + && l.iter().zip(r.iter()).all(|(a, b)| { + a.name() == b.name() && same_type(a.data_type(), b.data_type()) + }) + } + _ => left == right, + } +} + +/// The name Spark's `toSQLType` gives the Spark type that an Arrow type maps to, for error +/// messages. Arrow types without a Spark counterpart use their Arrow name. +pub(crate) fn spark_sql_type(data_type: &DataType) -> String { + use DataType::*; + match data_type { + Null => "VOID".to_string(), + Boolean => "BOOLEAN".to_string(), + Int8 => "TINYINT".to_string(), + Int16 => "SMALLINT".to_string(), + Int32 => "INT".to_string(), + Int64 => "BIGINT".to_string(), + Float32 => "FLOAT".to_string(), + Float64 => "DOUBLE".to_string(), + Decimal128(p, s) => format!("DECIMAL({p},{s})"), + Utf8 | LargeUtf8 | Utf8View => "STRING".to_string(), + Binary | LargeBinary | BinaryView => "BINARY".to_string(), + Date32 => "DATE".to_string(), + Timestamp(_, Some(_)) => "TIMESTAMP".to_string(), + Timestamp(_, None) => "TIMESTAMP_NTZ".to_string(), + Interval(IntervalUnit::MonthDayNano) => "INTERVAL".to_string(), + Interval(IntervalUnit::YearMonth) => "INTERVAL YEAR TO MONTH".to_string(), + Duration(_) => "INTERVAL DAY TO SECOND".to_string(), + // Arrow has no TIME precision. Spark's `make_time` returns TIME(6). + Time64(_) => "TIME(6)".to_string(), + List(field) | LargeList(field) | FixedSizeList(field, _) => { + format!("ARRAY<{}>", spark_sql_type(field.data_type())) + } + Struct(fields) => format!( + "STRUCT<{}>", + fields + .iter() + .map(|f| format!( + "{}: {}{}", + quote_if_needed(f.name()), + spark_sql_type(f.data_type()), + if f.is_nullable() { "" } else { " NOT NULL" } + )) + .collect::>() + .join(", ") + ), + Map(entries, _) => match entries.data_type() { + Struct(kv) if let [k, v] = &kv[..] => format!( + "MAP<{}, {}>", + spark_sql_type(k.data_type()), + spark_sql_type(v.data_type()) + ), + other => format!("MAP<{other}>"), + }, + Dictionary(_, value) => spark_sql_type(value), + other => other.to_string(), + } +} + +/// Spark's `QuotingUtils.quoteIfNeeded`, which `StructField.sql` uses for field names. +fn quote_if_needed(name: &str) -> String { + let mut chars = name.chars(); + let plain = chars + .next() + .is_some_and(|c| c.is_ascii_alphabetic() || c == '_') + && chars.all(|c| c.is_ascii_alphanumeric() || c == '_'); + if plain { + name.to_string() + } else { + format!("`{}`", name.replace('`', "``")) + } +} + +/// Unlike [`DataType::is_binary`], this leaves out `FixedSizeBinary`, which +/// [`wider_physical_type`] cannot widen. +fn is_binary(data_type: &DataType) -> bool { + matches!( + data_type, + DataType::Binary | DataType::LargeBinary | DataType::BinaryView + ) +} + +/// Of two physical layouts of the same Spark string or binary type, the one that can hold the +/// values of both. +fn wider_physical_type(left: &DataType, right: &DataType) -> DataType { + use DataType::*; + match (left, right) { + (LargeUtf8, _) | (_, LargeUtf8) => LargeUtf8, + (LargeBinary, _) | (_, LargeBinary) => LargeBinary, + (Utf8View, _) | (_, Utf8View) => Utf8View, + (BinaryView, _) | (_, BinaryView) => BinaryView, + _ => left.clone(), + } +} + +/// The precision of Spark's `DecimalType.forType` for an integral type. +fn integral_digits(data_type: &DataType) -> Option { + match data_type { + DataType::Int8 => Some(3), + DataType::Int16 => Some(5), + DataType::Int32 => Some(10), + DataType::Int64 => Some(20), + _ => None, + } +} + +/// Spark's `numericPrecedence`, which leaves out decimals. +fn numeric_precedence(data_type: &DataType) -> Option { + match data_type { + DataType::Int8 => Some(0), + DataType::Int16 => Some(1), + DataType::Int32 => Some(2), + DataType::Int64 => Some(3), + DataType::Float32 => Some(4), + DataType::Float64 => Some(5), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use DataType::*; + + fn list(item: DataType) -> DataType { + List(Arc::new(Field::new("item", item, true))) + } + + fn map(key: DataType, value: DataType) -> DataType { + let entries = Struct(Fields::from(vec![ + Field::new("key", key, false), + Field::new("value", value, true), + ])); + Map(Arc::new(Field::new("entries", entries, false)), false) + } + + #[test] + fn numbers_widen_and_ansi_widens_int_and_float_to_double() { + assert_eq!(tightest_common_type(&Int32, &Int64, false), Some(Int64)); + assert_eq!(tightest_common_type(&Int32, &Float32, false), Some(Float32)); + assert_eq!(tightest_common_type(&Int32, &Float32, true), Some(Float64)); + assert_eq!( + tightest_common_type(&Int32, &Decimal128(10, 0), false), + Some(Decimal128(10, 0)) + ); + assert_eq!( + tightest_common_type(&Int32, &Decimal128(10, 1), false), + None + ); + assert_eq!(tightest_common_type(&Int32, &Utf8, false), None); + } + + #[test] + fn string_layouts_widen_to_the_one_that_holds_both() { + assert_eq!( + tightest_common_type(&Utf8, &LargeUtf8, false), + Some(LargeUtf8) + ); + assert_eq!( + tightest_common_type(&LargeUtf8, &Utf8, false), + Some(LargeUtf8) + ); + assert_eq!( + tightest_common_type(&Utf8, &Utf8View, false), + Some(Utf8View) + ); + } + + #[test] + fn complex_types_widen_element_by_element() { + assert_eq!( + tightest_common_type(&list(Int32), &list(Int64), false), + Some(list(Int64)) + ); + assert_eq!( + tightest_common_type(&map(Int32, Int32), &map(Int64, Int16), false), + Some(map(Int64, Int32)) + ); + let a = Struct(Fields::from(vec![Field::new("a", Int32, false)])); + let b = Struct(Fields::from(vec![Field::new("b", Int32, false)])); + // Map entries named differently are matched by position. + let renamed = Struct(Fields::from(vec![ + Field::new("keys", Int64, false), + Field::new("values", Int16, true), + ])); + let renamed = Map(Arc::new(Field::new("entries", renamed, false)), false); + assert_eq!( + tightest_common_type(&map(Int32, Int32), &renamed, false), + Some(map(Int64, Int32)) + ); + let wide = Struct(Fields::from(vec![Field::new("a", Int64, true)])); + assert_eq!(tightest_common_type(&a, &wide, false), Some(wide)); + assert_eq!(tightest_common_type(&a, &b, false), None); + } + + #[test] + fn map_keys_do_not_widen_from_date_to_timestamp_ntz() { + let ntz = Timestamp(arrow::datatypes::TimeUnit::Microsecond, None); + assert_eq!( + tightest_common_type(&map(Date32, Int32), &map(ntz.clone(), Int32), false), + None + ); + assert_eq!( + tightest_common_type(&map(ntz.clone(), Int32), &map(Date32, Int32), true), + None + ); + assert_eq!( + tightest_common_type(&map(Int32, Date32), &map(Int32, ntz.clone()), false), + Some(map(Int32, ntz.clone())) + ); + let date_dictionary = Dictionary(Box::new(Int32), Box::new(Date32)); + assert_eq!( + tightest_common_type( + &map(date_dictionary, Int32), + &map(ntz.clone(), Int32), + false + ), + None + ); + let tz = Timestamp(arrow::datatypes::TimeUnit::Microsecond, Some("UTC".into())); + assert_eq!( + tightest_common_type(&map(Date32, Int32), &map(tz.clone(), Int32), false), + Some(map(tz, Int32)) + ); + } + + #[test] + fn struct_field_types_must_match_at_any_depth() { + let int = Struct(Fields::from(vec![Field::new("a", Int32, false)])); + let nullable_int = Struct(Fields::from(vec![Field::new("a", Int32, true)])); + let long = Struct(Fields::from(vec![Field::new("a", Int64, false)])); + assert!(struct_field_types_match(&int, &nullable_int)); + assert!(!struct_field_types_match(&int, &long)); + assert!(!struct_field_types_match( + &list(int.clone()), + &list(long.clone()) + )); + assert!(!struct_field_types_match( + &map(Int32, int), + &map(Int64, long) + )); + assert!(struct_field_types_match(&list(Int32), &list(Int64))); + } + + #[test] + fn dictionaries_have_the_type_of_their_values() { + let dictionary = |value: DataType| Dictionary(Box::new(Int32), Box::new(value)); + assert_eq!( + tightest_common_type(&dictionary(Utf8), &Utf8, false), + Some(Utf8) + ); + assert_eq!( + tightest_common_type(&Int64, &dictionary(Int32), false), + Some(Int64) + ); + assert_eq!(tightest_common_type(&dictionary(Utf8), &Int32, false), None); + let field = |data_type: DataType| { + Struct(Fields::from(vec![Field::new("a", data_type, true)])) + }; + assert!(struct_field_types_match( + &field(dictionary(Utf8)), + &field(Utf8) + )); + assert_eq!(spark_sql_type(&dictionary(Utf8)), "STRING"); + // Two dictionaries are unwrapped too, so the function sees the values' type. + assert_eq!( + tightest_common_type(&dictionary(Float64), &dictionary(Float64), false), + Some(Float64) + ); + // So are dictionaries inside two equal nested types. + assert_eq!( + tightest_common_type( + &list(dictionary(Float64)), + &list(dictionary(Float64)), + false + ), + Some(list(Float64)) + ); + assert_eq!( + tightest_common_type( + &field(dictionary(Float64)), + &field(dictionary(Float64)), + false + ), + Some(field(Float64)) + ); + } + + #[test] + fn spark_names_for_intervals_times_and_struct_fields() { + use arrow::datatypes::TimeUnit; + assert_eq!( + spark_sql_type(&Interval(IntervalUnit::YearMonth)), + "INTERVAL YEAR TO MONTH" + ); + assert_eq!( + spark_sql_type(&Duration(TimeUnit::Microsecond)), + "INTERVAL DAY TO SECOND" + ); + assert_eq!(spark_sql_type(&Time64(TimeUnit::Nanosecond)), "TIME(6)"); + let fields = Struct(Fields::from(vec![ + Field::new("a_1", Int32, true), + Field::new("a b", Int32, true), + Field::new("1a", Int32, true), + Field::new("a`b", Int32, false), + ])); + assert_eq!( + spark_sql_type(&fields), + "STRUCT" + ); + } + + #[test] + fn maps_and_calendar_intervals_are_not_orderable() { + let interval = Interval(IntervalUnit::MonthDayNano); + assert!(!is_orderable(&map(Int32, Int32))); + assert!(!is_orderable(&interval)); + assert!(!is_orderable(&list(interval.clone()))); + assert!(is_orderable(&list(Int32))); + assert_eq!(spark_sql_type(&list(interval)), "ARRAY"); + } +} diff --git a/crates/sqllogictest/test_files/array/array_position.slt b/crates/sqllogictest/test_files/array/array_position.slt index cf9ef3e8..7572fc2d 100644 --- a/crates/sqllogictest/test_files/array/array_position.slt +++ b/crates/sqllogictest/test_files/array/array_position.slt @@ -274,8 +274,8 @@ SELECT array_position(array(CAST(1.0 AS DOUBLE)), CAST(1 AS DECIMAL(5,1))); query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] SELECT array_position(array(1), '1'); -query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] -SELECT array_position(array('1'), 1); +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] Input to `array_position` should have been "ARRAY" followed by a value with same element type, but it's \["ARRAY", "INT"\]\. SQLSTATE: 42K09 +SELECT array_position(array('1'), CAST(1 AS INT)); query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] SELECT array_position(array(1), CAST(NULL AS STRING)); @@ -292,6 +292,24 @@ SELECT array_position(array(X'01'), '1'); query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] SELECT array_position(array(DATE '2020-01-01'), '2020-01-01'); +# A dictionary-encoded element or array has the type of its values +query II +SELECT array_position(array('a', 'b'), arrow_cast('b', 'Dictionary(Int32, Utf8)')), array_position(array(arrow_cast('a', 'Dictionary(Int32, Utf8)'), arrow_cast('b', 'Dictionary(Int32, Utf8)')), 'b'); +---- +2 2 + +# Dictionary-encoded floats are normalized, so -0.0 equals 0.0 +query I +SELECT array_position(arrow_cast(array(CAST('-0.0' AS DOUBLE), CAST(1.0 AS DOUBLE)), 'List(Dictionary(Int32, Float64))'), arrow_cast(CAST(0.0 AS DOUBLE), 'Dictionary(Int32, Float64)')); +---- +1 + +# So are dictionary-encoded floats inside nested types +query II +SELECT array_position(arrow_cast(array(array(CAST('-0.0' AS DOUBLE))), 'List(List(Dictionary(Int32, Float64)))'), arrow_cast(array(CAST(0.0 AS DOUBLE)), 'List(Dictionary(Int32, Float64))')), array_position(arrow_cast(array(named_struct('a', CAST('-0.0' AS DOUBLE))), 'List(Struct("a": Dictionary(Int32, Float64)))'), arrow_cast(named_struct('a', CAST(0.0 AS DOUBLE)), 'Struct("a": Dictionary(Int32, Float64))')); +---- +1 1 + # DATE and TIMESTAMP become TIMESTAMP query I SELECT array_position(array(DATE '2020-01-01'), TIMESTAMP '2020-01-01 00:00:00'); @@ -332,6 +350,38 @@ SELECT array_position(array(named_struct('m', map(1, 2))), named_struct('m', map query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] SELECT array_position(array(array(map(1, 2))), array(map(1, 2))); +# Map keys and values are widened, and the common map type is then rejected +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_position` does not support ordering on type "MAP"\. SQLSTATE: 42K09 +SELECT array_position(array(map(CAST(1 AS INT), CAST(1 AS INT))), map(1L, 1L)); + +# A map key does not widen from DATE to TIMESTAMP_NTZ, because Spark's cast between them can +# return NULL. A map value does. +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] Input to `array_position` should have been "ARRAY" followed by a value with same element type, but it's \["ARRAY>", "MAP"\]\. SQLSTATE: 42K09 +SELECT array_position(array(map(DATE '2020-01-01', CAST(1 AS INT))), map(arrow_cast(TIMESTAMP '2020-01-01 00:00:00', 'Timestamp(Microsecond, None)'), CAST(1 AS INT))); + +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_position` does not support ordering on type "MAP"\. SQLSTATE: 42K09 +SELECT array_position(array(map(CAST(1 AS INT), DATE '2020-01-01')), map(CAST(1 AS INT), arrow_cast(TIMESTAMP '2020-01-01 00:00:00', 'Timestamp(Microsecond, None)'))); + +# Spark's names for interval, time and struct types in errors +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] Input to `array_position` should have been "ARRAY" followed by a value with same element type, but it's \["ARRAY", "STRING"\]\. SQLSTATE: 42K09 +SELECT array_position(array(make_dt_interval(1)), 'x'); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] Input to `array_position` should have been "ARRAY" followed by a value with same element type, but it's \["ARRAY", "STRING"\]\. SQLSTATE: 42K09 +SELECT array_position(array(arrow_cast(1, 'Interval(YearMonth)')), 'x'); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] Input to `array_position` should have been "ARRAY" followed by a value with same element type, but it's \["ARRAY", "STRING"\]\. SQLSTATE: 42K09 +SELECT array_position(array(make_time(1, 2, 3)), 'x'); + +query error \[DATATYPE_MISMATCH\.ARRAY_FUNCTION_DIFF_TYPES\] Input to `array_position` should have been "ARRAY" followed by a value with same element type, but it's \["ARRAY>", "STRING"\]\. SQLSTATE: 42K09 +SELECT array_position(array(named_struct('a b', CAST(NULL AS INT))), 'x'); + +# Spark cannot order intervals +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_position` does not support ordering on type "INTERVAL"\. SQLSTATE: 42K09 +SELECT array_position(array(make_interval(1)), make_interval(2)); + +query error \[DATATYPE_MISMATCH\.INVALID_ORDERING_TYPE\] The `array_position` does not support ordering on type "ARRAY"\. SQLSTATE: 42K09 +SELECT array_position(array(array(make_interval(1))), array(make_interval(2))); + # Signed zero: -0.0 equals 0.0 query I SELECT array_position(array(CAST('-0.0' AS DOUBLE), CAST(1.0 AS DOUBLE)), CAST(0.0 AS DOUBLE));