From b6913d2f288b865618464fc488774d312831b748 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 11:44:20 -0700 Subject: [PATCH 1/5] feat(spark): cast function arguments to Spark's declared input types Add `function/implicit_cast.rs`, a port of Spark's `implicitCast` from `TypeCoercion` and `AnsiTypeCoercion`. A function declares its input types with `SparkInputType`, as Spark's `ImplicitCastInputTypes` does, and coerces its arguments through `SparkInputTypes`, which returns Spark's `UNEXPECTED_INPUT_TYPE` and `WRONG_NUM_ARGS` errors. `sha2` is the first function on it. It now accepts STRING, DOUBLE, FLOAT and DECIMAL bit lengths as Spark does. Part of #346. --- crates/spark/src/function/hash/sha2.rs | 46 +- crates/spark/src/function/implicit_cast.rs | 754 +++++++++++++++++++ crates/spark/src/function/mod.rs | 1 + crates/sqllogictest/test_files/hash/sha2.slt | 99 +++ 4 files changed, 879 insertions(+), 21 deletions(-) create mode 100644 crates/spark/src/function/implicit_cast.rs diff --git a/crates/spark/src/function/hash/sha2.rs b/crates/spark/src/function/hash/sha2.rs index 29e56f98..97d32b0d 100644 --- a/crates/spark/src/function/hash/sha2.rs +++ b/crates/spark/src/function/hash/sha2.rs @@ -6,19 +6,19 @@ use arrow::array::{ ArrayRef, AsArray, BinaryArrayType, BinaryBuilder, Int32Array, StringArray, }; use arrow::datatypes::{DataType, Int32Type}; -use datafusion_common::types::{ - NativeType, logical_binary, logical_int32, logical_string, -}; +use datafusion_common::config::ConfigOptions; use datafusion_common::utils::hex::{HexCase, encode_bytes_into}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, internal_err}; use datafusion_expr::{ - Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, - TypeSignatureClass, Volatility, + ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, + Signature, Volatility, }; use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; use sha2::{self, Digest}; + +use crate::function::implicit_cast::{SparkInputType, SparkInputTypes}; use std::sync::Arc; /// Spark-compatible `sha2` expression. @@ -45,9 +45,10 @@ NULL description = "Integer SHA-2 variant: 224, 256, 384 or 512. 0 is treated as 256. Any other value, or NULL, returns NULL." ) )] -#[derive(Debug, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct SparkSha2 { signature: Signature, + input_types: SparkInputTypes, } impl Default for SparkSha2 { @@ -59,21 +60,12 @@ impl Default for SparkSha2 { impl SparkSha2 { pub fn new() -> Self { Self { - signature: Signature::coercible( - vec![ - Coercion::new_implicit( - TypeSignatureClass::Native(logical_binary()), - vec![TypeSignatureClass::Native(logical_string())], - NativeType::Binary, - ), - Coercion::new_implicit( - TypeSignatureClass::Native(logical_int32()), - vec![TypeSignatureClass::Integer], - NativeType::Int32, - ), - ], - Volatility::Immutable, - ), + signature: Signature::user_defined(Volatility::Immutable), + // Spark's `Sha2.inputTypes` + input_types: SparkInputTypes::new(vec![ + SparkInputType::Binary, + SparkInputType::Integer, + ]), } } } @@ -87,6 +79,18 @@ impl ScalarUDFImpl for SparkSha2 { &self.signature } + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { + self.input_types.coerce(self.name(), arg_types) + } + + fn with_updated_config(&self, config: &ConfigOptions) -> Option { + let input_types = self.input_types.with_config(config)?; + Some(ScalarUDF::from(Self { + input_types, + ..self.clone() + })) + } + fn return_type(&self, _arg_types: &[DataType]) -> Result { Ok(DataType::Utf8) } diff --git a/crates/spark/src/function/implicit_cast.rs b/crates/spark/src/function/implicit_cast.rs new file mode 100644 index 00000000..69883771 --- /dev/null +++ b/crates/spark/src/function/implicit_cast.rs @@ -0,0 +1,754 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Casting a function's arguments to the input types it declares, as Spark does. +//! +//! Most Spark expressions mix in `ImplicitCastInputTypes` and declare +//! `inputTypes: Seq[AbstractDataType]`. The analyzer then casts each argument with +//! `implicitCast` from `TypeCoercion` (ANSI mode off) or `AnsiTypeCoercion` (ANSI mode on), +//! and `ExpectsInputTypes.checkInputDataTypes` raises `DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE` +//! for an argument that cannot be cast. +//! +//! A function here declares the same types with [`SparkInputType`], holds them in a +//! [`SparkInputTypes`], and uses `Signature::user_defined`: +//! +//! ```ignore +//! fn coerce_types(&self, arg_types: &[DataType]) -> Result> { +//! self.input_types.coerce(self.name(), arg_types) +//! } +//! +//! fn with_updated_config(&self, config: &ConfigOptions) -> Option { +//! let input_types = self.input_types.with_config(config)?; +//! Some(ScalarUDF::from(Self { input_types, ..self.clone() })) +//! } +//! ``` +//! +//! This module only chooses the target types. The cast that runs is DataFusion's, so how a +//! malformed string or an overflowing value is cast is part of the CAST work in #88. + +use arrow::datatypes::{DataType, IntervalUnit}; +use datafusion_common::config::ConfigOptions; +use datafusion_common::{Result, plan_err}; + +/// A type that a Spark function declares for one of its arguments: a concrete Spark type, +/// or one of Spark's abstract types such as `NumericType`. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[allow( + dead_code, + reason = "declared for the functions that move onto these rules after sha2 (#346)" +)] +pub(crate) enum SparkInputType { + /// `AnyDataType`: accepts every type and never casts. + Any, + /// `BooleanType` + Boolean, + /// `ByteType` (TINYINT) + Byte, + /// `ShortType` (SMALLINT) + Short, + /// `IntegerType` (INT) + Integer, + /// `LongType` (BIGINT) + Long, + /// `FloatType` + Float, + /// `DoubleType` + Double, + /// The abstract `NumericType`. Its default type is DOUBLE. + Numeric, + /// The abstract `IntegralType`. Its default type is INT. + Integral, + /// The abstract `DecimalType`: a decimal of any precision and scale. Its default type is + /// `DECIMAL(38,18)`. + Decimal, + /// `StringType` + String, + /// `BinaryType` + Binary, + /// `AbstractArrayType(element)`: an array whose elements are cast to `element`. + Array(Box), + /// `TypeCollection(types)`: the first type that the argument is accepted as, or else the + /// first that it can be cast to. + OneOf(Vec), +} + +/// The input types a function declares, and whether ANSI mode is on, which changes the casts +/// Spark allows. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub(crate) struct SparkInputTypes { + types: Vec, + ansi_mode: bool, +} + +impl SparkInputTypes { + /// Declares one input type per argument, with ANSI mode off. + pub(crate) fn new(types: Vec) -> Self { + Self { + types, + ansi_mode: false, + } + } + + /// Returns the input types for `config`, or `None` when ANSI mode is unchanged. + /// + /// Call it from `ScalarUDFImpl::with_updated_config`, which is how a function sees the + /// session's `datafusion.execution.enable_ansi_mode`. + pub(crate) fn with_config(&self, config: &ConfigOptions) -> Option { + let ansi_mode = config.execution.enable_ansi_mode; + (ansi_mode != self.ansi_mode).then(|| Self { + types: self.types.clone(), + ansi_mode, + }) + } + + /// Casts each of `arg_types` to its declared type, for `ScalarUDFImpl::coerce_types`. + /// + /// Returns Spark's `WRONG_NUM_ARGS` error for the wrong number of arguments, and + /// `DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE` for an argument that cannot be cast. + pub(crate) fn coerce( + &self, + name: &str, + arg_types: &[DataType], + ) -> Result> { + if arg_types.len() != self.types.len() { + return plan_err!( + "[WRONG_NUM_ARGS.WITHOUT_SUGGESTION] The `{name}` requires {} parameters but \ + the actual number is {}. Please, refer to \ + 'https://spark.apache.org/docs/latest/sql-ref-functions.html' for a fix. \ + SQLSTATE: 42605", + self.types.len(), + arg_types.len() + ); + } + arg_types + .iter() + .zip(&self.types) + .enumerate() + .map(|(idx, (arg_type, expected))| { + match implicit_cast(arg_type, expected, self.ansi_mode) { + Some(target) => Ok(target), + None => plan_err!( + "[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] The {} parameter of \ + `{name}` requires the {} type, however it has the type \"{}\". \ + SQLSTATE: 42K09", + ordinal_number(idx), + expected.sql(), + spark_sql_type(arg_type) + ), + } + }) + .collect() + } +} + +impl SparkInputType { + /// Spark's `AbstractDataType.acceptsType`: whether an argument of this type is used as it + /// is, without a cast. + fn accepts(&self, data_type: &DataType) -> bool { + use SparkInputType::*; + match self { + Any => true, + Boolean => matches!(data_type, DataType::Boolean), + Byte => matches!(data_type, DataType::Int8), + Short => matches!(data_type, DataType::Int16), + Integer => matches!(data_type, DataType::Int32), + Long => matches!(data_type, DataType::Int64), + Float => matches!(data_type, DataType::Float32), + Double => matches!(data_type, DataType::Float64), + Numeric => is_numeric(data_type), + Integral => is_integral(data_type), + Decimal => is_decimal(data_type), + String => is_string(data_type), + Binary => is_binary(data_type), + Array(element) => { + list_element(data_type).is_some_and(|item| element.accepts(item)) + } + OneOf(types) => types.iter().any(|t| t.accepts(data_type)), + } + } + + /// Spark's `AbstractDataType.defaultConcreteType`: the type an argument is cast to when + /// the declared type is abstract. `None` for [`SparkInputType::Any`], which never casts. + fn default_type(&self) -> Option { + use SparkInputType::*; + Some(match self { + Any => return None, + Boolean => DataType::Boolean, + Byte => DataType::Int8, + Short => DataType::Int16, + Integer | Integral => DataType::Int32, + Long => DataType::Int64, + Float => DataType::Float32, + Double | Numeric => DataType::Float64, + Decimal => DataType::Decimal128(38, 18), + String => DataType::Utf8, + Binary => DataType::Binary, + Array(element) => DataType::new_list(element.default_type()?, true), + OneOf(types) => return types.first()?.default_type(), + }) + } + + /// Whether this is a concrete Spark `AtomicType` (and not an abstract type). + fn is_concrete_atomic(&self) -> bool { + use SparkInputType::*; + matches!( + self, + Boolean | Byte | Short | Integer | Long | Float | Double | String | Binary + ) + } + + /// The concrete numeric type this declares, if it declares one. + fn concrete_numeric(&self) -> Option { + use SparkInputType::*; + match self { + Byte | Short | Integer | Long | Float | Double => self.default_type(), + _ => None, + } + } + + /// The type as Spark's `toSQLType` writes it in an error message, with the quotes. + fn sql(&self) -> String { + use SparkInputType::*; + match self { + OneOf(types) => format!( + "({})", + types + .iter() + .map(|t| t.sql()) + .collect::>() + .join(" or ") + ), + other => format!("\"{}\"", other.simple_string()), + } + } + + fn simple_string(&self) -> String { + use SparkInputType::*; + match self { + Any => "ANY".to_string(), + Boolean => "BOOLEAN".to_string(), + Byte => "TINYINT".to_string(), + Short => "SMALLINT".to_string(), + Integer => "INT".to_string(), + Long => "BIGINT".to_string(), + Float => "FLOAT".to_string(), + Double => "DOUBLE".to_string(), + Numeric => "NUMERIC".to_string(), + Integral => "INTEGRAL".to_string(), + Decimal => "DECIMAL".to_string(), + String => "STRING".to_string(), + Binary => "BINARY".to_string(), + Array(element) => format!("ARRAY<{}>", element.simple_string()), + OneOf(types) => types + .iter() + .map(|t| t.simple_string()) + .collect::>() + .join(" or "), + } + } +} + +/// The type to cast an argument of type `from` to, for a declared `expected` type, or `None` +/// if Spark does not cast it. +/// +/// A port of the private `implicitCast(DataType, AbstractDataType)` in Spark's +/// `TypeCoercion` (ANSI mode off) and `AnsiTypeCoercion` (ANSI mode on). Spark has no +/// dictionary types, so a dictionary argument has the type of its values (see #349). +fn implicit_cast( + from: &DataType, + expected: &SparkInputType, + ansi_mode: bool, +) -> Option { + let from = match from { + DataType::Dictionary(_, value) => value.as_ref(), + other => other, + }; + if expected.accepts(from) { + return Some(from.clone()); + } + if ansi_mode { + ansi_implicit_cast(from, expected) + } else { + legacy_implicit_cast(from, expected) + } +} + +/// `TypeCoercion.implicitCast`, with ANSI mode off. +fn legacy_implicit_cast(from: &DataType, expected: &SparkInputType) -> Option { + use SparkInputType as T; + match (from, expected) { + (DataType::Null, target) => target.default_type(), + // Hive's convention: a string where any number is accepted becomes a DOUBLE. + (f, T::Numeric) if is_string(f) => Some(DataType::Float64), + (f, T::Decimal) if is_numeric(f) => decimal_for_type(f), + (f, target) if is_numeric(f) && target.concrete_numeric().is_some() => { + target.concrete_numeric() + } + (f, T::Decimal) if is_string(f) => T::Decimal.default_type(), + (f, target) if is_string(f) && target.concrete_numeric().is_some() => { + target.concrete_numeric() + } + (f, T::Binary) if is_string(f) => Some(binary_for_string(f)), + (f, T::String) if is_atomic(f) => Some(DataType::Utf8), + (f, T::OneOf(types)) => types.iter().find_map(|t| implicit_cast(f, t, false)), + (f, T::Array(element)) => { + let item = list_element(f)?; + implicit_cast(item, element, false).map(|t| DataType::new_list(t, true)) + } + _ => None, + } +} + +/// `AnsiTypeCoercion.implicitCast`, with ANSI mode on. +fn ansi_implicit_cast(from: &DataType, expected: &SparkInputType) -> Option { + use SparkInputType as T; + match (from, expected) { + (f, T::Decimal) if is_numeric(f) => decimal_for_type(f), + (f, T::Integral) if is_fractional(f) => None, + // Spark allows a string to be cast to any primitive type, to keep existing queries + // working, although `Cast.canANSIStoreAssign` does not. + (f, target) + if is_string(f) + && (target.is_concrete_atomic() + || matches!(target, T::Numeric | T::Decimal)) => + { + match target { + T::Binary => Some(binary_for_string(f)), + _ => target.default_type(), + } + } + (f, T::Array(element)) if list_element(f).is_some() => { + let item = list_element(f)?; + implicit_cast(item, element, true).map(|t| DataType::new_list(t, true)) + } + (f, T::OneOf(types)) => types.iter().find_map(|t| implicit_cast(f, t, true)), + (f, target) => { + let concrete = target.default_type()?; + can_ansi_store_assign(f, &concrete).then_some(concrete) + } + } +} + +/// Spark's `Cast.canANSIStoreAssign`: whether a value of type `from` can be stored in a +/// column of type `to` under ANSI rules. +fn can_ansi_store_assign(from: &DataType, to: &DataType) -> bool { + match (from, to) { + _ if from == to => true, + (DataType::Null, _) => true, + (f, t) if is_numeric(f) && is_numeric(t) => true, + (f, t) if is_string(t) && (is_atomic(f) || is_calendar_interval(f)) => true, + (f, t) if is_datetime(f) && is_datetime(t) => true, + (f, t) if list_element(f).is_some() && list_element(t).is_some() => { + can_ansi_store_assign(list_element(f).unwrap(), list_element(t).unwrap()) + } + (DataType::Map(from_entries, _), DataType::Map(to_entries, _)) => { + can_ansi_store_assign(from_entries.data_type(), to_entries.data_type()) + } + (DataType::Struct(from_fields), DataType::Struct(to_fields)) => { + from_fields.len() == to_fields.len() + && from_fields.iter().zip(to_fields.iter()).all(|(f, t)| { + (t.is_nullable() || !f.is_nullable()) + && can_ansi_store_assign(f.data_type(), t.data_type()) + }) + } + _ => false, + } +} + +/// Spark's `DecimalType.forType`: the decimal that holds every value of a numeric type. +fn decimal_for_type(data_type: &DataType) -> Option { + use DataType::*; + let (precision, scale) = match data_type { + Int8 | UInt8 => (3, 0), + Int16 | UInt16 => (5, 0), + Int32 | UInt32 => (10, 0), + Int64 | UInt64 => (20, 0), + Float16 | Float32 => (14, 7), + Float64 => (30, 15), + _ => return None, + }; + Some(Decimal128(precision, scale)) +} + +/// The binary layout that matches a string layout, so the cast keeps the buffers. +fn binary_for_string(data_type: &DataType) -> DataType { + match data_type { + DataType::LargeUtf8 => DataType::LargeBinary, + DataType::Utf8View => DataType::BinaryView, + _ => DataType::Binary, + } +} + +fn list_element(data_type: &DataType) -> Option<&DataType> { + match data_type { + DataType::List(field) + | DataType::LargeList(field) + | DataType::FixedSizeList(field, _) => Some(field.data_type()), + _ => None, + } +} + +fn is_integral(data_type: &DataType) -> bool { + data_type.is_integer() +} + +fn is_fractional(data_type: &DataType) -> bool { + data_type.is_floating() || is_decimal(data_type) +} + +fn is_decimal(data_type: &DataType) -> bool { + data_type.is_decimal() +} + +fn is_numeric(data_type: &DataType) -> bool { + data_type.is_numeric() +} + +fn is_string(data_type: &DataType) -> bool { + matches!( + data_type, + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View + ) +} + +fn is_binary(data_type: &DataType) -> bool { + matches!( + data_type, + DataType::Binary + | DataType::LargeBinary + | DataType::BinaryView + | DataType::FixedSizeBinary(_) + ) +} + +/// Spark's `DatetimeType`: DATE, TIMESTAMP, TIMESTAMP_NTZ and TIME. +fn is_datetime(data_type: &DataType) -> bool { + matches!( + data_type, + DataType::Date32 + | DataType::Date64 + | DataType::Timestamp(_, _) + | DataType::Time32(_) + | DataType::Time64(_) + ) +} + +/// Spark's `CalendarIntervalType`, which is not an `AtomicType`. +fn is_calendar_interval(data_type: &DataType) -> bool { + matches!(data_type, DataType::Interval(IntervalUnit::MonthDayNano)) +} + +/// Spark's `AtomicType`: every type except NULL, arrays, maps, structs and +/// `CalendarIntervalType`. The year-month and day-time interval types are atomic. +fn is_atomic(data_type: &DataType) -> bool { + matches!( + data_type, + DataType::Boolean + | DataType::Interval(IntervalUnit::YearMonth | IntervalUnit::DayTime) + | DataType::Duration(_) + ) || is_numeric(data_type) + || is_string(data_type) + || is_binary(data_type) + || is_datetime(data_type) +} + +/// Spark's `ordinalNumber`, for "The first parameter ...". +fn ordinal_number(idx: usize) -> String { + const ORDINALS: [&str; 10] = [ + "first", "second", "third", "fourth", "fifth", "sixth", "seventh", "eighth", + "ninth", "tenth", + ]; + ORDINALS + .get(idx) + .map(|s| s.to_string()) + .unwrap_or_else(|| format!("{}th", idx + 1)) +} + +/// The name Spark's `toSQLType` gives a type, for error messages. +/// +/// The same function is in `type_coercion.rs` in #345; whichever lands second should use the +/// other's copy. +fn spark_sql_type(data_type: &DataType) -> String { + use DataType::*; + match data_type { + Null => "VOID".to_string(), + Boolean => "BOOLEAN".to_string(), + Int8 => "TINYINT".to_string(), + Int16 => "SMALLINT".to_string(), + Int32 => "INT".to_string(), + Int64 => "BIGINT".to_string(), + Float32 => "FLOAT".to_string(), + Float64 => "DOUBLE".to_string(), + Decimal32(p, s) | Decimal64(p, s) => format!("DECIMAL({p},{s})"), + Decimal128(p, s) | Decimal256(p, s) => format!("DECIMAL({p},{s})"), + Utf8 | LargeUtf8 | Utf8View => "STRING".to_string(), + Binary | LargeBinary | BinaryView | FixedSizeBinary(_) => "BINARY".to_string(), + Date32 | Date64 => "DATE".to_string(), + Timestamp(_, Some(_)) => "TIMESTAMP".to_string(), + Timestamp(_, None) => "TIMESTAMP_NTZ".to_string(), + Interval(IntervalUnit::MonthDayNano) => "INTERVAL".to_string(), + List(field) | LargeList(field) | FixedSizeList(field, _) => { + format!("ARRAY<{}>", spark_sql_type(field.data_type())) + } + Struct(fields) => format!( + "STRUCT<{}>", + fields + .iter() + .map(|f| format!( + "{}: {}{}", + f.name(), + spark_sql_type(f.data_type()), + if f.is_nullable() { "" } else { " NOT NULL" } + )) + .collect::>() + .join(", ") + ), + Map(entries, _) => match entries.data_type() { + Struct(kv) if kv.len() == 2 => format!( + "MAP<{}, {}>", + spark_sql_type(kv[0].data_type()), + spark_sql_type(kv[1].data_type()) + ), + other => format!("MAP<{other}>"), + }, + Dictionary(_, value) => spark_sql_type(value), + other => other.to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::SparkInputType as T; + use super::*; + + fn cast(from: DataType, expected: T) -> (Option, Option) { + ( + implicit_cast(&from, &expected, false), + implicit_cast(&from, &expected, true), + ) + } + + fn both(t: DataType) -> (Option, Option) { + (Some(t.clone()), Some(t)) + } + + fn list(item: DataType) -> DataType { + DataType::new_list(item, true) + } + + #[test] + fn accepted_types_are_not_cast() { + assert_eq!(cast(DataType::Int32, T::Integer), both(DataType::Int32)); + assert_eq!( + cast(DataType::LargeBinary, T::Binary), + both(DataType::LargeBinary) + ); + assert_eq!( + cast(DataType::Utf8View, T::String), + both(DataType::Utf8View) + ); + assert_eq!( + cast(DataType::Decimal128(10, 2), T::Numeric), + both(DataType::Decimal128(10, 2)) + ); + assert_eq!(cast(DataType::Int8, T::Integral), both(DataType::Int8)); + assert_eq!( + cast(list(DataType::Int32), T::Any), + both(list(DataType::Int32)) + ); + } + + #[test] + fn null_becomes_the_default_type() { + assert_eq!(cast(DataType::Null, T::Binary), both(DataType::Binary)); + assert_eq!(cast(DataType::Null, T::Numeric), both(DataType::Float64)); + assert_eq!( + cast(DataType::Null, T::OneOf(vec![T::String, T::Binary])), + both(DataType::Utf8) + ); + } + + #[test] + fn numbers_cast_to_each_other() { + assert_eq!(cast(DataType::Int64, T::Integer), both(DataType::Int32)); + assert_eq!(cast(DataType::Float64, T::Integer), both(DataType::Int32)); + assert_eq!( + cast(DataType::Decimal128(10, 2), T::Integer), + both(DataType::Int32) + ); + assert_eq!(cast(DataType::Int32, T::Double), both(DataType::Float64)); + assert_eq!(cast(DataType::UInt16, T::Integer), both(DataType::Int32)); + } + + #[test] + fn numbers_cast_to_the_decimal_that_holds_them() { + assert_eq!( + cast(DataType::Int32, T::Decimal), + both(DataType::Decimal128(10, 0)) + ); + assert_eq!( + cast(DataType::Int64, T::Decimal), + both(DataType::Decimal128(20, 0)) + ); + assert_eq!( + cast(DataType::Float64, T::Decimal), + both(DataType::Decimal128(30, 15)) + ); + assert_eq!( + cast(DataType::Utf8, T::Decimal), + both(DataType::Decimal128(38, 18)) + ); + } + + #[test] + fn strings_cast_to_numbers_and_binary() { + assert_eq!(cast(DataType::Utf8, T::Numeric), both(DataType::Float64)); + assert_eq!(cast(DataType::Utf8, T::Integer), both(DataType::Int32)); + assert_eq!(cast(DataType::Utf8, T::Binary), both(DataType::Binary)); + assert_eq!( + cast(DataType::LargeUtf8, T::Binary), + both(DataType::LargeBinary) + ); + assert_eq!( + cast(DataType::Utf8View, T::Binary), + both(DataType::BinaryView) + ); + } + + #[test] + fn atomic_types_cast_to_string() { + assert_eq!(cast(DataType::Int32, T::String), both(DataType::Utf8)); + assert_eq!(cast(DataType::Boolean, T::String), both(DataType::Utf8)); + assert_eq!(cast(DataType::Date32, T::String), both(DataType::Utf8)); + assert_eq!(cast(DataType::Binary, T::String), both(DataType::Utf8)); + assert_eq!(cast(list(DataType::Utf8), T::String), (None, None)); + } + + #[test] + fn types_that_are_never_cast() { + assert_eq!(cast(DataType::Int32, T::Binary), (None, None)); + assert_eq!(cast(DataType::Boolean, T::Integer), (None, None)); + assert_eq!(cast(DataType::Date32, T::Integer), (None, None)); + assert_eq!(cast(list(DataType::Utf8), T::Binary), (None, None)); + assert_eq!(cast(DataType::Utf8, T::Integral), (None, None)); + } + + #[test] + fn ansi_mode_changes_some_casts() { + // A string becomes a BOOLEAN only with ANSI mode on. + assert_eq!( + cast(DataType::Utf8, T::Boolean), + (None, Some(DataType::Boolean)) + ); + // Neither mode casts a fraction to INTEGRAL: the legacy rules cast only to concrete + // numeric types, and the ANSI rules reject fractions outright. + assert_eq!(cast(DataType::Decimal128(10, 2), T::Integral), (None, None)); + assert_eq!(cast(DataType::Float64, T::Integral), (None, None)); + assert_eq!(cast(DataType::Int64, T::Integral), both(DataType::Int64)); + // A CalendarInterval becomes a STRING only with ANSI mode on. + assert_eq!( + cast(DataType::Interval(IntervalUnit::MonthDayNano), T::String), + (None, Some(DataType::Utf8)) + ); + } + + #[test] + fn one_of_picks_the_first_type_that_works() { + let string_or_binary = T::OneOf(vec![T::String, T::Binary]); + assert_eq!( + cast(DataType::Binary, string_or_binary.clone()), + both(DataType::Binary) + ); + assert_eq!( + cast(DataType::Int32, string_or_binary), + both(DataType::Utf8) + ); + } + + #[test] + fn arrays_cast_element_by_element() { + let strings = T::Array(Box::new(T::String)); + assert_eq!( + cast(list(DataType::Int32), strings.clone()), + both(list(DataType::Utf8)) + ); + assert_eq!(cast(list(list(DataType::Int32)), strings), (None, None)); + } + + #[test] + fn dictionaries_have_the_type_of_their_values() { + let dict = + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)); + assert_eq!(cast(dict.clone(), T::String), both(DataType::Utf8)); + assert_eq!(cast(dict, T::Binary), both(DataType::Binary)); + } + + #[test] + fn coerce_reports_spark_errors() { + let types = SparkInputTypes::new(vec![T::Binary, T::Integer]); + assert_eq!( + types + .coerce("sha2", &[DataType::Utf8, DataType::Int64]) + .unwrap(), + vec![DataType::Binary, DataType::Int32] + ); + + let err = types + .coerce("sha2", &[DataType::Int32, DataType::Int32]) + .unwrap_err() + .strip_backtrace(); + assert!( + err.contains( + "[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] The first parameter of `sha2` \ + requires the \"BINARY\" type, however it has the type \"INT\". \ + SQLSTATE: 42K09" + ), + "{err}" + ); + + let err = types + .coerce("sha2", &[DataType::Utf8]) + .unwrap_err() + .strip_backtrace(); + assert!( + err.contains( + "[WRONG_NUM_ARGS.WITHOUT_SUGGESTION] The `sha2` requires 2 parameters but \ + the actual number is 1." + ), + "{err}" + ); + + let one_of = SparkInputTypes::new(vec![T::OneOf(vec![T::String, T::Binary])]); + let err = one_of + .coerce("f", &[list(DataType::Int32)]) + .unwrap_err() + .strip_backtrace(); + assert!( + err.contains("requires the (\"STRING\" or \"BINARY\") type"), + "{err}" + ); + } + + #[test] + fn with_config_follows_ansi_mode() { + let types = SparkInputTypes::new(vec![T::Boolean]); + let mut config = ConfigOptions::default(); + assert!(types.with_config(&config).is_none()); + + config.execution.enable_ansi_mode = true; + let ansi = types.with_config(&config).unwrap(); + assert!(types.coerce("f", &[DataType::Utf8]).is_err()); + assert_eq!( + ansi.coerce("f", &[DataType::Utf8]).unwrap(), + vec![DataType::Boolean] + ); + } + + #[test] + fn ordinals() { + assert_eq!(ordinal_number(0), "first"); + assert_eq!(ordinal_number(9), "tenth"); + assert_eq!(ordinal_number(10), "11th"); + } +} diff --git a/crates/spark/src/function/mod.rs b/crates/spark/src/function/mod.rs index 483f9b64..28e53829 100644 --- a/crates/spark/src/function/mod.rs +++ b/crates/spark/src/function/mod.rs @@ -15,6 +15,7 @@ pub mod error_utils; pub mod functions_nested_utils; pub mod generator; pub mod hash; +pub(crate) mod implicit_cast; pub mod json; pub mod lambda; pub mod map; diff --git a/crates/sqllogictest/test_files/hash/sha2.slt b/crates/sqllogictest/test_files/hash/sha2.slt index 0c0bf63c..d9683b8d 100644 --- a/crates/sqllogictest/test_files/hash/sha2.slt +++ b/crates/sqllogictest/test_files/hash/sha2.slt @@ -117,3 +117,102 @@ query T select sha2('a', null::int); ---- NULL + +# Spark declares the input types (BINARY, INT) and casts other arguments to them. + +# An integer literal is a BIGINT here, and it is cast to INT +query T +SELECT sha2('Spark', 256); +---- +529bc3b07127ecb7e53a4dcf1991d9152c24537d919178022b2c42657f79a26b + +query T +SELECT sha2('Spark', '256'); +---- +529bc3b07127ecb7e53a4dcf1991d9152c24537d919178022b2c42657f79a26b + +query T +SELECT sha2('Spark', 256::SMALLINT); +---- +529bc3b07127ecb7e53a4dcf1991d9152c24537d919178022b2c42657f79a26b + +query T +SELECT sha2('Spark', 0::TINYINT); +---- +529bc3b07127ecb7e53a4dcf1991d9152c24537d919178022b2c42657f79a26b + +query T +SELECT sha2('Spark', 256.7::DOUBLE); +---- +529bc3b07127ecb7e53a4dcf1991d9152c24537d919178022b2c42657f79a26b + +query T +SELECT sha2('Spark', 256.7::FLOAT); +---- +529bc3b07127ecb7e53a4dcf1991d9152c24537d919178022b2c42657f79a26b + +query T +SELECT sha2('Spark', 256.0::DECIMAL(10, 2)); +---- +529bc3b07127ecb7e53a4dcf1991d9152c24537d919178022b2c42657f79a26b + +query T +SELECT sha2(NULL::STRING, 256); +---- +NULL + +query T +SELECT sha2(arrow_cast('Spark', 'Dictionary(Int32, Utf8)'), 256); +---- +529bc3b07127ecb7e53a4dcf1991d9152c24537d919178022b2c42657f79a26b + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `sha2` requires the "BINARY" type, however it has the type "INT"\. SQLSTATE: 42K09 +SELECT sha2(1::INT, 256); + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `sha2` requires the "BINARY" type, however it has the type "DOUBLE"\. SQLSTATE: 42K09 +SELECT sha2(1.5::DOUBLE, 256); + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `sha2` requires the "BINARY" type, however it has the type "BOOLEAN"\. SQLSTATE: 42K09 +SELECT sha2(true, 256); + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `sha2` requires the "BINARY" type, however it has the type "DATE"\. SQLSTATE: 42K09 +SELECT sha2(DATE '2024-01-01', 256); + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `sha2` requires the "BINARY" type, however it has the type "ARRAY"\. SQLSTATE: 42K09 +SELECT sha2(array('a'), 256); + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The second parameter of `sha2` requires the "INT" type, however it has the type "BOOLEAN"\. SQLSTATE: 42K09 +SELECT sha2('Spark', true); + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The second parameter of `sha2` requires the "INT" type, however it has the type "DATE"\. SQLSTATE: 42K09 +SELECT sha2('Spark', DATE '2024-01-01'); + +query error \[WRONG_NUM_ARGS\.WITHOUT_SUGGESTION\] The `sha2` requires 2 parameters but the actual number is 1\. +SELECT sha2('Spark'); + +query error \[WRONG_NUM_ARGS\.WITHOUT_SUGGESTION\] The `sha2` requires 2 parameters but the actual number is 3\. +SELECT sha2('Spark', 256, 1); + +# The same arguments are cast with ANSI mode on + +statement ok +set datafusion.execution.enable_ansi_mode = true; + +query T +SELECT sha2('Spark', '256'); +---- +529bc3b07127ecb7e53a4dcf1991d9152c24537d919178022b2c42657f79a26b + +query T +SELECT sha2('Spark', 256.7::DOUBLE); +---- +529bc3b07127ecb7e53a4dcf1991d9152c24537d919178022b2c42657f79a26b + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `sha2` requires the "BINARY" type, however it has the type "INT"\. SQLSTATE: 42K09 +SELECT sha2(1::INT, 256); + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The second parameter of `sha2` requires the "INT" type, however it has the type "BOOLEAN"\. SQLSTATE: 42K09 +SELECT sha2('Spark', true); + +statement ok +set datafusion.execution.enable_ansi_mode = false; From 0599e811d4680d078c932c4f2893f387fe614cc6 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 11:49:58 -0700 Subject: [PATCH 2/5] fix(spark): cast a fixed-size binary argument to BINARY Spark has no fixed-size binary, and sha2 has no kernel for it, so FixedSizeBinary arguments failed with an internal error. The implicit cast now gives them Spark's BINARY layout, as it does for dictionaries. Also test that sha2 picks up ANSI mode through with_updated_config, and say in its docs that the bit length is cast to INT. --- crates/spark/src/function/hash/sha2.rs | 24 +++++++++++++-- crates/spark/src/function/implicit_cast.rs | 32 ++++++++++++++++---- crates/sqllogictest/test_files/hash/sha2.slt | 12 ++++++++ 3 files changed, 60 insertions(+), 8 deletions(-) diff --git a/crates/spark/src/function/hash/sha2.rs b/crates/spark/src/function/hash/sha2.rs index 97d32b0d..888e72d3 100644 --- a/crates/spark/src/function/hash/sha2.rs +++ b/crates/spark/src/function/hash/sha2.rs @@ -17,9 +17,9 @@ use datafusion_expr::{ use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; use sha2::{self, Digest}; +use std::sync::Arc; use crate::function::implicit_cast::{SparkInputType, SparkInputTypes}; -use std::sync::Arc; /// Spark-compatible `sha2` expression. /// @@ -42,7 +42,7 @@ NULL ), argument( name = "bit_length", - description = "Integer SHA-2 variant: 224, 256, 384 or 512. 0 is treated as 256. Any other value, or NULL, returns NULL." + description = "SHA-2 variant: 224, 256, 384 or 512, cast to INT as in Spark. 0 is treated as 256. Any other value, or NULL, returns NULL." ) )] #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -319,3 +319,23 @@ where }; Arc::new(str_array) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ansi_mode_reaches_coercion() { + let mut config = ConfigOptions::default(); + assert!(SparkSha2::new().with_updated_config(&config).is_none()); + + config.execution.enable_ansi_mode = true; + let ansi = SparkSha2::new().with_updated_config(&config).unwrap(); + assert_eq!( + ansi.coerce_types(&[DataType::Utf8, DataType::Utf8]) + .unwrap(), + vec![DataType::Binary, DataType::Int32] + ); + assert!(ansi.inner().with_updated_config(&config).is_none()); + } +} diff --git a/crates/spark/src/function/implicit_cast.rs b/crates/spark/src/function/implicit_cast.rs index 69883771..8a13a253 100644 --- a/crates/spark/src/function/implicit_cast.rs +++ b/crates/spark/src/function/implicit_cast.rs @@ -253,17 +253,13 @@ impl SparkInputType { /// if Spark does not cast it. /// /// A port of the private `implicitCast(DataType, AbstractDataType)` in Spark's -/// `TypeCoercion` (ANSI mode off) and `AnsiTypeCoercion` (ANSI mode on). Spark has no -/// dictionary types, so a dictionary argument has the type of its values (see #349). +/// `TypeCoercion` (ANSI mode off) and `AnsiTypeCoercion` (ANSI mode on). fn implicit_cast( from: &DataType, expected: &SparkInputType, ansi_mode: bool, ) -> Option { - let from = match from { - DataType::Dictionary(_, value) => value.as_ref(), - other => other, - }; + let from = &spark_layout(from); if expected.accepts(from) { return Some(from.clone()); } @@ -274,6 +270,18 @@ fn implicit_cast( } } +/// The layout a function receives for an argument of a type Spark does not have. +/// +/// A dictionary has the type of its values (see #349), and a fixed-size binary becomes +/// BINARY, so that a function only has to handle the layouts of Spark's own types. +fn spark_layout(data_type: &DataType) -> DataType { + match data_type { + DataType::Dictionary(_, value) => spark_layout(value), + DataType::FixedSizeBinary(_) => DataType::Binary, + other => other.clone(), + } +} + /// `TypeCoercion.implicitCast`, with ANSI mode off. fn legacy_implicit_cast(from: &DataType, expected: &SparkInputType) -> Option { use SparkInputType as T; @@ -684,6 +692,18 @@ mod tests { assert_eq!(cast(dict, T::Binary), both(DataType::Binary)); } + #[test] + fn fixed_size_binary_becomes_binary() { + assert_eq!( + cast(DataType::FixedSizeBinary(5), T::Binary), + both(DataType::Binary) + ); + assert_eq!( + cast(DataType::FixedSizeBinary(5), T::String), + both(DataType::Utf8) + ); + } + #[test] fn coerce_reports_spark_errors() { let types = SparkInputTypes::new(vec![T::Binary, T::Integer]); diff --git a/crates/sqllogictest/test_files/hash/sha2.slt b/crates/sqllogictest/test_files/hash/sha2.slt index d9683b8d..dbf5ccdf 100644 --- a/crates/sqllogictest/test_files/hash/sha2.slt +++ b/crates/sqllogictest/test_files/hash/sha2.slt @@ -166,6 +166,18 @@ SELECT sha2(arrow_cast('Spark', 'Dictionary(Int32, Utf8)'), 256); ---- 529bc3b07127ecb7e53a4dcf1991d9152c24537d919178022b2c42657f79a26b +# Spark has no fixed-size binary, so it is cast to BINARY +query T +SELECT sha2(arrow_cast(X'537061726b', 'FixedSizeBinary(5)'), 256); +---- +529bc3b07127ecb7e53a4dcf1991d9152c24537d919178022b2c42657f79a26b + +query T +SELECT sha2(c, 256) FROM (VALUES (arrow_cast(X'537061726b', 'FixedSizeBinary(5)')), (arrow_cast(X'666f6f2062', 'FixedSizeBinary(5)'))) AS t(c); +---- +529bc3b07127ecb7e53a4dcf1991d9152c24537d919178022b2c42657f79a26b +bf83b9ea169e80880e11056e2e3e7624635321a1d3805fbc5e62e82fc2568b05 + query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `sha2` requires the "BINARY" type, however it has the type "INT"\. SQLSTATE: 42K09 SELECT sha2(1::INT, 256); From 2e10f260e009e19ec163d36fef778fd69acd4e36 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 11:52:11 -0700 Subject: [PATCH 3/5] refactor(spark): simplify implicit_cast and let SparkInputTypes own the signature SparkInputTypes now builds the user_defined Signature, so a function cannot pair declared input types with a different signature. Use arrow's DataType predicates instead of local copies, and flatten a few match arms. --- crates/spark/src/function/hash/sha2.rs | 19 +-- crates/spark/src/function/implicit_cast.rs | 174 ++++++++++----------- 2 files changed, 86 insertions(+), 107 deletions(-) diff --git a/crates/spark/src/function/hash/sha2.rs b/crates/spark/src/function/hash/sha2.rs index 888e72d3..8dc122cd 100644 --- a/crates/spark/src/function/hash/sha2.rs +++ b/crates/spark/src/function/hash/sha2.rs @@ -45,9 +45,8 @@ NULL description = "SHA-2 variant: 224, 256, 384 or 512, cast to INT as in Spark. 0 is treated as 256. Any other value, or NULL, returns NULL." ) )] -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkSha2 { - signature: Signature, input_types: SparkInputTypes, } @@ -60,12 +59,11 @@ impl Default for SparkSha2 { impl SparkSha2 { pub fn new() -> Self { Self { - signature: Signature::user_defined(Volatility::Immutable), // Spark's `Sha2.inputTypes` - input_types: SparkInputTypes::new(vec![ - SparkInputType::Binary, - SparkInputType::Integer, - ]), + input_types: SparkInputTypes::new( + vec![SparkInputType::Binary, SparkInputType::Integer], + Volatility::Immutable, + ), } } } @@ -76,7 +74,7 @@ impl ScalarUDFImpl for SparkSha2 { } fn signature(&self) -> &Signature { - &self.signature + self.input_types.signature() } fn coerce_types(&self, arg_types: &[DataType]) -> Result> { @@ -85,10 +83,7 @@ impl ScalarUDFImpl for SparkSha2 { fn with_updated_config(&self, config: &ConfigOptions) -> Option { let input_types = self.input_types.with_config(config)?; - Some(ScalarUDF::from(Self { - input_types, - ..self.clone() - })) + Some(ScalarUDF::from(Self { input_types })) } fn return_type(&self, _arg_types: &[DataType]) -> Result { diff --git a/crates/spark/src/function/implicit_cast.rs b/crates/spark/src/function/implicit_cast.rs index 8a13a253..7ea1b3d3 100644 --- a/crates/spark/src/function/implicit_cast.rs +++ b/crates/spark/src/function/implicit_cast.rs @@ -11,16 +11,20 @@ //! for an argument that cannot be cast. //! //! A function here declares the same types with [`SparkInputType`], holds them in a -//! [`SparkInputTypes`], and uses `Signature::user_defined`: +//! [`SparkInputTypes`], and takes its signature from it: //! //! ```ignore +//! fn signature(&self) -> &Signature { +//! self.input_types.signature() +//! } +//! //! fn coerce_types(&self, arg_types: &[DataType]) -> Result> { //! self.input_types.coerce(self.name(), arg_types) //! } //! //! fn with_updated_config(&self, config: &ConfigOptions) -> Option { //! let input_types = self.input_types.with_config(config)?; -//! Some(ScalarUDF::from(Self { input_types, ..self.clone() })) +//! Some(ScalarUDF::from(Self { input_types })) //! } //! ``` //! @@ -30,6 +34,7 @@ use arrow::datatypes::{DataType, IntervalUnit}; use datafusion_common::config::ConfigOptions; use datafusion_common::{Result, plan_err}; +use datafusion_expr::{Signature, Volatility}; /// A type that a Spark function declares for one of its arguments: a concrete Spark type, /// or one of Spark's abstract types such as `NumericType`. @@ -79,17 +84,24 @@ pub(crate) enum SparkInputType { pub(crate) struct SparkInputTypes { types: Vec, ansi_mode: bool, + signature: Signature, } impl SparkInputTypes { /// Declares one input type per argument, with ANSI mode off. - pub(crate) fn new(types: Vec) -> Self { + pub(crate) fn new(types: Vec, volatility: Volatility) -> Self { Self { types, ansi_mode: false, + signature: Signature::user_defined(volatility), } } + /// The function's `Signature`, which leaves coercion to [`Self::coerce`]. + pub(crate) fn signature(&self) -> &Signature { + &self.signature + } + /// Returns the input types for `config`, or `None` when ANSI mode is unchanged. /// /// Call it from `ScalarUDFImpl::with_updated_config`, which is how a function sees the @@ -97,8 +109,8 @@ impl SparkInputTypes { pub(crate) fn with_config(&self, config: &ConfigOptions) -> Option { let ansi_mode = config.execution.enable_ansi_mode; (ansi_mode != self.ansi_mode).then(|| Self { - types: self.types.clone(), ansi_mode, + ..self.clone() }) } @@ -156,11 +168,11 @@ impl SparkInputType { Long => matches!(data_type, DataType::Int64), Float => matches!(data_type, DataType::Float32), Double => matches!(data_type, DataType::Float64), - Numeric => is_numeric(data_type), - Integral => is_integral(data_type), - Decimal => is_decimal(data_type), - String => is_string(data_type), - Binary => is_binary(data_type), + Numeric => data_type.is_numeric(), + Integral => data_type.is_integer(), + Decimal => data_type.is_decimal(), + String => data_type.is_string(), + Binary => data_type.is_binary(), Array(element) => { list_element(data_type).is_some_and(|item| element.accepts(item)) } @@ -192,19 +204,13 @@ impl SparkInputType { /// Whether this is a concrete Spark `AtomicType` (and not an abstract type). fn is_concrete_atomic(&self) -> bool { use SparkInputType::*; - matches!( - self, - Boolean | Byte | Short | Integer | Long | Float | Double | String | Binary - ) + self.is_concrete_numeric() || matches!(self, Boolean | String | Binary) } - /// The concrete numeric type this declares, if it declares one. - fn concrete_numeric(&self) -> Option { + /// Whether this is a concrete Spark `NumericType` (and not `Numeric` or `Decimal`). + fn is_concrete_numeric(&self) -> bool { use SparkInputType::*; - match self { - Byte | Short | Integer | Long | Float | Double => self.default_type(), - _ => None, - } + matches!(self, Byte | Short | Integer | Long | Float | Double) } /// The type as Spark's `toSQLType` writes it in an error message, with the quotes. @@ -226,26 +232,29 @@ impl SparkInputType { fn simple_string(&self) -> String { use SparkInputType::*; match self { - Any => "ANY".to_string(), - Boolean => "BOOLEAN".to_string(), - Byte => "TINYINT".to_string(), - Short => "SMALLINT".to_string(), - Integer => "INT".to_string(), - Long => "BIGINT".to_string(), - Float => "FLOAT".to_string(), - Double => "DOUBLE".to_string(), - Numeric => "NUMERIC".to_string(), - Integral => "INTEGRAL".to_string(), - Decimal => "DECIMAL".to_string(), - String => "STRING".to_string(), - Binary => "BINARY".to_string(), - Array(element) => format!("ARRAY<{}>", element.simple_string()), - OneOf(types) => types - .iter() - .map(|t| t.simple_string()) - .collect::>() - .join(" or "), + Any => "ANY", + Boolean => "BOOLEAN", + Byte => "TINYINT", + Short => "SMALLINT", + Integer => "INT", + Long => "BIGINT", + Float => "FLOAT", + Double => "DOUBLE", + Numeric => "NUMERIC", + Integral => "INTEGRAL", + Decimal => "DECIMAL", + String => "STRING", + Binary => "BINARY", + Array(element) => return format!("ARRAY<{}>", element.simple_string()), + OneOf(types) => { + return types + .iter() + .map(|t| t.simple_string()) + .collect::>() + .join(" or "); + } } + .to_string() } } @@ -288,16 +297,16 @@ fn legacy_implicit_cast(from: &DataType, expected: &SparkInputType) -> Option target.default_type(), // Hive's convention: a string where any number is accepted becomes a DOUBLE. - (f, T::Numeric) if is_string(f) => Some(DataType::Float64), - (f, T::Decimal) if is_numeric(f) => decimal_for_type(f), - (f, target) if is_numeric(f) && target.concrete_numeric().is_some() => { - target.concrete_numeric() + (f, T::Numeric) if f.is_string() => Some(DataType::Float64), + (f, T::Decimal) if f.is_numeric() => decimal_for_type(f), + (f, target) if f.is_numeric() && target.is_concrete_numeric() => { + target.default_type() } - (f, T::Decimal) if is_string(f) => T::Decimal.default_type(), - (f, target) if is_string(f) && target.concrete_numeric().is_some() => { - target.concrete_numeric() + (f, T::Decimal) if f.is_string() => T::Decimal.default_type(), + (f, target) if f.is_string() && target.is_concrete_numeric() => { + target.default_type() } - (f, T::Binary) if is_string(f) => Some(binary_for_string(f)), + (f, T::Binary) if f.is_string() => Some(binary_for_string(f)), (f, T::String) if is_atomic(f) => Some(DataType::Utf8), (f, T::OneOf(types)) => types.iter().find_map(|t| implicit_cast(f, t, false)), (f, T::Array(element)) => { @@ -312,19 +321,17 @@ fn legacy_implicit_cast(from: &DataType, expected: &SparkInputType) -> Option Option { use SparkInputType as T; match (from, expected) { - (f, T::Decimal) if is_numeric(f) => decimal_for_type(f), + (f, T::Decimal) if f.is_numeric() => decimal_for_type(f), (f, T::Integral) if is_fractional(f) => None, // Spark allows a string to be cast to any primitive type, to keep existing queries // working, although `Cast.canANSIStoreAssign` does not. + (f, T::Binary) if f.is_string() => Some(binary_for_string(f)), (f, target) - if is_string(f) + if f.is_string() && (target.is_concrete_atomic() || matches!(target, T::Numeric | T::Decimal)) => { - match target { - T::Binary => Some(binary_for_string(f)), - _ => target.default_type(), - } + target.default_type() } (f, T::Array(element)) if list_element(f).is_some() => { let item = list_element(f)?; @@ -341,15 +348,15 @@ fn ansi_implicit_cast(from: &DataType, expected: &SparkInputType) -> Option bool { + if let (Some(from_item), Some(to_item)) = (list_element(from), list_element(to)) { + return can_ansi_store_assign(from_item, to_item); + } match (from, to) { _ if from == to => true, (DataType::Null, _) => true, - (f, t) if is_numeric(f) && is_numeric(t) => true, - (f, t) if is_string(t) && (is_atomic(f) || is_calendar_interval(f)) => true, + (f, t) if f.is_numeric() && t.is_numeric() => true, + (f, t) if t.is_string() && (is_atomic(f) || is_calendar_interval(f)) => true, (f, t) if is_datetime(f) && is_datetime(t) => true, - (f, t) if list_element(f).is_some() && list_element(t).is_some() => { - can_ansi_store_assign(list_element(f).unwrap(), list_element(t).unwrap()) - } (DataType::Map(from_entries, _), DataType::Map(to_entries, _)) => { can_ansi_store_assign(from_entries.data_type(), to_entries.data_type()) } @@ -397,37 +404,9 @@ fn list_element(data_type: &DataType) -> Option<&DataType> { } } -fn is_integral(data_type: &DataType) -> bool { - data_type.is_integer() -} - +/// Spark's `FractionalType`: FLOAT, DOUBLE and DECIMAL. fn is_fractional(data_type: &DataType) -> bool { - data_type.is_floating() || is_decimal(data_type) -} - -fn is_decimal(data_type: &DataType) -> bool { - data_type.is_decimal() -} - -fn is_numeric(data_type: &DataType) -> bool { - data_type.is_numeric() -} - -fn is_string(data_type: &DataType) -> bool { - matches!( - data_type, - DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View - ) -} - -fn is_binary(data_type: &DataType) -> bool { - matches!( - data_type, - DataType::Binary - | DataType::LargeBinary - | DataType::BinaryView - | DataType::FixedSizeBinary(_) - ) + data_type.is_floating() || data_type.is_decimal() } /// Spark's `DatetimeType`: DATE, TIMESTAMP, TIMESTAMP_NTZ and TIME. @@ -455,9 +434,9 @@ fn is_atomic(data_type: &DataType) -> bool { DataType::Boolean | DataType::Interval(IntervalUnit::YearMonth | IntervalUnit::DayTime) | DataType::Duration(_) - ) || is_numeric(data_type) - || is_string(data_type) - || is_binary(data_type) + ) || data_type.is_numeric() + || data_type.is_string() + || data_type.is_binary() || is_datetime(data_type) } @@ -488,8 +467,9 @@ fn spark_sql_type(data_type: &DataType) -> String { Int64 => "BIGINT".to_string(), Float32 => "FLOAT".to_string(), Float64 => "DOUBLE".to_string(), - Decimal32(p, s) | Decimal64(p, s) => format!("DECIMAL({p},{s})"), - Decimal128(p, s) | Decimal256(p, s) => format!("DECIMAL({p},{s})"), + Decimal32(p, s) | Decimal64(p, s) | Decimal128(p, s) | Decimal256(p, s) => { + format!("DECIMAL({p},{s})") + } Utf8 | LargeUtf8 | Utf8View => "STRING".to_string(), Binary | LargeBinary | BinaryView | FixedSizeBinary(_) => "BINARY".to_string(), Date32 | Date64 => "DATE".to_string(), @@ -706,7 +686,8 @@ mod tests { #[test] fn coerce_reports_spark_errors() { - let types = SparkInputTypes::new(vec![T::Binary, T::Integer]); + let types = + SparkInputTypes::new(vec![T::Binary, T::Integer], Volatility::Immutable); assert_eq!( types .coerce("sha2", &[DataType::Utf8, DataType::Int64]) @@ -739,7 +720,10 @@ mod tests { "{err}" ); - let one_of = SparkInputTypes::new(vec![T::OneOf(vec![T::String, T::Binary])]); + let one_of = SparkInputTypes::new( + vec![T::OneOf(vec![T::String, T::Binary])], + Volatility::Immutable, + ); let err = one_of .coerce("f", &[list(DataType::Int32)]) .unwrap_err() @@ -752,7 +736,7 @@ mod tests { #[test] fn with_config_follows_ansi_mode() { - let types = SparkInputTypes::new(vec![T::Boolean]); + let types = SparkInputTypes::new(vec![T::Boolean], Volatility::Immutable); let mut config = ConfigOptions::default(); assert!(types.with_config(&config).is_none()); From 57e82562f617cc22e9fe9aca18e6fc7c01ae6076 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 12:48:13 -0700 Subject: [PATCH 4/5] fix(spark): match Spark's ordinals and nested type-collection names in errors Spark spells out only the first three ordinals ("4th parameter"), and writes a type collection inside an array as ARRAY<(STRING OR BINARY)>. --- crates/spark/src/function/implicit_cast.rs | 46 +++++++++++++++------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/crates/spark/src/function/implicit_cast.rs b/crates/spark/src/function/implicit_cast.rs index 7ea1b3d3..520bdf85 100644 --- a/crates/spark/src/function/implicit_cast.rs +++ b/crates/spark/src/function/implicit_cast.rs @@ -246,12 +246,17 @@ impl SparkInputType { String => "STRING", Binary => "BINARY", Array(element) => return format!("ARRAY<{}>", element.simple_string()), + // `toSQLType` upper-cases the whole `simpleString` of an abstract type, so a + // collection inside an array reads `ARRAY<(STRING OR BINARY)>`. OneOf(types) => { - return types - .iter() - .map(|t| t.simple_string()) - .collect::>() - .join(" or "); + return format!( + "({})", + types + .iter() + .map(|t| t.simple_string()) + .collect::>() + .join(" OR ") + ); } } .to_string() @@ -440,16 +445,15 @@ fn is_atomic(data_type: &DataType) -> bool { || is_datetime(data_type) } -/// Spark's `ordinalNumber`, for "The first parameter ...". +/// Spark's `ordinalNumber`, for "The first parameter ...". Spark spells out only the first +/// three. fn ordinal_number(idx: usize) -> String { - const ORDINALS: [&str; 10] = [ - "first", "second", "third", "fourth", "fifth", "sixth", "seventh", "eighth", - "ninth", "tenth", - ]; - ORDINALS - .get(idx) - .map(|s| s.to_string()) - .unwrap_or_else(|| format!("{}th", idx + 1)) + match idx { + 0 => "first".to_string(), + 1 => "second".to_string(), + 2 => "third".to_string(), + _ => format!("{}th", idx + 1), + } } /// The name Spark's `toSQLType` gives a type, for error messages. @@ -752,7 +756,19 @@ mod tests { #[test] fn ordinals() { assert_eq!(ordinal_number(0), "first"); - assert_eq!(ordinal_number(9), "tenth"); + assert_eq!(ordinal_number(2), "third"); + assert_eq!(ordinal_number(3), "4th"); assert_eq!(ordinal_number(10), "11th"); } + + #[test] + fn type_names_match_to_sql_type() { + let string_or_binary = T::OneOf(vec![T::String, T::Binary]); + assert_eq!(string_or_binary.sql(), "(\"STRING\" or \"BINARY\")"); + assert_eq!( + T::Array(Box::new(string_or_binary)).sql(), + "\"ARRAY<(STRING OR BINARY)>\"" + ); + assert_eq!(T::Array(Box::new(T::Long)).sql(), "\"ARRAY\""); + } } From d3f29ae25b64df69ecc3fc0323d040d69fdeea0d Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 18 Nov 2026 08:27:49 -0700 Subject: [PATCH 5/5] fix(spark): match #345's spark_sql_type names in implicit_cast errors Name interval, duration and time types as Spark does, quote struct field names that need it, and use Arrow names for types Spark lacks. Co-Authored-By: Claude --- crates/spark/src/function/implicit_cast.rs | 62 +++++++++++++++++--- crates/sqllogictest/test_files/hash/sha2.slt | 9 +++ 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/crates/spark/src/function/implicit_cast.rs b/crates/spark/src/function/implicit_cast.rs index 520bdf85..3453eaf1 100644 --- a/crates/spark/src/function/implicit_cast.rs +++ b/crates/spark/src/function/implicit_cast.rs @@ -471,15 +471,17 @@ fn spark_sql_type(data_type: &DataType) -> String { Int64 => "BIGINT".to_string(), Float32 => "FLOAT".to_string(), Float64 => "DOUBLE".to_string(), - Decimal32(p, s) | Decimal64(p, s) | Decimal128(p, s) | Decimal256(p, s) => { - format!("DECIMAL({p},{s})") - } + Decimal128(p, s) => format!("DECIMAL({p},{s})"), Utf8 | LargeUtf8 | Utf8View => "STRING".to_string(), - Binary | LargeBinary | BinaryView | FixedSizeBinary(_) => "BINARY".to_string(), - Date32 | Date64 => "DATE".to_string(), + Binary | LargeBinary | BinaryView => "BINARY".to_string(), + Date32 => "DATE".to_string(), Timestamp(_, Some(_)) => "TIMESTAMP".to_string(), Timestamp(_, None) => "TIMESTAMP_NTZ".to_string(), Interval(IntervalUnit::MonthDayNano) => "INTERVAL".to_string(), + Interval(IntervalUnit::YearMonth) => "INTERVAL YEAR TO MONTH".to_string(), + Duration(_) => "INTERVAL DAY TO SECOND".to_string(), + // Arrow has no TIME precision. Spark's `make_time` returns TIME(6). + Time64(_) => "TIME(6)".to_string(), List(field) | LargeList(field) | FixedSizeList(field, _) => { format!("ARRAY<{}>", spark_sql_type(field.data_type())) } @@ -489,7 +491,7 @@ fn spark_sql_type(data_type: &DataType) -> String { .iter() .map(|f| format!( "{}: {}{}", - f.name(), + quote_if_needed(f.name()), spark_sql_type(f.data_type()), if f.is_nullable() { "" } else { " NOT NULL" } )) @@ -497,10 +499,10 @@ fn spark_sql_type(data_type: &DataType) -> String { .join(", ") ), Map(entries, _) => match entries.data_type() { - Struct(kv) if kv.len() == 2 => format!( + Struct(kv) if let [k, v] = &kv[..] => format!( "MAP<{}, {}>", - spark_sql_type(kv[0].data_type()), - spark_sql_type(kv[1].data_type()) + spark_sql_type(k.data_type()), + spark_sql_type(v.data_type()) ), other => format!("MAP<{other}>"), }, @@ -509,6 +511,20 @@ fn spark_sql_type(data_type: &DataType) -> String { } } +/// Spark's `QuotingUtils.quoteIfNeeded`, which `StructField.sql` uses for field names. +fn quote_if_needed(name: &str) -> String { + let mut chars = name.chars(); + let plain = chars + .next() + .is_some_and(|c| c.is_ascii_alphabetic() || c == '_') + && chars.all(|c| c.is_ascii_alphanumeric() || c == '_'); + if plain { + name.to_string() + } else { + format!("`{}`", name.replace('`', "``")) + } +} + #[cfg(test)] mod tests { use super::SparkInputType as T; @@ -771,4 +787,32 @@ mod tests { ); assert_eq!(T::Array(Box::new(T::Long)).sql(), "\"ARRAY\""); } + + #[test] + fn arrow_type_names_match_to_sql_type() { + use arrow::datatypes::{Field, TimeUnit}; + let name = spark_sql_type; + assert_eq!( + name(&DataType::Interval(IntervalUnit::YearMonth)), + "INTERVAL YEAR TO MONTH" + ); + assert_eq!( + name(&DataType::Interval(IntervalUnit::MonthDayNano)), + "INTERVAL" + ); + assert_eq!( + name(&DataType::Duration(TimeUnit::Microsecond)), + "INTERVAL DAY TO SECOND" + ); + assert_eq!(name(&DataType::Time64(TimeUnit::Nanosecond)), "TIME(6)"); + let fields = vec![ + Field::new("a", DataType::Int32, true), + Field::new("a b", DataType::Int32, false), + Field::new("c`d", DataType::Utf8, true), + ]; + assert_eq!( + name(&DataType::Struct(fields.into())), + "STRUCT" + ); + } } diff --git a/crates/sqllogictest/test_files/hash/sha2.slt b/crates/sqllogictest/test_files/hash/sha2.slt index dbf5ccdf..6426c6d1 100644 --- a/crates/sqllogictest/test_files/hash/sha2.slt +++ b/crates/sqllogictest/test_files/hash/sha2.slt @@ -199,6 +199,15 @@ SELECT sha2('Spark', true); query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The second parameter of `sha2` requires the "INT" type, however it has the type "DATE"\. SQLSTATE: 42K09 SELECT sha2('Spark', DATE '2024-01-01'); +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `sha2` requires the "BINARY" type, however it has the type "INTERVAL DAY TO SECOND"\. SQLSTATE: 42K09 +SELECT sha2(make_dt_interval(1), 256); + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `sha2` requires the "BINARY" type, however it has the type "INTERVAL YEAR TO MONTH"\. SQLSTATE: 42K09 +SELECT sha2(arrow_cast(1, 'Interval(YearMonth)'), 256); + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `sha2` requires the "BINARY" type, however it has the type "STRUCT<`a b`: INT>"\. SQLSTATE: 42K09 +SELECT sha2(named_struct('a b', CAST(NULL AS INT)), 256); + query error \[WRONG_NUM_ARGS\.WITHOUT_SUGGESTION\] The `sha2` requires 2 parameters but the actual number is 1\. SELECT sha2('Spark');