From 2547d65b8ea08cebb9d9184ff0a3b50bad7d80cf Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 18 Nov 2026 09:34:52 -0700 Subject: [PATCH 1/2] feat(spark): parse Spark type names in spark_cast and call the cast kernel - Parse the target type of `spark_cast(expr, '')` with Spark's SQL type grammar, including decimals, CHAR/VARCHAR, TIMESTAMP variants and ARRAY, MAP and STRUCT types, and reject the types this crate does not support. - Accept any input type, and call the Spark-compatible cast kernel with the eval mode and session time zone from the config. - Port Comet's integer and float to TIMESTAMP casts into the kernel. In LEGACY mode an out-of-range float saturates, as Spark's `doubleToTimestamp` does, where Comet returns NULL. --- .../src/function/conversion/cast/data_type.rs | 543 ++++++++ .../spark/src/function/conversion/cast/mod.rs | 110 +- .../src/function/conversion/cast/numeric.rs | 336 +++++ .../spark/src/function/conversion/cast/udf.rs | 1095 ++++------------- .../conversion/cast_float_to_timestamp.slt | 17 +- .../conversion/cast_int_to_timestamp.slt | 8 +- .../conversion/cast_target_type.slt | 89 ++ 7 files changed, 1305 insertions(+), 893 deletions(-) create mode 100644 crates/spark/src/function/conversion/cast/data_type.rs create mode 100644 crates/spark/src/function/conversion/cast/numeric.rs create mode 100644 crates/sqllogictest/test_files/conversion/cast_target_type.slt diff --git a/crates/spark/src/function/conversion/cast/data_type.rs b/crates/spark/src/function/conversion/cast/data_type.rs new file mode 100644 index 00000000..1e27442f --- /dev/null +++ b/crates/spark/src/function/conversion/cast/data_type.rs @@ -0,0 +1,543 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Parses Spark type names with Spark's SQL type grammar, the `dataType` rule +//! of +//! [`SqlBaseParser.g4`](https://github.com/apache/spark/blob/v4.2.0/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4#L1477-L1525). + +use std::sync::Arc; + +use arrow::datatypes::{DECIMAL128_MAX_PRECISION, DataType, Field, Fields, TimeUnit}; +use datafusion_common::{DataFusionError, Result, plan_datafusion_err, plan_err}; + +/// Parses a Spark type name, such as `int`, `decimal(10, 2)` or +/// `struct>`, into the Arrow type that represents it. +/// +/// Keywords are case-insensitive, as in Spark. `TIMESTAMP` is +/// `TIMESTAMP_LTZ`, Spark's default `spark.sql.timestampType`, and is tagged +/// with `session_timezone`. `CHAR(n)` and `VARCHAR(n)` are `STRING`, as Spark +/// casts them by default. Types Spark can name but this crate does not support, +/// such as intervals, `TIME`, `VARIANT` and collated strings, are an error. +pub(crate) fn parse_spark_type( + type_name: &str, + session_timezone: &str, +) -> Result { + let mut parser = Parser { + tokens: tokenize(type_name)?, + position: 0, + type_name, + session_timezone, + }; + let data_type = parser.data_type()?; + match parser.peek() { + None => Ok(data_type), + Some(token) => Err(parser.error(&format!("unexpected {token}"))), + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum Token { + /// An unquoted identifier or keyword. + Word(String), + /// A backquoted identifier, with its quotes removed. + Quoted(String), + /// A single-quoted string, as in `COMMENT 'text'`. + Text, + Integer(u64), + Symbol(char), +} + +impl std::fmt::Display for Token { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Token::Word(word) => write!(f, "'{word}'"), + Token::Quoted(name) => write!(f, "`{name}`"), + Token::Text => write!(f, "string literal"), + Token::Integer(value) => write!(f, "'{value}'"), + Token::Symbol(symbol) => write!(f, "'{symbol}'"), + } + } +} + +fn invalid(type_name: &str, message: &str) -> DataFusionError { + plan_datafusion_err!("Invalid spark_cast target type '{type_name}': {message}") +} + +fn tokenize(type_name: &str) -> Result> { + let error = |message: &str| invalid(type_name, message); + let mut tokens = vec![]; + let mut chars = type_name.chars().peekable(); + while let Some(&c) = chars.peek() { + if c.is_whitespace() { + chars.next(); + } else if c.is_ascii_alphabetic() || c == '_' { + let mut word = String::new(); + while let Some(&c) = chars.peek() + && (c.is_ascii_alphanumeric() || c == '_') + { + word.push(c); + chars.next(); + } + tokens.push(Token::Word(word)); + } else if c.is_ascii_digit() { + let mut digits = String::new(); + while let Some(&c) = chars.peek() + && c.is_ascii_digit() + { + digits.push(c); + chars.next(); + } + let value = digits + .parse() + .map_err(|_| error(&format!("'{digits}' is out of range")))?; + tokens.push(Token::Integer(value)); + } else if c == '`' || c == '\'' { + // A doubled quote inside a quoted identifier or string is a literal + // quote. + chars.next(); + let mut text = String::new(); + loop { + match chars.next() { + Some(q) if q == c => { + if chars.peek() == Some(&c) { + text.push(c); + chars.next(); + } else { + break; + } + } + Some(other) => text.push(other), + None => return Err(error("unterminated quote")), + } + } + tokens.push(if c == '`' { + Token::Quoted(text) + } else { + Token::Text + }); + } else if "<>(),:".contains(c) { + tokens.push(Token::Symbol(c)); + chars.next(); + } else { + return Err(error(&format!("unexpected character '{c}'"))); + } + } + Ok(tokens) +} + +const TIMESTAMP_NTZ: DataType = DataType::Timestamp(TimeUnit::Microsecond, None); + +struct Parser<'a> { + tokens: Vec, + position: usize, + type_name: &'a str, + session_timezone: &'a str, +} + +impl Parser<'_> { + fn error(&self, message: &str) -> DataFusionError { + invalid(self.type_name, message) + } + + fn unsupported(&self, what: &str) -> DataFusionError { + plan_datafusion_err!( + "Unsupported spark_cast target type '{}': {what} is not supported", + self.type_name + ) + } + + fn peek(&self) -> Option<&Token> { + self.tokens.get(self.position) + } + + /// Applies `f` to the next token, and consumes it if `f` returns `Some`. + fn eat_map(&mut self, f: impl FnOnce(&Token) -> Option) -> Option { + let value = self.peek().and_then(f); + if value.is_some() { + self.position += 1; + } + value + } + + fn peek_keyword(&self, keyword: &str) -> bool { + matches!(self.peek(), Some(Token::Word(word)) if word.eq_ignore_ascii_case(keyword)) + } + + fn eat_keyword(&mut self, keyword: &str) -> bool { + let found = self.peek_keyword(keyword); + if found { + self.position += 1; + } + found + } + + fn expect_keyword(&mut self, keyword: &str) -> Result<()> { + if self.eat_keyword(keyword) { + Ok(()) + } else { + Err(self.expected(&format!("'{keyword}'"))) + } + } + + fn eat_symbol(&mut self, symbol: char) -> bool { + let found = self.peek() == Some(&Token::Symbol(symbol)); + if found { + self.position += 1; + } + found + } + + fn expect_symbol(&mut self, symbol: char) -> Result<()> { + if self.eat_symbol(symbol) { + Ok(()) + } else { + Err(self.expected(&format!("'{symbol}'"))) + } + } + + fn expected(&self, what: &str) -> DataFusionError { + match self.peek() { + Some(token) => self.error(&format!("expected {what}, found {token}")), + None => self.error(&format!("expected {what}")), + } + } + + fn integer(&mut self) -> Result { + self.eat_map(|token| match token { + Token::Integer(value) => Some(*value), + _ => None, + }) + .ok_or_else(|| self.expected("an integer")) + } + + /// The parenthesized integers after a type name, if any. + fn parameters(&mut self) -> Result> { + let mut parameters = vec![]; + if self.eat_symbol('(') { + loop { + parameters.push(self.integer()?); + if !self.eat_symbol(',') { + break; + } + } + self.expect_symbol(')')?; + } + Ok(parameters) + } + + fn data_type(&mut self) -> Result { + let Some(word) = self.eat_map(|token| match token { + Token::Word(word) => Some(word.to_ascii_lowercase()), + _ => None, + }) else { + return Err(self.expected("a type")); + }; + let data_type = match word.as_str() { + "array" => { + self.expect_symbol('<')?; + let element = self.data_type()?; + self.expect_symbol('>')?; + DataType::List(Arc::new(Field::new_list_field(element, true))) + } + "map" => { + self.expect_symbol('<')?; + let key = self.data_type()?; + self.expect_symbol(',')?; + let value = self.data_type()?; + self.expect_symbol('>')?; + // The field names Spark's `ArrowUtils` uses for a map. + let entries = Field::new_struct( + "entries", + vec![ + Field::new("key", key, false), + Field::new("value", value, true), + ], + false, + ); + DataType::Map(Arc::new(entries), false) + } + "struct" => DataType::Struct(self.struct_fields()?), + "boolean" => DataType::Boolean, + "tinyint" | "byte" => DataType::Int8, + "smallint" | "short" => DataType::Int16, + "int" | "integer" => DataType::Int32, + "bigint" | "long" => DataType::Int64, + "float" | "real" => DataType::Float32, + "double" => DataType::Float64, + "date" => DataType::Date32, + "binary" => DataType::Binary, + "timestamp_ltz" => self.timestamp_ltz(), + "timestamp_ntz" => TIMESTAMP_NTZ, + "timestamp" => { + if self.eat_keyword("with") { + for keyword in ["local", "time", "zone"] { + self.expect_keyword(keyword)?; + } + self.timestamp_ltz() + } else if self.eat_keyword("without") { + for keyword in ["time", "zone"] { + self.expect_keyword(keyword)?; + } + TIMESTAMP_NTZ + } else { + self.timestamp_ltz() + } + } + "string" => DataType::Utf8, + "char" | "character" | "varchar" => { + match self.parameters()?.len() { + 1 => DataType::Utf8, + // Spark requires a length. + 0 => { + let name = word.to_ascii_uppercase(); + return plan_err!( + "[DATATYPE_MISSING_SIZE] DataType \"{name}\" requires a length \ + parameter, for example \"{name}\"(10). Please specify the length. \ + SQLSTATE: 42K01" + ); + } + _ => return Err(self.error(&format!("{word} takes one length"))), + } + } + "decimal" | "dec" | "numeric" => self.decimal()?, + "interval" | "time" | "variant" | "void" | "geography" | "geometry" => { + return Err(self.unsupported(&word.to_ascii_uppercase())); + } + _ => return Err(self.error(&format!("unknown type '{word}'"))), + }; + if self.peek_keyword("collate") { + return Err(self.unsupported("a collation")); + } + Ok(data_type) + } + + fn timestamp_ltz(&self) -> DataType { + DataType::Timestamp(TimeUnit::Microsecond, Some(self.session_timezone.into())) + } + + fn decimal(&mut self) -> Result { + let (precision, scale) = match self.parameters()?.as_slice() { + // Spark's default for a decimal without a precision. + [] => (10, 0), + [precision] => (*precision, 0), + [precision, scale] => (*precision, *scale), + _ => return Err(self.error("decimal takes a precision and a scale")), + }; + if !(1..=u64::from(DECIMAL128_MAX_PRECISION)).contains(&precision) { + return Err(self.error(&format!( + "decimal precision {precision} is not between 1 and \ + {DECIMAL128_MAX_PRECISION}" + ))); + } + if scale > precision { + return Err(self.error(&format!( + "decimal scale {scale} is greater than its precision {precision}" + ))); + } + // Both fit, since precision is at most 38. + Ok(DataType::Decimal128(precision as u8, scale as i8)) + } + + /// `struct<>`, or `struct`. + fn struct_fields(&mut self) -> Result { + self.expect_symbol('<')?; + let mut fields = vec![]; + if self.eat_symbol('>') { + return Ok(fields.into()); + } + loop { + let Some(name) = self.eat_map(|token| match token { + Token::Word(name) | Token::Quoted(name) => Some(name.clone()), + _ => None, + }) else { + return Err(self.expected("a field name")); + }; + self.eat_symbol(':'); + let data_type = self.data_type()?; + let nullable = if self.eat_keyword("not") { + self.expect_keyword("null")?; + false + } else { + true + }; + if self.eat_keyword("comment") + && self + .eat_map(|token| (token == &Token::Text).then_some(())) + .is_none() + { + return Err(self.expected("a comment string")); + } + fields.push(Field::new(name, data_type, nullable)); + if !self.eat_symbol(',') { + break; + } + } + self.expect_symbol('>')?; + Ok(fields.into()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse(type_name: &str) -> DataType { + parse_spark_type(type_name, "America/New_York").unwrap() + } + + fn parse_error(type_name: &str) -> String { + parse_spark_type(type_name, "UTC").unwrap_err().to_string() + } + + fn list(element: DataType) -> DataType { + DataType::List(Arc::new(Field::new_list_field(element, true))) + } + + #[test] + fn test_primitive_types() { + let ltz = + DataType::Timestamp(TimeUnit::Microsecond, Some("America/New_York".into())); + let ntz = DataType::Timestamp(TimeUnit::Microsecond, None); + for (type_name, expected) in [ + ("boolean", DataType::Boolean), + ("tinyint", DataType::Int8), + ("byte", DataType::Int8), + ("smallint", DataType::Int16), + ("short", DataType::Int16), + ("int", DataType::Int32), + ("INTEGER", DataType::Int32), + ("bigint", DataType::Int64), + ("Long", DataType::Int64), + ("float", DataType::Float32), + ("real", DataType::Float32), + ("double", DataType::Float64), + ("date", DataType::Date32), + ("binary", DataType::Binary), + ("string", DataType::Utf8), + ("char(3)", DataType::Utf8), + ("varchar(10)", DataType::Utf8), + ("timestamp", ltz.clone()), + ("timestamp_ltz", ltz.clone()), + ("timestamp with local time zone", ltz), + ("timestamp_ntz", ntz.clone()), + ("TIMESTAMP WITHOUT TIME ZONE", ntz), + ("decimal", DataType::Decimal128(10, 0)), + ("dec(5)", DataType::Decimal128(5, 0)), + ("numeric(38, 38)", DataType::Decimal128(38, 38)), + (" decimal ( 10 , 2 ) ", DataType::Decimal128(10, 2)), + ] { + assert_eq!(parse(type_name), expected, "{type_name}"); + } + } + + #[test] + fn test_complex_types() { + assert_eq!(parse("array"), list(DataType::Int32)); + assert_eq!(parse("ARRAY>"), list(list(DataType::Utf8))); + + let DataType::Map(entries, false) = parse("map>") else { + panic!("expected a map"); + }; + let DataType::Struct(fields) = entries.data_type() else { + panic!("expected struct entries"); + }; + assert_eq!( + fields[0].as_ref(), + &Field::new("key", DataType::Utf8, false) + ); + assert_eq!( + fields[1].as_ref(), + &Field::new("value", list(DataType::Int32), true) + ); + + assert_eq!( + parse( + "struct, a bigint>" + ), + DataType::Struct(Fields::from(vec![ + Field::new("a", DataType::Int32, true), + Field::new("b", DataType::Utf8, false), + Field::new( + "c d", + DataType::Struct(Fields::from(vec![Field::new( + "a", + DataType::Decimal128(4, 1), + true + )])), + true + ), + // Spark allows duplicate field names. + Field::new("a", DataType::Int64, true), + ])) + ); + assert_eq!(parse("struct<>"), DataType::Struct(Fields::empty())); + } + + /// Spark requires a length for CHAR and VARCHAR. + #[test] + fn test_char_and_varchar_require_a_length() { + for (type_name, name) in [ + ("char", "CHAR"), + ("Character", "CHARACTER"), + ("varchar", "VARCHAR"), + ] { + assert_eq!( + parse_error(type_name), + format!( + "Error during planning: [DATATYPE_MISSING_SIZE] DataType \"{name}\" requires a \ + length parameter, for example \"{name}\"(10). Please specify the length. \ + SQLSTATE: 42K01" + ) + ); + } + } + + #[test] + fn test_invalid_types() { + for (type_name, message) in [ + ("", "expected a type"), + ("integer2", "unknown type 'integer2'"), + ("int int", "unexpected 'int'"), + ("array", "expected '<'"), + ("array'"), + ("map", "expected ','"), + ("struct", "expected a type, found '>'"), + ("struct", "expected a field name, found '>'"), + ("struct<`a int>", "unterminated quote"), + ("decimal(0)", "precision 0 is not between 1 and 38"), + ("decimal(39)", "precision 39 is not between 1 and 38"), + ("decimal(5, 6)", "scale 6 is greater than its precision 5"), + ("decimal(1, 2, 3)", "decimal takes a precision and a scale"), + ("decimal(-1)", "unexpected character '-'"), + ("timestamp with time zone", "expected 'local', found 'time'"), + ] { + let error = parse_error(type_name); + assert!( + error.contains("Invalid spark_cast target type") + && error.contains(message), + "{type_name}: {error}" + ); + } + } + + #[test] + fn test_unsupported_types() { + for type_name in [ + "interval", + "interval day to second", + "time", + "variant", + "void", + "string collate UTF8_LCASE", + "array", + ] { + let error = parse_error(type_name); + assert!( + error.contains("Unsupported spark_cast target type"), + "{type_name}: {error}" + ); + } + } +} diff --git a/crates/spark/src/function/conversion/cast/mod.rs b/crates/spark/src/function/conversion/cast/mod.rs index f6a96682..130c6c36 100644 --- a/crates/spark/src/function/conversion/cast/mod.rs +++ b/crates/spark/src/function/conversion/cast/mod.rs @@ -15,6 +15,8 @@ //! than falling through to Arrow's cast and producing a result that differs //! from Spark's. +mod data_type; +mod numeric; mod udf; mod utils; @@ -32,8 +34,10 @@ use datafusion_expr::ColumnarValue; use crate::function::timezone_utils::{local_to_timestamp, timestamp_to_local}; use crate::{EvalMode, SparkTimeZone}; +use numeric::{cast_float_to_timestamp, cast_int_to_timestamp}; use utils::remove_trailing_zeroes; +pub(crate) use data_type::parse_spark_type; pub use udf::SparkCast; /// The format Spark uses for a timestamp cast to a string. `%.f` writes the @@ -54,6 +58,8 @@ pub struct SparkCastOptions { /// `TIMESTAMP` to `TIMESTAMP_NTZ`, `STRING` or `DATE`, or from /// `TIMESTAMP_NTZ` to `TIMESTAMP`, returns an error without one. pub timezone: Option, + /// Why `timezone` is missing, for the error of a cast that needs it. + missing_timezone_reason: Option, } impl SparkCastOptions { @@ -62,20 +68,39 @@ impl SparkCastOptions { Self { eval_mode, timezone: None, + missing_timezone_reason: None, } } pub fn with_timezone(self, timezone: SparkTimeZone) -> Self { Self { timezone: Some(timezone), + missing_timezone_reason: None, + ..self + } + } + + /// Options with no session time zone because the configured one could not + /// be used, for `reason`. + pub(crate) fn without_timezone_because(self, reason: String) -> Self { + Self { + timezone: None, + missing_timezone_reason: Some(reason), ..self } } fn session_timezone(&self) -> Result<&SparkTimeZone> { - self.timezone.as_ref().ok_or_else(|| { - exec_datafusion_err!("Casting a timestamp requires a session time zone") - }) + self.timezone + .as_ref() + .ok_or_else(|| match &self.missing_timezone_reason { + Some(reason) => exec_datafusion_err!( + "Casting a timestamp requires a session time zone: {reason}" + ), + None => exec_datafusion_err!( + "Casting a timestamp requires a session time zone" + ), + }) } } @@ -159,6 +184,15 @@ pub(crate) fn cast_array( // A NULL array has no values, so Arrow's cast only changes the type. (Null, _) => arrow_cast(&array), + // Numbers are seconds since the epoch. Spark has no cast from a number + // to TIMESTAMP_NTZ. + (Int8 | Int16 | Int32 | Int64, Timestamp(_, tz @ Some(_))) => { + cast_int_to_timestamp(&array, tz) + } + (Float32 | Float64, Timestamp(_, tz @ Some(_))) => { + cast_float_to_timestamp(&array, tz, eval_mode) + } + // Timestamps. Arrow's casts ignore the session time zone. (Timestamp(_, Some(_)), Int64) => { // Spark returns whole seconds, rounded down. @@ -213,6 +247,38 @@ pub(crate) fn cast_array( } // Spark replaces invalid UTF-8, and Arrow raises an error. (Binary, Utf8) => not_yet_ported(from_type, to_type, eval_mode), + // Casts Spark supports that Arrow does not do the same way. + ( + Utf8 | LargeUtf8 | Utf8View, + Boolean + | Int8 + | Int16 + | Int32 + | Int64 + | Float32 + | Float64 + | Decimal128(_, _) + | Date32 + | Timestamp(_, _), + ) + | (Boolean | Decimal128(_, _), Timestamp(_, Some(_))) + | (Boolean | Int8 | Int16 | Int32 | Int64, Decimal128(_, _)) + | (Int8 | Int16 | Int32 | Int64, Binary) + | (Float32 | Float64, Decimal128(_, _) | Utf8) + | (Decimal128(_, _), Boolean | Float32 | Float64) + | (Date32, Timestamp(_, _)) + | ( + Timestamp(_, Some(_)), + Boolean | Int8 | Int16 | Int32 | Float32 | Float64 | Decimal128(_, _), + ) + | ( + Date32, + Boolean | Int8 | Int16 | Int64 | Float32 | Float64 | Decimal128(_, _), + ) + | (Struct(_) | List(_) | Map(_, _), Utf8) + | (Struct(_), Struct(_)) + | (List(_), List(_)) + | (Map(_, _), Map(_, _)) => not_yet_ported(from_type, to_type, eval_mode), // LEGACY returns NULL and ANSI rejects the cast, while Arrow reinterprets // the days since the epoch. (Date32, Int32) => not_yet_ported(from_type, to_type, eval_mode), @@ -321,9 +387,10 @@ mod tests { #[test] fn test_unsupported_cast_is_an_error() { + // Spark has no cast from DATE to BINARY. let error = cast_array( - Arc::new(StringArray::from(vec!["1"])), - &DataType::Int32, + Arc::new(arrow::array::Date32Array::from(vec![1])), + &DataType::Binary, &legacy(), ) .unwrap_err(); @@ -334,9 +401,40 @@ mod tests { assert!( error .to_string() - .contains("cast from Utf8 to Int32 is not supported"), + .contains("cast from Date32 to Binary is not supported"), "{error}" ); + + // Spark supports a cast from STRING to INT, which is not ported yet. + let error = cast_array( + Arc::new(StringArray::from(vec!["1"])), + &DataType::Int32, + &legacy(), + ) + .unwrap_err(); + assert!(error.to_string().contains("not implemented yet"), "{error}"); + } + + /// Casts from TIMESTAMP and DATE to numbers and BOOLEAN exist in Spark, so + /// they are reported as not implemented rather than not supported. + #[test] + fn test_timestamp_and_date_to_numbers_are_pending() { + let timestamp: ArrayRef = + Arc::new(TimestampMicrosecondArray::from(vec![0]).with_timezone("UTC")); + let date: ArrayRef = Arc::new(arrow::array::Date32Array::from(vec![0])); + let targets = [ + DataType::Boolean, + DataType::Int16, + DataType::Float64, + DataType::Decimal128(20, 2), + ]; + for (array, target) in [×tamp, &date] + .into_iter() + .flat_map(|array| targets.iter().map(move |t| (array, t))) + { + let error = cast_array(Arc::clone(array), target, &legacy()).unwrap_err(); + assert!(error.to_string().contains("not implemented yet"), "{error}"); + } } /// Arrow reinterprets the days since the epoch, while Spark returns NULL diff --git a/crates/spark/src/function/conversion/cast/numeric.rs b/crates/spark/src/function/conversion/cast/numeric.rs new file mode 100644 index 00000000..8b447caa --- /dev/null +++ b/crates/spark/src/function/conversion/cast/numeric.rs @@ -0,0 +1,336 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Casts from numeric types. +//! +//! Ported from Comet's `native/spark-expr/src/conversion_funcs/numeric.rs`. + +use std::sync::Arc; + +use arrow::array::{Array, ArrayRef, ArrowPrimitiveType, AsArray, PrimitiveArray}; +use arrow::buffer::{BooleanBuffer, NullBuffer}; +use arrow::datatypes::{ + DataType, Float32Type, Float64Type, Int8Type, Int16Type, Int32Type, Int64Type, + TimestampMicrosecondType, +}; +use arrow::temporal_conversions::MICROSECONDS; +use datafusion_common::{DataFusionError, Result, internal_err}; + +use crate::{EvalMode, SparkError}; + +/// Casts integral seconds since the epoch to a timestamp, like Spark's +/// `SECONDS.toMicros`, which saturates on overflow in every eval mode. +pub(super) fn cast_int_to_timestamp( + array: &ArrayRef, + target_tz: &Option>, +) -> Result { + fn micros(array: &ArrayRef) -> PrimitiveArray + where + T: ArrowPrimitiveType, + T::Native: Into, + { + array + .as_primitive::() + .unary(|seconds| seconds.into().saturating_mul(MICROSECONDS)) + } + + let micros = match array.data_type() { + DataType::Int8 => micros::(array), + DataType::Int16 => micros::(array), + DataType::Int32 => micros::(array), + DataType::Int64 => micros::(array), + other => return internal_err!("cast_int_to_timestamp: unexpected {other}"), + }; + Ok(Arc::new(micros.with_timezone_opt(target_tz.clone()))) +} + +/// Casts fractional seconds since the epoch to a timestamp. Spark widens a +/// `FLOAT` to `DOUBLE` first. +/// +/// | Mode | NaN or infinite | Out of range | +/// |--------|----------------------|--------------------------------| +/// | LEGACY | NULL | Saturates, like Scala's `toLong` | +/// | ANSI | `CAST_INVALID_INPUT` | `CAST_OVERFLOW` | +/// | TRY | NULL | NULL | +/// +/// Comet returns NULL for an out-of-range value in LEGACY mode, while Spark's +/// `doubleToTimestamp` saturates. +pub(super) fn cast_float_to_timestamp( + array: &ArrayRef, + target_tz: &Option>, + eval_mode: EvalMode, +) -> Result { + let micros = match array.data_type() { + DataType::Float32 => { + float_to_timestamp(array.as_primitive::(), eval_mode)? + } + DataType::Float64 => { + float_to_timestamp(array.as_primitive::(), eval_mode)? + } + other => return internal_err!("cast_float_to_timestamp: unexpected {other}"), + }; + Ok(Arc::new(micros.with_timezone_opt(target_tz.clone()))) +} + +fn float_to_timestamp( + array: &PrimitiveArray, + eval_mode: EvalMode, +) -> Result> +where + T: ArrowPrimitiveType, + T::Native: Into, +{ + Ok(match eval_mode { + EvalMode::Legacy => float_to_micros_nulling(array, f64::is_finite), + EvalMode::Try => float_to_micros_nulling(array, |seconds| { + float_timestamp_micros(seconds).is_some() + }), + // `try_unary` skips null slots, so a NaN under a null is not raised. + EvalMode::Ansi => array.try_unary(|value| { + let seconds = value.into(); + float_timestamp_micros(seconds) + .ok_or_else(|| DataFusionError::from(float_timestamp_error(seconds))) + })?, + }) +} + +/// Converts seconds to microseconds, and nulls the rows where `is_valid` is +/// false. `as` saturates, like Scala's `Double.toLong`. +/// +/// The conversion and the validity check are separate passes, so both +/// vectorize, and an array where nothing becomes null keeps its null buffer. +fn float_to_micros_nulling( + array: &PrimitiveArray, + is_valid: impl Fn(f64) -> bool, +) -> PrimitiveArray +where + T: ArrowPrimitiveType, + T::Native: Into, +{ + let micros = array.unary(|value| (value.into() * MICROSECONDS as f64) as i64); + let values = array.values(); + let valid = BooleanBuffer::collect_bool(array.len(), |i| is_valid(values[i].into())); + if valid.count_set_bits() == array.len() { + return micros; + } + let nulls = NullBuffer::union(array.nulls(), Some(&NullBuffer::new(valid))); + let (_, values, _) = micros.into_parts(); + PrimitiveArray::new(values, nulls) +} + +/// Seconds since the epoch as microseconds, or `None` when the value is NaN, +/// infinite, or out of range, like Spark's `doubleToTimestampAnsi`. +fn float_timestamp_micros(seconds: f64) -> Option { + // Spark's check from `DoubleExactNumeric.toLong` is + // `floor(x) <= Long.MaxValue && ceil(x) >= Long.MinValue`, with the bounds + // converted to double. Doubles that large are whole numbers, so it is a + // bounds check, which NaN and infinity fail. A value that passes is + // truncated toward zero. + let micros = seconds * MICROSECONDS as f64; + (i64::MIN as f64..=i64::MAX as f64) + .contains(µs) + .then_some(micros as i64) +} + +/// The ANSI error for a value [`float_timestamp_micros`] rejects. The source +/// type is `DOUBLE` even for a `FLOAT` array, because Spark widens it first. +fn float_timestamp_error(seconds: f64) -> SparkError { + if !seconds.is_finite() { + SparkError::InvalidInputInCastToDatetime { + value: java_double_literal(seconds), + from_type: "DOUBLE".to_string(), + to_type: "TIMESTAMP".to_string(), + } + } else { + // The overflow is in the conversion of the product to a long. + SparkError::CastOverflow { + value: java_double_literal(seconds * MICROSECONDS as f64), + from_type: "DOUBLE".to_string(), + to_type: "BIGINT".to_string(), + } + } +} + +/// A double that is NaN, infinite, or out of the range of a long, as Spark's +/// `toSQLValue` writes it, such as `NaN` or `1.0E25D`. Java's `Double.toString` +/// uses scientific notation from 10^7, so every value out of the range of a +/// long is in it. +fn java_double_literal(value: f64) -> String { + if value.is_nan() { + return "NaN".to_string(); + } + if value.is_infinite() { + return if value > 0.0 { "Infinity" } else { "-Infinity" }.to_string(); + } + let scientific = format!("{value:e}"); + let (mantissa, exponent) = scientific.split_once('e').unwrap_or((&scientific, "0")); + if mantissa.contains('.') { + format!("{mantissa}E{exponent}D") + } else { + format!("{mantissa}.0E{exponent}D") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{ + Float32Array, Float64Array, Int8Array, Int64Array, TimestampMicrosecondArray, + }; + + fn utc() -> Option> { + Some("UTC".into()) + } + + fn micros(array: &ArrayRef) -> &TimestampMicrosecondArray { + array.as_primitive::() + } + + #[test] + fn test_int_to_timestamp() { + let array: ArrayRef = Arc::new(Int8Array::from(vec![Some(-128), None, Some(1)])); + let result = cast_int_to_timestamp(&array, &utc()).unwrap(); + assert_eq!( + micros(&result), + &TimestampMicrosecondArray::from(vec![ + Some(-128_000_000), + None, + Some(1_000_000) + ]) + .with_timezone("UTC") + ); + } + + #[test] + fn test_int_to_timestamp_saturates() { + let array: ArrayRef = Arc::new(Int64Array::from(vec![i64::MAX, i64::MIN])); + let result = cast_int_to_timestamp(&array, &None).unwrap(); + assert_eq!(micros(&result).values(), &[i64::MAX, i64::MIN]); + assert_eq!( + result.data_type(), + &DataType::Timestamp(arrow::datatypes::TimeUnit::Microsecond, None) + ); + } + + fn float_input() -> ArrayRef { + Arc::new(Float64Array::from(vec![ + Some(1.5), + Some(-1.0000005), + None, + Some(f64::NAN), + Some(f64::INFINITY), + Some(1e19), + Some(-1e19), + ])) + } + + #[test] + fn test_float_to_timestamp_legacy() { + let result = + cast_float_to_timestamp(&float_input(), &utc(), EvalMode::Legacy).unwrap(); + assert_eq!( + micros(&result), + &TimestampMicrosecondArray::from(vec![ + Some(1_500_000), + // Truncated toward zero. + Some(-1_000_000), + None, + None, + None, + Some(i64::MAX), + Some(i64::MIN), + ]) + .with_timezone("UTC") + ); + } + + #[test] + fn test_float_to_timestamp_try() { + let result = + cast_float_to_timestamp(&float_input(), &utc(), EvalMode::Try).unwrap(); + assert_eq!( + micros(&result), + &TimestampMicrosecondArray::from(vec![ + Some(1_500_000), + Some(-1_000_000), + None, + None, + None, + None, + None, + ]) + .with_timezone("UTC") + ); + } + + #[test] + fn test_float_to_timestamp_ansi() { + let cast = |value: f64| { + let array: ArrayRef = Arc::new(Float64Array::from(vec![value])); + cast_float_to_timestamp(&array, &utc(), EvalMode::Ansi) + }; + assert_eq!(micros(&cast(1.5).unwrap()).value(0), 1_500_000); + + let error = cast(f64::NAN).unwrap_err(); + assert_eq!( + SparkError::find(&error), + Some(&SparkError::InvalidInputInCastToDatetime { + value: "NaN".to_string(), + from_type: "DOUBLE".to_string(), + to_type: "TIMESTAMP".to_string(), + }), + "{error}" + ); + let error = cast(f64::NEG_INFINITY).unwrap_err(); + assert!( + error + .to_string() + .contains("The value -Infinity of the type \"DOUBLE\""), + "{error}" + ); + + let error = cast(1e19).unwrap_err(); + assert_eq!( + SparkError::find(&error), + Some(&SparkError::CastOverflow { + value: "1.0E25D".to_string(), + from_type: "DOUBLE".to_string(), + to_type: "BIGINT".to_string(), + }), + "{error}" + ); + + // A NaN under a null is not an error. + let array: ArrayRef = Arc::new(Float64Array::new( + vec![f64::NAN].into(), + Some(vec![false].into()), + )); + let result = cast_float_to_timestamp(&array, &utc(), EvalMode::Ansi).unwrap(); + assert!(result.is_null(0)); + } + + #[test] + fn test_float_to_timestamp_keeps_null_buffer_when_nothing_becomes_null() { + let array: ArrayRef = Arc::new(Float64Array::from(vec![1.0, 2.0])); + for eval_mode in [EvalMode::Legacy, EvalMode::Try] { + let result = cast_float_to_timestamp(&array, &None, eval_mode).unwrap(); + assert!(result.nulls().is_none(), "{eval_mode:?}"); + } + } + + #[test] + fn test_float32_widens_to_double() { + let array: ArrayRef = Arc::new(Float32Array::from(vec![0.1f32])); + let result = cast_float_to_timestamp(&array, &None, EvalMode::Legacy).unwrap(); + // 0.1f32 is 0.100000001490116... as a double. + assert_eq!(micros(&result).value(0), 100_000); + } + + #[test] + fn test_java_double_literal() { + assert_eq!(java_double_literal(1e25), "1.0E25D"); + assert_eq!(java_double_literal(-1.2345e19), "-1.2345E19D"); + assert_eq!(java_double_literal(f64::INFINITY), "Infinity"); + } +} diff --git a/crates/spark/src/function/conversion/cast/udf.rs b/crates/spark/src/function/conversion/cast/udf.rs index 7ea73c45..3eda2c0b 100644 --- a/crates/spark/src/function/conversion/cast/udf.rs +++ b/crates/spark/src/function/conversion/cast/udf.rs @@ -2,98 +2,73 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use arrow::array::{Array, ArrayRef, AsArray, TimestampMicrosecondBuilder}; -use arrow::datatypes::{ - ArrowPrimitiveType, DataType, Field, FieldRef, Float32Type, Float64Type, Int8Type, - Int16Type, Int32Type, Int64Type, TimeUnit, -}; +use std::sync::Arc; + +use arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::config::ConfigOptions; -use datafusion_common::types::{ - logical_float32, logical_float64, logical_int8, logical_int16, logical_int32, - logical_int64, logical_string, -}; -use datafusion_common::{Result, ScalarValue, exec_err, internal_err}; -use datafusion_expr::{Coercion, TypeSignatureClass}; +use datafusion_common::utils::take_function_args; +use datafusion_common::{Result, internal_err, plan_err}; use datafusion_expr::{ - ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, - Signature, TypeSignature, Volatility, + ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF, + ScalarUDFImpl, Signature, Volatility, }; -use std::sync::Arc; - -const MICROS_PER_SECOND: i64 = 1_000_000; - -/// Convert integer seconds to microseconds with saturating overflow behavior -#[inline] -fn secs_to_micros(secs: i64) -> i64 { - secs.saturating_mul(MICROS_PER_SECOND) -} +use datafusion_macros::user_doc; -/// Convert float seconds to microseconds -/// Returns None for NaN/Infinity in non-ANSI mode, error in ANSI mode -/// Saturates to i64::MAX/MIN for overflow -#[inline] -fn float_secs_to_micros(val: f64, enable_ansi_mode: bool) -> Result> { - if val.is_nan() || val.is_infinite() { - if enable_ansi_mode { - let display_val = if val.is_nan() { - "NaN" - } else if val.is_sign_positive() { - "Infinity" - } else { - "-Infinity" - }; - return exec_err!("Cannot cast {} to TIMESTAMP", display_val); - } - return Ok(None); - } - let micros = val * MICROS_PER_SECOND as f64; +use super::{SparkCastOptions, parse_spark_type, spark_cast}; +use crate::{EvalMode, SparkTimeZone}; - // Bounds check for i64 range. - // Note on precision: i64::MIN (-2^63) is exactly representable in f64, - // but i64::MAX (2^63 - 1) is not - it rounds up to 2^63 (i64::MAX + 1). - // We use strict `<` for the upper bound to reject values >= 2^63, - // which correctly handles the precision loss edge case. - if micros >= i64::MIN as f64 && micros < i64::MAX as f64 { - Ok(Some(micros as i64)) - } else { - if enable_ansi_mode { - return exec_err!("Overflow casting {} to TIMESTAMP", val); - } - // Saturate to i64::MAX or i64::MIN like Spark does for overflow - if micros.is_sign_negative() { - Ok(Some(i64::MIN)) - } else { - Ok(Some(i64::MAX)) - } - } -} +/// The time zone DataFusion uses when `datafusion.execution.time_zone` is not +/// set. +const DEFAULT_TIMEZONE: &str = "UTC"; -/// Spark-compatible `cast` function for type conversions +/// `spark_cast(expr, '')`: Spark's `CAST(expr AS )`. /// -/// This implements Spark's CAST expression with a target type parameter -/// -/// # Usage -/// ```sql -/// SELECT spark_cast(value, 'timestamp') -/// ``` -/// -/// # Currently supported conversions -/// - Int8/Int16/Int32/Int64/Float32/Float64 -> Timestamp (target_type = 'timestamp') -/// -/// The integer value is interpreted as seconds since the Unix epoch (1970-01-01 00:00:00 UTC) -/// and converted to a timestamp with microsecond precision (matches spark's spec). Same is the case -/// with Float but with higher precision to support micro / nanoseconds. -/// -/// # Overflow behavior -/// Uses saturating multiplication to handle overflow - values that would overflow -/// i64 when multiplied by 1,000,000 are clamped to i64::MAX or i64::MIN +/// The target type is a string literal in Spark's SQL type syntax, such as +/// `'int'`, `'decimal(10, 2)'` or `'array'`. The cast follows +/// `datafusion.execution.enable_ansi_mode`, as Spark's follows +/// `spark.sql.ansi.enabled`, and uses `datafusion.execution.time_zone` as the +/// session time zone. /// /// # References /// - +#[user_doc( + doc_section(label = "Other Functions"), + description = "Casts `expr` to `type` as Spark's `CAST(expr AS type)` does. The target type \ + is a string literal in Spark's SQL type syntax: BOOLEAN, TINYINT (BYTE), SMALLINT \ + (SHORT), INT (INTEGER), BIGINT (LONG), FLOAT (REAL), DOUBLE, DECIMAL(p, s) (DEC, \ + NUMERIC), STRING, CHAR(n), VARCHAR(n), BINARY, DATE, TIMESTAMP (TIMESTAMP_LTZ), \ + TIMESTAMP_NTZ, and ARRAY<...>, MAP<..., ...> and STRUCT of these. \ + Type names are case-insensitive. CHAR and VARCHAR require a length and cast to STRING. \ + INTERVAL, TIME, VARIANT, VOID and the geospatial types are not supported. The cast \ + follows `datafusion.execution.enable_ansi_mode`, as Spark's follows \ + `spark.sql.ansi.enabled`, and uses `datafusion.execution.time_zone` as the session \ + time zone. Casts are being ported from Comet \ + (https://github.com/apache/datafusion-spark/issues/88): a cast that Spark supports \ + but this function does not yet returns a \"not implemented yet\" error, and a cast \ + that Spark does not support returns a \"not supported\" error. Returns NULL if `expr` \ + is NULL.", + syntax_example = "spark_cast(expr, type)", + sql_example = r#"```sql +> SELECT spark_cast(true, 'int'); +1 +> SELECT spark_cast(0, 'timestamp'); +1970-01-01T00:00:00Z +```"#, + argument(name = "expr", description = "Expression to cast."), + argument( + name = "type", + description = "The target type, as a string literal in Spark's SQL type syntax, such as 'int', 'decimal(10, 2)' or 'array'." + ) +)] #[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkCast { signature: Signature, - timezone: Option>, + /// The session time zone when the function was planned. A `TIMESTAMP` + /// target is tagged with it. + timezone: Arc, + /// `timezone` parsed with Spark's rules, or why it could not be. Only the + /// casts that use the session time zone fail when it cannot be parsed. + session_timezone: Result, } impl Default for SparkCast { @@ -108,121 +83,26 @@ impl SparkCast { } pub fn new_with_config(config: &ConfigOptions) -> Self { - // First arg: value to cast - // Second arg: target datatype as Utf8 string literal (ex : 'timestamp') - let string_arg = - Coercion::new_exact(TypeSignatureClass::Native(logical_string())); - - // Supported input types: signed integers and floats - let input_type_signatures = [ - logical_int8(), - logical_int16(), - logical_int32(), - logical_int64(), - logical_float32(), - logical_float64(), - ] - .map(|input_type| { - TypeSignature::Coercible(vec![ - Coercion::new_exact(TypeSignatureClass::Native(input_type)), - string_arg.clone(), - ]) - }); - - Self { - signature: Signature::new( - TypeSignature::OneOf(Vec::from(input_type_signatures)), - Volatility::Stable, - ), - timezone: config + Self::with_session_timezone( + config .execution .time_zone .as_ref() - .map(|tz| Arc::from(tz.as_str())) - .or_else(|| Some(Arc::from("UTC"))), - } + .map_or(DEFAULT_TIMEZONE, |tz| tz.as_str()), + ) } -} -/// Parse target type string into a DataType -fn parse_target_type(type_str: &str, timezone: Option>) -> Result { - match type_str.to_lowercase().as_str() { - // further data type support in future - "timestamp" => Ok(DataType::Timestamp(TimeUnit::Microsecond, timezone)), - other => exec_err!( - "Unsupported spark_cast target type '{}'. Supported types: timestamp", - other - ), - } -} - -/// Extract target type string from scalar arguments -fn get_target_type_from_scalar_args( - scalar_args: &[Option<&ScalarValue>], - timezone: Option>, -) -> Result { - let type_arg = scalar_args.get(1).and_then(|opt| *opt); - - match type_arg { - Some(ScalarValue::Utf8(Some(s))) - | Some(ScalarValue::LargeUtf8(Some(s))) - | Some(ScalarValue::Utf8View(Some(s))) => parse_target_type(s, timezone), - _ => exec_err!( - "spark_cast requires second argument to be a string of target data type ex: timestamp" - ), - } -} - -fn cast_int_to_timestamp( - array: &ArrayRef, - timezone: Option>, -) -> Result -where - T::Native: Into, -{ - let arr = array.as_primitive::(); - let mut builder = TimestampMicrosecondBuilder::with_capacity(arr.len()); - - for i in 0..arr.len() { - if arr.is_null(i) { - builder.append_null(); - } else { - // spark saturates to i64 min/max - let micros = secs_to_micros(arr.value(i).into()); - builder.append_value(micros); - } - } - - Ok(Arc::new(builder.finish().with_timezone_opt(timezone))) -} - -/// Cast float to timestamp -/// Float value represents seconds (with fractional part) since Unix epoch -/// NaN and Infinity: error in ANSI mode, NULL in non-ANSI mode -fn cast_float_to_timestamp( - array: &ArrayRef, - timezone: Option>, - enable_ansi_mode: bool, -) -> Result -where - T::Native: Into, -{ - let arr = array.as_primitive::(); - let mut builder = TimestampMicrosecondBuilder::with_capacity(arr.len()); - - for i in 0..arr.len() { - if arr.is_null(i) { - builder.append_null(); - } else { - let val: f64 = arr.value(i).into(); - match float_secs_to_micros(val, enable_ansi_mode)? { - Some(micros) => builder.append_value(micros), - None => builder.append_null(), - } + fn with_session_timezone(timezone: &str) -> Self { + Self { + // The second argument must be a string literal, which + // `return_field_from_args` checks. + signature: Signature::any(2, Volatility::Stable), + timezone: timezone.into(), + session_timezone: SparkTimeZone::parse(timezone).map_err(|e| { + format!("the session time zone '{timezone}' is not supported: {e}") + }), } } - - Ok(Arc::new(builder.finish().with_timezone_opt(timezone))) } impl ScalarUDFImpl for SparkCast { @@ -243,752 +123,211 @@ impl ScalarUDFImpl for SparkCast { } fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { - let return_type = get_target_type_from_scalar_args( - args.scalar_arguments, - self.timezone.clone(), - )?; + let Some(Some(type_name)) = args + .scalar_arguments + .get(1) + .and_then(|arg| arg.and_then(|arg| arg.try_as_str())) + else { + return plan_err!( + "spark_cast requires a string literal target type, such as 'int'" + ); + }; + let return_type = parse_spark_type(type_name, &self.timezone)?; Ok(Arc::new(Field::new(self.name(), return_type, true))) } - fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { - let enable_ansi_mode = args.config_options.execution.enable_ansi_mode; - let target_type = args.return_field.data_type(); - match target_type { - DataType::Timestamp(TimeUnit::Microsecond, tz) => { - cast_to_timestamp(&args.args[0], tz.clone(), enable_ansi_mode) - } - other => exec_err!("Unsupported spark_cast target type: {:?}", other), - } + fn documentation(&self) -> Option<&Documentation> { + self.doc() } -} -/// Cast value to timestamp internal function -fn cast_to_timestamp( - input: &ColumnarValue, - timezone: Option>, - enable_ansi_mode: bool, -) -> Result { - match input { - ColumnarValue::Array(array) => match array.data_type() { - DataType::Null => Ok(ColumnarValue::Array(Arc::new( - arrow::array::TimestampMicrosecondArray::new_null(array.len()) - .with_timezone_opt(timezone), - ))), - DataType::Int8 => Ok(ColumnarValue::Array( - cast_int_to_timestamp::(array, timezone)?, - )), - DataType::Int16 => Ok(ColumnarValue::Array(cast_int_to_timestamp::< - Int16Type, - >(array, timezone)?)), - DataType::Int32 => Ok(ColumnarValue::Array(cast_int_to_timestamp::< - Int32Type, - >(array, timezone)?)), - DataType::Int64 => Ok(ColumnarValue::Array(cast_int_to_timestamp::< - Int64Type, - >(array, timezone)?)), - DataType::Float32 => Ok(ColumnarValue::Array(cast_float_to_timestamp::< - Float32Type, - >( - array, - timezone, - enable_ansi_mode, - )?)), - DataType::Float64 => Ok(ColumnarValue::Array(cast_float_to_timestamp::< - Float64Type, - >( - array, - timezone, - enable_ansi_mode, - )?)), - other => exec_err!("Unsupported cast from {:?} to timestamp", other), - }, - ColumnarValue::Scalar(scalar) => { - let micros = match scalar { - ScalarValue::Null - | ScalarValue::Int8(None) - | ScalarValue::Int16(None) - | ScalarValue::Int32(None) - | ScalarValue::Int64(None) - | ScalarValue::Float32(None) - | ScalarValue::Float64(None) => None, - ScalarValue::Int8(Some(v)) => Some(secs_to_micros((*v).into())), - ScalarValue::Int16(Some(v)) => Some(secs_to_micros((*v).into())), - ScalarValue::Int32(Some(v)) => Some(secs_to_micros((*v).into())), - ScalarValue::Int64(Some(v)) => Some(secs_to_micros(*v)), - ScalarValue::Float32(Some(v)) => { - float_secs_to_micros(*v as f64, enable_ansi_mode)? - } - ScalarValue::Float64(Some(v)) => { - float_secs_to_micros(*v, enable_ansi_mode)? - } - other => { - return exec_err!("Unsupported cast from {:?} to timestamp", other); - } - }; - Ok(ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond( - micros, timezone, - ))) - } + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + let options = SparkCastOptions::new(EvalMode::from_config(&args.config_options)); + let options = match &self.session_timezone { + Ok(timezone) => options.with_timezone(timezone.clone()), + Err(reason) => options.without_timezone_because(reason.clone()), + }; + let [value, _] = take_function_args(self.name(), args.args)?; + spark_cast(value, args.return_field.data_type(), &options) } } #[cfg(test)] mod tests { use super::*; - use arrow::array::{ - Float32Array, Float64Array, Int8Array, Int16Array, Int32Array, Int64Array, - }; - use arrow::datatypes::TimestampMicrosecondType; - - // helpers to make testing easier - fn make_args(input: ColumnarValue, target_type: &str) -> ScalarFunctionArgs { - make_args_with_timezone(input, target_type, Some("UTC")) + use arrow::array::{Array, ArrayRef, AsArray, Int32Array}; + use arrow::datatypes::{TimeUnit, TimestampMicrosecondType}; + use datafusion_common::ScalarValue; + + fn return_field( + udf: &SparkCast, + input: DataType, + type_name: &str, + ) -> Result { + let type_name = ScalarValue::Utf8(Some(type_name.to_string())); + udf.return_field_from_args(ReturnFieldArgs { + arg_fields: &[ + Arc::new(Field::new("a", input, true)), + Arc::new(Field::new("b", DataType::Utf8, false)), + ], + scalar_arguments: &[None, Some(&type_name)], + }) } - fn make_args_with_timezone( + fn invoke( + udf: &SparkCast, + config: ConfigOptions, input: ColumnarValue, - target_type: &str, - timezone: Option<&str>, - ) -> ScalarFunctionArgs { - let return_field = Arc::new(Field::new( - "result", - DataType::Timestamp( - TimeUnit::Microsecond, - Some(Arc::from(timezone.unwrap())), - ), - true, - )); - let mut config = ConfigOptions::default(); - if let Some(tz) = timezone { - config.execution.time_zone = Some(tz.parse().unwrap()); - } - ScalarFunctionArgs { + type_name: &str, + ) -> Result { + let return_field = return_field(udf, input.data_type(), type_name)?; + udf.invoke_with_args(ScalarFunctionArgs { args: vec![ input, - ColumnarValue::Scalar(ScalarValue::Utf8(Some(target_type.to_string()))), + ColumnarValue::Scalar(ScalarValue::Utf8(Some(type_name.to_string()))), ], arg_fields: vec![], - number_rows: 0, + number_rows: 1, return_field, config_options: Arc::new(config), - } - } - - fn assert_scalar_timestamp(result: ColumnarValue, expected: i64) { - assert_scalar_timestamp_with_tz(result, expected, "UTC"); + }) } - fn assert_scalar_timestamp_with_tz( - result: ColumnarValue, - expected: i64, - expected_tz: &str, - ) { - match result { - ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond( - Some(val), - Some(tz), - )) => { - assert_eq!(val, expected); - assert_eq!(tz.as_ref(), expected_tz); - } - _ => { - panic!( - "Expected scalar timestamp with value {expected} and {expected_tz} timezone" - ) - } - } - } - - fn assert_scalar_null(result: ColumnarValue) { - assert_scalar_null_with_tz(result, "UTC"); - } - - fn assert_scalar_null_with_tz(result: ColumnarValue, expected_tz: &str) { - match result { - ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond(None, Some(tz))) => { - assert_eq!(tz.as_ref(), expected_tz); - } - _ => panic!("Expected null scalar timestamp with {expected_tz} timezone"), - } - } - - #[test] - fn test_cast_int8_array_to_timestamp() { - let array: ArrayRef = Arc::new(Int8Array::from(vec![ - Some(0), - Some(1), - Some(-1), - Some(127), - Some(-128), - None, - ])); - - let cast = SparkCast::new(); - let args = make_args(ColumnarValue::Array(array), "timestamp"); - let result = cast.invoke_with_args(args).unwrap(); - - match result { - ColumnarValue::Array(result_array) => { - let ts_array = result_array.as_primitive::(); - assert_eq!(ts_array.value(0), 0); - assert_eq!(ts_array.value(1), 1_000_000); - assert_eq!(ts_array.value(2), -1_000_000); - assert_eq!(ts_array.value(3), 127_000_000); - assert_eq!(ts_array.value(4), -128_000_000); - assert!(ts_array.is_null(5)); - } - _ => panic!("Expected array result"), - } - } - - #[test] - fn test_cast_int16_array_to_timestamp() { - let array: ArrayRef = Arc::new(Int16Array::from(vec![ - Some(0), - Some(32767), - Some(-32768), - None, - ])); - - let cast = SparkCast::new(); - let args = make_args(ColumnarValue::Array(array), "timestamp"); - let result = cast.invoke_with_args(args).unwrap(); - - match result { - ColumnarValue::Array(result_array) => { - let ts_array = result_array.as_primitive::(); - assert_eq!(ts_array.value(0), 0); - assert_eq!(ts_array.value(1), 32_767_000_000); - assert_eq!(ts_array.value(2), -32_768_000_000); - assert!(ts_array.is_null(3)); - } - _ => panic!("Expected array result"), - } - } - - #[test] - fn test_cast_int32_array_to_timestamp() { - let array: ArrayRef = - Arc::new(Int32Array::from(vec![Some(0), Some(1704067200), None])); - - let cast = SparkCast::new(); - let args = make_args(ColumnarValue::Array(array), "timestamp"); - let result = cast.invoke_with_args(args).unwrap(); - - match result { - ColumnarValue::Array(result_array) => { - let ts_array = result_array.as_primitive::(); - assert_eq!(ts_array.value(0), 0); - assert_eq!(ts_array.value(1), 1_704_067_200_000_000); - assert!(ts_array.is_null(2)); - } - _ => panic!("Expected array result"), - } - } - - #[test] - fn test_cast_int64_array_overflow() { - let array: ArrayRef = - Arc::new(Int64Array::from(vec![Some(i64::MAX), Some(i64::MIN)])); - - let cast = SparkCast::new(); - let args = make_args(ColumnarValue::Array(array), "timestamp"); - let result = cast.invoke_with_args(args).unwrap(); - - match result { - ColumnarValue::Array(result_array) => { - let ts_array = result_array.as_primitive::(); - // saturating_mul clamps to i64::MAX/MIN - assert_eq!(ts_array.value(0), i64::MAX); - assert_eq!(ts_array.value(1), i64::MIN); - } - _ => panic!("Expected array result"), - } - } - - #[test] - fn test_cast_int64_array_to_timestamp() { - let array: ArrayRef = Arc::new(Int64Array::from(vec![ - Some(0), - Some(1704067200), - Some(-86400), - None, - ])); - - let cast = SparkCast::new(); - let args = make_args(ColumnarValue::Array(array), "timestamp"); - let result = cast.invoke_with_args(args).unwrap(); - - match result { - ColumnarValue::Array(result_array) => { - let ts_array = result_array.as_primitive::(); - assert_eq!(ts_array.value(0), 0); - assert_eq!(ts_array.value(1), 1_704_067_200_000_000); - assert_eq!(ts_array.value(2), -86_400_000_000); // -1 day - assert!(ts_array.is_null(3)); - } - _ => panic!("Expected array result"), - } - } - - #[test] - fn test_cast_scalar_int8() { - let cast = SparkCast::new(); - let args = make_args( - ColumnarValue::Scalar(ScalarValue::Int8(Some(100))), - "timestamp", - ); - let result = cast.invoke_with_args(args).unwrap(); - assert_scalar_timestamp(result, 100_000_000); - } - - #[test] - fn test_cast_scalar_int16() { - let cast = SparkCast::new(); - let args = make_args( - ColumnarValue::Scalar(ScalarValue::Int16(Some(100))), - "timestamp", - ); - let result = cast.invoke_with_args(args).unwrap(); - assert_scalar_timestamp(result, 100_000_000); - } - - #[test] - fn test_cast_scalar_int32() { - let cast = SparkCast::new(); - let args = make_args( - ColumnarValue::Scalar(ScalarValue::Int32(Some(1704067200))), - "timestamp", - ); - let result = cast.invoke_with_args(args).unwrap(); - assert_scalar_timestamp(result, 1_704_067_200_000_000); + /// Invokes `spark_cast` planned and run with `config`. + fn invoke_with( + config: ConfigOptions, + input: ColumnarValue, + type_name: &str, + ) -> Result { + invoke( + &SparkCast::new_with_config(&config), + config, + input, + type_name, + ) } - #[test] - fn test_cast_scalar_int64() { - let cast = SparkCast::new(); - let args = make_args( - ColumnarValue::Scalar(ScalarValue::Int64(Some(1704067200))), - "timestamp", - ); - let result = cast.invoke_with_args(args).unwrap(); - assert_scalar_timestamp(result, 1_704_067_200_000_000); + fn config(time_zone: Option<&str>, ansi: bool) -> ConfigOptions { + let mut config = ConfigOptions::default(); + config.execution.time_zone = time_zone.map(|tz| tz.parse().unwrap()); + config.execution.enable_ansi_mode = ansi; + config } #[test] - fn test_cast_scalar_negative() { - let cast = SparkCast::new(); - let args = make_args( - ColumnarValue::Scalar(ScalarValue::Int32(Some(-86400))), - "timestamp", + fn test_return_field_tags_timestamp_with_session_timezone() { + let udf = SparkCast::new_with_config(&config(Some("America/Los_Angeles"), false)); + let field = return_field(&udf, DataType::Int64, "timestamp").unwrap(); + assert_eq!( + field.data_type(), + &DataType::Timestamp( + TimeUnit::Microsecond, + Some("America/Los_Angeles".into()) + ) ); - let result = cast.invoke_with_args(args).unwrap(); - // -86400 seconds = -1 day before epoch - assert_scalar_timestamp(result, -86_400_000_000); - } - - #[test] - fn test_cast_scalar_null() { - let cast = SparkCast::new(); - let args = - make_args(ColumnarValue::Scalar(ScalarValue::Int64(None)), "timestamp"); - let result = cast.invoke_with_args(args).unwrap(); - assert_scalar_null(result); - } + assert!(field.is_nullable()); - #[test] - fn test_cast_scalar_int64_overflow() { - let cast = SparkCast::new(); - let args = make_args( - ColumnarValue::Scalar(ScalarValue::Int64(Some(i64::MAX))), - "timestamp", + let udf = SparkCast::new_with_config(&config(None, false)); + let field = return_field(&udf, DataType::Int64, "timestamp").unwrap(); + assert_eq!( + field.data_type(), + &DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())) ); - let result = cast.invoke_with_args(args).unwrap(); - // saturating_mul clamps to i64::MAX - assert_scalar_timestamp(result, i64::MAX); } #[test] - fn test_unsupported_target_type() { - let cast = SparkCast::new(); - // invoke_with_args uses return_field which would be set correctly by planning - // For this test, we need to check return_field_from_args - let arg_fields: Vec = - vec![Arc::new(Field::new("a", DataType::Int64, true))]; - let target_type = ScalarValue::Utf8(Some("string".to_string())); - let scalar_arguments: Vec> = vec![None, Some(&target_type)]; - let return_field_args = ReturnFieldArgs { - arg_fields: &arg_fields, - scalar_arguments: &scalar_arguments, - }; - let result = cast.return_field_from_args(return_field_args); - assert!(result.is_err()); + fn test_target_type_must_be_a_string_literal() { + let udf = SparkCast::new(); + let error = udf + .return_field_from_args(ReturnFieldArgs { + arg_fields: &[ + Arc::new(Field::new("a", DataType::Int32, true)), + Arc::new(Field::new("b", DataType::Utf8, true)), + ], + scalar_arguments: &[None, None], + }) + .unwrap_err(); assert!( - result - .unwrap_err() - .to_string() - .contains("Unsupported spark_cast target type") + error.to_string().contains("string literal target type"), + "{error}" ); - } - #[test] - fn test_unsupported_source_type() { - let cast = SparkCast::new(); - let args = make_args( - ColumnarValue::Scalar(ScalarValue::Utf8(Some("2024-01-01".to_string()))), - "timestamp", - ); - let result = cast.invoke_with_args(args); - assert!(result.is_err()); + let error = return_field(&udf, DataType::Int32, "integer2").unwrap_err(); assert!( - result - .unwrap_err() - .to_string() - .contains("Unsupported cast from") - ); - } - - #[test] - fn test_cast_null_to_timestamp() { - let cast = SparkCast::new(); - let args = make_args(ColumnarValue::Scalar(ScalarValue::Null), "timestamp"); - let result = cast.invoke_with_args(args).unwrap(); - assert_scalar_null(result); - } - - #[test] - fn test_cast_null_array_to_timestamp() { - let array: ArrayRef = Arc::new(arrow::array::NullArray::new(3)); - - let cast = SparkCast::new(); - let args = make_args(ColumnarValue::Array(array), "timestamp"); - let result = cast.invoke_with_args(args).unwrap(); - - match result { - ColumnarValue::Array(result_array) => { - let ts_array = result_array.as_primitive::(); - assert_eq!(ts_array.len(), 3); - assert!(ts_array.is_null(0)); - assert!(ts_array.is_null(1)); - assert!(ts_array.is_null(2)); - } - _ => panic!("Expected array result"), - } - } - - #[test] - fn test_cast_int_to_timestamp_with_timezones() { - // Test with various timezones like Comet does - let timezones = [ - "UTC", - "America/New_York", - "America/Los_Angeles", - "Europe/London", - "Asia/Tokyo", - "Australia/Sydney", - ]; - - let cast = SparkCast::new(); - let test_value: i64 = 1704067200; // 2024-01-01 00:00:00 UTC - let expected_micros = test_value * MICROS_PER_SECOND; - - for tz in timezones { - // scalar - let args = make_args_with_timezone( - ColumnarValue::Scalar(ScalarValue::Int64(Some(test_value))), - "timestamp", - Some(tz), - ); - let result = cast.invoke_with_args(args).unwrap(); - assert_scalar_timestamp_with_tz(result, expected_micros, tz); - - // array input - let array: ArrayRef = - Arc::new(Int64Array::from(vec![Some(test_value), None])); - let args = make_args_with_timezone( - ColumnarValue::Array(array), - "timestamp", - Some(tz), - ); - let result = cast.invoke_with_args(args).unwrap(); - - match result { - ColumnarValue::Array(result_array) => { - let ts_array = - result_array.as_primitive::(); - assert_eq!(ts_array.value(0), expected_micros); - assert!(ts_array.is_null(1)); - assert_eq!(ts_array.timezone(), Some(tz)); - } - _ => panic!("Expected array result for timezone {tz}"), - } - } - } - - #[test] - fn test_cast_int_to_timestamp_default_timezone() { - let cast = SparkCast::new(); - let args = make_args( - ColumnarValue::Scalar(ScalarValue::Int64(Some(0))), - "timestamp", - ); - let result = cast.invoke_with_args(args).unwrap(); - // Defaults to UTC - assert_scalar_timestamp_with_tz(result, 0, "UTC"); - } - - fn make_args_with_ansi_mode( - input: ColumnarValue, - target_type: &str, - enable_ansi_mode: bool, - ) -> ScalarFunctionArgs { - let return_field = Arc::new(Field::new( - "result", - DataType::Timestamp(TimeUnit::Microsecond, Some(Arc::from("UTC"))), - true, - )); - let mut config = ConfigOptions::default(); - config.execution.time_zone = Some("UTC".parse().unwrap()); - config.execution.enable_ansi_mode = enable_ansi_mode; - ScalarFunctionArgs { - args: vec![ - input, - ColumnarValue::Scalar(ScalarValue::Utf8(Some(target_type.to_string()))), - ], - arg_fields: vec![], - number_rows: 0, - return_field, - config_options: Arc::new(config), - } - } - - #[test] - fn test_cast_float64_array_to_timestamp() { - let array: ArrayRef = Arc::new(Float64Array::from(vec![ - Some(0.0), - Some(1.5), - Some(-1.5), - Some(1704067200.123456), - None, - ])); - - let cast = SparkCast::new(); - let args = make_args(ColumnarValue::Array(array), "timestamp"); - let result = cast.invoke_with_args(args).unwrap(); - - match result { - ColumnarValue::Array(result_array) => { - let ts_array = result_array.as_primitive::(); - assert_eq!(ts_array.value(0), 0); - assert_eq!(ts_array.value(1), 1_500_000); // 1.5 seconds - assert_eq!(ts_array.value(2), -1_500_000); // -1.5 seconds - assert_eq!(ts_array.value(3), 1_704_067_200_123_456); // with fractional - assert!(ts_array.is_null(4)); - } - _ => panic!("Expected array result"), - } - } - - #[test] - fn test_cast_float32_array_to_timestamp() { - let array: ArrayRef = Arc::new(Float32Array::from(vec![ - Some(0.0f32), - Some(1.5f32), - Some(-1.5f32), - None, - ])); - - let cast = SparkCast::new(); - let args = make_args(ColumnarValue::Array(array), "timestamp"); - let result = cast.invoke_with_args(args).unwrap(); - - match result { - ColumnarValue::Array(result_array) => { - let ts_array = result_array.as_primitive::(); - assert_eq!(ts_array.value(0), 0); - assert_eq!(ts_array.value(1), 1_500_000); // 1.5 seconds - assert_eq!(ts_array.value(2), -1_500_000); // -1.5 seconds - assert!(ts_array.is_null(3)); - } - _ => panic!("Expected array result"), - } - } - - #[test] - fn test_cast_scalar_float64() { - let cast = SparkCast::new(); - let args = make_args( - ColumnarValue::Scalar(ScalarValue::Float64(Some(1.5))), - "timestamp", - ); - let result = cast.invoke_with_args(args).unwrap(); - assert_scalar_timestamp(result, 1_500_000); - } - - #[test] - fn test_cast_scalar_float32() { - let cast = SparkCast::new(); - let args = make_args( - ColumnarValue::Scalar(ScalarValue::Float32(Some(1.5f32))), - "timestamp", + error.to_string().contains("unknown type 'integer2'"), + "{error}" ); - let result = cast.invoke_with_args(args).unwrap(); - assert_scalar_timestamp(result, 1_500_000); } #[test] - fn test_cast_float_nan_non_ansi_mode() { - // In non-ANSI mode, NaN should return NULL - let cast = SparkCast::new(); - let args = make_args_with_ansi_mode( - ColumnarValue::Scalar(ScalarValue::Float64(Some(f64::NAN))), + fn test_invoke_array() { + let input: ArrayRef = Arc::new(Int32Array::from(vec![Some(1), None])); + let result = invoke_with( + config(Some("America/Los_Angeles"), false), + ColumnarValue::Array(input), "timestamp", - false, - ); - let result = cast.invoke_with_args(args).unwrap(); - assert_scalar_null(result); + ) + .unwrap(); + let ColumnarValue::Array(result) = result else { + panic!("expected an array, got {result:?}"); + }; + let result = result.as_primitive::(); + assert_eq!(result.value(0), 1_000_000); + assert!(result.is_null(1)); + assert_eq!(result.timezone(), Some("America/Los_Angeles")); } #[test] - fn test_cast_float_infinity_non_ansi_mode() { - // In non-ANSI mode, Infinity should return NULL - let cast = SparkCast::new(); - - // Positive infinity - let args = make_args_with_ansi_mode( - ColumnarValue::Scalar(ScalarValue::Float64(Some(f64::INFINITY))), - "timestamp", - false, - ); - let result = cast.invoke_with_args(args).unwrap(); - assert_scalar_null(result); - - // Negative infinity - let args = make_args_with_ansi_mode( - ColumnarValue::Scalar(ScalarValue::Float64(Some(f64::NEG_INFINITY))), - "timestamp", - false, - ); - let result = cast.invoke_with_args(args).unwrap(); - assert_scalar_null(result); + fn test_invoke_scalar() { + let result = invoke_with( + config(None, false), + ColumnarValue::Scalar(ScalarValue::Int32(Some(42))), + "string", + ) + .unwrap(); + let ColumnarValue::Scalar(scalar) = result else { + panic!("expected a scalar, got {result:?}"); + }; + assert_eq!(scalar, ScalarValue::Utf8(Some("42".to_string()))); } #[test] - fn test_cast_float_nan_ansi_mode() { - // In ANSI mode, NaN should error - let cast = SparkCast::new(); - let args = make_args_with_ansi_mode( - ColumnarValue::Scalar(ScalarValue::Float64(Some(f64::NAN))), - "timestamp", - true, - ); - let result = cast.invoke_with_args(args); - assert!(result.is_err()); - assert!(result.unwrap_err().to_string().contains("Cannot cast NaN")); - } + fn test_unsupported_session_timezone_only_affects_casts_that_use_it() { + // DataFusion accepts any time zone Arrow can parse, and Spark's parser + // is stricter, so the session time zone can be one Spark rejects. + let udf = SparkCast::with_session_timezone("+0530x"); + assert!(udf.session_timezone.is_err()); + let cast = |input: ScalarValue, type_name: &str| { + invoke( + &udf, + ConfigOptions::default(), + ColumnarValue::Scalar(input), + type_name, + ) + }; + assert!(cast(ScalarValue::Int32(Some(1)), "string").is_ok()); - #[test] - fn test_cast_float_infinity_ansi_mode() { - // In ANSI mode, Infinity should error - let cast = SparkCast::new(); - let args = make_args_with_ansi_mode( - ColumnarValue::Scalar(ScalarValue::Float64(Some(f64::INFINITY))), - "timestamp", - true, - ); - let result = cast.invoke_with_args(args); - assert!(result.is_err()); + let timestamp = ScalarValue::TimestampMicrosecond(Some(0), Some("UTC".into())); + let error = cast(timestamp, "string").unwrap_err().to_string(); + assert!(error.contains("requires a session time zone"), "{error}"); assert!( - result - .unwrap_err() - .to_string() - .contains("Cannot cast Infinity") - ); - } - - #[test] - fn test_cast_float_overflow_non_ansi_mode() { - // Value too large to fit in i64 microseconds - should saturate to i64::MAX like Spark - let cast = SparkCast::new(); - let large_value = 1e19; // Way too large for i64 microseconds - let args = make_args_with_ansi_mode( - ColumnarValue::Scalar(ScalarValue::Float64(Some(large_value))), - "timestamp", - false, - ); - let result = cast.invoke_with_args(args).unwrap(); - // Spark saturates overflow to i64::MAX - assert_scalar_timestamp(result, i64::MAX); - } - - #[test] - fn test_cast_float_negative_overflow_non_ansi_mode() { - // Large negative value - should saturate to i64::MIN like Spark - let cast = SparkCast::new(); - let large_value = -1e19; // Way too large negative for i64 microseconds - let args = make_args_with_ansi_mode( - ColumnarValue::Scalar(ScalarValue::Float64(Some(large_value))), - "timestamp", - false, + error.contains("the session time zone '+0530x' is not supported"), + "{error}" ); - let result = cast.invoke_with_args(args).unwrap(); - // Spark saturates negative overflow to i64::MIN - assert_scalar_timestamp(result, i64::MIN); } #[test] - fn test_cast_float_overflow_ansi_mode() { - // Value too large to fit in i64 microseconds - should error in ANSI mode - let cast = SparkCast::new(); - let large_value = 1e19; // Way too large for i64 microseconds - let args = make_args_with_ansi_mode( - ColumnarValue::Scalar(ScalarValue::Float64(Some(large_value))), - "timestamp", - true, - ); - let result = cast.invoke_with_args(args); - assert!(result.is_err()); - assert!(result.unwrap_err().to_string().contains("Overflow")); - } - - #[test] - fn test_cast_float_array_with_nan_and_infinity() { - // Array with NaN and Infinity in non-ANSI mode - let array: ArrayRef = Arc::new(Float64Array::from(vec![ - Some(1.0), - Some(f64::NAN), - Some(f64::INFINITY), - Some(f64::NEG_INFINITY), - Some(2.0), - ])); - - let cast = SparkCast::new(); - let args = - make_args_with_ansi_mode(ColumnarValue::Array(array), "timestamp", false); - let result = cast.invoke_with_args(args).unwrap(); - - match result { - ColumnarValue::Array(result_array) => { - let ts_array = result_array.as_primitive::(); - assert_eq!(ts_array.value(0), 1_000_000); - assert!(ts_array.is_null(1)); // NaN -> NULL - assert!(ts_array.is_null(2)); // Infinity -> NULL - assert!(ts_array.is_null(3)); // -Infinity -> NULL - assert_eq!(ts_array.value(4), 2_000_000); - } - _ => panic!("Expected array result"), - } - } + fn test_invoke_reads_ansi_mode() { + let nan = || ColumnarValue::Scalar(ScalarValue::Float64(Some(f64::NAN))); + let result = invoke_with(config(None, false), nan(), "timestamp").unwrap(); + let ColumnarValue::Scalar(scalar) = result else { + panic!("expected a scalar, got {result:?}"); + }; + assert!(scalar.is_null()); - #[test] - fn test_cast_float_negative_values() { - let cast = SparkCast::new(); - let args = make_args( - ColumnarValue::Scalar(ScalarValue::Float64(Some(-86400.5))), - "timestamp", + let error = invoke_with(config(None, true), nan(), "timestamp").unwrap_err(); + assert!( + error.to_string().contains("[CAST_INVALID_INPUT]"), + "{error}" ); - let result = cast.invoke_with_args(args).unwrap(); - // -86400.5 seconds = -86400500000 microseconds (1 day and 0.5 seconds before epoch) - assert_scalar_timestamp(result, -86_400_500_000); } } diff --git a/crates/sqllogictest/test_files/conversion/cast_float_to_timestamp.slt b/crates/sqllogictest/test_files/conversion/cast_float_to_timestamp.slt index fe885d86..f4361450 100644 --- a/crates/sqllogictest/test_files/conversion/cast_float_to_timestamp.slt +++ b/crates/sqllogictest/test_files/conversion/cast_float_to_timestamp.slt @@ -175,10 +175,13 @@ DROP TABLE float_test; statement ok DROP TABLE float_special; -# Note: Overflow saturation tests (1e19, -1e19) are not included here because -# DataFusion's timestamp formatter cannot display i64::MAX/MIN microsecond values. -# The saturation behavior (matching Spark) is verified in unit tests: -# test_cast_float_overflow_non_ansi_mode and test_cast_float_negative_overflow_non_ansi_mode +# An out-of-range value saturates, as in Spark. DataFusion cannot display the +# saturated timestamps, so cast them to seconds. +query II +SELECT spark_cast(spark_cast(1e19, 'timestamp'), 'bigint'), + spark_cast(spark_cast(-1e19, 'timestamp'), 'bigint'); +---- +9223372036854 -9223372036855 ############################# # ANSI Mode Tests @@ -189,15 +192,15 @@ statement ok SET datafusion.execution.enable_ansi_mode = true; # NaN should error in ANSI mode -statement error +statement error \[CAST_INVALID_INPUT\] The value NaN of the type "DOUBLE" cannot be cast to "TIMESTAMP" because it is malformed SELECT spark_cast(arrow_cast('NaN', 'Float64'), 'timestamp'); # Infinity should error in ANSI mode -statement error +statement error \[CAST_INVALID_INPUT\] The value Infinity of the type "DOUBLE" cannot be cast to "TIMESTAMP" SELECT spark_cast(arrow_cast('Infinity', 'Float64'), 'timestamp'); # Very large value should error due to overflow -statement error +statement error \[CAST_OVERFLOW\] The value 1.0E25D of the type "DOUBLE" cannot be cast to "BIGINT" due to an overflow SELECT spark_cast(1e19, 'timestamp'); # Normal values should still work in ANSI mode diff --git a/crates/sqllogictest/test_files/conversion/cast_int_to_timestamp.slt b/crates/sqllogictest/test_files/conversion/cast_int_to_timestamp.slt index 05f3fd47..fa40d8a0 100644 --- a/crates/sqllogictest/test_files/conversion/cast_int_to_timestamp.slt +++ b/crates/sqllogictest/test_files/conversion/cast_int_to_timestamp.slt @@ -113,8 +113,12 @@ statement error SELECT spark_cast('2024-01-01', 'timestamp'); # Test unsupported target type - should error -statement error -SELECT spark_cast(100, 'string'); +statement error Unsupported spark_cast target type 'interval': INTERVAL is not supported +SELECT spark_cast(100, 'interval'); + +# Spark has no cast from a number to TIMESTAMP_NTZ +statement error Spark-compatible cast from Int64 to Timestamp\(µs\) is not supported +SELECT spark_cast(100::bigint, 'timestamp_ntz'); # Test with different session timezones to verify simplify() picks up config diff --git a/crates/sqllogictest/test_files/conversion/cast_target_type.slt b/crates/sqllogictest/test_files/conversion/cast_target_type.slt new file mode 100644 index 00000000..1b04a8f5 --- /dev/null +++ b/crates/sqllogictest/test_files/conversion/cast_target_type.slt @@ -0,0 +1,89 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +# Tests for the target type of spark_cast, which uses Spark's SQL type syntax + +query TTTT +SELECT arrow_typeof(spark_cast(NULL, 'tinyint')), arrow_typeof(spark_cast(NULL, 'SMALLINT')), + arrow_typeof(spark_cast(NULL, 'Int')), arrow_typeof(spark_cast(NULL, 'long')); +---- +Int8 Int16 Int32 Int64 + +query TTT +SELECT arrow_typeof(spark_cast(NULL, 'float')), arrow_typeof(spark_cast(NULL, 'double')), + arrow_typeof(spark_cast(NULL, 'boolean')); +---- +Float32 Float64 Boolean + +query TT +SELECT arrow_typeof(spark_cast(NULL, 'string')), arrow_typeof(spark_cast(NULL, 'varchar(3)')); +---- +Utf8 Utf8 + +# CHAR and VARCHAR require a length, as in Spark +statement error \[DATATYPE_MISSING_SIZE\] DataType "CHAR" requires a length parameter, for example "CHAR"\(10\)\. Please specify the length\. SQLSTATE: 42K01 +SELECT spark_cast(NULL, 'char'); + +query TT +SELECT arrow_typeof(spark_cast(NULL, 'decimal')), arrow_typeof(spark_cast(NULL, 'decimal(38, 2)')); +---- +Decimal128(10, 0) Decimal128(38, 2) + +query TT +SELECT arrow_typeof(spark_cast(NULL, 'timestamp')), arrow_typeof(spark_cast(NULL, 'timestamp_ntz')); +---- +Timestamp(µs, "UTC") Timestamp(µs) + +# TIMESTAMP is tagged with the session time zone +statement ok +SET datafusion.execution.time_zone = 'America/Los_Angeles'; + +query T +SELECT arrow_typeof(spark_cast(NULL, 'timestamp')); +---- +Timestamp(µs, "America/Los_Angeles") + +statement ok +RESET datafusion.execution.time_zone; + +query TTT +SELECT arrow_typeof(spark_cast(NULL, 'array')), + arrow_typeof(spark_cast(NULL, 'map')), + arrow_typeof(spark_cast(NULL, 'struct')); +---- +List(Int32) Map("entries": non-null Struct("key": non-null Utf8, "value": Int64), unsorted) Struct("a": Int32, "b": non-null Utf8) + +# Casts that Arrow already does the way Spark does +query TIR +SELECT spark_cast(42, 'string'), spark_cast(true, 'int'), spark_cast(3, 'double'); +---- +42 1 3 + +# Invalid and unsupported target types +statement error Invalid spark_cast target type 'integer2': unknown type 'integer2' +SELECT spark_cast(1, 'integer2'); + +statement error Invalid spark_cast target type 'decimal\(39\)': decimal precision 39 is not between 1 and 38 +SELECT spark_cast(1, 'decimal(39)'); + +statement error Invalid spark_cast target type 'array' +SELECT spark_cast(1, 'array Date: Wed, 18 Nov 2026 09:37:23 -0700 Subject: [PATCH 2/2] refactor(spark): name types the Spark way in spark_cast errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use spark_sql_type (#345) for the "not supported" and "not implemented yet" errors, so they read "cast from BIGINT to TIMESTAMP_NTZ" rather than "cast from Int64 to Timestamp(µs)", and check that parse_spark_type round-trips with spark_sql_type. --- .../src/function/conversion/cast/data_type.rs | 29 +++++++++++++++++++ .../spark/src/function/conversion/cast/mod.rs | 14 ++++++--- .../conversion/cast_int_to_timestamp.slt | 2 +- .../conversion/cast_target_type.slt | 6 ++-- 4 files changed, 43 insertions(+), 8 deletions(-) diff --git a/crates/spark/src/function/conversion/cast/data_type.rs b/crates/spark/src/function/conversion/cast/data_type.rs index 1e27442f..021dcdeb 100644 --- a/crates/spark/src/function/conversion/cast/data_type.rs +++ b/crates/spark/src/function/conversion/cast/data_type.rs @@ -475,6 +475,35 @@ mod tests { assert_eq!(parse("struct<>"), DataType::Struct(Fields::empty())); } + /// The canonical Spark name of each type parses to a type that + /// [`spark_sql_type`] names the same way. + #[test] + fn test_round_trips_with_spark_sql_type() { + use crate::function::type_coercion::spark_sql_type; + for type_name in [ + "BOOLEAN", + "TINYINT", + "SMALLINT", + "INT", + "BIGINT", + "FLOAT", + "DOUBLE", + "DECIMAL(10,0)", + "DECIMAL(38,18)", + "STRING", + "BINARY", + "DATE", + "TIMESTAMP", + "TIMESTAMP_NTZ", + "ARRAY>", + "MAP>", + "STRUCT>", + "STRUCT<>", + ] { + assert_eq!(spark_sql_type(&parse(type_name)), type_name); + } + } + /// Spark requires a length for CHAR and VARCHAR. #[test] fn test_char_and_varchar_require_a_length() { diff --git a/crates/spark/src/function/conversion/cast/mod.rs b/crates/spark/src/function/conversion/cast/mod.rs index 130c6c36..8b3b20d0 100644 --- a/crates/spark/src/function/conversion/cast/mod.rs +++ b/crates/spark/src/function/conversion/cast/mod.rs @@ -33,6 +33,7 @@ use datafusion_common::{ use datafusion_expr::ColumnarValue; use crate::function::timezone_utils::{local_to_timestamp, timestamp_to_local}; +use crate::function::type_coercion::spark_sql_type; use crate::{EvalMode, SparkTimeZone}; use numeric::{cast_float_to_timestamp, cast_int_to_timestamp}; use utils::remove_trailing_zeroes; @@ -289,7 +290,11 @@ pub(crate) fn cast_array( } fn not_supported(from_type: &DataType, to_type: &DataType) -> Result { - not_impl_err!("Spark-compatible cast from {from_type} to {to_type} is not supported") + not_impl_err!( + "Spark-compatible cast from {} to {} is not supported", + spark_sql_type(from_type), + spark_sql_type(to_type) + ) } fn not_yet_ported( @@ -298,8 +303,9 @@ fn not_yet_ported( eval_mode: EvalMode, ) -> Result { not_impl_err!( - "Spark-compatible cast from {from_type} to {to_type} in {eval_mode:?} mode is \ - not implemented yet" + "Spark-compatible cast from {} to {} in {eval_mode:?} mode is not implemented yet", + spark_sql_type(from_type), + spark_sql_type(to_type) ) } @@ -401,7 +407,7 @@ mod tests { assert!( error .to_string() - .contains("cast from Date32 to Binary is not supported"), + .contains("cast from DATE to BINARY is not supported"), "{error}" ); diff --git a/crates/sqllogictest/test_files/conversion/cast_int_to_timestamp.slt b/crates/sqllogictest/test_files/conversion/cast_int_to_timestamp.slt index fa40d8a0..3a98ef7e 100644 --- a/crates/sqllogictest/test_files/conversion/cast_int_to_timestamp.slt +++ b/crates/sqllogictest/test_files/conversion/cast_int_to_timestamp.slt @@ -117,7 +117,7 @@ statement error Unsupported spark_cast target type 'interval': INTERVAL is not s SELECT spark_cast(100, 'interval'); # Spark has no cast from a number to TIMESTAMP_NTZ -statement error Spark-compatible cast from Int64 to Timestamp\(µs\) is not supported +statement error Spark-compatible cast from BIGINT to TIMESTAMP_NTZ is not supported SELECT spark_cast(100::bigint, 'timestamp_ntz'); # Test with different session timezones to verify simplify() picks up config diff --git a/crates/sqllogictest/test_files/conversion/cast_target_type.slt b/crates/sqllogictest/test_files/conversion/cast_target_type.slt index 1b04a8f5..1c5e46e7 100644 --- a/crates/sqllogictest/test_files/conversion/cast_target_type.slt +++ b/crates/sqllogictest/test_files/conversion/cast_target_type.slt @@ -78,12 +78,12 @@ SELECT spark_cast(1, column1) FROM (VALUES ('int')); # Casts that are not implemented yet return an error instead of a result that # differs from Spark's -statement error Spark-compatible cast from Utf8 to Int32 in Legacy mode is not implemented yet +statement error Spark-compatible cast from STRING to INT in Legacy mode is not implemented yet SELECT spark_cast('1', 'int'); # Spark has no cast from DATE to BINARY -statement error Spark-compatible cast from Date32 to Binary is not supported +statement error Spark-compatible cast from DATE to BINARY is not supported SELECT spark_cast(arrow_cast(1, 'Date32'), 'binary'); -statement error Spark-compatible cast from Int64 to Int32 in Legacy mode is not implemented yet +statement error Spark-compatible cast from BIGINT to INT in Legacy mode is not implemented yet SELECT spark_cast(2147483648, 'int');