From 72529419948b82a989a44a45c9e84bf9681bb82d Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 13 Nov 2026 06:43:43 -0700 Subject: [PATCH 1/7] feat(spark): add EvalMode and use it for parse_url and try_parse_url Add a public `EvalMode { Legacy, Ansi, Try }` that mirrors Spark's `EvalMode`, with `from_config` and `from_ansi_enabled` constructors. `parse_url` and `try_parse_url` now share one public `spark_parse_url(args, EvalMode)` instead of passing their own error handlers. Their behavior does not change. Co-Authored-By: Claude Opus 5.5 --- crates/spark/src/eval_mode.rs | 73 +++++++++++++++++++ crates/spark/src/function/url/parse_url.rs | 38 +++++++--- .../spark/src/function/url/try_parse_url.rs | 15 ++-- crates/spark/src/lib.rs | 3 + 4 files changed, 108 insertions(+), 21 deletions(-) create mode 100644 crates/spark/src/eval_mode.rs diff --git a/crates/spark/src/eval_mode.rs b/crates/spark/src/eval_mode.rs new file mode 100644 index 00000000..84311630 --- /dev/null +++ b/crates/spark/src/eval_mode.rs @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use datafusion_common::config::ConfigOptions; + +/// How a Spark expression handles a value it cannot evaluate, such as an +/// overflow or a string that does not parse. +/// +/// Mirrors Spark's +/// [`EvalMode`](https://github.com/apache/spark/blob/v4.1.1/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/EvalMode.scala). +/// An expression that supports more than one mode takes an `EvalMode` rather +/// than a `bool`, so that a `try_*` function can share its implementation. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub enum EvalMode { + /// Spark's behavior when `spark.sql.ansi.enabled` is `false`, the default + /// before Spark 4.0. An invalid value usually produces `NULL` or a wrapped + /// result instead of an error. + #[default] + Legacy, + /// Spark's behavior when `spark.sql.ansi.enabled` is `true`, the default + /// since Spark 4.0. An invalid value raises an error. + Ansi, + /// The mode of Spark's `try_*` functions, such as `try_cast` and + /// `try_parse_url`. It is the same as [`EvalMode::Ansi`], except that an + /// error on a value produces `NULL` instead. + Try, +} + +impl EvalMode { + /// Returns [`EvalMode::Ansi`] when `ansi_enabled` is true, and + /// [`EvalMode::Legacy`] otherwise, like Spark's `EvalMode.fromBoolean`. + pub fn from_ansi_enabled(ansi_enabled: bool) -> Self { + if ansi_enabled { + Self::Ansi + } else { + Self::Legacy + } + } + + /// Returns the mode for a session, from + /// `datafusion.execution.enable_ansi_mode`, like Spark's + /// `EvalMode.fromSQLConf`. A session never selects [`EvalMode::Try`]; only a + /// `try_*` function does. + pub fn from_config(config: &ConfigOptions) -> Self { + Self::from_ansi_enabled(config.execution.enable_ansi_mode) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_default_is_legacy() { + assert_eq!(EvalMode::default(), EvalMode::Legacy); + } + + #[test] + fn test_from_ansi_enabled() { + assert_eq!(EvalMode::from_ansi_enabled(false), EvalMode::Legacy); + assert_eq!(EvalMode::from_ansi_enabled(true), EvalMode::Ansi); + } + + #[test] + fn test_from_config() { + let mut config = ConfigOptions::default(); + assert_eq!(EvalMode::from_config(&config), EvalMode::Legacy); + + config.execution.enable_ansi_mode = true; + assert_eq!(EvalMode::from_config(&config), EvalMode::Ansi); + } +} diff --git a/crates/spark/src/function/url/parse_url.rs b/crates/spark/src/function/url/parse_url.rs index edb967bc..5f78d4bc 100644 --- a/crates/spark/src/function/url/parse_url.rs +++ b/crates/spark/src/function/url/parse_url.rs @@ -20,6 +20,8 @@ use datafusion_expr::{ use datafusion_functions::utils::make_scalar_function; use url::{ParseError, Url}; +use crate::EvalMode; + #[derive(Debug, PartialEq, Eq, Hash)] pub struct ParseUrl { signature: Signature, @@ -180,9 +182,9 @@ impl ScalarUDFImpl for ParseUrl { } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { - let enable_ansi_mode = args.config_options.execution.enable_ansi_mode; + let eval_mode = EvalMode::from_config(&args.config_options); make_scalar_function( - move |arrays: &[ArrayRef]| spark_parse_url(arrays, enable_ansi_mode), + move |arrays: &[ArrayRef]| spark_parse_url(arrays, eval_mode), vec![], )(&args.args) } @@ -203,9 +205,14 @@ impl ScalarUDFImpl for ParseUrl { /// - A string array with extracted URL components /// - `None` values where extraction failed or component doesn't exist /// - The output array type (StringArray or LargeStringArray) is determined by input types -fn spark_parse_url(args: &[ArrayRef], enable_ansi_mode: bool) -> Result { +/// +/// A URL that cannot be parsed produces `NULL` in [`EvalMode::Legacy`] (Spark's +/// `parse_url` with ANSI mode off) and [`EvalMode::Try`] (`try_parse_url`), and +/// an error in [`EvalMode::Ansi`]. An invalid number of arguments or a +/// non-string argument is an error in every mode. +pub fn spark_parse_url(args: &[ArrayRef], eval_mode: EvalMode) -> Result { spark_handled_parse_url(args, |x| match x { - Err(_) if !enable_ansi_mode => Ok(None), + Err(_) if eval_mode != EvalMode::Ansi => Ok(None), result => result, }) } @@ -637,17 +644,24 @@ mod tests { ]); let parts = sa(&[Some("HOST"), Some("HOST")]); - let out = spark_parse_url(&[Arc::clone(&urls), Arc::clone(&parts)], false)?; - let out_sa = out.as_any().downcast_ref::().unwrap(); - assert_eq!(out_sa.value(0), "example.com"); - assert!(out_sa.is_null(1)); + for eval_mode in [EvalMode::Legacy, EvalMode::Try] { + let out = + spark_parse_url(&[Arc::clone(&urls), Arc::clone(&parts)], eval_mode)?; + let out_sa = out.as_any().downcast_ref::().unwrap(); + assert_eq!(out_sa.value(0), "example.com"); + assert!(out_sa.is_null(1)); + } - let err = spark_parse_url(&[Arc::clone(&urls), parts], true).unwrap_err(); + let err = + spark_parse_url(&[Arc::clone(&urls), Arc::clone(&parts)], EvalMode::Ansi) + .unwrap_err(); assert!(format!("{err}").contains("The url is invalid")); - // Argument errors are not turned into NULL - let err = spark_parse_url(from_ref(&urls), false).unwrap_err(); - assert!(format!("{err}").contains("expects 2 or 3 arguments")); + // Argument errors are not turned into NULL in any mode + for eval_mode in [EvalMode::Legacy, EvalMode::Ansi, EvalMode::Try] { + let err = spark_parse_url(from_ref(&urls), eval_mode).unwrap_err(); + assert!(format!("{err}").contains("expects 2 or 3 arguments")); + } Ok(()) } } diff --git a/crates/spark/src/function/url/try_parse_url.rs b/crates/spark/src/function/url/try_parse_url.rs index ecfce671..e5b57881 100644 --- a/crates/spark/src/function/url/try_parse_url.rs +++ b/crates/spark/src/function/url/try_parse_url.rs @@ -2,7 +2,8 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -use crate::function::url::parse_url::{ParseUrl, spark_handled_parse_url}; +use crate::EvalMode; +use crate::function::url::parse_url::{ParseUrl, spark_parse_url}; use arrow::array::ArrayRef; use arrow::datatypes::DataType; use datafusion_common::Result; @@ -52,13 +53,9 @@ impl ScalarUDFImpl for TryParseUrl { fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { let ScalarFunctionArgs { args, .. } = args; - make_scalar_function(spark_try_parse_url, vec![])(&args) + make_scalar_function( + |arrays: &[ArrayRef]| spark_parse_url(arrays, EvalMode::Try), + vec![], + )(&args) } } - -fn spark_try_parse_url(args: &[ArrayRef]) -> Result { - spark_handled_parse_url(args, |x| match x { - Err(_) => Ok(None), - result => result, - }) -} diff --git a/crates/spark/src/lib.rs b/crates/spark/src/lib.rs index 9c5a6737..ca37d668 100644 --- a/crates/spark/src/lib.rs +++ b/crates/spark/src/lib.rs @@ -115,9 +115,12 @@ //! Then use the extension trait - see [`SessionStateBuilderSpark::with_spark_features`] //! for an example. +mod eval_mode; pub mod function; pub mod planner; +pub use eval_mode::EvalMode; + #[cfg(feature = "core")] mod session_state; From ff1a4c4a301ed4054be15af4da07562cd5c5bf31 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 16 Nov 2026 04:57:50 -0700 Subject: [PATCH 2/7] feat(spark): add the Spark-compatible cast framework Add the kernel and dispatcher that Comet's Spark-compatible CAST will be ported into, one source type family at a time (#88): - `SparkError`, a typed error that renders Spark 4.x's messages for the cast error classes and reaches callers as `DataFusionError::External`. - `SparkCastOptions` and the `spark_cast` kernel, with a `cast_array` dispatcher that handles dictionary targets, NULL input, dictionary string and binary input, and TIMESTAMP to TIMESTAMP_NTZ and back. - An allowlist of casts where Arrow's result matches Spark's. Casts that Comet implements itself, although Arrow supports them, are rejected until they are ported, so they cannot silently diverge from Spark. - Comet's post-processing of Arrow's timestamp casts and its session time zone helpers. Co-Authored-By: Claude Opus 5.5 --- crates/spark/src/error.rs | 243 +++++++++ .../spark/src/function/conversion/cast/mod.rs | 512 ++++++++++++++++++ .../conversion/{cast.rs => cast/udf.rs} | 0 .../src/function/conversion/cast/utils.rs | 191 +++++++ crates/spark/src/function/conversion/mod.rs | 2 +- crates/spark/src/function/mod.rs | 1 + crates/spark/src/function/timezone_utils.rs | 424 +++++++++++++++ crates/spark/src/lib.rs | 2 + 8 files changed, 1374 insertions(+), 1 deletion(-) create mode 100644 crates/spark/src/error.rs create mode 100644 crates/spark/src/function/conversion/cast/mod.rs rename crates/spark/src/function/conversion/{cast.rs => cast/udf.rs} (100%) create mode 100644 crates/spark/src/function/conversion/cast/utils.rs create mode 100644 crates/spark/src/function/timezone_utils.rs diff --git a/crates/spark/src/error.rs b/crates/spark/src/error.rs new file mode 100644 index 00000000..77100652 --- /dev/null +++ b/crates/spark/src/error.rs @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Typed Spark errors. +//! +//! A [`SparkError`] renders the message Spark raises for the same condition, +//! including its error class (such as `[CAST_OVERFLOW]`). It reaches callers as +//! [`DataFusionError::External`], so an engine that maps errors back to Spark +//! exceptions, such as Comet, can recover the variant with +//! [`SparkError::find`] instead of parsing the message. + +use std::error::Error; +use std::fmt::{Display, Formatter}; + +use datafusion_common::DataFusionError; + +/// An error that Spark raises with a specific error class. +/// +/// The messages follow Spark 4.x's +/// [`error-conditions.json`](https://github.com/apache/spark/blob/v4.2.0/common/utils/src/main/resources/error/error-conditions.json). +/// Spark 3.x appended `If necessary set "spark.sql.ansi.enabled" to "false" to +/// bypass this error.` to the `CAST_INVALID_INPUT` and `CAST_OVERFLOW` +/// messages, and 4.0 dropped it. +/// Spark quotes values and types in the forms its `toSQLValue` and `toSQLType` +/// produce. The caller passes them already formatted, such as `2147483648L` and +/// `"BIGINT"` without the quotes, since only the caller knows the Spark type. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum SparkError { + /// `CAST_INVALID_INPUT`: a value that cannot be parsed as the target type, + /// in ANSI mode. Spark raises it as a `SparkNumberFormatException`. + CastInvalidValue { + value: String, + from_type: String, + to_type: String, + }, + /// The same error class and message as [`SparkError::CastInvalidValue`], + /// for a cast to a date or timestamp. Spark raises it as a + /// `SparkDateTimeException` rather than a `SparkNumberFormatException`. + InvalidInputInCastToDatetime { + value: String, + from_type: String, + to_type: String, + }, + /// `CAST_OVERFLOW`: a value outside the range of the target type, in ANSI + /// mode. + CastOverflow { + value: String, + from_type: String, + to_type: String, + }, + /// `NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION`: a value that does not fit + /// the target decimal precision and scale, in ANSI mode. + NumericValueOutOfRange { + value: String, + precision: u8, + scale: i8, + }, + /// `NUMERIC_OUT_OF_SUPPORTED_RANGE`: a string with more than 38 digits cast + /// to a decimal. + NumericOutOfSupportedRange { value: String }, +} + +impl SparkError { + /// The Spark error class, such as `CAST_OVERFLOW`. + pub fn error_class(&self) -> &'static str { + match self { + Self::CastInvalidValue { .. } | Self::InvalidInputInCastToDatetime { .. } => { + "CAST_INVALID_INPUT" + } + Self::CastOverflow { .. } => "CAST_OVERFLOW", + Self::NumericValueOutOfRange { .. } => { + "NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION" + } + Self::NumericOutOfSupportedRange { .. } => "NUMERIC_OUT_OF_SUPPORTED_RANGE", + } + } + + /// Finds a `SparkError` in `error`, looking through the + /// [`DataFusionError::Context`], [`DataFusionError::Shared`] and + /// [`DataFusionError::ArrowError`] wrappers that DataFusion and Arrow + /// kernels add around an external error. + pub fn find(error: &DataFusionError) -> Option<&SparkError> { + match error { + DataFusionError::External(e) => e.downcast_ref::(), + DataFusionError::ArrowError(e, _) => match e.as_ref() { + arrow::error::ArrowError::ExternalError(e) => e.downcast_ref(), + _ => None, + }, + DataFusionError::Context(_, e) => Self::find(e), + DataFusionError::Shared(e) => Self::find(e), + _ => None, + } + } +} + +impl Display for SparkError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let class = self.error_class(); + match self { + Self::CastInvalidValue { + value, + from_type, + to_type, + } + | Self::InvalidInputInCastToDatetime { + value, + from_type, + to_type, + } => write!( + f, + "[{class}] The value '{value}' of the type \"{from_type}\" cannot be cast to \ + \"{to_type}\" because it is malformed. Correct the value as per the syntax, or \ + change its target type. Use `try_cast` to tolerate malformed input and return \ + NULL instead." + ), + Self::CastOverflow { + value, + from_type, + to_type, + } => write!( + f, + "[{class}] The value {value} of the type \"{from_type}\" cannot be cast to \ + \"{to_type}\" due to an overflow. Use `try_cast` to tolerate overflow and return \ + NULL instead." + ), + Self::NumericValueOutOfRange { + value, + precision, + scale, + } => write!( + f, + "[{class}] {value} cannot be represented as Decimal({precision}, {scale}). If \ + necessary set \"spark.sql.ansi.enabled\" to \"false\" to bypass this error, and \ + return NULL instead." + ), + Self::NumericOutOfSupportedRange { value } => write!( + f, + "[{class}] The value {value} cannot be interpreted as a numeric since it has \ + more than 38 digits." + ), + } + } +} + +impl Error for SparkError {} + +impl From for DataFusionError { + fn from(error: SparkError) -> Self { + DataFusionError::External(Box::new(error)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::error::ArrowError; + use std::sync::Arc; + + fn overflow() -> SparkError { + SparkError::CastOverflow { + value: "2147483648L".to_string(), + from_type: "BIGINT".to_string(), + to_type: "INT".to_string(), + } + } + + // Messages from Spark 4.2.0's error-conditions.json. + #[test] + fn test_messages_match_spark() { + assert_eq!( + overflow().to_string(), + "[CAST_OVERFLOW] The value 2147483648L of the type \"BIGINT\" cannot be cast to \ + \"INT\" due to an overflow. Use `try_cast` to tolerate overflow and return NULL \ + instead." + ); + let invalid = SparkError::CastInvalidValue { + value: "abc".to_string(), + from_type: "STRING".to_string(), + to_type: "INT".to_string(), + }; + assert_eq!( + invalid.to_string(), + "[CAST_INVALID_INPUT] The value 'abc' of the type \"STRING\" cannot be cast to \ + \"INT\" because it is malformed. Correct the value as per the syntax, or change its \ + target type. Use `try_cast` to tolerate malformed input and return NULL instead." + ); + let datetime = SparkError::InvalidInputInCastToDatetime { + value: "abc".to_string(), + from_type: "STRING".to_string(), + to_type: "DATE".to_string(), + }; + assert!(datetime.to_string().starts_with( + "[CAST_INVALID_INPUT] The value 'abc' of the type \"STRING\" cannot be cast to \"DATE\"" + )); + let out_of_range = SparkError::NumericValueOutOfRange { + value: "123.45".to_string(), + precision: 4, + scale: 2, + }; + assert_eq!( + out_of_range.to_string(), + "[NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION] 123.45 cannot be represented as \ + Decimal(4, 2). If necessary set \"spark.sql.ansi.enabled\" to \"false\" to bypass \ + this error, and return NULL instead." + ); + let too_many_digits = SparkError::NumericOutOfSupportedRange { + value: "1".repeat(39), + }; + assert_eq!( + too_many_digits.to_string(), + format!( + "[NUMERIC_OUT_OF_SUPPORTED_RANGE] The value {} cannot be interpreted as a \ + numeric since it has more than 38 digits.", + "1".repeat(39) + ) + ); + } + + #[test] + fn test_find_through_wrappers() { + let external = DataFusionError::from(overflow()); + assert_eq!(SparkError::find(&external), Some(&overflow())); + assert!(external.to_string().contains(&overflow().to_string())); + + let context = external.context("while evaluating"); + assert_eq!(SparkError::find(&context), Some(&overflow())); + + let shared = DataFusionError::Shared(Arc::new(context)); + assert_eq!(SparkError::find(&shared), Some(&overflow())); + + // Arrow's `try_unary` kernels can only return an `ArrowError`. + let arrow = + DataFusionError::from(ArrowError::ExternalError(Box::new(overflow()))); + assert_eq!(SparkError::find(&arrow), Some(&overflow())); + + assert_eq!( + SparkError::find(&DataFusionError::Execution("x".to_string())), + None + ); + } +} diff --git a/crates/spark/src/function/conversion/cast/mod.rs b/crates/spark/src/function/conversion/cast/mod.rs new file mode 100644 index 00000000..b6fb116e --- /dev/null +++ b/crates/spark/src/function/conversion/cast/mod.rs @@ -0,0 +1,512 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Spark-compatible `CAST`. +//! +//! [`spark_cast`] is the kernel, for engines that build physical plans +//! themselves, such as Comet. The `spark_cast(expr, '')` SQL function is +//! built on it. +//! +//! The kernel is being ported from Comet's +//! `native/spark-expr/src/conversion_funcs`, one source type family at a time +//! (). A cast with no +//! Spark-compatible implementation yet returns a "not implemented" error, rather +//! than falling through to Arrow's cast and producing a result that differs +//! from Spark's. + +mod udf; +mod utils; + +use std::sync::Arc; + +use arrow::array::{Array, ArrayRef, AsArray}; +use arrow::compute::{CastOptions, cast_with_options}; +use arrow::datatypes::DataType; +use arrow::util::display::FormatOptions; +use datafusion_common::{Result, ScalarValue, internal_err, not_impl_err}; +use datafusion_expr::ColumnarValue; + +use crate::EvalMode; +use crate::function::timezone_utils::{ + array_with_timezone, cast_timestamp_to_ntz, timestamp_ntz_to_timestamp, +}; +use utils::spark_cast_postprocess; + +pub use udf::SparkCast; + +/// The format Spark uses for a timestamp cast to a string. `%.f` writes the +/// fraction of a second only when it is not zero, and +/// [`spark_cast_postprocess`] then drops its trailing zeros. +const TIMESTAMP_FORMAT: Option<&str> = Some("%Y-%m-%d %H:%M:%S%.f"); + +/// Options for [`spark_cast`]. +/// +/// More options, such as the target Spark version, are added as the casts that +/// depend on them are ported. +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +#[non_exhaustive] +pub struct SparkCastOptions { + /// How a value that cannot be cast is handled. + pub eval_mode: EvalMode, + /// The session time zone (`spark.sql.session.timeZone`), as an IANA name + /// such as `America/New_York` or an offset such as `+05:00`. Casts between + /// `TIMESTAMP` and `TIMESTAMP_NTZ`, `STRING` or `DATE` need it. It is empty + /// when the caller has no time zone, and those casts then return an error. + pub timezone: String, +} + +impl SparkCastOptions { + pub fn new(eval_mode: EvalMode, timezone: &str) -> Self { + Self { + eval_mode, + timezone: timezone.to_string(), + } + } + + /// Options for a cast that does not involve a time zone. + pub fn new_without_timezone(eval_mode: EvalMode) -> Self { + Self::new(eval_mode, "") + } +} + +/// Casts `arg` to `data_type` the way Spark's `Cast` expression does. +/// +/// An error is returned for a cast that Spark does not support, or that is not +/// implemented yet. Spark can never name a dictionary type, so a dictionary +/// `data_type` is an internal error. +pub fn spark_cast( + arg: ColumnarValue, + data_type: &DataType, + cast_options: &SparkCastOptions, +) -> Result { + match arg { + ColumnarValue::Array(array) => Ok(ColumnarValue::Array(cast_array( + array, + data_type, + cast_options, + )?)), + ColumnarValue::Scalar(scalar) => { + let array = cast_array(scalar.to_array()?, data_type, cast_options)?; + Ok(ColumnarValue::Scalar(ScalarValue::try_from_array( + &array, 0, + )?)) + } + } +} + +pub(crate) fn cast_array( + array: ArrayRef, + to_type: &DataType, + cast_options: &SparkCastOptions, +) -> Result { + use DataType::*; + let from_type = array.data_type().clone(); + + // Spark's SQL type grammar cannot express a dictionary as a cast target: + // https://github.com/apache/spark/blob/v4.2.0/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4#L1477-L1525 + if matches!(to_type, Dictionary(_, _)) { + return internal_err!("Spark cannot specify dictionary types as cast targets"); + } + + if &from_type == to_type { + return Ok(array); + } + + let array = array_with_timezone(array, cast_options.timezone.clone(), Some(to_type))?; + let eval_mode = cast_options.eval_mode; + + let native_cast_options = CastOptions { + // Arrow returns NULL for a value it cannot cast when `safe` is set, and + // an error otherwise. + safe: eval_mode != EvalMode::Ansi, + format_options: FormatOptions::new() + .with_timestamp_tz_format(TIMESTAMP_FORMAT) + .with_timestamp_format(TIMESTAMP_FORMAT), + }; + + // Engines such as Comet can read a string or binary Parquet column as a + // dictionary, although Spark sees its value type. Cast the dictionary values + // and unpack the result. + if let Dictionary(_, value_type) = &from_type + && matches!(value_type.as_ref(), Utf8 | LargeUtf8 | Binary | LargeBinary) + { + let dictionary = array.as_any_dictionary(); + let values = cast_array(Arc::clone(dictionary.values()), to_type, cast_options)?; + let dictionary = dictionary.with_values(values); + let unpacked = cast_with_options(&dictionary, to_type, &native_cast_options)?; + return spark_cast_postprocess(unpacked, &from_type, to_type); + } + + let result = match (&from_type, to_type) { + // A NULL array has no values, so Arrow's cast only changes the type. + (Null, _) => cast_with_options(&array, to_type, &native_cast_options)?, + // `array_with_timezone` has already converted the local date-times to + // UTC. This arm must come before the Arrow fallthrough, which would copy + // the values without converting them. + (Timestamp(_, None), Timestamp(_, Some(target_tz))) => { + timestamp_ntz_to_timestamp( + array, + &cast_options.timezone, + Some(target_tz.as_ref()), + )? + } + (Timestamp(_, Some(_)), Timestamp(_, None)) => { + cast_timestamp_to_ntz(array, &cast_options.timezone)? + } + _ if is_pending_port(&from_type, to_type, eval_mode) => { + return not_impl_err!( + "Spark-compatible cast from {from_type} to {to_type} in {eval_mode:?} mode \ + is not implemented yet" + ); + } + _ if is_datafusion_spark_compatible(&from_type, to_type) => { + cast_with_options(&array, to_type, &native_cast_options)? + } + _ => { + return not_impl_err!( + "Spark-compatible cast from {from_type} to {to_type} is not supported" + ); + } + }; + + spark_cast_postprocess(result, &from_type, to_type) +} + +/// Whether Spark's result for the cast differs from Arrow's, although +/// [`is_datafusion_spark_compatible`] accepts it, and the Spark-compatible +/// implementation has not been ported from Comet yet. Each family removes its +/// cases from here as it is ported. +fn is_pending_port( + from_type: &DataType, + to_type: &DataType, + eval_mode: EvalMode, +) -> bool { + use DataType::*; + match (from_type, to_type) { + // LEGACY wraps on overflow and ANSI raises CAST_OVERFLOW, while Arrow + // returns NULL. TRY matches Arrow. + (Int64, Int32 | Int16 | Int8) | (Int32, Int16 | Int8) | (Int16, Int8) => { + eval_mode != EvalMode::Try + } + (Float32 | Float64 | Decimal128(_, _), Int8 | Int16 | Int32 | Int64) => { + eval_mode != EvalMode::Try + } + // ANSI raises NUMERIC_VALUE_OUT_OF_RANGE, with Spark's message. + (Decimal128(_, _), Decimal128(_, _)) => eval_mode == EvalMode::Ansi, + // LEGACY uses Java's BigDecimal.toString, as in "0E-18". + (Decimal128(_, _), Utf8) => eval_mode == EvalMode::Legacy, + // Spark replaces invalid UTF-8, and Arrow raises an error. + (Binary, Utf8) => true, + _ => false, + } +} + +/// Whether Arrow's cast from `from_type` to `to_type` matches Spark's, for every +/// eval mode, once [`spark_cast_postprocess`] has run. +fn is_datafusion_spark_compatible(from_type: &DataType, to_type: &DataType) -> bool { + use DataType::*; + if from_type == to_type { + return true; + } + match from_type { + Null => matches!(to_type, List(_)), + Boolean => matches!( + to_type, + Int8 | Int16 | Int32 | Int64 | Float32 | Float64 | Utf8 + ), + Int8 | Int16 | Int32 | Int64 => matches!( + to_type, + Boolean | Int8 | Int16 | Int32 | Int64 | Float32 | Float64 | Utf8 + ), + Float32 | Float64 => matches!( + to_type, + Boolean | Int8 | Int16 | Int32 | Int64 | Float32 | Float64 + ), + // Float32 and Float64 are absent: Arrow computes + // `(unscaled as f64) / 10^scale`, which rounds twice and can differ from + // Spark's correctly rounded BigDecimal.doubleValue by one ulp. Boolean is + // absent because Arrow does not implement that cast. + Decimal128(_, _) | Decimal256(_, _) => matches!( + to_type, + Int8 | Int16 | Int32 | Int64 | Decimal128(_, _) | Decimal256(_, _) | Utf8 + ), + Utf8 => matches!(to_type, Binary), + // Arrow reinterprets the days since the epoch, as Spark does. + Date32 => matches!(to_type, Int32 | Utf8), + Timestamp(_, _) => matches!(to_type, Int64 | Date32 | Utf8 | Timestamp(_, _)), + Binary => matches!(to_type, Utf8), + _ => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{ + BinaryArray, DictionaryArray, Int32Array, Int64Array, NullArray, StringArray, + TimestampMicrosecondArray, + }; + use arrow::datatypes::{Field, Int32Type, Int64Type, TimeUnit}; + use datafusion_common::DataFusionError; + + fn legacy() -> SparkCastOptions { + SparkCastOptions::new(EvalMode::Legacy, "UTC") + } + + fn timestamp(tz: Option<&str>) -> DataType { + DataType::Timestamp(TimeUnit::Microsecond, tz.map(Into::into)) + } + + #[test] + fn test_cast_to_dictionary_is_rejected() { + let error = cast_array( + Arc::new(StringArray::from(vec!["a"])), + &DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)), + &legacy(), + ) + .unwrap_err(); + assert!( + error + .to_string() + .contains("Spark cannot specify dictionary types as cast targets"), + "{error}" + ); + } + + #[test] + fn test_cast_to_same_type_returns_input() { + let array: ArrayRef = Arc::new(Int32Array::from(vec![Some(1), None])); + let result = cast_array(Arc::clone(&array), &DataType::Int32, &legacy()).unwrap(); + assert!(Arc::ptr_eq(&array, &result)); + } + + #[test] + fn test_unsupported_cast_is_an_error() { + let error = cast_array( + Arc::new(StringArray::from(vec!["1"])), + &DataType::Int32, + &legacy(), + ) + .unwrap_err(); + assert!( + matches!(error, DataFusionError::NotImplemented(_)), + "{error}" + ); + assert!( + error + .to_string() + .contains("cast from Utf8 to Int32 is not supported"), + "{error}" + ); + } + + /// A cast whose Spark-compatible implementation has not been ported must + /// not fall through to Arrow's cast, whose result differs from Spark's. + #[test] + fn test_pending_cast_does_not_fall_through_to_arrow() { + let array: ArrayRef = Arc::new(Int64Array::from(vec![i64::from(i32::MAX) + 1])); + for eval_mode in [EvalMode::Legacy, EvalMode::Ansi] { + let error = cast_array( + Arc::clone(&array), + &DataType::Int32, + &SparkCastOptions::new_without_timezone(eval_mode), + ) + .unwrap_err(); + assert!( + matches!(error, DataFusionError::NotImplemented(_)), + "{error}" + ); + assert!(error.to_string().contains("not implemented yet"), "{error}"); + } + + // TRY matches Arrow's safe cast, which returns NULL on overflow. + let result = cast_array( + array, + &DataType::Int32, + &SparkCastOptions::new_without_timezone(EvalMode::Try), + ) + .unwrap(); + assert!(result.is_null(0)); + } + + #[test] + fn test_arrow_fallthrough() { + // A widening cast cannot fail in any mode. + for eval_mode in [EvalMode::Legacy, EvalMode::Ansi, EvalMode::Try] { + let result = cast_array( + Arc::new(Int32Array::from(vec![Some(-1), None])), + &DataType::Int64, + &SparkCastOptions::new_without_timezone(eval_mode), + ) + .unwrap(); + assert_eq!( + result.as_primitive::(), + &Int64Array::from(vec![Some(-1), None]) + ); + } + + let result = cast_array( + Arc::new(Int32Array::from(vec![Some(0), Some(7), None])), + &DataType::Boolean, + &legacy(), + ) + .unwrap(); + let result = result.as_boolean(); + assert!(!result.value(0) && result.value(1) && result.is_null(2)); + } + + #[test] + fn test_scalar() { + let result = spark_cast( + ColumnarValue::Scalar(ScalarValue::Int32(Some(42))), + &DataType::Utf8, + &legacy(), + ) + .unwrap(); + let ColumnarValue::Scalar(scalar) = result else { + panic!("expected a scalar, got {result:?}"); + }; + assert_eq!(scalar, ScalarValue::Utf8(Some("42".to_string()))); + + let result = spark_cast( + ColumnarValue::Scalar(ScalarValue::Int32(None)), + &DataType::Int64, + &legacy(), + ) + .unwrap(); + let ColumnarValue::Scalar(scalar) = result else { + panic!("expected a scalar, got {result:?}"); + }; + assert_eq!(scalar, ScalarValue::Int64(None)); + } + + #[test] + fn test_cast_null_array() { + for to_type in [ + DataType::Int32, + DataType::Utf8, + DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))), + ] { + let result = + cast_array(Arc::new(NullArray::new(3)), &to_type, &legacy()).unwrap(); + assert_eq!(result.data_type(), &to_type); + assert_eq!(result.null_count(), 3); + } + } + + #[test] + fn test_cast_timestamp_to_long_rounds_down() { + let array: ArrayRef = Arc::new( + TimestampMicrosecondArray::from(vec![1_500_000, -1_500_000]) + .with_timezone("UTC"), + ); + let result = cast_array(array, &DataType::Int64, &legacy()).unwrap(); + assert_eq!( + result.as_primitive::(), + &Int64Array::from(vec![1, -2]) + ); + } + + #[test] + fn test_cast_timestamp_to_string_in_session_timezone() { + // 2024-01-15 15:30:00.1 UTC + let array: ArrayRef = Arc::new( + TimestampMicrosecondArray::from(vec![Some(1_705_332_600_100_000), None]) + .with_timezone("UTC"), + ); + let options = SparkCastOptions::new(EvalMode::Legacy, "America/New_York"); + let result = cast_array(array, &DataType::Utf8, &options).unwrap(); + assert_eq!( + result.as_string::(), + &StringArray::from(vec![Some("2024-01-15 10:30:00.1"), None]) + ); + } + + #[test] + fn test_cast_between_timestamp_and_timestamp_ntz() { + // 2024-01-15 15:30:00 UTC is 10:30:00 in New York. + let utc_micros = 1_705_332_600_000_000; + let local_micros = 1_705_314_600_000_000; + let options = SparkCastOptions::new(EvalMode::Legacy, "America/New_York"); + + let ltz: ArrayRef = Arc::new( + TimestampMicrosecondArray::from(vec![utc_micros]).with_timezone("UTC"), + ); + let ntz = cast_array(ltz, ×tamp(None), &options).unwrap(); + assert_eq!(ntz.data_type(), ×tamp(None)); + assert_eq!( + ntz.as_primitive::() + .value(0), + local_micros + ); + + let ltz = cast_array(ntz, ×tamp(Some("UTC")), &options).unwrap(); + assert_eq!(ltz.data_type(), ×tamp(Some("UTC"))); + assert_eq!( + ltz.as_primitive::() + .value(0), + utc_micros + ); + } + + #[test] + fn test_cast_timestamp_without_timezone_is_an_error() { + let array: ArrayRef = + Arc::new(TimestampMicrosecondArray::from(vec![0]).with_timezone("UTC")); + let options = SparkCastOptions::new_without_timezone(EvalMode::Legacy); + let error = cast_array(array, &DataType::Utf8, &options).unwrap_err(); + assert!(error.to_string().contains("requires a timezone"), "{error}"); + } + + #[test] + fn test_cast_invalid_timezone() { + let array: ArrayRef = + Arc::new(TimestampMicrosecondArray::from(vec![0]).with_timezone("UTC")); + let options = SparkCastOptions::new(EvalMode::Legacy, "Not a valid timezone"); + assert!(cast_array(array, &DataType::Date32, &options).is_err()); + } + + #[test] + fn test_cast_unsupported_timestamp_to_date() { + // chrono cannot represent every value of a microsecond timestamp. + let array: ArrayRef = Arc::new( + TimestampMicrosecondArray::from(vec![i64::MAX]) + .with_timezone("Europe/Copenhagen"), + ); + assert!(cast_array(array, &DataType::Date32, &legacy()).is_err()); + } + + #[test] + fn test_cast_dictionary_string_unpacks_result() { + let dictionary: ArrayRef = Arc::new( + DictionaryArray::::try_new( + Int32Array::from(vec![Some(1), None, Some(0), Some(1)]), + Arc::new(StringArray::from(vec!["a", "bc"])), + ) + .unwrap(), + ); + let result = cast_array(dictionary, &DataType::Binary, &legacy()).unwrap(); + assert_eq!( + result.as_binary::(), + &BinaryArray::from(vec![ + Some(b"bc".as_slice()), + None, + Some(b"a".as_slice()), + Some(b"bc".as_slice()), + ]) + ); + } + + #[test] + fn test_cast_dictionary_of_other_types_is_unsupported() { + let dictionary: ArrayRef = Arc::new( + DictionaryArray::::try_new( + Int32Array::from(vec![0]), + Arc::new(Int32Array::from(vec![1])), + ) + .unwrap(), + ); + assert!(cast_array(dictionary, &DataType::Int64, &legacy()).is_err()); + } +} diff --git a/crates/spark/src/function/conversion/cast.rs b/crates/spark/src/function/conversion/cast/udf.rs similarity index 100% rename from crates/spark/src/function/conversion/cast.rs rename to crates/spark/src/function/conversion/cast/udf.rs diff --git a/crates/spark/src/function/conversion/cast/utils.rs b/crates/spark/src/function/conversion/cast/utils.rs new file mode 100644 index 00000000..65147e12 --- /dev/null +++ b/crates/spark/src/function/conversion/cast/utils.rs @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Fix-ups applied after Arrow's cast kernel, where its result differs from +//! Spark's. +//! +//! Ported from Comet's `native/spark-expr/src/conversion_funcs/utils.rs`. + +use std::sync::Arc; + +use arrow::array::{ + Array, ArrayRef, ArrowPrimitiveType, AsArray, GenericStringArray, PrimitiveArray, +}; +use arrow::compute::unary; +use arrow::datatypes::{DataType, Int64Type}; +use arrow::error::ArrowError; +use datafusion_common::Result; + +pub(crate) const MICROS_PER_SECOND: i64 = 1_000_000; + +/// Applies `op` to every value of a primitive array, or to the values of a +/// dictionary array of that type. +fn unary_dyn(array: &ArrayRef, op: F) -> Result +where + T: ArrowPrimitiveType, + F: Fn(T::Native) -> T::Native, +{ + if let Some(d) = array.as_any_dictionary_opt() { + let new_values = unary_dyn::(d.values(), op)?; + return Ok(Arc::new(d.with_values(Arc::new(new_values)))); + } + + match array.as_primitive_opt::() { + Some(a) if PrimitiveArray::::is_compatible(a.data_type()) => { + Ok(Arc::new(unary::(a, op))) + } + _ => Err(ArrowError::NotYetImplemented(format!( + "Cannot perform unary operation of type {} on array of type {}", + T::DATA_TYPE, + array.data_type() + ))), + } +} + +/// Corrects the result of Arrow's cast from `from_type` to `to_type` where it +/// differs from Spark's: +/// +/// - `TIMESTAMP` to `BIGINT`: Arrow reinterprets the microseconds, while Spark +/// returns whole seconds, rounded down. +/// - `TIMESTAMP` to `STRING`: Arrow pads the fraction of a second to 3, 6 or 9 +/// digits, while Spark drops trailing zeros. +/// +/// `from_type` is the type before the cast, so it can still be a dictionary. +pub(crate) fn spark_cast_postprocess( + array: ArrayRef, + from_type: &DataType, + to_type: &DataType, +) -> Result { + let is_timestamp = match from_type { + DataType::Dictionary(_, value_type) => { + matches!(value_type.as_ref(), DataType::Timestamp(_, _)) + } + other => matches!(other, DataType::Timestamp(_, _)), + }; + if !is_timestamp { + return Ok(array); + } + match to_type { + DataType::Int64 => Ok(unary_dyn::<_, Int64Type>(&array, |v| { + v.div_euclid(MICROS_PER_SECOND) + })?), + DataType::Utf8 => Ok(remove_trailing_zeroes(&array)), + _ => Ok(array), + } +} + +/// Removes trailing zeros from the fraction of a second, as Spark does: +/// +/// | Arrow | Spark | +/// |---------------------------|-------------------------| +/// | `1970-01-01 05:29:59.900` | `1970-01-01 05:29:59.9` | +/// | `1970-01-01 05:29:59.990` | `1970-01-01 05:29:59.99`| +/// | `1970-01-01 05:30:00` | `1970-01-01 05:30:00` | +/// | `1970-01-01 05:30:00.001` | `1970-01-01 05:30:00.001`| +fn remove_trailing_zeroes(array: &ArrayRef) -> ArrayRef { + let string_array = array.as_string::(); + let result = string_array + .iter() + .map(|s| s.map(trim_fraction_zeroes)) + .collect::>(); + Arc::new(result) +} + +fn trim_fraction_zeroes(s: &str) -> &str { + if s.contains('.') { + s.trim_end_matches('0') + } else { + s + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{ + DictionaryArray, Int32Array, Int64Array, StringArray, TimestampMicrosecondArray, + }; + use arrow::datatypes::{Int32Type, TimeUnit}; + + const TS: DataType = DataType::Timestamp(TimeUnit::Microsecond, None); + + #[test] + fn test_timestamp_to_long_rounds_down() { + // Arrow's cast kernel reinterprets the microseconds as the BIGINT. + let micros: ArrayRef = Arc::new(Int64Array::from(vec![ + Some(1_500_000), + Some(-1_500_000), + Some(-1), + None, + ])); + let result = spark_cast_postprocess(micros, &TS, &DataType::Int64).unwrap(); + assert_eq!( + result.as_primitive::(), + &Int64Array::from(vec![Some(1), Some(-2), Some(-1), None]) + ); + } + + #[test] + fn test_dictionary_timestamp_to_long_rounds_down() { + let values = Arc::new(Int64Array::from(vec![-1_500_000, 2_000_000])); + let dict: ArrayRef = Arc::new( + DictionaryArray::::try_new( + Int32Array::from(vec![1, 0, 1]), + values, + ) + .unwrap(), + ); + let from = DataType::Dictionary(Box::new(DataType::Int32), Box::new(TS)); + let result = spark_cast_postprocess(dict, &from, &DataType::Int64).unwrap(); + let result = result.as_dictionary::(); + assert_eq!( + result.values().as_primitive::(), + &Int64Array::from(vec![-2, 2]) + ); + } + + #[test] + fn test_timestamp_to_string_drops_trailing_zeroes() { + let strings: ArrayRef = Arc::new(StringArray::from(vec![ + Some("1970-01-01 05:29:59.900"), + Some("1970-01-01 05:29:59.990"), + Some("1970-01-01 05:29:59.999"), + Some("1970-01-01 05:30:00"), + Some("1970-01-01 05:30:00.001"), + Some("1970-01-01 05:30:00.123400"), + None, + ])); + let result = spark_cast_postprocess(strings, &TS, &DataType::Utf8).unwrap(); + assert_eq!( + result.as_string::(), + &StringArray::from(vec![ + Some("1970-01-01 05:29:59.9"), + Some("1970-01-01 05:29:59.99"), + Some("1970-01-01 05:29:59.999"), + Some("1970-01-01 05:30:00"), + Some("1970-01-01 05:30:00.001"), + Some("1970-01-01 05:30:00.1234"), + None, + ]) + ); + } + + #[test] + fn test_other_casts_are_unchanged() { + let array: ArrayRef = Arc::new(Int64Array::from(vec![1_500_000])); + let result = spark_cast_postprocess( + Arc::clone(&array), + &DataType::Int32, + &DataType::Int64, + ) + .unwrap(); + assert!(Arc::ptr_eq(&array, &result)); + + let ts = TimestampMicrosecondArray::from(vec![1_500_000]); + let array: ArrayRef = Arc::new(ts); + let result = + spark_cast_postprocess(Arc::clone(&array), &TS, &DataType::Date32).unwrap(); + assert!(Arc::ptr_eq(&array, &result)); + } +} diff --git a/crates/spark/src/function/conversion/mod.rs b/crates/spark/src/function/conversion/mod.rs index 6d0f1b0c..86008a09 100644 --- a/crates/spark/src/function/conversion/mod.rs +++ b/crates/spark/src/function/conversion/mod.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -mod cast; +pub mod cast; use datafusion_expr::ScalarUDF; use datafusion_functions::make_udf_function_with_config; diff --git a/crates/spark/src/function/mod.rs b/crates/spark/src/function/mod.rs index 483f9b64..f4fd2020 100644 --- a/crates/spark/src/function/mod.rs +++ b/crates/spark/src/function/mod.rs @@ -25,6 +25,7 @@ pub mod predicate; pub mod string; pub mod r#struct; pub mod table; +pub(crate) mod timezone_utils; pub mod url; pub mod utils; pub mod window; diff --git a/crates/spark/src/function/timezone_utils.rs b/crates/spark/src/function/timezone_utils.rs new file mode 100644 index 00000000..34faade0 --- /dev/null +++ b/crates/spark/src/function/timezone_utils.rs @@ -0,0 +1,424 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Helpers for applying Spark's session time zone to timestamp arrays. +//! +//! Spark stores a `TIMESTAMP` as microseconds since the epoch in UTC and a +//! `TIMESTAMP_NTZ` as microseconds since the epoch of a local date-time. Moving +//! between them, or rendering a `TIMESTAMP` as a local string or date, needs +//! the session time zone. +//! +//! Ported from Comet's `native/spark-expr/src/utils.rs`. + +use std::sync::Arc; + +use arrow::array::timezone::Tz; +use arrow::array::types::{ + Int32Type, TimestampMicrosecondType, TimestampMillisecondType, +}; +use arrow::array::{ + Array, ArrayRef, PrimitiveArray, as_dictionary_array, as_primitive_array, +}; +use arrow::datatypes::{DataType, TimeUnit}; +use arrow::error::ArrowError; +use arrow::temporal_conversions::as_datetime; +use chrono::{DateTime, LocalResult, NaiveDateTime, Offset, TimeZone}; + +/// Prepares a timestamp array for a cast to `to_type`, using the session +/// `timezone`. +/// +/// | Conversion | Input array | Output array | +/// |------------------------------------|-------------------------|-------------------------------------------------| +/// | `TIMESTAMP` to string or date | UTC | Shifted to local time, with the time zone removed | +/// | `TIMESTAMP` to `TIMESTAMP` | UTC | The input, tagged with `timezone` | +/// | `TIMESTAMP_NTZ` to string or date | Local time | The input | +/// | `TIMESTAMP_NTZ` to `TIMESTAMP` | Local time in `timezone`| UTC, tagged with the target time zone | +/// | `TIMESTAMP_NTZ` to any other type | | An error | +/// +/// Arrays of other types are returned unchanged. +pub(crate) fn array_with_timezone( + array: ArrayRef, + timezone: String, + to_type: Option<&DataType>, +) -> Result { + match array.data_type() { + DataType::Timestamp(_, None) => { + require_timezone(&timezone)?; + match to_type { + Some(DataType::Utf8) | Some(DataType::Date32) => Ok(array), + Some(DataType::Timestamp(_, Some(target_tz))) => { + // Interpret NTZ as local time in the session time zone, and tag the + // output with the target time zone the caller expects. + timestamp_ntz_to_timestamp( + array, + timezone.as_str(), + Some(target_tz.as_ref()), + ) + } + _ => Err(ArrowError::CastError(format!( + "Cannot convert from {:?} to {:?}", + array.data_type(), + to_type + ))), + } + } + DataType::Timestamp(TimeUnit::Microsecond, Some(_)) => { + require_timezone(&timezone)?; + let array = as_primitive_array::(&array); + let array = + Arc::new(array.clone().with_timezone(timezone.clone())) as ArrayRef; + match to_type { + Some(DataType::Utf8) | Some(DataType::Date32) => { + pre_timestamp_cast(array, timezone) + } + _ => Ok(array), + } + } + DataType::Timestamp(TimeUnit::Millisecond, Some(_)) => { + require_timezone(&timezone)?; + let array = as_primitive_array::(&array); + let array = + Arc::new(array.clone().with_timezone(timezone.clone())) as ArrayRef; + match to_type { + Some(DataType::Utf8) | Some(DataType::Date32) => { + pre_timestamp_cast(array, timezone) + } + _ => Ok(array), + } + } + DataType::Dictionary(key_type, value_type) + if key_type.as_ref() == &DataType::Int32 + && matches!(value_type.as_ref(), DataType::Timestamp(_, _)) => + { + let dict = as_dictionary_array::(&array); + let values = + array_with_timezone(Arc::clone(dict.values()), timezone, to_type)?; + Ok(Arc::new(dict.with_values(values))) + } + _ => Ok(array), + } +} + +/// Converting a timestamp needs the session time zone. An empty one means the +/// caller built the cast without it, which is reported as an error rather than +/// a panic. +fn require_timezone(timezone: &str) -> Result<(), ArrowError> { + if timezone.is_empty() { + return Err(ArrowError::InvalidArgumentError( + "Converting a timestamp requires a timezone, but none was given".to_string(), + )); + } + Ok(()) +} + +fn datetime_cast_err(value: i64) -> ArrowError { + ArrowError::CastError(format!( + "Cannot convert timestamp {value} to a date-time. Only dates between \ + Jan 1, 262145 BCE and Dec 31, 262143 CE are supported", + )) +} + +/// Resolves a local date-time in `tz` to an instant, the way Spark's +/// `LocalDateTime.atZone` does. +/// +/// An ambiguous time, in the hour repeated when clocks go back, resolves to the +/// earlier offset. A time in the gap when clocks go forward is shifted forward +/// by the length of the gap, by applying the offset in effect before it. +pub(crate) fn resolve_local_datetime( + tz: &Tz, + local_datetime: NaiveDateTime, +) -> DateTime { + match tz.from_local_datetime(&local_datetime) { + LocalResult::Single(dt) => dt, + LocalResult::Ambiguous(dt, _) => dt, + LocalResult::None => { + // Find the offset in effect before the transition. + let probe = local_datetime - chrono::Duration::hours(3); + let pre_offset = match tz.from_local_datetime(&probe) { + LocalResult::Single(dt) => dt.offset().fix(), + LocalResult::Ambiguous(dt, _) => dt.offset().fix(), + LocalResult::None => { + // The offset cannot be determined, so fall back to UTC. + return local_datetime.and_utc().with_timezone(tz); + } + }; + let offset_secs = pre_offset.local_minus_utc() as i64; + let utc_naive = local_datetime - chrono::Duration::seconds(offset_secs); + utc_naive.and_utc().with_timezone(tz) + } + } +} + +/// Converts a `TIMESTAMP_NTZ` array, whose values are local date-times in `tz`, +/// to a `TIMESTAMP` array in UTC tagged with `to_timezone`. +/// +/// Arrays that are not `TIMESTAMP_NTZ` in milliseconds or microseconds are +/// returned unchanged. +pub(crate) fn timestamp_ntz_to_timestamp( + array: ArrayRef, + tz: &str, + to_timezone: Option<&str>, +) -> Result { + fn convert( + array: &ArrayRef, + tz: &str, + to_timezone: Option<&str>, + to_value: impl Fn(DateTime) -> i64, + ) -> Result { + let array = as_primitive_array::(array); + let tz: Tz = tz.parse()?; + let array: PrimitiveArray = array.try_unary(|value| { + as_datetime::(value) + .ok_or_else(|| datetime_cast_err(value)) + .map(|local_datetime| { + to_value(resolve_local_datetime(&tz, local_datetime)) + }) + })?; + Ok(Arc::new(match to_timezone { + Some(to_tz) => array.with_timezone(to_tz), + None => array, + })) + } + + match array.data_type() { + DataType::Timestamp(TimeUnit::Microsecond, None) => { + convert::(&array, tz, to_timezone, |dt| { + dt.timestamp_micros() + }) + } + DataType::Timestamp(TimeUnit::Millisecond, None) => { + convert::(&array, tz, to_timezone, |dt| { + dt.timestamp_millis() + }) + } + _ => Ok(array), + } +} + +/// Converts a `TIMESTAMP` array in UTC to a `TIMESTAMP_NTZ` array of local +/// date-times in the session `timezone`, like Spark's +/// `convertTz(ts, ZoneOffset.UTC, zoneId)`. +pub(crate) fn cast_timestamp_to_ntz( + array: ArrayRef, + timezone: &str, +) -> Result { + let tz: Tz = timezone.parse()?; + match array.data_type() { + DataType::Timestamp(TimeUnit::Microsecond, Some(_)) => { + let array = as_primitive_array::(&array); + let result: PrimitiveArray = + array.try_unary(|value| { + as_datetime::(value) + .ok_or_else(|| datetime_cast_err(value)) + .map(|utc_naive| { + // Store the local date-time as if it were UTC, which is how + // NTZ values are encoded. + let local_dt = tz.from_utc_datetime(&utc_naive); + local_dt.naive_local().and_utc().timestamp_micros() + }) + })?; + // No time zone on the output type means TIMESTAMP_NTZ. + Ok(Arc::new(result)) + } + _ => Err(ArrowError::CastError(format!( + "cast_timestamp_to_ntz: unexpected input type {:?}", + array.data_type() + ))), + } +} + +/// Shifts a `TIMESTAMP` array to local time in `timezone` and drops its time +/// zone, before a cast to string or date. Spark renders the local time without +/// an offset, while Arrow's cast kernel appends one when the type has a time +/// zone. +fn pre_timestamp_cast(array: ArrayRef, timezone: String) -> Result { + fn shift( + array: &ArrayRef, + tz: &Tz, + to_value: impl Fn(NaiveDateTime) -> i64, + ) -> Result { + let array = as_primitive_array::(array); + let array: PrimitiveArray = array.try_unary(|value| { + as_datetime::(value) + .ok_or_else(|| datetime_cast_err(value)) + .map(|datetime| { + let offset = tz.offset_from_utc_datetime(&datetime).fix(); + to_value(datetime + offset) + }) + })?; + Ok(Arc::new(array)) + } + + let tz: Tz = timezone.parse()?; + match array.data_type() { + DataType::Timestamp(TimeUnit::Microsecond, _) => { + shift::(&array, &tz, |dt| { + dt.and_utc().timestamp_micros() + }) + } + DataType::Timestamp(TimeUnit::Millisecond, _) => { + shift::(&array, &tz, |dt| { + dt.and_utc().timestamp_millis() + }) + } + _ => Ok(array), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{TimestampMicrosecondArray, TimestampMillisecondArray}; + + fn micros_for(datetime: &str) -> i64 { + NaiveDateTime::parse_from_str(datetime, "%Y-%m-%d %H:%M:%S") + .unwrap() + .and_utc() + .timestamp_micros() + } + + /// A TIMESTAMP_NTZ array holding one local date-time. + fn ntz(local_datetime: &str) -> ArrayRef { + Arc::new(TimestampMicrosecondArray::from(vec![micros_for( + local_datetime, + )])) + } + + /// A TIMESTAMP array holding one UTC date-time, tagged with `tz`. + fn ts_with_tz(utc_datetime: &str, tz: &str) -> ArrayRef { + Arc::new( + TimestampMicrosecondArray::from(vec![micros_for(utc_datetime)]) + .with_timezone(tz.to_string()), + ) + } + + fn first_value(array: &ArrayRef) -> i64 { + as_primitive_array::(array).value(0) + } + + #[test] + fn test_timestamp_ntz_to_timestamp_handles_non_existent_time() { + // 01:30 does not exist in London on 2024-03-31. Spark moves it forward an + // hour, to 02:30 BST, which is 01:30 UTC. + let output = + timestamp_ntz_to_timestamp(ntz("2024-03-31 01:30:00"), "Europe/London", None) + .unwrap(); + assert_eq!(first_value(&output), micros_for("2024-03-31 01:30:00")); + } + + #[test] + fn test_timestamp_ntz_to_timestamp_handles_ambiguous_time() { + // 01:30 happens twice in London on 2024-10-27. Spark picks the earlier + // offset, BST, which is 00:30 UTC. + let output = + timestamp_ntz_to_timestamp(ntz("2024-10-27 01:30:00"), "Europe/London", None) + .unwrap(); + assert_eq!(first_value(&output), micros_for("2024-10-27 00:30:00")); + } + + #[test] + fn test_timestamp_ntz_to_timestamp_tags_target_timezone() { + let output = timestamp_ntz_to_timestamp( + ntz("2024-01-15 10:30:00"), + "America/New_York", + Some("UTC"), + ) + .unwrap(); + assert_eq!(first_value(&output), micros_for("2024-01-15 15:30:00")); + assert_eq!( + output.data_type(), + &DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())) + ); + } + + #[test] + fn test_timestamp_ntz_to_timestamp_milliseconds() { + let millis = micros_for("2024-01-15 10:30:00") / 1000; + let input: ArrayRef = Arc::new(TimestampMillisecondArray::from(vec![millis])); + let output = timestamp_ntz_to_timestamp(input, "America/New_York", None).unwrap(); + assert_eq!( + as_primitive_array::(&output).value(0), + micros_for("2024-01-15 15:30:00") / 1000 + ); + } + + #[test] + fn test_array_with_timezone_requires_a_timezone() { + let ltz = ts_with_tz("2024-01-15 10:30:00", "UTC"); + let err = + array_with_timezone(ltz, String::new(), Some(&DataType::Utf8)).unwrap_err(); + assert!(err.to_string().contains("requires a timezone"), "{err}"); + + let utc = DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())); + assert!( + array_with_timezone(ntz("2024-01-15 10:30:00"), String::new(), Some(&utc)) + .is_err() + ); + } + + #[test] + fn test_array_with_timezone_shifts_timestamp_for_string() { + let input = ts_with_tz("2024-01-15 15:30:00", "UTC"); + let output = array_with_timezone( + input, + "America/New_York".to_string(), + Some(&DataType::Utf8), + ) + .unwrap(); + assert_eq!(first_value(&output), micros_for("2024-01-15 10:30:00")); + } + + #[test] + fn test_array_with_timezone_rejects_ntz_to_other_types() { + let err = array_with_timezone( + ntz("2024-01-15 10:30:00"), + "UTC".to_string(), + Some(&DataType::Int64), + ) + .unwrap_err(); + assert!(err.to_string().contains("Cannot convert"), "{err}"); + } + + #[test] + fn test_cast_timestamp_to_ntz_utc() { + let result = + cast_timestamp_to_ntz(ts_with_tz("2024-01-15 10:30:00", "UTC"), "UTC") + .unwrap(); + assert_eq!(first_value(&result), micros_for("2024-01-15 10:30:00")); + assert_eq!( + result.data_type(), + &DataType::Timestamp(TimeUnit::Microsecond, None) + ); + } + + #[test] + fn test_cast_timestamp_to_ntz_offset_timezone() { + // New York is UTC-5 in January. + let result = cast_timestamp_to_ntz( + ts_with_tz("2024-01-15 15:30:00", "UTC"), + "America/New_York", + ) + .unwrap(); + assert_eq!(first_value(&result), micros_for("2024-01-15 10:30:00")); + } + + #[test] + fn test_cast_timestamp_to_ntz_dst() { + // New York is UTC-4 in July. + let result = cast_timestamp_to_ntz( + ts_with_tz("2024-07-04 16:30:00", "UTC"), + "America/New_York", + ) + .unwrap(); + assert_eq!(first_value(&result), micros_for("2024-07-04 12:30:00")); + } + + #[test] + fn test_out_of_range_timestamp_is_an_error() { + let input: ArrayRef = Arc::new( + TimestampMicrosecondArray::from(vec![i64::MAX]).with_timezone("UTC"), + ); + assert!(cast_timestamp_to_ntz(input, "Europe/Copenhagen").is_err()); + } +} diff --git a/crates/spark/src/lib.rs b/crates/spark/src/lib.rs index ca37d668..4c92f5e2 100644 --- a/crates/spark/src/lib.rs +++ b/crates/spark/src/lib.rs @@ -115,10 +115,12 @@ //! Then use the extension trait - see [`SessionStateBuilderSpark::with_spark_features`] //! for an example. +mod error; mod eval_mode; pub mod function; pub mod planner; +pub use error::SparkError; pub use eval_mode::EvalMode; #[cfg(feature = "core")] From 9c6369a304b8af597c71c1f160965eb7afe7a8a2 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 16 Nov 2026 05:12:54 -0700 Subject: [PATCH 3/7] refactor(spark): parse the cast session time zone with Spark's rules - Add a public `SparkTimeZone`, wrapping the crate's Spark-style time zone parser (Java short IDs, GMT offsets, SystemV zones), and store it in `SparkCastOptions` instead of a string, so it is parsed once. - Add `local_to_utc`, which resolves daylight saving gaps and overlaps like Java's `LocalDateTime.atZone`, including for SystemV zones. - Give each timestamp cast its own dispatch arm, replacing the time zone pre-pass and the post-processing step, and add fast paths for UTC and fixed offsets. - Make the not-yet-ported casts placeholder arms in the dispatch match. - Remove branches that cannot be reached. Co-Authored-By: Claude Opus 5.5 --- .../spark/src/function/conversion/cast/mod.rs | 302 +++++++---- .../src/function/conversion/cast/utils.rs | 155 +----- .../function/datetime/from_utc_timestamp.rs | 6 +- crates/spark/src/function/datetime/mod.rs | 2 +- .../spark/src/function/datetime/timezone.rs | 149 +++++- crates/spark/src/function/timezone_utils.rs | 476 ++++-------------- crates/spark/src/lib.rs | 1 + 7 files changed, 461 insertions(+), 630 deletions(-) diff --git a/crates/spark/src/function/conversion/cast/mod.rs b/crates/spark/src/function/conversion/cast/mod.rs index b6fb116e..084e6ae2 100644 --- a/crates/spark/src/function/conversion/cast/mod.rs +++ b/crates/spark/src/function/conversion/cast/mod.rs @@ -22,22 +22,23 @@ use std::sync::Arc; use arrow::array::{Array, ArrayRef, AsArray}; use arrow::compute::{CastOptions, cast_with_options}; -use arrow::datatypes::DataType; +use arrow::datatypes::{DataType, Int64Type, TimeUnit, TimestampMicrosecondType}; +use arrow::temporal_conversions::MICROSECONDS; use arrow::util::display::FormatOptions; -use datafusion_common::{Result, ScalarValue, internal_err, not_impl_err}; +use datafusion_common::{ + Result, ScalarValue, exec_datafusion_err, internal_err, not_impl_err, +}; use datafusion_expr::ColumnarValue; -use crate::EvalMode; -use crate::function::timezone_utils::{ - array_with_timezone, cast_timestamp_to_ntz, timestamp_ntz_to_timestamp, -}; -use utils::spark_cast_postprocess; +use crate::function::timezone_utils::{local_to_timestamp, timestamp_to_local}; +use crate::{EvalMode, SparkTimeZone}; +use utils::remove_trailing_zeroes; pub use udf::SparkCast; /// The format Spark uses for a timestamp cast to a string. `%.f` writes the -/// fraction of a second only when it is not zero, and -/// [`spark_cast_postprocess`] then drops its trailing zeros. +/// fraction of a second only when it is not zero, padded to 3, 6 or 9 digits, +/// and [`remove_trailing_zeroes`] then drops the padding. const TIMESTAMP_FORMAT: Option<&str> = Some("%Y-%m-%d %H:%M:%S%.f"); /// Options for [`spark_cast`]. @@ -49,24 +50,32 @@ const TIMESTAMP_FORMAT: Option<&str> = Some("%Y-%m-%d %H:%M:%S%.f"); pub struct SparkCastOptions { /// How a value that cannot be cast is handled. pub eval_mode: EvalMode, - /// The session time zone (`spark.sql.session.timeZone`), as an IANA name - /// such as `America/New_York` or an offset such as `+05:00`. Casts between - /// `TIMESTAMP` and `TIMESTAMP_NTZ`, `STRING` or `DATE` need it. It is empty - /// when the caller has no time zone, and those casts then return an error. - pub timezone: String, + /// The session time zone (`spark.sql.session.timeZone`). A cast from + /// `TIMESTAMP` to `TIMESTAMP_NTZ`, `STRING` or `DATE`, or from + /// `TIMESTAMP_NTZ` to `TIMESTAMP`, returns an error without one. + pub timezone: Option, } impl SparkCastOptions { - pub fn new(eval_mode: EvalMode, timezone: &str) -> Self { + /// Options with no session time zone. + pub fn new(eval_mode: EvalMode) -> Self { Self { eval_mode, - timezone: timezone.to_string(), + timezone: None, + } + } + + pub fn with_timezone(self, timezone: SparkTimeZone) -> Self { + Self { + timezone: Some(timezone), + ..self } } - /// Options for a cast that does not involve a time zone. - pub fn new_without_timezone(eval_mode: EvalMode) -> Self { - Self::new(eval_mode, "") + fn session_timezone(&self) -> Result<&SparkTimeZone> { + self.timezone.as_ref().ok_or_else(|| { + exec_datafusion_err!("Casting a timestamp requires a session time zone") + }) } } @@ -101,7 +110,8 @@ pub(crate) fn cast_array( cast_options: &SparkCastOptions, ) -> Result { use DataType::*; - let from_type = array.data_type().clone(); + use EvalMode::*; + let from_type = array.data_type(); // Spark's SQL type grammar cannot express a dictionary as a cast target: // https://github.com/apache/spark/blob/v4.2.0/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4#L1477-L1525 @@ -109,108 +119,127 @@ pub(crate) fn cast_array( return internal_err!("Spark cannot specify dictionary types as cast targets"); } - if &from_type == to_type { + if from_type == to_type { return Ok(array); } - let array = array_with_timezone(array, cast_options.timezone.clone(), Some(to_type))?; - let eval_mode = cast_options.eval_mode; + // Spark's TIMESTAMP and TIMESTAMP_NTZ are microseconds since the epoch, and + // the timestamp casts below assume that unit. + let is_other_timestamp_unit = + |t: &DataType| matches!(t, Timestamp(unit, _) if *unit != TimeUnit::Microsecond); + if is_other_timestamp_unit(from_type) || is_other_timestamp_unit(to_type) { + return not_supported(from_type, to_type); + } + let eval_mode = cast_options.eval_mode; let native_cast_options = CastOptions { // Arrow returns NULL for a value it cannot cast when `safe` is set, and // an error otherwise. - safe: eval_mode != EvalMode::Ansi, - format_options: FormatOptions::new() - .with_timestamp_tz_format(TIMESTAMP_FORMAT) - .with_timestamp_format(TIMESTAMP_FORMAT), + safe: eval_mode != Ansi, + format_options: FormatOptions::new().with_timestamp_format(TIMESTAMP_FORMAT), }; + let arrow_cast = + |array: &dyn Array| Ok(cast_with_options(array, to_type, &native_cast_options)?); // Engines such as Comet can read a string or binary Parquet column as a // dictionary, although Spark sees its value type. Cast the dictionary values // and unpack the result. - if let Dictionary(_, value_type) = &from_type + // + // TODO: this casts values that no row refers to, so once a cast from string + // can fail, ANSI mode could raise an error that Spark would not. + if let Dictionary(_, value_type) = from_type && matches!(value_type.as_ref(), Utf8 | LargeUtf8 | Binary | LargeBinary) { let dictionary = array.as_any_dictionary(); let values = cast_array(Arc::clone(dictionary.values()), to_type, cast_options)?; - let dictionary = dictionary.with_values(values); - let unpacked = cast_with_options(&dictionary, to_type, &native_cast_options)?; - return spark_cast_postprocess(unpacked, &from_type, to_type); + return arrow_cast(&dictionary.with_values(values)); } - let result = match (&from_type, to_type) { + match (from_type, to_type) { // A NULL array has no values, so Arrow's cast only changes the type. - (Null, _) => cast_with_options(&array, to_type, &native_cast_options)?, - // `array_with_timezone` has already converted the local date-times to - // UTC. This arm must come before the Arrow fallthrough, which would copy - // the values without converting them. - (Timestamp(_, None), Timestamp(_, Some(target_tz))) => { - timestamp_ntz_to_timestamp( - array, - &cast_options.timezone, - Some(target_tz.as_ref()), - )? - } - (Timestamp(_, Some(_)), Timestamp(_, None)) => { - cast_timestamp_to_ntz(array, &cast_options.timezone)? - } - _ if is_pending_port(&from_type, to_type, eval_mode) => { - return not_impl_err!( - "Spark-compatible cast from {from_type} to {to_type} in {eval_mode:?} mode \ - is not implemented yet" - ); + (Null, _) => arrow_cast(&array), + + // Timestamps. Arrow's casts ignore the session time zone. + (Timestamp(_, Some(_)), Int64) => { + // Spark returns whole seconds, rounded down. + let micros = array.as_primitive::(); + Ok(Arc::new( + micros.unary::<_, Int64Type>(|v| v.div_euclid(MICROSECONDS)), + )) } - _ if is_datafusion_spark_compatible(&from_type, to_type) => { - cast_with_options(&array, to_type, &native_cast_options)? + (Timestamp(_, Some(_)), Utf8 | Date32 | Timestamp(_, None)) => { + let local = timestamp_to_local( + array.as_primitive(), + cast_options.session_timezone()?, + )?; + match to_type { + Utf8 => Ok(remove_trailing_zeroes(&arrow_cast(&local)?)), + Date32 => arrow_cast(&local), + _ => Ok(Arc::new(local)), + } } - _ => { - return not_impl_err!( - "Spark-compatible cast from {from_type} to {to_type} is not supported" - ); + (Timestamp(_, None), Utf8) => Ok(remove_trailing_zeroes(&arrow_cast(&array)?)), + (Timestamp(_, None), Timestamp(_, Some(target_tz))) => { + let utc = local_to_timestamp( + array.as_primitive(), + cast_options.session_timezone()?, + )?; + Ok(Arc::new(utc.with_timezone(Arc::clone(target_tz)))) } - }; - - spark_cast_postprocess(result, &from_type, to_type) -} -/// Whether Spark's result for the cast differs from Arrow's, although -/// [`is_datafusion_spark_compatible`] accepts it, and the Spark-compatible -/// implementation has not been ported from Comet yet. Each family removes its -/// cases from here as it is ported. -fn is_pending_port( - from_type: &DataType, - to_type: &DataType, - eval_mode: EvalMode, -) -> bool { - use DataType::*; - match (from_type, to_type) { + // Casts that Arrow supports, but where Spark's result differs, and that + // have not been ported from Comet yet. Each is replaced by its + // implementation as its family is ported. + // // LEGACY wraps on overflow and ANSI raises CAST_OVERFLOW, while Arrow // returns NULL. TRY matches Arrow. - (Int64, Int32 | Int16 | Int8) | (Int32, Int16 | Int8) | (Int16, Int8) => { - eval_mode != EvalMode::Try + (Int64, Int32 | Int16 | Int8) | (Int32, Int16 | Int8) | (Int16, Int8) + if eval_mode != Try => + { + not_yet_ported(from_type, to_type, eval_mode) } - (Float32 | Float64 | Decimal128(_, _), Int8 | Int16 | Int32 | Int64) => { - eval_mode != EvalMode::Try + (Float32 | Float64 | Decimal128(_, _), Int8 | Int16 | Int32 | Int64) + if eval_mode != Try => + { + not_yet_ported(from_type, to_type, eval_mode) } // ANSI raises NUMERIC_VALUE_OUT_OF_RANGE, with Spark's message. - (Decimal128(_, _), Decimal128(_, _)) => eval_mode == EvalMode::Ansi, + (Decimal128(_, _), Decimal128(_, _)) if eval_mode == Ansi => { + not_yet_ported(from_type, to_type, eval_mode) + } // LEGACY uses Java's BigDecimal.toString, as in "0E-18". - (Decimal128(_, _), Utf8) => eval_mode == EvalMode::Legacy, + (Decimal128(_, _), Utf8) if eval_mode == Legacy => { + not_yet_ported(from_type, to_type, eval_mode) + } // Spark replaces invalid UTF-8, and Arrow raises an error. - (Binary, Utf8) => true, - _ => false, + (Binary, Utf8) => not_yet_ported(from_type, to_type, eval_mode), + + _ if is_datafusion_spark_compatible(from_type, to_type) => arrow_cast(&array), + _ => not_supported(from_type, to_type), } } -/// Whether Arrow's cast from `from_type` to `to_type` matches Spark's, for every -/// eval mode, once [`spark_cast_postprocess`] has run. +fn not_supported(from_type: &DataType, to_type: &DataType) -> Result { + not_impl_err!("Spark-compatible cast from {from_type} to {to_type} is not supported") +} + +fn not_yet_ported( + from_type: &DataType, + to_type: &DataType, + eval_mode: EvalMode, +) -> Result { + not_impl_err!( + "Spark-compatible cast from {from_type} to {to_type} in {eval_mode:?} mode is \ + not implemented yet" + ) +} + +/// Whether Arrow's cast from `from_type` to `to_type` matches Spark's in every +/// eval mode. [`cast_array`] handles the casts it needs to before consulting +/// this. fn is_datafusion_spark_compatible(from_type: &DataType, to_type: &DataType) -> bool { use DataType::*; - if from_type == to_type { - return true; - } match from_type { - Null => matches!(to_type, List(_)), Boolean => matches!( to_type, Int8 | Int16 | Int32 | Int64 | Float32 | Float64 | Utf8 @@ -234,8 +263,9 @@ fn is_datafusion_spark_compatible(from_type: &DataType, to_type: &DataType) -> b Utf8 => matches!(to_type, Binary), // Arrow reinterprets the days since the epoch, as Spark does. Date32 => matches!(to_type, Int32 | Utf8), - Timestamp(_, _) => matches!(to_type, Int64 | Date32 | Utf8 | Timestamp(_, _)), - Binary => matches!(to_type, Utf8), + // The instant is unchanged, so Arrow only replaces the time zone. + Timestamp(_, Some(_)) => matches!(to_type, Timestamp(_, Some(_))), + Timestamp(_, None) => matches!(to_type, Date32), _ => false, } } @@ -243,15 +273,21 @@ fn is_datafusion_spark_compatible(from_type: &DataType, to_type: &DataType) -> b #[cfg(test)] mod tests { use super::*; + use arrow::array::TimestampMillisecondArray; use arrow::array::{ BinaryArray, DictionaryArray, Int32Array, Int64Array, NullArray, StringArray, TimestampMicrosecondArray, }; - use arrow::datatypes::{Field, Int32Type, Int64Type, TimeUnit}; + use arrow::datatypes::{Field, Int32Type, Int64Type}; use datafusion_common::DataFusionError; + fn options(eval_mode: EvalMode, timezone: &str) -> SparkCastOptions { + SparkCastOptions::new(eval_mode) + .with_timezone(SparkTimeZone::parse(timezone).unwrap()) + } + fn legacy() -> SparkCastOptions { - SparkCastOptions::new(EvalMode::Legacy, "UTC") + options(EvalMode::Legacy, "UTC") } fn timestamp(tz: Option<&str>) -> DataType { @@ -310,7 +346,7 @@ mod tests { let error = cast_array( Arc::clone(&array), &DataType::Int32, - &SparkCastOptions::new_without_timezone(eval_mode), + &SparkCastOptions::new(eval_mode), ) .unwrap_err(); assert!( @@ -324,7 +360,7 @@ mod tests { let result = cast_array( array, &DataType::Int32, - &SparkCastOptions::new_without_timezone(EvalMode::Try), + &SparkCastOptions::new(EvalMode::Try), ) .unwrap(); assert!(result.is_null(0)); @@ -337,7 +373,7 @@ mod tests { let result = cast_array( Arc::new(Int32Array::from(vec![Some(-1), None])), &DataType::Int64, - &SparkCastOptions::new_without_timezone(eval_mode), + &SparkCastOptions::new(eval_mode), ) .unwrap(); assert_eq!( @@ -415,7 +451,7 @@ mod tests { TimestampMicrosecondArray::from(vec![Some(1_705_332_600_100_000), None]) .with_timezone("UTC"), ); - let options = SparkCastOptions::new(EvalMode::Legacy, "America/New_York"); + let options = options(EvalMode::Legacy, "America/New_York"); let result = cast_array(array, &DataType::Utf8, &options).unwrap(); assert_eq!( result.as_string::(), @@ -423,12 +459,30 @@ mod tests { ); } + #[test] + fn test_cast_timestamp_to_date_in_session_timezone() { + // 2024-01-15 03:00:00 UTC is still 2024-01-14 in Los Angeles. + let array: ArrayRef = Arc::new( + TimestampMicrosecondArray::from(vec![1_705_287_600_000_000]) + .with_timezone("UTC"), + ); + let result = + cast_array(array, &DataType::Date32, &options(EvalMode::Legacy, "PST")) + .unwrap(); + assert_eq!( + result + .as_primitive::() + .value(0), + 19_736 + ); + } + #[test] fn test_cast_between_timestamp_and_timestamp_ntz() { // 2024-01-15 15:30:00 UTC is 10:30:00 in New York. let utc_micros = 1_705_332_600_000_000; let local_micros = 1_705_314_600_000_000; - let options = SparkCastOptions::new(EvalMode::Legacy, "America/New_York"); + let options = options(EvalMode::Legacy, "America/New_York"); let ltz: ArrayRef = Arc::new( TimestampMicrosecondArray::from(vec![utc_micros]).with_timezone("UTC"), @@ -436,45 +490,75 @@ mod tests { let ntz = cast_array(ltz, ×tamp(None), &options).unwrap(); assert_eq!(ntz.data_type(), ×tamp(None)); assert_eq!( - ntz.as_primitive::() - .value(0), + ntz.as_primitive::().value(0), local_micros ); let ltz = cast_array(ntz, ×tamp(Some("UTC")), &options).unwrap(); assert_eq!(ltz.data_type(), ×tamp(Some("UTC"))); assert_eq!( - ltz.as_primitive::() - .value(0), + ltz.as_primitive::().value(0), utc_micros ); } + #[test] + fn test_cast_timestamp_in_other_units_is_unsupported() { + let millis: ArrayRef = Arc::new(TimestampMillisecondArray::from(vec![1_500])); + let error = cast_array(millis, &DataType::Int64, &legacy()).unwrap_err(); + assert!( + matches!(error, DataFusionError::NotImplemented(_)), + "{error}" + ); + + let micros: ArrayRef = Arc::new(TimestampMicrosecondArray::from(vec![0])); + let nanos = DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())); + let error = cast_array(micros, &nanos, &legacy()).unwrap_err(); + assert!( + matches!(error, DataFusionError::NotImplemented(_)), + "{error}" + ); + } + #[test] fn test_cast_timestamp_without_timezone_is_an_error() { let array: ArrayRef = Arc::new(TimestampMicrosecondArray::from(vec![0]).with_timezone("UTC")); - let options = SparkCastOptions::new_without_timezone(EvalMode::Legacy); + let options = SparkCastOptions::new(EvalMode::Legacy); let error = cast_array(array, &DataType::Utf8, &options).unwrap_err(); - assert!(error.to_string().contains("requires a timezone"), "{error}"); + assert!( + error.to_string().contains("requires a session time zone"), + "{error}" + ); } #[test] - fn test_cast_invalid_timezone() { - let array: ArrayRef = - Arc::new(TimestampMicrosecondArray::from(vec![0]).with_timezone("UTC")); - let options = SparkCastOptions::new(EvalMode::Legacy, "Not a valid timezone"); - assert!(cast_array(array, &DataType::Date32, &options).is_err()); + fn test_cast_timestamp_ntz() { + // 2024-01-15 10:30:00.5 local time + let ntz: ArrayRef = + Arc::new(TimestampMicrosecondArray::from(vec![1_705_314_600_500_000])); + // A local date-time renders without the session time zone. + let options = SparkCastOptions::new(EvalMode::Legacy); + let result = cast_array(Arc::clone(&ntz), &DataType::Utf8, &options).unwrap(); + assert_eq!( + result.as_string::(), + &StringArray::from(vec!["2024-01-15 10:30:00.5"]) + ); + let error = cast_array(ntz, &DataType::Int64, &legacy()).unwrap_err(); + assert!( + matches!(error, DataFusionError::NotImplemented(_)), + "{error}" + ); } #[test] fn test_cast_unsupported_timestamp_to_date() { // chrono cannot represent every value of a microsecond timestamp. let array: ArrayRef = Arc::new( - TimestampMicrosecondArray::from(vec![i64::MAX]) - .with_timezone("Europe/Copenhagen"), + TimestampMicrosecondArray::from(vec![i64::MAX]).with_timezone("UTC"), ); - assert!(cast_array(array, &DataType::Date32, &legacy()).is_err()); + let options = options(EvalMode::Legacy, "Europe/Copenhagen"); + assert!(cast_array(array, &DataType::Date32, &options).is_err()); } #[test] diff --git a/crates/spark/src/function/conversion/cast/utils.rs b/crates/spark/src/function/conversion/cast/utils.rs index 65147e12..ca0c64ae 100644 --- a/crates/spark/src/function/conversion/cast/utils.rs +++ b/crates/spark/src/function/conversion/cast/utils.rs @@ -2,90 +2,22 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -//! Fix-ups applied after Arrow's cast kernel, where its result differs from -//! Spark's. -//! -//! Ported from Comet's `native/spark-expr/src/conversion_funcs/utils.rs`. - use std::sync::Arc; -use arrow::array::{ - Array, ArrayRef, ArrowPrimitiveType, AsArray, GenericStringArray, PrimitiveArray, -}; -use arrow::compute::unary; -use arrow::datatypes::{DataType, Int64Type}; -use arrow::error::ArrowError; -use datafusion_common::Result; - -pub(crate) const MICROS_PER_SECOND: i64 = 1_000_000; - -/// Applies `op` to every value of a primitive array, or to the values of a -/// dictionary array of that type. -fn unary_dyn(array: &ArrayRef, op: F) -> Result -where - T: ArrowPrimitiveType, - F: Fn(T::Native) -> T::Native, -{ - if let Some(d) = array.as_any_dictionary_opt() { - let new_values = unary_dyn::(d.values(), op)?; - return Ok(Arc::new(d.with_values(Arc::new(new_values)))); - } - - match array.as_primitive_opt::() { - Some(a) if PrimitiveArray::::is_compatible(a.data_type()) => { - Ok(Arc::new(unary::(a, op))) - } - _ => Err(ArrowError::NotYetImplemented(format!( - "Cannot perform unary operation of type {} on array of type {}", - T::DATA_TYPE, - array.data_type() - ))), - } -} - -/// Corrects the result of Arrow's cast from `from_type` to `to_type` where it -/// differs from Spark's: -/// -/// - `TIMESTAMP` to `BIGINT`: Arrow reinterprets the microseconds, while Spark -/// returns whole seconds, rounded down. -/// - `TIMESTAMP` to `STRING`: Arrow pads the fraction of a second to 3, 6 or 9 -/// digits, while Spark drops trailing zeros. -/// -/// `from_type` is the type before the cast, so it can still be a dictionary. -pub(crate) fn spark_cast_postprocess( - array: ArrayRef, - from_type: &DataType, - to_type: &DataType, -) -> Result { - let is_timestamp = match from_type { - DataType::Dictionary(_, value_type) => { - matches!(value_type.as_ref(), DataType::Timestamp(_, _)) - } - other => matches!(other, DataType::Timestamp(_, _)), - }; - if !is_timestamp { - return Ok(array); - } - match to_type { - DataType::Int64 => Ok(unary_dyn::<_, Int64Type>(&array, |v| { - v.div_euclid(MICROS_PER_SECOND) - })?), - DataType::Utf8 => Ok(remove_trailing_zeroes(&array)), - _ => Ok(array), - } -} +use arrow::array::{ArrayRef, AsArray, GenericStringArray}; -/// Removes trailing zeros from the fraction of a second, as Spark does: +/// Removes trailing zeros from the fraction of a second in timestamps that +/// Arrow formatted, as Spark does: /// -/// | Arrow | Spark | -/// |---------------------------|-------------------------| -/// | `1970-01-01 05:29:59.900` | `1970-01-01 05:29:59.9` | -/// | `1970-01-01 05:29:59.990` | `1970-01-01 05:29:59.99`| -/// | `1970-01-01 05:30:00` | `1970-01-01 05:30:00` | +/// | Arrow | Spark | +/// |---------------------------|--------------------------| +/// | `1970-01-01 05:29:59.900` | `1970-01-01 05:29:59.9` | +/// | `1970-01-01 05:29:59.990` | `1970-01-01 05:29:59.99` | +/// | `1970-01-01 05:30:00` | `1970-01-01 05:30:00` | /// | `1970-01-01 05:30:00.001` | `1970-01-01 05:30:00.001`| -fn remove_trailing_zeroes(array: &ArrayRef) -> ArrayRef { - let string_array = array.as_string::(); - let result = string_array +pub(super) fn remove_trailing_zeroes(array: &ArrayRef) -> ArrayRef { + let result = array + .as_string::() .iter() .map(|s| s.map(trim_fraction_zeroes)) .collect::>(); @@ -103,50 +35,10 @@ fn trim_fraction_zeroes(s: &str) -> &str { #[cfg(test)] mod tests { use super::*; - use arrow::array::{ - DictionaryArray, Int32Array, Int64Array, StringArray, TimestampMicrosecondArray, - }; - use arrow::datatypes::{Int32Type, TimeUnit}; - - const TS: DataType = DataType::Timestamp(TimeUnit::Microsecond, None); + use arrow::array::StringArray; #[test] - fn test_timestamp_to_long_rounds_down() { - // Arrow's cast kernel reinterprets the microseconds as the BIGINT. - let micros: ArrayRef = Arc::new(Int64Array::from(vec![ - Some(1_500_000), - Some(-1_500_000), - Some(-1), - None, - ])); - let result = spark_cast_postprocess(micros, &TS, &DataType::Int64).unwrap(); - assert_eq!( - result.as_primitive::(), - &Int64Array::from(vec![Some(1), Some(-2), Some(-1), None]) - ); - } - - #[test] - fn test_dictionary_timestamp_to_long_rounds_down() { - let values = Arc::new(Int64Array::from(vec![-1_500_000, 2_000_000])); - let dict: ArrayRef = Arc::new( - DictionaryArray::::try_new( - Int32Array::from(vec![1, 0, 1]), - values, - ) - .unwrap(), - ); - let from = DataType::Dictionary(Box::new(DataType::Int32), Box::new(TS)); - let result = spark_cast_postprocess(dict, &from, &DataType::Int64).unwrap(); - let result = result.as_dictionary::(); - assert_eq!( - result.values().as_primitive::(), - &Int64Array::from(vec![-2, 2]) - ); - } - - #[test] - fn test_timestamp_to_string_drops_trailing_zeroes() { + fn test_drops_trailing_zeroes() { let strings: ArrayRef = Arc::new(StringArray::from(vec![ Some("1970-01-01 05:29:59.900"), Some("1970-01-01 05:29:59.990"), @@ -156,9 +48,8 @@ mod tests { Some("1970-01-01 05:30:00.123400"), None, ])); - let result = spark_cast_postprocess(strings, &TS, &DataType::Utf8).unwrap(); assert_eq!( - result.as_string::(), + remove_trailing_zeroes(&strings).as_string::(), &StringArray::from(vec![ Some("1970-01-01 05:29:59.9"), Some("1970-01-01 05:29:59.99"), @@ -170,22 +61,4 @@ mod tests { ]) ); } - - #[test] - fn test_other_casts_are_unchanged() { - let array: ArrayRef = Arc::new(Int64Array::from(vec![1_500_000])); - let result = spark_cast_postprocess( - Arc::clone(&array), - &DataType::Int32, - &DataType::Int64, - ) - .unwrap(); - assert!(Arc::ptr_eq(&array, &result)); - - let ts = TimestampMicrosecondArray::from(vec![1_500_000]); - let array: ArrayRef = Arc::new(ts); - let result = - spark_cast_postprocess(Arc::clone(&array), &TS, &DataType::Date32).unwrap(); - assert!(Arc::ptr_eq(&array, &result)); - } } diff --git a/crates/spark/src/function/datetime/from_utc_timestamp.rs b/crates/spark/src/function/datetime/from_utc_timestamp.rs index c4ad439b..3eb80ff0 100644 --- a/crates/spark/src/function/datetime/from_utc_timestamp.rs +++ b/crates/spark/src/function/datetime/from_utc_timestamp.rs @@ -6,7 +6,7 @@ use std::fmt::{Display, Formatter}; use std::hash::{Hash, Hasher}; use std::sync::Arc; -use super::timezone::SparkTimeZone; +use super::timezone::TimeZoneRules; use arrow::array::{ Array, ArrayRef, AsArray, BooleanArray, PrimitiveBuilder, StringArrayType, new_empty_array, @@ -202,8 +202,8 @@ where Ok(Arc::new(builder.finish())) } -fn parse_timezone(timezone: &str) -> Result { - SparkTimeZone::parse(timezone).map_err(|e| { +fn parse_timezone(timezone: &str) -> Result { + TimeZoneRules::parse(timezone).map_err(|e| { exec_datafusion_err!("`from_utc_timestamp`: invalid timezone '{timezone}': {e}") }) } diff --git a/crates/spark/src/function/datetime/mod.rs b/crates/spark/src/function/datetime/mod.rs index 787e7f99..b4416218 100644 --- a/crates/spark/src/function/datetime/mod.rs +++ b/crates/spark/src/function/datetime/mod.rs @@ -16,7 +16,7 @@ pub mod make_interval; pub mod monthname; pub mod next_day; pub mod time_trunc; -mod timezone; +pub(crate) mod timezone; pub mod to_utc_timestamp; pub mod trunc; pub mod unix; diff --git a/crates/spark/src/function/datetime/timezone.rs b/crates/spark/src/function/datetime/timezone.rs index 1490b334..588ec84f 100644 --- a/crates/spark/src/function/datetime/timezone.rs +++ b/crates/spark/src/function/datetime/timezone.rs @@ -2,26 +2,85 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use std::fmt::{Display, Formatter}; +use std::hash::{Hash, Hasher}; +use std::sync::Arc; + use arrow::array::timezone::Tz; use arrow::datatypes::{ArrowTimestampType, TimeUnit}; -use chrono::{DateTime, Datelike, FixedOffset, NaiveDate, Utc}; +use chrono::{ + DateTime, Datelike, FixedOffset, LocalResult, NaiveDate, NaiveDateTime, Offset, + TimeZone, Utc, +}; use datafusion_common::{Result, exec_datafusion_err, internal_datafusion_err}; use datafusion_functions::datetime::to_local_time::adjust_to_local_time; -/// A timezone resolved using Spark's `getZoneId` spelling rules. +/// A Spark session time zone (`spark.sql.session.timeZone`). +/// +/// [`SparkTimeZone::parse`] accepts the IDs Spark's `getZoneId` accepts: IANA +/// region IDs, Java's short IDs such as `PST`, offsets such as `+05:30` and +/// `GMT-8`, and Java's `SystemV` zones. Two time zones are equal when their IDs +/// are, so `PST` and `America/Los_Angeles` are different time zones with the +/// same rules. +#[derive(Debug, Clone)] +pub struct SparkTimeZone { + id: Arc, + rules: TimeZoneRules, +} + +impl SparkTimeZone { + /// Parses a Spark time zone ID. + pub fn parse(id: &str) -> Result { + Ok(Self { + id: id.into(), + rules: TimeZoneRules::parse(id)?, + }) + } + + /// The ID this time zone was parsed from. + pub fn id(&self) -> &str { + &self.id + } + + pub(crate) fn rules(&self) -> TimeZoneRules { + self.rules + } +} + +impl PartialEq for SparkTimeZone { + fn eq(&self, other: &Self) -> bool { + self.id == other.id + } +} + +impl Eq for SparkTimeZone {} + +impl Hash for SparkTimeZone { + fn hash(&self, state: &mut H) { + self.id.hash(state); + } +} + +impl Display for SparkTimeZone { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.id) + } +} + +/// The offset rules of a [`SparkTimeZone`]. /// /// Fixed offsets are kept as seconds to preserve Spark's second-resolution offsets. /// Region IDs use Arrow's Chrono-backed timezone database. /// Java's legacy SystemV zones require their own rules because that database omits them. #[derive(Debug, Clone, Copy)] -pub(super) enum SparkTimeZone { +pub(crate) enum TimeZoneRules { FixedOffset(i32), Region(Tz), SystemV(i32), } -impl SparkTimeZone { - pub(super) fn parse(timezone: &str) -> Result { +impl TimeZoneRules { + pub(crate) fn parse(timezone: &str) -> Result { // These are the exact aliases in java.time.ZoneId.SHORT_IDS. The regional // aliases retain their daylight saving rules; EST, HST, and MST are fixed. let timezone = match timezone { @@ -93,7 +152,17 @@ impl SparkTimeZone { Ok(Self::Region(timezone.parse()?)) } - pub(super) fn adjust_to_local_time( + /// The offset from UTC in seconds when it is the same at every instant. + pub(crate) fn fixed_offset_seconds(self) -> Option { + match self { + Self::FixedOffset(seconds) => Some(seconds), + Self::Region(_) | Self::SystemV(_) => None, + } + } + + /// Converts `ts`, in UTC, to the local date-time in this time zone, encoded + /// as if it were UTC. + pub(crate) fn adjust_to_local_time( self, ts: i64, ) -> Result { @@ -105,6 +174,68 @@ impl SparkTimeZone { } } } + + /// Converts `local`, a local date-time in this time zone encoded as if it + /// were UTC, to UTC, the way Java's `LocalDateTime.atZone` does. The + /// inverse of [`Self::adjust_to_local_time`]. + /// + /// A local time that happens twice, when clocks go back, resolves to the + /// earlier offset. A local time in the gap when clocks go forward moves + /// forward by the length of the gap. + pub(crate) fn local_to_utc(self, local: i64) -> Result { + let units = units_per_second::(); + let shift = |offset_seconds: i32| { + local + .checked_sub(i64::from(offset_seconds) * units) + .ok_or_else(|| exec_datafusion_err!("Timestamp {local} overflows")) + }; + match self { + Self::FixedOffset(seconds) => shift(seconds), + Self::Region(tz) => { + let local_datetime = + DateTime::::from_timestamp(local.div_euclid(units), 0) + .ok_or_else(|| { + exec_datafusion_err!( + "Timestamp is outside Chrono's calendar range" + ) + })? + .naive_utc(); + let offset = region_offset_at_local(&tz, local_datetime); + shift(offset) + } + Self::SystemV(standard_offset) => { + // Use daylight time if the result is in daylight time. That picks + // the earlier offset in the overlap, and standard time, the + // offset before the transition, in the gap. + let daylight_offset = standard_offset + 3600; + let utc = shift(daylight_offset)?; + if system_v_offset::(utc, standard_offset)? == daylight_offset { + Ok(utc) + } else { + shift(standard_offset) + } + } + } + } +} + +/// The offset of `tz`, in seconds, at the local date-time `local`, resolved as +/// Java's `LocalDateTime.atZone` does. The fraction of a second does not affect +/// the offset, so `local` is in whole seconds. +fn region_offset_at_local(tz: &Tz, local: NaiveDateTime) -> i32 { + let offset_of = |result: LocalResult>| match result { + LocalResult::Single(dt) | LocalResult::Ambiguous(dt, _) => { + Some(dt.offset().fix().local_minus_utc()) + } + LocalResult::None => None, + }; + offset_of(tz.from_local_datetime(&local)) + // In a gap, apply the offset in effect before it, which moves the time + // forward by the length of the gap. + .or_else(|| { + offset_of(tz.from_local_datetime(&(local - chrono::Duration::hours(3)))) + }) + .unwrap_or(0) } /// Parses Java's signed offsets after Spark's single-digit hour/minute normalization. @@ -236,7 +367,7 @@ mod tests { use arrow::datatypes::TimestampMicrosecondType; fn shift(timezone: &str, timestamp: i64) -> i64 { - SparkTimeZone::parse(timezone) + TimeZoneRules::parse(timezone) .unwrap() .adjust_to_local_time::(timestamp) .unwrap() @@ -307,7 +438,7 @@ mod tests { "--01", ] { let timezone = format!("{prefix}{offset}"); - assert!(SparkTimeZone::parse(&timezone).is_err(), "{timezone}"); + assert!(TimeZoneRules::parse(&timezone).is_err(), "{timezone}"); } } for timezone in [ @@ -321,7 +452,7 @@ mod tests { "SystemV/PST9PDT", " UTC", ] { - assert!(SparkTimeZone::parse(timezone).is_err(), "{timezone}"); + assert!(TimeZoneRules::parse(timezone).is_err(), "{timezone}"); } } diff --git a/crates/spark/src/function/timezone_utils.rs b/crates/spark/src/function/timezone_utils.rs index 34faade0..1361af38 100644 --- a/crates/spark/src/function/timezone_utils.rs +++ b/crates/spark/src/function/timezone_utils.rs @@ -2,423 +2,165 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -//! Helpers for applying Spark's session time zone to timestamp arrays. +//! Conversions between Spark's `TIMESTAMP` and `TIMESTAMP_NTZ` arrays. //! -//! Spark stores a `TIMESTAMP` as microseconds since the epoch in UTC and a -//! `TIMESTAMP_NTZ` as microseconds since the epoch of a local date-time. Moving -//! between them, or rendering a `TIMESTAMP` as a local string or date, needs -//! the session time zone. -//! -//! Ported from Comet's `native/spark-expr/src/utils.rs`. - -use std::sync::Arc; - -use arrow::array::timezone::Tz; -use arrow::array::types::{ - Int32Type, TimestampMicrosecondType, TimestampMillisecondType, -}; -use arrow::array::{ - Array, ArrayRef, PrimitiveArray, as_dictionary_array, as_primitive_array, -}; -use arrow::datatypes::{DataType, TimeUnit}; -use arrow::error::ArrowError; -use arrow::temporal_conversions::as_datetime; -use chrono::{DateTime, LocalResult, NaiveDateTime, Offset, TimeZone}; - -/// Prepares a timestamp array for a cast to `to_type`, using the session -/// `timezone`. -/// -/// | Conversion | Input array | Output array | -/// |------------------------------------|-------------------------|-------------------------------------------------| -/// | `TIMESTAMP` to string or date | UTC | Shifted to local time, with the time zone removed | -/// | `TIMESTAMP` to `TIMESTAMP` | UTC | The input, tagged with `timezone` | -/// | `TIMESTAMP_NTZ` to string or date | Local time | The input | -/// | `TIMESTAMP_NTZ` to `TIMESTAMP` | Local time in `timezone`| UTC, tagged with the target time zone | -/// | `TIMESTAMP_NTZ` to any other type | | An error | -/// -/// Arrays of other types are returned unchanged. -pub(crate) fn array_with_timezone( - array: ArrayRef, - timezone: String, - to_type: Option<&DataType>, -) -> Result { - match array.data_type() { - DataType::Timestamp(_, None) => { - require_timezone(&timezone)?; - match to_type { - Some(DataType::Utf8) | Some(DataType::Date32) => Ok(array), - Some(DataType::Timestamp(_, Some(target_tz))) => { - // Interpret NTZ as local time in the session time zone, and tag the - // output with the target time zone the caller expects. - timestamp_ntz_to_timestamp( - array, - timezone.as_str(), - Some(target_tz.as_ref()), - ) - } - _ => Err(ArrowError::CastError(format!( - "Cannot convert from {:?} to {:?}", - array.data_type(), - to_type - ))), - } - } - DataType::Timestamp(TimeUnit::Microsecond, Some(_)) => { - require_timezone(&timezone)?; - let array = as_primitive_array::(&array); - let array = - Arc::new(array.clone().with_timezone(timezone.clone())) as ArrayRef; - match to_type { - Some(DataType::Utf8) | Some(DataType::Date32) => { - pre_timestamp_cast(array, timezone) - } - _ => Ok(array), - } - } - DataType::Timestamp(TimeUnit::Millisecond, Some(_)) => { - require_timezone(&timezone)?; - let array = as_primitive_array::(&array); - let array = - Arc::new(array.clone().with_timezone(timezone.clone())) as ArrayRef; - match to_type { - Some(DataType::Utf8) | Some(DataType::Date32) => { - pre_timestamp_cast(array, timezone) - } - _ => Ok(array), - } - } - DataType::Dictionary(key_type, value_type) - if key_type.as_ref() == &DataType::Int32 - && matches!(value_type.as_ref(), DataType::Timestamp(_, _)) => - { - let dict = as_dictionary_array::(&array); - let values = - array_with_timezone(Arc::clone(dict.values()), timezone, to_type)?; - Ok(Arc::new(dict.with_values(values))) - } - _ => Ok(array), - } -} - -/// Converting a timestamp needs the session time zone. An empty one means the -/// caller built the cast without it, which is reported as an error rather than -/// a panic. -fn require_timezone(timezone: &str) -> Result<(), ArrowError> { - if timezone.is_empty() { - return Err(ArrowError::InvalidArgumentError( - "Converting a timestamp requires a timezone, but none was given".to_string(), - )); +//! Spark stores a `TIMESTAMP` as microseconds since the epoch in UTC, and a +//! `TIMESTAMP_NTZ` as a local date-time, in microseconds since the epoch as if +//! it were UTC. Converting between them needs the session time zone. + +use arrow::array::{PrimitiveArray, TimestampMicrosecondArray}; +use arrow::datatypes::TimestampMicrosecondType; +use arrow::temporal_conversions::MICROSECONDS; +use datafusion_common::{Result, exec_datafusion_err}; + +use crate::SparkTimeZone; + +/// Converts UTC timestamps to local date-times in `tz`, like Spark's +/// `convertTz(ts, ZoneOffset.UTC, zoneId)`. The result has no time zone. +pub(crate) fn timestamp_to_local( + array: &TimestampMicrosecondArray, + tz: &SparkTimeZone, +) -> Result { + let rules = tz.rules(); + match rules.fixed_offset_seconds() { + Some(seconds) => add_micros(array, i64::from(seconds) * MICROSECONDS), + None => array + .try_unary(|ts| rules.adjust_to_local_time::(ts)), } - Ok(()) -} - -fn datetime_cast_err(value: i64) -> ArrowError { - ArrowError::CastError(format!( - "Cannot convert timestamp {value} to a date-time. Only dates between \ - Jan 1, 262145 BCE and Dec 31, 262143 CE are supported", - )) } -/// Resolves a local date-time in `tz` to an instant, the way Spark's -/// `LocalDateTime.atZone` does. -/// -/// An ambiguous time, in the hour repeated when clocks go back, resolves to the -/// earlier offset. A time in the gap when clocks go forward is shifted forward -/// by the length of the gap, by applying the offset in effect before it. -pub(crate) fn resolve_local_datetime( - tz: &Tz, - local_datetime: NaiveDateTime, -) -> DateTime { - match tz.from_local_datetime(&local_datetime) { - LocalResult::Single(dt) => dt, - LocalResult::Ambiguous(dt, _) => dt, - LocalResult::None => { - // Find the offset in effect before the transition. - let probe = local_datetime - chrono::Duration::hours(3); - let pre_offset = match tz.from_local_datetime(&probe) { - LocalResult::Single(dt) => dt.offset().fix(), - LocalResult::Ambiguous(dt, _) => dt.offset().fix(), - LocalResult::None => { - // The offset cannot be determined, so fall back to UTC. - return local_datetime.and_utc().with_timezone(tz); - } - }; - let offset_secs = pre_offset.local_minus_utc() as i64; - let utc_naive = local_datetime - chrono::Duration::seconds(offset_secs); - utc_naive.and_utc().with_timezone(tz) - } +/// Converts local date-times in `tz` to UTC timestamps, the way Java's +/// `LocalDateTime.atZone` resolves daylight saving gaps and overlaps. The +/// result has no time zone. +pub(crate) fn local_to_timestamp( + array: &TimestampMicrosecondArray, + tz: &SparkTimeZone, +) -> Result { + let rules = tz.rules(); + match rules.fixed_offset_seconds() { + Some(seconds) => add_micros(array, -i64::from(seconds) * MICROSECONDS), + None => array.try_unary(|ts| rules.local_to_utc::(ts)), } } -/// Converts a `TIMESTAMP_NTZ` array, whose values are local date-times in `tz`, -/// to a `TIMESTAMP` array in UTC tagged with `to_timezone`. -/// -/// Arrays that are not `TIMESTAMP_NTZ` in milliseconds or microseconds are -/// returned unchanged. -pub(crate) fn timestamp_ntz_to_timestamp( - array: ArrayRef, - tz: &str, - to_timezone: Option<&str>, -) -> Result { - fn convert( - array: &ArrayRef, - tz: &str, - to_timezone: Option<&str>, - to_value: impl Fn(DateTime) -> i64, - ) -> Result { - let array = as_primitive_array::(array); - let tz: Tz = tz.parse()?; - let array: PrimitiveArray = array.try_unary(|value| { - as_datetime::(value) - .ok_or_else(|| datetime_cast_err(value)) - .map(|local_datetime| { - to_value(resolve_local_datetime(&tz, local_datetime)) - }) - })?; - Ok(Arc::new(match to_timezone { - Some(to_tz) => array.with_timezone(to_tz), - None => array, - })) - } - - match array.data_type() { - DataType::Timestamp(TimeUnit::Microsecond, None) => { - convert::(&array, tz, to_timezone, |dt| { - dt.timestamp_micros() - }) - } - DataType::Timestamp(TimeUnit::Millisecond, None) => { - convert::(&array, tz, to_timezone, |dt| { - dt.timestamp_millis() - }) - } - _ => Ok(array), - } -} - -/// Converts a `TIMESTAMP` array in UTC to a `TIMESTAMP_NTZ` array of local -/// date-times in the session `timezone`, like Spark's -/// `convertTz(ts, ZoneOffset.UTC, zoneId)`. -pub(crate) fn cast_timestamp_to_ntz( - array: ArrayRef, - timezone: &str, -) -> Result { - let tz: Tz = timezone.parse()?; - match array.data_type() { - DataType::Timestamp(TimeUnit::Microsecond, Some(_)) => { - let array = as_primitive_array::(&array); - let result: PrimitiveArray = - array.try_unary(|value| { - as_datetime::(value) - .ok_or_else(|| datetime_cast_err(value)) - .map(|utc_naive| { - // Store the local date-time as if it were UTC, which is how - // NTZ values are encoded. - let local_dt = tz.from_utc_datetime(&utc_naive); - local_dt.naive_local().and_utc().timestamp_micros() - }) - })?; - // No time zone on the output type means TIMESTAMP_NTZ. - Ok(Arc::new(result)) - } - _ => Err(ArrowError::CastError(format!( - "cast_timestamp_to_ntz: unexpected input type {:?}", - array.data_type() - ))), - } -} - -/// Shifts a `TIMESTAMP` array to local time in `timezone` and drops its time -/// zone, before a cast to string or date. Spark renders the local time without -/// an offset, while Arrow's cast kernel appends one when the type has a time -/// zone. -fn pre_timestamp_cast(array: ArrayRef, timezone: String) -> Result { - fn shift( - array: &ArrayRef, - tz: &Tz, - to_value: impl Fn(NaiveDateTime) -> i64, - ) -> Result { - let array = as_primitive_array::(array); - let array: PrimitiveArray = array.try_unary(|value| { - as_datetime::(value) - .ok_or_else(|| datetime_cast_err(value)) - .map(|datetime| { - let offset = tz.offset_from_utc_datetime(&datetime).fix(); - to_value(datetime + offset) - }) - })?; - Ok(Arc::new(array)) - } - - let tz: Tz = timezone.parse()?; - match array.data_type() { - DataType::Timestamp(TimeUnit::Microsecond, _) => { - shift::(&array, &tz, |dt| { - dt.and_utc().timestamp_micros() - }) - } - DataType::Timestamp(TimeUnit::Millisecond, _) => { - shift::(&array, &tz, |dt| { - dt.and_utc().timestamp_millis() - }) - } - _ => Ok(array), - } +/// Adds `delta` to every value, dropping the array's time zone. +fn add_micros( + array: &TimestampMicrosecondArray, + delta: i64, +) -> Result { + if delta == 0 { + let (_, values, nulls) = array.clone().into_parts(); + return Ok(PrimitiveArray::new(values, nulls)); + } + array.try_unary(|ts| { + ts.checked_add(delta) + .ok_or_else(|| exec_datafusion_err!("Timestamp {ts} overflows")) + }) } #[cfg(test)] mod tests { use super::*; - use arrow::array::{TimestampMicrosecondArray, TimestampMillisecondArray}; + use arrow::array::Array; + use chrono::NaiveDateTime; - fn micros_for(datetime: &str) -> i64 { - NaiveDateTime::parse_from_str(datetime, "%Y-%m-%d %H:%M:%S") + fn micros(datetime: &str) -> i64 { + NaiveDateTime::parse_from_str(datetime, "%Y-%m-%d %H:%M:%S%.f") .unwrap() .and_utc() .timestamp_micros() } - /// A TIMESTAMP_NTZ array holding one local date-time. - fn ntz(local_datetime: &str) -> ArrayRef { - Arc::new(TimestampMicrosecondArray::from(vec![micros_for( - local_datetime, - )])) - } - - /// A TIMESTAMP array holding one UTC date-time, tagged with `tz`. - fn ts_with_tz(utc_datetime: &str, tz: &str) -> ArrayRef { - Arc::new( - TimestampMicrosecondArray::from(vec![micros_for(utc_datetime)]) - .with_timezone(tz.to_string()), - ) + fn array(datetime: &str) -> TimestampMicrosecondArray { + TimestampMicrosecondArray::from(vec![Some(micros(datetime)), None]) } - fn first_value(array: &ArrayRef) -> i64 { - as_primitive_array::(array).value(0) + fn tz(id: &str) -> SparkTimeZone { + SparkTimeZone::parse(id).unwrap() } #[test] - fn test_timestamp_ntz_to_timestamp_handles_non_existent_time() { + fn test_local_to_timestamp_non_existent_time() { // 01:30 does not exist in London on 2024-03-31. Spark moves it forward an // hour, to 02:30 BST, which is 01:30 UTC. - let output = - timestamp_ntz_to_timestamp(ntz("2024-03-31 01:30:00"), "Europe/London", None) + let result = + local_to_timestamp(&array("2024-03-31 01:30:00"), &tz("Europe/London")) .unwrap(); - assert_eq!(first_value(&output), micros_for("2024-03-31 01:30:00")); + assert_eq!(result.value(0), micros("2024-03-31 01:30:00")); + assert!(result.is_null(1)); } #[test] - fn test_timestamp_ntz_to_timestamp_handles_ambiguous_time() { + fn test_local_to_timestamp_ambiguous_time() { // 01:30 happens twice in London on 2024-10-27. Spark picks the earlier // offset, BST, which is 00:30 UTC. - let output = - timestamp_ntz_to_timestamp(ntz("2024-10-27 01:30:00"), "Europe/London", None) + let result = + local_to_timestamp(&array("2024-10-27 01:30:00"), &tz("Europe/London")) .unwrap(); - assert_eq!(first_value(&output), micros_for("2024-10-27 00:30:00")); + assert_eq!(result.value(0), micros("2024-10-27 00:30:00")); } #[test] - fn test_timestamp_ntz_to_timestamp_tags_target_timezone() { - let output = timestamp_ntz_to_timestamp( - ntz("2024-01-15 10:30:00"), - "America/New_York", - Some("UTC"), + fn test_local_to_timestamp_keeps_fraction() { + let result = local_to_timestamp( + &array("2024-01-15 10:30:00.123456"), + &tz("America/New_York"), ) .unwrap(); - assert_eq!(first_value(&output), micros_for("2024-01-15 15:30:00")); - assert_eq!( - output.data_type(), - &DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())) - ); - } - - #[test] - fn test_timestamp_ntz_to_timestamp_milliseconds() { - let millis = micros_for("2024-01-15 10:30:00") / 1000; - let input: ArrayRef = Arc::new(TimestampMillisecondArray::from(vec![millis])); - let output = timestamp_ntz_to_timestamp(input, "America/New_York", None).unwrap(); - assert_eq!( - as_primitive_array::(&output).value(0), - micros_for("2024-01-15 15:30:00") / 1000 - ); + assert_eq!(result.value(0), micros("2024-01-15 15:30:00.123456")); } #[test] - fn test_array_with_timezone_requires_a_timezone() { - let ltz = ts_with_tz("2024-01-15 10:30:00", "UTC"); - let err = - array_with_timezone(ltz, String::new(), Some(&DataType::Utf8)).unwrap_err(); - assert!(err.to_string().contains("requires a timezone"), "{err}"); - - let utc = DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())); - assert!( - array_with_timezone(ntz("2024-01-15 10:30:00"), String::new(), Some(&utc)) - .is_err() - ); + fn test_system_v_daylight_saving() { + // SystemV/PST8PDT is UTC-7 in summer and UTC-8 in winter. + let pst8pdt = tz("SystemV/PST8PDT"); + let summer = local_to_timestamp(&array("2024-07-04 12:00:00"), &pst8pdt).unwrap(); + assert_eq!(summer.value(0), micros("2024-07-04 19:00:00")); + let winter = local_to_timestamp(&array("2024-01-15 12:00:00"), &pst8pdt).unwrap(); + assert_eq!(winter.value(0), micros("2024-01-15 20:00:00")); + // 01:30 happens twice on 2024-10-27, and resolves to daylight time. + let overlap = + local_to_timestamp(&array("2024-10-27 01:30:00"), &pst8pdt).unwrap(); + assert_eq!(overlap.value(0), micros("2024-10-27 08:30:00")); + // 02:30 does not exist on 2024-04-28, and moves forward to 03:30 PDT. + let gap = local_to_timestamp(&array("2024-04-28 02:30:00"), &pst8pdt).unwrap(); + assert_eq!(gap.value(0), micros("2024-04-28 10:30:00")); } #[test] - fn test_array_with_timezone_shifts_timestamp_for_string() { - let input = ts_with_tz("2024-01-15 15:30:00", "UTC"); - let output = array_with_timezone( - input, - "America/New_York".to_string(), - Some(&DataType::Utf8), - ) - .unwrap(); - assert_eq!(first_value(&output), micros_for("2024-01-15 10:30:00")); + fn test_timestamp_to_local() { + // New York is UTC-5 in January and UTC-4 in July. + let new_york = tz("America/New_York"); + let winter = + timestamp_to_local(&array("2024-01-15 15:30:00"), &new_york).unwrap(); + assert_eq!(winter.value(0), micros("2024-01-15 10:30:00")); + assert!(winter.is_null(1)); + let summer = + timestamp_to_local(&array("2024-07-04 16:30:00"), &new_york).unwrap(); + assert_eq!(summer.value(0), micros("2024-07-04 12:30:00")); + // Java's short IDs are accepted, as in Spark. + let pst = timestamp_to_local(&array("2024-01-15 20:00:00"), &tz("PST")).unwrap(); + assert_eq!(pst.value(0), micros("2024-01-15 12:00:00")); } #[test] - fn test_array_with_timezone_rejects_ntz_to_other_types() { - let err = array_with_timezone( - ntz("2024-01-15 10:30:00"), - "UTC".to_string(), - Some(&DataType::Int64), - ) - .unwrap_err(); - assert!(err.to_string().contains("Cannot convert"), "{err}"); - } + fn test_fixed_offsets() { + let input = array("2024-01-15 15:30:00"); + let utc = timestamp_to_local(&input, &tz("UTC")).unwrap(); + assert_eq!(utc.values(), input.values()); + assert_eq!(utc.nulls(), input.nulls()); - #[test] - fn test_cast_timestamp_to_ntz_utc() { - let result = - cast_timestamp_to_ntz(ts_with_tz("2024-01-15 10:30:00", "UTC"), "UTC") - .unwrap(); - assert_eq!(first_value(&result), micros_for("2024-01-15 10:30:00")); - assert_eq!( - result.data_type(), - &DataType::Timestamp(TimeUnit::Microsecond, None) - ); - } - - #[test] - fn test_cast_timestamp_to_ntz_offset_timezone() { - // New York is UTC-5 in January. - let result = cast_timestamp_to_ntz( - ts_with_tz("2024-01-15 15:30:00", "UTC"), - "America/New_York", - ) - .unwrap(); - assert_eq!(first_value(&result), micros_for("2024-01-15 10:30:00")); - } - - #[test] - fn test_cast_timestamp_to_ntz_dst() { - // New York is UTC-4 in July. - let result = cast_timestamp_to_ntz( - ts_with_tz("2024-07-04 16:30:00", "UTC"), - "America/New_York", - ) - .unwrap(); - assert_eq!(first_value(&result), micros_for("2024-07-04 12:30:00")); + let plus_5_30 = tz("+05:30"); + let local = timestamp_to_local(&input, &plus_5_30).unwrap(); + assert_eq!(local.value(0), micros("2024-01-15 21:00:00")); + let back = local_to_timestamp(&local, &plus_5_30).unwrap(); + assert_eq!(back.value(0), input.value(0)); } #[test] fn test_out_of_range_timestamp_is_an_error() { - let input: ArrayRef = Arc::new( - TimestampMicrosecondArray::from(vec![i64::MAX]).with_timezone("UTC"), - ); - assert!(cast_timestamp_to_ntz(input, "Europe/Copenhagen").is_err()); + let input = TimestampMicrosecondArray::from(vec![i64::MAX]); + assert!(timestamp_to_local(&input, &tz("Europe/Copenhagen")).is_err()); + assert!(timestamp_to_local(&input, &tz("+01:00")).is_err()); } } diff --git a/crates/spark/src/lib.rs b/crates/spark/src/lib.rs index 4c92f5e2..0785628e 100644 --- a/crates/spark/src/lib.rs +++ b/crates/spark/src/lib.rs @@ -122,6 +122,7 @@ pub mod planner; pub use error::SparkError; pub use eval_mode::EvalMode; +pub use function::datetime::timezone::SparkTimeZone; #[cfg(feature = "core")] mod session_state; From ba244381a84b7251679b21304153fa06a7937934 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 11:01:45 -0700 Subject: [PATCH 4/7] fix(spark): render SparkError messages as Spark does Append the SQLSTATE to every message, let callers pass the value already formatted (with SparkError::sql_string for strings, so doubles stay unquoted and quotes are escaped), put two spaces after NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION, and mark each variant non_exhaustive. --- crates/spark/src/error.rs | 80 +++++++++++++++++++++++++++++++-------- 1 file changed, 65 insertions(+), 15 deletions(-) diff --git a/crates/spark/src/error.rs b/crates/spark/src/error.rs index 77100652..0c4525ae 100644 --- a/crates/spark/src/error.rs +++ b/crates/spark/src/error.rs @@ -23,13 +23,19 @@ use datafusion_common::DataFusionError; /// bypass this error.` to the `CAST_INVALID_INPUT` and `CAST_OVERFLOW` /// messages, and 4.0 dropped it. /// Spark quotes values and types in the forms its `toSQLValue` and `toSQLType` -/// produce. The caller passes them already formatted, such as `2147483648L` and -/// `"BIGINT"` without the quotes, since only the caller knows the Spark type. +/// produce. The caller passes them already formatted, such as `2147483648L`, +/// `NaN`, or `'abc'` from [`SparkError::sql_string`], and `"BIGINT"` without the +/// quotes, since only the caller knows the Spark type. Every message ends with +/// the error's SQLSTATE, as Spark's do. +/// +/// The variants are `#[non_exhaustive]` so that fields can be added later. +/// Code outside this crate matches them with `..`. #[derive(Debug, Clone, PartialEq, Eq)] #[non_exhaustive] pub enum SparkError { /// `CAST_INVALID_INPUT`: a value that cannot be parsed as the target type, /// in ANSI mode. Spark raises it as a `SparkNumberFormatException`. + #[non_exhaustive] CastInvalidValue { value: String, from_type: String, @@ -38,6 +44,7 @@ pub enum SparkError { /// The same error class and message as [`SparkError::CastInvalidValue`], /// for a cast to a date or timestamp. Spark raises it as a /// `SparkDateTimeException` rather than a `SparkNumberFormatException`. + #[non_exhaustive] InvalidInputInCastToDatetime { value: String, from_type: String, @@ -45,6 +52,7 @@ pub enum SparkError { }, /// `CAST_OVERFLOW`: a value outside the range of the target type, in ANSI /// mode. + #[non_exhaustive] CastOverflow { value: String, from_type: String, @@ -52,6 +60,7 @@ pub enum SparkError { }, /// `NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION`: a value that does not fit /// the target decimal precision and scale, in ANSI mode. + #[non_exhaustive] NumericValueOutOfRange { value: String, precision: u8, @@ -59,6 +68,7 @@ pub enum SparkError { }, /// `NUMERIC_OUT_OF_SUPPORTED_RANGE`: a string with more than 38 digits cast /// to a decimal. + #[non_exhaustive] NumericOutOfSupportedRange { value: String }, } @@ -77,6 +87,24 @@ impl SparkError { } } + /// The SQLSTATE that Spark reports with the error class. + pub fn sql_state(&self) -> &'static str { + match self { + Self::CastInvalidValue { .. } | Self::InvalidInputInCastToDatetime { .. } => { + "22018" + } + Self::CastOverflow { .. } + | Self::NumericValueOutOfRange { .. } + | Self::NumericOutOfSupportedRange { .. } => "22003", + } + } + + /// A string value as Spark's `toSQLValue` renders it in a message: quoted, + /// with backslashes and quotes escaped, so `a'b` becomes `'a\'b'`. + pub fn sql_string(value: &str) -> String { + format!("'{}'", value.replace('\\', "\\\\").replace('\'', "\\'")) + } + /// Finds a `SparkError` in `error`, looking through the /// [`DataFusionError::Context`], [`DataFusionError::Shared`] and /// [`DataFusionError::ArrowError`] wrappers that DataFusion and Arrow @@ -98,6 +126,7 @@ impl SparkError { impl Display for SparkError { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { let class = self.error_class(); + let sql_state = self.sql_state(); match self { Self::CastInvalidValue { value, @@ -110,10 +139,10 @@ impl Display for SparkError { to_type, } => write!( f, - "[{class}] The value '{value}' of the type \"{from_type}\" cannot be cast to \ + "[{class}] The value {value} of the type \"{from_type}\" cannot be cast to \ \"{to_type}\" because it is malformed. Correct the value as per the syntax, or \ change its target type. Use `try_cast` to tolerate malformed input and return \ - NULL instead." + NULL instead. SQLSTATE: {sql_state}" ), Self::CastOverflow { value, @@ -123,7 +152,7 @@ impl Display for SparkError { f, "[{class}] The value {value} of the type \"{from_type}\" cannot be cast to \ \"{to_type}\" due to an overflow. Use `try_cast` to tolerate overflow and return \ - NULL instead." + NULL instead. SQLSTATE: {sql_state}" ), Self::NumericValueOutOfRange { value, @@ -131,14 +160,16 @@ impl Display for SparkError { scale, } => write!( f, - "[{class}] {value} cannot be represented as Decimal({precision}, {scale}). If \ + // Spark renders the empty parent template, a space, then the + // subclass template, so there are two spaces after the class. + "[{class}] {value} cannot be represented as Decimal({precision}, {scale}). If \ necessary set \"spark.sql.ansi.enabled\" to \"false\" to bypass this error, and \ - return NULL instead." + return NULL instead. SQLSTATE: {sql_state}" ), Self::NumericOutOfSupportedRange { value } => write!( f, "[{class}] The value {value} cannot be interpreted as a numeric since it has \ - more than 38 digits." + more than 38 digits. SQLSTATE: {sql_state}" ), } } @@ -173,10 +204,10 @@ mod tests { overflow().to_string(), "[CAST_OVERFLOW] The value 2147483648L of the type \"BIGINT\" cannot be cast to \ \"INT\" due to an overflow. Use `try_cast` to tolerate overflow and return NULL \ - instead." + instead. SQLSTATE: 22003" ); let invalid = SparkError::CastInvalidValue { - value: "abc".to_string(), + value: SparkError::sql_string("abc"), from_type: "STRING".to_string(), to_type: "INT".to_string(), }; @@ -184,10 +215,11 @@ mod tests { invalid.to_string(), "[CAST_INVALID_INPUT] The value 'abc' of the type \"STRING\" cannot be cast to \ \"INT\" because it is malformed. Correct the value as per the syntax, or change its \ - target type. Use `try_cast` to tolerate malformed input and return NULL instead." + target type. Use `try_cast` to tolerate malformed input and return NULL instead. \ + SQLSTATE: 22018" ); let datetime = SparkError::InvalidInputInCastToDatetime { - value: "abc".to_string(), + value: SparkError::sql_string("abc"), from_type: "STRING".to_string(), to_type: "DATE".to_string(), }; @@ -201,9 +233,9 @@ mod tests { }; assert_eq!( out_of_range.to_string(), - "[NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION] 123.45 cannot be represented as \ + "[NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION] 123.45 cannot be represented as \ Decimal(4, 2). If necessary set \"spark.sql.ansi.enabled\" to \"false\" to bypass \ - this error, and return NULL instead." + this error, and return NULL instead. SQLSTATE: 22003" ); let too_many_digits = SparkError::NumericOutOfSupportedRange { value: "1".repeat(39), @@ -212,12 +244,30 @@ mod tests { too_many_digits.to_string(), format!( "[NUMERIC_OUT_OF_SUPPORTED_RANGE] The value {} cannot be interpreted as a \ - numeric since it has more than 38 digits.", + numeric since it has more than 38 digits. SQLSTATE: 22003", "1".repeat(39) ) ); } + /// Spark's `toSQLValue` for strings, and a double, which it leaves unquoted. + #[test] + fn test_values_are_formatted_by_the_caller() { + assert_eq!(SparkError::sql_string("abc"), "'abc'"); + assert_eq!(SparkError::sql_string("a'b"), r"'a\'b'"); + assert_eq!(SparkError::sql_string(r"a\b"), r"'a\\b'"); + let nan = SparkError::InvalidInputInCastToDatetime { + value: "NaN".to_string(), + from_type: "DOUBLE".to_string(), + to_type: "TIMESTAMP".to_string(), + }; + assert!( + nan.to_string() + .starts_with("[CAST_INVALID_INPUT] The value NaN of the type \"DOUBLE\""), + "{nan}" + ); + } + #[test] fn test_find_through_wrappers() { let external = DataFusionError::from(overflow()); From ffc46a20ef5e87964962bb4ca164022c10efeaf6 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 11:01:45 -0700 Subject: [PATCH 5/7] fix(spark): do not reinterpret DATE as INT in the cast allowlist Spark returns NULL with ANSI mode off and rejects the cast with it on, so move (Date32, Int32) to the not-yet-ported arm. --- .../spark/src/function/conversion/cast/mod.rs | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/crates/spark/src/function/conversion/cast/mod.rs b/crates/spark/src/function/conversion/cast/mod.rs index 084e6ae2..f6a96682 100644 --- a/crates/spark/src/function/conversion/cast/mod.rs +++ b/crates/spark/src/function/conversion/cast/mod.rs @@ -213,6 +213,9 @@ pub(crate) fn cast_array( } // Spark replaces invalid UTF-8, and Arrow raises an error. (Binary, Utf8) => not_yet_ported(from_type, to_type, eval_mode), + // LEGACY returns NULL and ANSI rejects the cast, while Arrow reinterprets + // the days since the epoch. + (Date32, Int32) => not_yet_ported(from_type, to_type, eval_mode), _ if is_datafusion_spark_compatible(from_type, to_type) => arrow_cast(&array), _ => not_supported(from_type, to_type), @@ -261,8 +264,7 @@ fn is_datafusion_spark_compatible(from_type: &DataType, to_type: &DataType) -> b Int8 | Int16 | Int32 | Int64 | Decimal128(_, _) | Decimal256(_, _) | Utf8 ), Utf8 => matches!(to_type, Binary), - // Arrow reinterprets the days since the epoch, as Spark does. - Date32 => matches!(to_type, Int32 | Utf8), + Date32 => matches!(to_type, Utf8), // The instant is unchanged, so Arrow only replaces the time zone. Timestamp(_, Some(_)) => matches!(to_type, Timestamp(_, Some(_))), Timestamp(_, None) => matches!(to_type, Date32), @@ -337,6 +339,22 @@ mod tests { ); } + /// Arrow reinterprets the days since the epoch, while Spark returns NULL + /// (LEGACY) or rejects the cast (ANSI), so DATE to INT is not ported yet. + #[test] + fn test_date_to_int_is_not_reinterpreted() { + let array: ArrayRef = Arc::new(arrow::array::Date32Array::from(vec![18262])); + for eval_mode in [EvalMode::Legacy, EvalMode::Ansi, EvalMode::Try] { + let error = cast_array( + Arc::clone(&array), + &DataType::Int32, + &SparkCastOptions::new(eval_mode), + ) + .unwrap_err(); + assert!(error.to_string().contains("not implemented yet"), "{error}"); + } + } + /// A cast whose Spark-compatible implementation has not been ported must /// not fall through to Arrow's cast, whose result differs from Spark's. #[test] From 21c490c1451af0ec6ea1b205f41614640e8fae55 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 11:01:45 -0700 Subject: [PATCH 6/7] fix(spark): resolve local times in day-long time zone gaps Take the offset before a gap from the UTC side, 14 hours back, so a gap that skips a whole day (Pacific/Apia, Pacific/Kwajalein) no longer falls back to offset 0. --- .../spark/src/function/datetime/timezone.rs | 37 ++++++++++++++++--- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/crates/spark/src/function/datetime/timezone.rs b/crates/spark/src/function/datetime/timezone.rs index 588ec84f..c1aaae1d 100644 --- a/crates/spark/src/function/datetime/timezone.rs +++ b/crates/spark/src/function/datetime/timezone.rs @@ -229,13 +229,15 @@ fn region_offset_at_local(tz: &Tz, local: NaiveDateTime) -> i32 { } LocalResult::None => None, }; - offset_of(tz.from_local_datetime(&local)) + offset_of(tz.from_local_datetime(&local)).unwrap_or_else(|| { // In a gap, apply the offset in effect before it, which moves the time - // forward by the length of the gap. - .or_else(|| { - offset_of(tz.from_local_datetime(&(local - chrono::Duration::hours(3)))) - }) - .unwrap_or(0) + // forward by the length of the gap. No zone is more than 14 hours ahead + // of UTC after a transition, so `local - 14h`, read as UTC, is an instant + // before the transition, even in a gap that skips a whole day. + tz.offset_from_utc_datetime(&(local - chrono::Duration::hours(14))) + .fix() + .local_minus_utc() + }) } /// Parses Java's signed offsets after Spark's single-digit hour/minute normalization. @@ -373,6 +375,29 @@ mod tests { .unwrap() } + /// Local times in a gap move forward by the length of the gap, as Java's + /// `LocalDateTime.atZone` does. Expected values from Spark 4.2.0's + /// `CAST(CAST(TIMESTAMP_NTZ'...' AS TIMESTAMP) AS BIGINT)`. + #[test] + fn local_times_in_a_gap_use_the_offset_before_it() { + let cases = [ + // An hour skipped for daylight saving time. + ("America/New_York", "2024-03-10T02:30:00", 1710055800), + // Whole days skipped when the zone crossed the date line. + ("Pacific/Apia", "2011-12-30T12:00:00", 1325282400), + ("Pacific/Kwajalein", "1993-08-21T12:00:00", 745977600), + ]; + for (timezone, local, expected) in cases { + let local: NaiveDateTime = local.parse().unwrap(); + let micros = local.and_utc().timestamp() * 1_000_000; + let utc = TimeZoneRules::parse(timezone) + .unwrap() + .local_to_utc::(micros) + .unwrap(); + assert_eq!(utc, expected * 1_000_000, "{timezone} {local}"); + } + } + #[test] fn spark_offset_spellings_preserve_sign_and_seconds() { let offsets = [ From 259b798b290fdbe66264e0df514cefe5fb8c6100 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 16:53:31 -0700 Subject: [PATCH 7/7] refactor(spark): drop SparkError::sql_string until a cast calls it --- crates/spark/src/error.rs | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/crates/spark/src/error.rs b/crates/spark/src/error.rs index 0c4525ae..38061e0a 100644 --- a/crates/spark/src/error.rs +++ b/crates/spark/src/error.rs @@ -24,9 +24,9 @@ use datafusion_common::DataFusionError; /// messages, and 4.0 dropped it. /// Spark quotes values and types in the forms its `toSQLValue` and `toSQLType` /// produce. The caller passes them already formatted, such as `2147483648L`, -/// `NaN`, or `'abc'` from [`SparkError::sql_string`], and `"BIGINT"` without the -/// quotes, since only the caller knows the Spark type. Every message ends with -/// the error's SQLSTATE, as Spark's do. +/// `NaN`, or `'abc'` with its quotes, and `"BIGINT"` without the quotes, since +/// only the caller knows the Spark type. Every message ends with the error's +/// SQLSTATE, as Spark's do. /// /// The variants are `#[non_exhaustive]` so that fields can be added later. /// Code outside this crate matches them with `..`. @@ -99,12 +99,6 @@ impl SparkError { } } - /// A string value as Spark's `toSQLValue` renders it in a message: quoted, - /// with backslashes and quotes escaped, so `a'b` becomes `'a\'b'`. - pub fn sql_string(value: &str) -> String { - format!("'{}'", value.replace('\\', "\\\\").replace('\'', "\\'")) - } - /// Finds a `SparkError` in `error`, looking through the /// [`DataFusionError::Context`], [`DataFusionError::Shared`] and /// [`DataFusionError::ArrowError`] wrappers that DataFusion and Arrow @@ -207,7 +201,7 @@ mod tests { instead. SQLSTATE: 22003" ); let invalid = SparkError::CastInvalidValue { - value: SparkError::sql_string("abc"), + value: "'abc'".to_string(), from_type: "STRING".to_string(), to_type: "INT".to_string(), }; @@ -219,7 +213,7 @@ mod tests { SQLSTATE: 22018" ); let datetime = SparkError::InvalidInputInCastToDatetime { - value: SparkError::sql_string("abc"), + value: "'abc'".to_string(), from_type: "STRING".to_string(), to_type: "DATE".to_string(), }; @@ -250,12 +244,10 @@ mod tests { ); } - /// Spark's `toSQLValue` for strings, and a double, which it leaves unquoted. + /// The message uses the value as given, so a double that Spark's + /// `toSQLValue` leaves unquoted, such as `NaN`, stays unquoted. #[test] fn test_values_are_formatted_by_the_caller() { - assert_eq!(SparkError::sql_string("abc"), "'abc'"); - assert_eq!(SparkError::sql_string("a'b"), r"'a\'b'"); - assert_eq!(SparkError::sql_string(r"a\b"), r"'a\\b'"); let nan = SparkError::InvalidInputInCastToDatetime { value: "NaN".to_string(), from_type: "DOUBLE".to_string(),