diff --git a/crates/spark/src/error.rs b/crates/spark/src/error.rs new file mode 100644 index 00000000..38061e0a --- /dev/null +++ b/crates/spark/src/error.rs @@ -0,0 +1,285 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Typed Spark errors. +//! +//! A [`SparkError`] renders the message Spark raises for the same condition, +//! including its error class (such as `[CAST_OVERFLOW]`). It reaches callers as +//! [`DataFusionError::External`], so an engine that maps errors back to Spark +//! exceptions, such as Comet, can recover the variant with +//! [`SparkError::find`] instead of parsing the message. + +use std::error::Error; +use std::fmt::{Display, Formatter}; + +use datafusion_common::DataFusionError; + +/// An error that Spark raises with a specific error class. +/// +/// The messages follow Spark 4.x's +/// [`error-conditions.json`](https://github.com/apache/spark/blob/v4.2.0/common/utils/src/main/resources/error/error-conditions.json). +/// Spark 3.x appended `If necessary set "spark.sql.ansi.enabled" to "false" to +/// bypass this error.` to the `CAST_INVALID_INPUT` and `CAST_OVERFLOW` +/// messages, and 4.0 dropped it. +/// Spark quotes values and types in the forms its `toSQLValue` and `toSQLType` +/// produce. The caller passes them already formatted, such as `2147483648L`, +/// `NaN`, or `'abc'` with its quotes, and `"BIGINT"` without the quotes, since +/// only the caller knows the Spark type. Every message ends with the error's +/// SQLSTATE, as Spark's do. +/// +/// The variants are `#[non_exhaustive]` so that fields can be added later. +/// Code outside this crate matches them with `..`. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum SparkError { + /// `CAST_INVALID_INPUT`: a value that cannot be parsed as the target type, + /// in ANSI mode. Spark raises it as a `SparkNumberFormatException`. + #[non_exhaustive] + CastInvalidValue { + value: String, + from_type: String, + to_type: String, + }, + /// The same error class and message as [`SparkError::CastInvalidValue`], + /// for a cast to a date or timestamp. Spark raises it as a + /// `SparkDateTimeException` rather than a `SparkNumberFormatException`. + #[non_exhaustive] + InvalidInputInCastToDatetime { + value: String, + from_type: String, + to_type: String, + }, + /// `CAST_OVERFLOW`: a value outside the range of the target type, in ANSI + /// mode. + #[non_exhaustive] + CastOverflow { + value: String, + from_type: String, + to_type: String, + }, + /// `NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION`: a value that does not fit + /// the target decimal precision and scale, in ANSI mode. + #[non_exhaustive] + NumericValueOutOfRange { + value: String, + precision: u8, + scale: i8, + }, + /// `NUMERIC_OUT_OF_SUPPORTED_RANGE`: a string with more than 38 digits cast + /// to a decimal. + #[non_exhaustive] + NumericOutOfSupportedRange { value: String }, +} + +impl SparkError { + /// The Spark error class, such as `CAST_OVERFLOW`. + pub fn error_class(&self) -> &'static str { + match self { + Self::CastInvalidValue { .. } | Self::InvalidInputInCastToDatetime { .. } => { + "CAST_INVALID_INPUT" + } + Self::CastOverflow { .. } => "CAST_OVERFLOW", + Self::NumericValueOutOfRange { .. } => { + "NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION" + } + Self::NumericOutOfSupportedRange { .. } => "NUMERIC_OUT_OF_SUPPORTED_RANGE", + } + } + + /// The SQLSTATE that Spark reports with the error class. + pub fn sql_state(&self) -> &'static str { + match self { + Self::CastInvalidValue { .. } | Self::InvalidInputInCastToDatetime { .. } => { + "22018" + } + Self::CastOverflow { .. } + | Self::NumericValueOutOfRange { .. } + | Self::NumericOutOfSupportedRange { .. } => "22003", + } + } + + /// Finds a `SparkError` in `error`, looking through the + /// [`DataFusionError::Context`], [`DataFusionError::Shared`] and + /// [`DataFusionError::ArrowError`] wrappers that DataFusion and Arrow + /// kernels add around an external error. + pub fn find(error: &DataFusionError) -> Option<&SparkError> { + match error { + DataFusionError::External(e) => e.downcast_ref::(), + DataFusionError::ArrowError(e, _) => match e.as_ref() { + arrow::error::ArrowError::ExternalError(e) => e.downcast_ref(), + _ => None, + }, + DataFusionError::Context(_, e) => Self::find(e), + DataFusionError::Shared(e) => Self::find(e), + _ => None, + } + } +} + +impl Display for SparkError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let class = self.error_class(); + let sql_state = self.sql_state(); + match self { + Self::CastInvalidValue { + value, + from_type, + to_type, + } + | Self::InvalidInputInCastToDatetime { + value, + from_type, + to_type, + } => write!( + f, + "[{class}] The value {value} of the type \"{from_type}\" cannot be cast to \ + \"{to_type}\" because it is malformed. Correct the value as per the syntax, or \ + change its target type. Use `try_cast` to tolerate malformed input and return \ + NULL instead. SQLSTATE: {sql_state}" + ), + Self::CastOverflow { + value, + from_type, + to_type, + } => write!( + f, + "[{class}] The value {value} of the type \"{from_type}\" cannot be cast to \ + \"{to_type}\" due to an overflow. Use `try_cast` to tolerate overflow and return \ + NULL instead. SQLSTATE: {sql_state}" + ), + Self::NumericValueOutOfRange { + value, + precision, + scale, + } => write!( + f, + // Spark renders the empty parent template, a space, then the + // subclass template, so there are two spaces after the class. + "[{class}] {value} cannot be represented as Decimal({precision}, {scale}). If \ + necessary set \"spark.sql.ansi.enabled\" to \"false\" to bypass this error, and \ + return NULL instead. SQLSTATE: {sql_state}" + ), + Self::NumericOutOfSupportedRange { value } => write!( + f, + "[{class}] The value {value} cannot be interpreted as a numeric since it has \ + more than 38 digits. SQLSTATE: {sql_state}" + ), + } + } +} + +impl Error for SparkError {} + +impl From for DataFusionError { + fn from(error: SparkError) -> Self { + DataFusionError::External(Box::new(error)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::error::ArrowError; + use std::sync::Arc; + + fn overflow() -> SparkError { + SparkError::CastOverflow { + value: "2147483648L".to_string(), + from_type: "BIGINT".to_string(), + to_type: "INT".to_string(), + } + } + + // Messages from Spark 4.2.0's error-conditions.json. + #[test] + fn test_messages_match_spark() { + assert_eq!( + overflow().to_string(), + "[CAST_OVERFLOW] The value 2147483648L of the type \"BIGINT\" cannot be cast to \ + \"INT\" due to an overflow. Use `try_cast` to tolerate overflow and return NULL \ + instead. SQLSTATE: 22003" + ); + let invalid = SparkError::CastInvalidValue { + value: "'abc'".to_string(), + from_type: "STRING".to_string(), + to_type: "INT".to_string(), + }; + assert_eq!( + invalid.to_string(), + "[CAST_INVALID_INPUT] The value 'abc' of the type \"STRING\" cannot be cast to \ + \"INT\" because it is malformed. Correct the value as per the syntax, or change its \ + target type. Use `try_cast` to tolerate malformed input and return NULL instead. \ + SQLSTATE: 22018" + ); + let datetime = SparkError::InvalidInputInCastToDatetime { + value: "'abc'".to_string(), + from_type: "STRING".to_string(), + to_type: "DATE".to_string(), + }; + assert!(datetime.to_string().starts_with( + "[CAST_INVALID_INPUT] The value 'abc' of the type \"STRING\" cannot be cast to \"DATE\"" + )); + let out_of_range = SparkError::NumericValueOutOfRange { + value: "123.45".to_string(), + precision: 4, + scale: 2, + }; + assert_eq!( + out_of_range.to_string(), + "[NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION] 123.45 cannot be represented as \ + Decimal(4, 2). If necessary set \"spark.sql.ansi.enabled\" to \"false\" to bypass \ + this error, and return NULL instead. SQLSTATE: 22003" + ); + let too_many_digits = SparkError::NumericOutOfSupportedRange { + value: "1".repeat(39), + }; + assert_eq!( + too_many_digits.to_string(), + format!( + "[NUMERIC_OUT_OF_SUPPORTED_RANGE] The value {} cannot be interpreted as a \ + numeric since it has more than 38 digits. SQLSTATE: 22003", + "1".repeat(39) + ) + ); + } + + /// The message uses the value as given, so a double that Spark's + /// `toSQLValue` leaves unquoted, such as `NaN`, stays unquoted. + #[test] + fn test_values_are_formatted_by_the_caller() { + let nan = SparkError::InvalidInputInCastToDatetime { + value: "NaN".to_string(), + from_type: "DOUBLE".to_string(), + to_type: "TIMESTAMP".to_string(), + }; + assert!( + nan.to_string() + .starts_with("[CAST_INVALID_INPUT] The value NaN of the type \"DOUBLE\""), + "{nan}" + ); + } + + #[test] + fn test_find_through_wrappers() { + let external = DataFusionError::from(overflow()); + assert_eq!(SparkError::find(&external), Some(&overflow())); + assert!(external.to_string().contains(&overflow().to_string())); + + let context = external.context("while evaluating"); + assert_eq!(SparkError::find(&context), Some(&overflow())); + + let shared = DataFusionError::Shared(Arc::new(context)); + assert_eq!(SparkError::find(&shared), Some(&overflow())); + + // Arrow's `try_unary` kernels can only return an `ArrowError`. + let arrow = + DataFusionError::from(ArrowError::ExternalError(Box::new(overflow()))); + assert_eq!(SparkError::find(&arrow), Some(&overflow())); + + assert_eq!( + SparkError::find(&DataFusionError::Execution("x".to_string())), + None + ); + } +} diff --git a/crates/spark/src/function/conversion/cast/mod.rs b/crates/spark/src/function/conversion/cast/mod.rs new file mode 100644 index 00000000..f6a96682 --- /dev/null +++ b/crates/spark/src/function/conversion/cast/mod.rs @@ -0,0 +1,614 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Spark-compatible `CAST`. +//! +//! [`spark_cast`] is the kernel, for engines that build physical plans +//! themselves, such as Comet. The `spark_cast(expr, '')` SQL function is +//! built on it. +//! +//! The kernel is being ported from Comet's +//! `native/spark-expr/src/conversion_funcs`, one source type family at a time +//! (). A cast with no +//! Spark-compatible implementation yet returns a "not implemented" error, rather +//! than falling through to Arrow's cast and producing a result that differs +//! from Spark's. + +mod udf; +mod utils; + +use std::sync::Arc; + +use arrow::array::{Array, ArrayRef, AsArray}; +use arrow::compute::{CastOptions, cast_with_options}; +use arrow::datatypes::{DataType, Int64Type, TimeUnit, TimestampMicrosecondType}; +use arrow::temporal_conversions::MICROSECONDS; +use arrow::util::display::FormatOptions; +use datafusion_common::{ + Result, ScalarValue, exec_datafusion_err, internal_err, not_impl_err, +}; +use datafusion_expr::ColumnarValue; + +use crate::function::timezone_utils::{local_to_timestamp, timestamp_to_local}; +use crate::{EvalMode, SparkTimeZone}; +use utils::remove_trailing_zeroes; + +pub use udf::SparkCast; + +/// The format Spark uses for a timestamp cast to a string. `%.f` writes the +/// fraction of a second only when it is not zero, padded to 3, 6 or 9 digits, +/// and [`remove_trailing_zeroes`] then drops the padding. +const TIMESTAMP_FORMAT: Option<&str> = Some("%Y-%m-%d %H:%M:%S%.f"); + +/// Options for [`spark_cast`]. +/// +/// More options, such as the target Spark version, are added as the casts that +/// depend on them are ported. +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +#[non_exhaustive] +pub struct SparkCastOptions { + /// How a value that cannot be cast is handled. + pub eval_mode: EvalMode, + /// The session time zone (`spark.sql.session.timeZone`). A cast from + /// `TIMESTAMP` to `TIMESTAMP_NTZ`, `STRING` or `DATE`, or from + /// `TIMESTAMP_NTZ` to `TIMESTAMP`, returns an error without one. + pub timezone: Option, +} + +impl SparkCastOptions { + /// Options with no session time zone. + pub fn new(eval_mode: EvalMode) -> Self { + Self { + eval_mode, + timezone: None, + } + } + + pub fn with_timezone(self, timezone: SparkTimeZone) -> Self { + Self { + timezone: Some(timezone), + ..self + } + } + + fn session_timezone(&self) -> Result<&SparkTimeZone> { + self.timezone.as_ref().ok_or_else(|| { + exec_datafusion_err!("Casting a timestamp requires a session time zone") + }) + } +} + +/// Casts `arg` to `data_type` the way Spark's `Cast` expression does. +/// +/// An error is returned for a cast that Spark does not support, or that is not +/// implemented yet. Spark can never name a dictionary type, so a dictionary +/// `data_type` is an internal error. +pub fn spark_cast( + arg: ColumnarValue, + data_type: &DataType, + cast_options: &SparkCastOptions, +) -> Result { + match arg { + ColumnarValue::Array(array) => Ok(ColumnarValue::Array(cast_array( + array, + data_type, + cast_options, + )?)), + ColumnarValue::Scalar(scalar) => { + let array = cast_array(scalar.to_array()?, data_type, cast_options)?; + Ok(ColumnarValue::Scalar(ScalarValue::try_from_array( + &array, 0, + )?)) + } + } +} + +pub(crate) fn cast_array( + array: ArrayRef, + to_type: &DataType, + cast_options: &SparkCastOptions, +) -> Result { + use DataType::*; + use EvalMode::*; + let from_type = array.data_type(); + + // Spark's SQL type grammar cannot express a dictionary as a cast target: + // https://github.com/apache/spark/blob/v4.2.0/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4#L1477-L1525 + if matches!(to_type, Dictionary(_, _)) { + return internal_err!("Spark cannot specify dictionary types as cast targets"); + } + + if from_type == to_type { + return Ok(array); + } + + // Spark's TIMESTAMP and TIMESTAMP_NTZ are microseconds since the epoch, and + // the timestamp casts below assume that unit. + let is_other_timestamp_unit = + |t: &DataType| matches!(t, Timestamp(unit, _) if *unit != TimeUnit::Microsecond); + if is_other_timestamp_unit(from_type) || is_other_timestamp_unit(to_type) { + return not_supported(from_type, to_type); + } + + let eval_mode = cast_options.eval_mode; + let native_cast_options = CastOptions { + // Arrow returns NULL for a value it cannot cast when `safe` is set, and + // an error otherwise. + safe: eval_mode != Ansi, + format_options: FormatOptions::new().with_timestamp_format(TIMESTAMP_FORMAT), + }; + let arrow_cast = + |array: &dyn Array| Ok(cast_with_options(array, to_type, &native_cast_options)?); + + // Engines such as Comet can read a string or binary Parquet column as a + // dictionary, although Spark sees its value type. Cast the dictionary values + // and unpack the result. + // + // TODO: this casts values that no row refers to, so once a cast from string + // can fail, ANSI mode could raise an error that Spark would not. + if let Dictionary(_, value_type) = from_type + && matches!(value_type.as_ref(), Utf8 | LargeUtf8 | Binary | LargeBinary) + { + let dictionary = array.as_any_dictionary(); + let values = cast_array(Arc::clone(dictionary.values()), to_type, cast_options)?; + return arrow_cast(&dictionary.with_values(values)); + } + + match (from_type, to_type) { + // A NULL array has no values, so Arrow's cast only changes the type. + (Null, _) => arrow_cast(&array), + + // Timestamps. Arrow's casts ignore the session time zone. + (Timestamp(_, Some(_)), Int64) => { + // Spark returns whole seconds, rounded down. + let micros = array.as_primitive::(); + Ok(Arc::new( + micros.unary::<_, Int64Type>(|v| v.div_euclid(MICROSECONDS)), + )) + } + (Timestamp(_, Some(_)), Utf8 | Date32 | Timestamp(_, None)) => { + let local = timestamp_to_local( + array.as_primitive(), + cast_options.session_timezone()?, + )?; + match to_type { + Utf8 => Ok(remove_trailing_zeroes(&arrow_cast(&local)?)), + Date32 => arrow_cast(&local), + _ => Ok(Arc::new(local)), + } + } + (Timestamp(_, None), Utf8) => Ok(remove_trailing_zeroes(&arrow_cast(&array)?)), + (Timestamp(_, None), Timestamp(_, Some(target_tz))) => { + let utc = local_to_timestamp( + array.as_primitive(), + cast_options.session_timezone()?, + )?; + Ok(Arc::new(utc.with_timezone(Arc::clone(target_tz)))) + } + + // Casts that Arrow supports, but where Spark's result differs, and that + // have not been ported from Comet yet. Each is replaced by its + // implementation as its family is ported. + // + // LEGACY wraps on overflow and ANSI raises CAST_OVERFLOW, while Arrow + // returns NULL. TRY matches Arrow. + (Int64, Int32 | Int16 | Int8) | (Int32, Int16 | Int8) | (Int16, Int8) + if eval_mode != Try => + { + not_yet_ported(from_type, to_type, eval_mode) + } + (Float32 | Float64 | Decimal128(_, _), Int8 | Int16 | Int32 | Int64) + if eval_mode != Try => + { + not_yet_ported(from_type, to_type, eval_mode) + } + // ANSI raises NUMERIC_VALUE_OUT_OF_RANGE, with Spark's message. + (Decimal128(_, _), Decimal128(_, _)) if eval_mode == Ansi => { + not_yet_ported(from_type, to_type, eval_mode) + } + // LEGACY uses Java's BigDecimal.toString, as in "0E-18". + (Decimal128(_, _), Utf8) if eval_mode == Legacy => { + not_yet_ported(from_type, to_type, eval_mode) + } + // Spark replaces invalid UTF-8, and Arrow raises an error. + (Binary, Utf8) => not_yet_ported(from_type, to_type, eval_mode), + // LEGACY returns NULL and ANSI rejects the cast, while Arrow reinterprets + // the days since the epoch. + (Date32, Int32) => not_yet_ported(from_type, to_type, eval_mode), + + _ if is_datafusion_spark_compatible(from_type, to_type) => arrow_cast(&array), + _ => not_supported(from_type, to_type), + } +} + +fn not_supported(from_type: &DataType, to_type: &DataType) -> Result { + not_impl_err!("Spark-compatible cast from {from_type} to {to_type} is not supported") +} + +fn not_yet_ported( + from_type: &DataType, + to_type: &DataType, + eval_mode: EvalMode, +) -> Result { + not_impl_err!( + "Spark-compatible cast from {from_type} to {to_type} in {eval_mode:?} mode is \ + not implemented yet" + ) +} + +/// Whether Arrow's cast from `from_type` to `to_type` matches Spark's in every +/// eval mode. [`cast_array`] handles the casts it needs to before consulting +/// this. +fn is_datafusion_spark_compatible(from_type: &DataType, to_type: &DataType) -> bool { + use DataType::*; + match from_type { + Boolean => matches!( + to_type, + Int8 | Int16 | Int32 | Int64 | Float32 | Float64 | Utf8 + ), + Int8 | Int16 | Int32 | Int64 => matches!( + to_type, + Boolean | Int8 | Int16 | Int32 | Int64 | Float32 | Float64 | Utf8 + ), + Float32 | Float64 => matches!( + to_type, + Boolean | Int8 | Int16 | Int32 | Int64 | Float32 | Float64 + ), + // Float32 and Float64 are absent: Arrow computes + // `(unscaled as f64) / 10^scale`, which rounds twice and can differ from + // Spark's correctly rounded BigDecimal.doubleValue by one ulp. Boolean is + // absent because Arrow does not implement that cast. + Decimal128(_, _) | Decimal256(_, _) => matches!( + to_type, + Int8 | Int16 | Int32 | Int64 | Decimal128(_, _) | Decimal256(_, _) | Utf8 + ), + Utf8 => matches!(to_type, Binary), + Date32 => matches!(to_type, Utf8), + // The instant is unchanged, so Arrow only replaces the time zone. + Timestamp(_, Some(_)) => matches!(to_type, Timestamp(_, Some(_))), + Timestamp(_, None) => matches!(to_type, Date32), + _ => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::TimestampMillisecondArray; + use arrow::array::{ + BinaryArray, DictionaryArray, Int32Array, Int64Array, NullArray, StringArray, + TimestampMicrosecondArray, + }; + use arrow::datatypes::{Field, Int32Type, Int64Type}; + use datafusion_common::DataFusionError; + + fn options(eval_mode: EvalMode, timezone: &str) -> SparkCastOptions { + SparkCastOptions::new(eval_mode) + .with_timezone(SparkTimeZone::parse(timezone).unwrap()) + } + + fn legacy() -> SparkCastOptions { + options(EvalMode::Legacy, "UTC") + } + + fn timestamp(tz: Option<&str>) -> DataType { + DataType::Timestamp(TimeUnit::Microsecond, tz.map(Into::into)) + } + + #[test] + fn test_cast_to_dictionary_is_rejected() { + let error = cast_array( + Arc::new(StringArray::from(vec!["a"])), + &DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)), + &legacy(), + ) + .unwrap_err(); + assert!( + error + .to_string() + .contains("Spark cannot specify dictionary types as cast targets"), + "{error}" + ); + } + + #[test] + fn test_cast_to_same_type_returns_input() { + let array: ArrayRef = Arc::new(Int32Array::from(vec![Some(1), None])); + let result = cast_array(Arc::clone(&array), &DataType::Int32, &legacy()).unwrap(); + assert!(Arc::ptr_eq(&array, &result)); + } + + #[test] + fn test_unsupported_cast_is_an_error() { + let error = cast_array( + Arc::new(StringArray::from(vec!["1"])), + &DataType::Int32, + &legacy(), + ) + .unwrap_err(); + assert!( + matches!(error, DataFusionError::NotImplemented(_)), + "{error}" + ); + assert!( + error + .to_string() + .contains("cast from Utf8 to Int32 is not supported"), + "{error}" + ); + } + + /// Arrow reinterprets the days since the epoch, while Spark returns NULL + /// (LEGACY) or rejects the cast (ANSI), so DATE to INT is not ported yet. + #[test] + fn test_date_to_int_is_not_reinterpreted() { + let array: ArrayRef = Arc::new(arrow::array::Date32Array::from(vec![18262])); + for eval_mode in [EvalMode::Legacy, EvalMode::Ansi, EvalMode::Try] { + let error = cast_array( + Arc::clone(&array), + &DataType::Int32, + &SparkCastOptions::new(eval_mode), + ) + .unwrap_err(); + assert!(error.to_string().contains("not implemented yet"), "{error}"); + } + } + + /// A cast whose Spark-compatible implementation has not been ported must + /// not fall through to Arrow's cast, whose result differs from Spark's. + #[test] + fn test_pending_cast_does_not_fall_through_to_arrow() { + let array: ArrayRef = Arc::new(Int64Array::from(vec![i64::from(i32::MAX) + 1])); + for eval_mode in [EvalMode::Legacy, EvalMode::Ansi] { + let error = cast_array( + Arc::clone(&array), + &DataType::Int32, + &SparkCastOptions::new(eval_mode), + ) + .unwrap_err(); + assert!( + matches!(error, DataFusionError::NotImplemented(_)), + "{error}" + ); + assert!(error.to_string().contains("not implemented yet"), "{error}"); + } + + // TRY matches Arrow's safe cast, which returns NULL on overflow. + let result = cast_array( + array, + &DataType::Int32, + &SparkCastOptions::new(EvalMode::Try), + ) + .unwrap(); + assert!(result.is_null(0)); + } + + #[test] + fn test_arrow_fallthrough() { + // A widening cast cannot fail in any mode. + for eval_mode in [EvalMode::Legacy, EvalMode::Ansi, EvalMode::Try] { + let result = cast_array( + Arc::new(Int32Array::from(vec![Some(-1), None])), + &DataType::Int64, + &SparkCastOptions::new(eval_mode), + ) + .unwrap(); + assert_eq!( + result.as_primitive::(), + &Int64Array::from(vec![Some(-1), None]) + ); + } + + let result = cast_array( + Arc::new(Int32Array::from(vec![Some(0), Some(7), None])), + &DataType::Boolean, + &legacy(), + ) + .unwrap(); + let result = result.as_boolean(); + assert!(!result.value(0) && result.value(1) && result.is_null(2)); + } + + #[test] + fn test_scalar() { + let result = spark_cast( + ColumnarValue::Scalar(ScalarValue::Int32(Some(42))), + &DataType::Utf8, + &legacy(), + ) + .unwrap(); + let ColumnarValue::Scalar(scalar) = result else { + panic!("expected a scalar, got {result:?}"); + }; + assert_eq!(scalar, ScalarValue::Utf8(Some("42".to_string()))); + + let result = spark_cast( + ColumnarValue::Scalar(ScalarValue::Int32(None)), + &DataType::Int64, + &legacy(), + ) + .unwrap(); + let ColumnarValue::Scalar(scalar) = result else { + panic!("expected a scalar, got {result:?}"); + }; + assert_eq!(scalar, ScalarValue::Int64(None)); + } + + #[test] + fn test_cast_null_array() { + for to_type in [ + DataType::Int32, + DataType::Utf8, + DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))), + ] { + let result = + cast_array(Arc::new(NullArray::new(3)), &to_type, &legacy()).unwrap(); + assert_eq!(result.data_type(), &to_type); + assert_eq!(result.null_count(), 3); + } + } + + #[test] + fn test_cast_timestamp_to_long_rounds_down() { + let array: ArrayRef = Arc::new( + TimestampMicrosecondArray::from(vec![1_500_000, -1_500_000]) + .with_timezone("UTC"), + ); + let result = cast_array(array, &DataType::Int64, &legacy()).unwrap(); + assert_eq!( + result.as_primitive::(), + &Int64Array::from(vec![1, -2]) + ); + } + + #[test] + fn test_cast_timestamp_to_string_in_session_timezone() { + // 2024-01-15 15:30:00.1 UTC + let array: ArrayRef = Arc::new( + TimestampMicrosecondArray::from(vec![Some(1_705_332_600_100_000), None]) + .with_timezone("UTC"), + ); + let options = options(EvalMode::Legacy, "America/New_York"); + let result = cast_array(array, &DataType::Utf8, &options).unwrap(); + assert_eq!( + result.as_string::(), + &StringArray::from(vec![Some("2024-01-15 10:30:00.1"), None]) + ); + } + + #[test] + fn test_cast_timestamp_to_date_in_session_timezone() { + // 2024-01-15 03:00:00 UTC is still 2024-01-14 in Los Angeles. + let array: ArrayRef = Arc::new( + TimestampMicrosecondArray::from(vec![1_705_287_600_000_000]) + .with_timezone("UTC"), + ); + let result = + cast_array(array, &DataType::Date32, &options(EvalMode::Legacy, "PST")) + .unwrap(); + assert_eq!( + result + .as_primitive::() + .value(0), + 19_736 + ); + } + + #[test] + fn test_cast_between_timestamp_and_timestamp_ntz() { + // 2024-01-15 15:30:00 UTC is 10:30:00 in New York. + let utc_micros = 1_705_332_600_000_000; + let local_micros = 1_705_314_600_000_000; + let options = options(EvalMode::Legacy, "America/New_York"); + + let ltz: ArrayRef = Arc::new( + TimestampMicrosecondArray::from(vec![utc_micros]).with_timezone("UTC"), + ); + let ntz = cast_array(ltz, ×tamp(None), &options).unwrap(); + assert_eq!(ntz.data_type(), ×tamp(None)); + assert_eq!( + ntz.as_primitive::().value(0), + local_micros + ); + + let ltz = cast_array(ntz, ×tamp(Some("UTC")), &options).unwrap(); + assert_eq!(ltz.data_type(), ×tamp(Some("UTC"))); + assert_eq!( + ltz.as_primitive::().value(0), + utc_micros + ); + } + + #[test] + fn test_cast_timestamp_in_other_units_is_unsupported() { + let millis: ArrayRef = Arc::new(TimestampMillisecondArray::from(vec![1_500])); + let error = cast_array(millis, &DataType::Int64, &legacy()).unwrap_err(); + assert!( + matches!(error, DataFusionError::NotImplemented(_)), + "{error}" + ); + + let micros: ArrayRef = Arc::new(TimestampMicrosecondArray::from(vec![0])); + let nanos = DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())); + let error = cast_array(micros, &nanos, &legacy()).unwrap_err(); + assert!( + matches!(error, DataFusionError::NotImplemented(_)), + "{error}" + ); + } + + #[test] + fn test_cast_timestamp_without_timezone_is_an_error() { + let array: ArrayRef = + Arc::new(TimestampMicrosecondArray::from(vec![0]).with_timezone("UTC")); + let options = SparkCastOptions::new(EvalMode::Legacy); + let error = cast_array(array, &DataType::Utf8, &options).unwrap_err(); + assert!( + error.to_string().contains("requires a session time zone"), + "{error}" + ); + } + + #[test] + fn test_cast_timestamp_ntz() { + // 2024-01-15 10:30:00.5 local time + let ntz: ArrayRef = + Arc::new(TimestampMicrosecondArray::from(vec![1_705_314_600_500_000])); + // A local date-time renders without the session time zone. + let options = SparkCastOptions::new(EvalMode::Legacy); + let result = cast_array(Arc::clone(&ntz), &DataType::Utf8, &options).unwrap(); + assert_eq!( + result.as_string::(), + &StringArray::from(vec!["2024-01-15 10:30:00.5"]) + ); + let error = cast_array(ntz, &DataType::Int64, &legacy()).unwrap_err(); + assert!( + matches!(error, DataFusionError::NotImplemented(_)), + "{error}" + ); + } + + #[test] + fn test_cast_unsupported_timestamp_to_date() { + // chrono cannot represent every value of a microsecond timestamp. + let array: ArrayRef = Arc::new( + TimestampMicrosecondArray::from(vec![i64::MAX]).with_timezone("UTC"), + ); + let options = options(EvalMode::Legacy, "Europe/Copenhagen"); + assert!(cast_array(array, &DataType::Date32, &options).is_err()); + } + + #[test] + fn test_cast_dictionary_string_unpacks_result() { + let dictionary: ArrayRef = Arc::new( + DictionaryArray::::try_new( + Int32Array::from(vec![Some(1), None, Some(0), Some(1)]), + Arc::new(StringArray::from(vec!["a", "bc"])), + ) + .unwrap(), + ); + let result = cast_array(dictionary, &DataType::Binary, &legacy()).unwrap(); + assert_eq!( + result.as_binary::(), + &BinaryArray::from(vec![ + Some(b"bc".as_slice()), + None, + Some(b"a".as_slice()), + Some(b"bc".as_slice()), + ]) + ); + } + + #[test] + fn test_cast_dictionary_of_other_types_is_unsupported() { + let dictionary: ArrayRef = Arc::new( + DictionaryArray::::try_new( + Int32Array::from(vec![0]), + Arc::new(Int32Array::from(vec![1])), + ) + .unwrap(), + ); + assert!(cast_array(dictionary, &DataType::Int64, &legacy()).is_err()); + } +} diff --git a/crates/spark/src/function/conversion/cast.rs b/crates/spark/src/function/conversion/cast/udf.rs similarity index 100% rename from crates/spark/src/function/conversion/cast.rs rename to crates/spark/src/function/conversion/cast/udf.rs diff --git a/crates/spark/src/function/conversion/cast/utils.rs b/crates/spark/src/function/conversion/cast/utils.rs new file mode 100644 index 00000000..ca0c64ae --- /dev/null +++ b/crates/spark/src/function/conversion/cast/utils.rs @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use std::sync::Arc; + +use arrow::array::{ArrayRef, AsArray, GenericStringArray}; + +/// Removes trailing zeros from the fraction of a second in timestamps that +/// Arrow formatted, as Spark does: +/// +/// | Arrow | Spark | +/// |---------------------------|--------------------------| +/// | `1970-01-01 05:29:59.900` | `1970-01-01 05:29:59.9` | +/// | `1970-01-01 05:29:59.990` | `1970-01-01 05:29:59.99` | +/// | `1970-01-01 05:30:00` | `1970-01-01 05:30:00` | +/// | `1970-01-01 05:30:00.001` | `1970-01-01 05:30:00.001`| +pub(super) fn remove_trailing_zeroes(array: &ArrayRef) -> ArrayRef { + let result = array + .as_string::() + .iter() + .map(|s| s.map(trim_fraction_zeroes)) + .collect::>(); + Arc::new(result) +} + +fn trim_fraction_zeroes(s: &str) -> &str { + if s.contains('.') { + s.trim_end_matches('0') + } else { + s + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::StringArray; + + #[test] + fn test_drops_trailing_zeroes() { + let strings: ArrayRef = Arc::new(StringArray::from(vec![ + Some("1970-01-01 05:29:59.900"), + Some("1970-01-01 05:29:59.990"), + Some("1970-01-01 05:29:59.999"), + Some("1970-01-01 05:30:00"), + Some("1970-01-01 05:30:00.001"), + Some("1970-01-01 05:30:00.123400"), + None, + ])); + assert_eq!( + remove_trailing_zeroes(&strings).as_string::(), + &StringArray::from(vec![ + Some("1970-01-01 05:29:59.9"), + Some("1970-01-01 05:29:59.99"), + Some("1970-01-01 05:29:59.999"), + Some("1970-01-01 05:30:00"), + Some("1970-01-01 05:30:00.001"), + Some("1970-01-01 05:30:00.1234"), + None, + ]) + ); + } +} diff --git a/crates/spark/src/function/conversion/mod.rs b/crates/spark/src/function/conversion/mod.rs index bced69f5..128b8ab0 100644 --- a/crates/spark/src/function/conversion/mod.rs +++ b/crates/spark/src/function/conversion/mod.rs @@ -4,7 +4,7 @@ //! Spark type conversion functions, including Spark-compatible `cast` semantics. -mod cast; +pub mod cast; use datafusion_expr::ScalarUDF; use datafusion_functions::make_udf_function_with_config; diff --git a/crates/spark/src/function/datetime/from_utc_timestamp.rs b/crates/spark/src/function/datetime/from_utc_timestamp.rs index a518a299..f40dba59 100644 --- a/crates/spark/src/function/datetime/from_utc_timestamp.rs +++ b/crates/spark/src/function/datetime/from_utc_timestamp.rs @@ -6,7 +6,7 @@ use std::fmt::{Display, Formatter}; use std::hash::{Hash, Hasher}; use std::sync::Arc; -use super::timezone::SparkTimeZone; +use super::timezone::TimeZoneRules; use arrow::array::{ Array, ArrayRef, AsArray, BooleanArray, PrimitiveBuilder, StringArrayType, new_empty_array, @@ -221,8 +221,8 @@ where Ok(Arc::new(builder.finish())) } -fn parse_timezone(timezone: &str) -> Result { - SparkTimeZone::parse(timezone).map_err(|e| { +fn parse_timezone(timezone: &str) -> Result { + TimeZoneRules::parse(timezone).map_err(|e| { exec_datafusion_err!("`from_utc_timestamp`: invalid timezone '{timezone}': {e}") }) } diff --git a/crates/spark/src/function/datetime/mod.rs b/crates/spark/src/function/datetime/mod.rs index 4364d2a9..d5b34332 100644 --- a/crates/spark/src/function/datetime/mod.rs +++ b/crates/spark/src/function/datetime/mod.rs @@ -23,7 +23,7 @@ pub mod make_time; pub mod monthname; pub mod next_day; pub mod time_trunc; -mod timezone; +pub(crate) mod timezone; pub mod to_utc_timestamp; pub mod trunc; pub mod unix; diff --git a/crates/spark/src/function/datetime/timezone.rs b/crates/spark/src/function/datetime/timezone.rs index 1490b334..c1aaae1d 100644 --- a/crates/spark/src/function/datetime/timezone.rs +++ b/crates/spark/src/function/datetime/timezone.rs @@ -2,26 +2,85 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use std::fmt::{Display, Formatter}; +use std::hash::{Hash, Hasher}; +use std::sync::Arc; + use arrow::array::timezone::Tz; use arrow::datatypes::{ArrowTimestampType, TimeUnit}; -use chrono::{DateTime, Datelike, FixedOffset, NaiveDate, Utc}; +use chrono::{ + DateTime, Datelike, FixedOffset, LocalResult, NaiveDate, NaiveDateTime, Offset, + TimeZone, Utc, +}; use datafusion_common::{Result, exec_datafusion_err, internal_datafusion_err}; use datafusion_functions::datetime::to_local_time::adjust_to_local_time; -/// A timezone resolved using Spark's `getZoneId` spelling rules. +/// A Spark session time zone (`spark.sql.session.timeZone`). +/// +/// [`SparkTimeZone::parse`] accepts the IDs Spark's `getZoneId` accepts: IANA +/// region IDs, Java's short IDs such as `PST`, offsets such as `+05:30` and +/// `GMT-8`, and Java's `SystemV` zones. Two time zones are equal when their IDs +/// are, so `PST` and `America/Los_Angeles` are different time zones with the +/// same rules. +#[derive(Debug, Clone)] +pub struct SparkTimeZone { + id: Arc, + rules: TimeZoneRules, +} + +impl SparkTimeZone { + /// Parses a Spark time zone ID. + pub fn parse(id: &str) -> Result { + Ok(Self { + id: id.into(), + rules: TimeZoneRules::parse(id)?, + }) + } + + /// The ID this time zone was parsed from. + pub fn id(&self) -> &str { + &self.id + } + + pub(crate) fn rules(&self) -> TimeZoneRules { + self.rules + } +} + +impl PartialEq for SparkTimeZone { + fn eq(&self, other: &Self) -> bool { + self.id == other.id + } +} + +impl Eq for SparkTimeZone {} + +impl Hash for SparkTimeZone { + fn hash(&self, state: &mut H) { + self.id.hash(state); + } +} + +impl Display for SparkTimeZone { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.id) + } +} + +/// The offset rules of a [`SparkTimeZone`]. /// /// Fixed offsets are kept as seconds to preserve Spark's second-resolution offsets. /// Region IDs use Arrow's Chrono-backed timezone database. /// Java's legacy SystemV zones require their own rules because that database omits them. #[derive(Debug, Clone, Copy)] -pub(super) enum SparkTimeZone { +pub(crate) enum TimeZoneRules { FixedOffset(i32), Region(Tz), SystemV(i32), } -impl SparkTimeZone { - pub(super) fn parse(timezone: &str) -> Result { +impl TimeZoneRules { + pub(crate) fn parse(timezone: &str) -> Result { // These are the exact aliases in java.time.ZoneId.SHORT_IDS. The regional // aliases retain their daylight saving rules; EST, HST, and MST are fixed. let timezone = match timezone { @@ -93,7 +152,17 @@ impl SparkTimeZone { Ok(Self::Region(timezone.parse()?)) } - pub(super) fn adjust_to_local_time( + /// The offset from UTC in seconds when it is the same at every instant. + pub(crate) fn fixed_offset_seconds(self) -> Option { + match self { + Self::FixedOffset(seconds) => Some(seconds), + Self::Region(_) | Self::SystemV(_) => None, + } + } + + /// Converts `ts`, in UTC, to the local date-time in this time zone, encoded + /// as if it were UTC. + pub(crate) fn adjust_to_local_time( self, ts: i64, ) -> Result { @@ -105,6 +174,70 @@ impl SparkTimeZone { } } } + + /// Converts `local`, a local date-time in this time zone encoded as if it + /// were UTC, to UTC, the way Java's `LocalDateTime.atZone` does. The + /// inverse of [`Self::adjust_to_local_time`]. + /// + /// A local time that happens twice, when clocks go back, resolves to the + /// earlier offset. A local time in the gap when clocks go forward moves + /// forward by the length of the gap. + pub(crate) fn local_to_utc(self, local: i64) -> Result { + let units = units_per_second::(); + let shift = |offset_seconds: i32| { + local + .checked_sub(i64::from(offset_seconds) * units) + .ok_or_else(|| exec_datafusion_err!("Timestamp {local} overflows")) + }; + match self { + Self::FixedOffset(seconds) => shift(seconds), + Self::Region(tz) => { + let local_datetime = + DateTime::::from_timestamp(local.div_euclid(units), 0) + .ok_or_else(|| { + exec_datafusion_err!( + "Timestamp is outside Chrono's calendar range" + ) + })? + .naive_utc(); + let offset = region_offset_at_local(&tz, local_datetime); + shift(offset) + } + Self::SystemV(standard_offset) => { + // Use daylight time if the result is in daylight time. That picks + // the earlier offset in the overlap, and standard time, the + // offset before the transition, in the gap. + let daylight_offset = standard_offset + 3600; + let utc = shift(daylight_offset)?; + if system_v_offset::(utc, standard_offset)? == daylight_offset { + Ok(utc) + } else { + shift(standard_offset) + } + } + } + } +} + +/// The offset of `tz`, in seconds, at the local date-time `local`, resolved as +/// Java's `LocalDateTime.atZone` does. The fraction of a second does not affect +/// the offset, so `local` is in whole seconds. +fn region_offset_at_local(tz: &Tz, local: NaiveDateTime) -> i32 { + let offset_of = |result: LocalResult>| match result { + LocalResult::Single(dt) | LocalResult::Ambiguous(dt, _) => { + Some(dt.offset().fix().local_minus_utc()) + } + LocalResult::None => None, + }; + offset_of(tz.from_local_datetime(&local)).unwrap_or_else(|| { + // In a gap, apply the offset in effect before it, which moves the time + // forward by the length of the gap. No zone is more than 14 hours ahead + // of UTC after a transition, so `local - 14h`, read as UTC, is an instant + // before the transition, even in a gap that skips a whole day. + tz.offset_from_utc_datetime(&(local - chrono::Duration::hours(14))) + .fix() + .local_minus_utc() + }) } /// Parses Java's signed offsets after Spark's single-digit hour/minute normalization. @@ -236,12 +369,35 @@ mod tests { use arrow::datatypes::TimestampMicrosecondType; fn shift(timezone: &str, timestamp: i64) -> i64 { - SparkTimeZone::parse(timezone) + TimeZoneRules::parse(timezone) .unwrap() .adjust_to_local_time::(timestamp) .unwrap() } + /// Local times in a gap move forward by the length of the gap, as Java's + /// `LocalDateTime.atZone` does. Expected values from Spark 4.2.0's + /// `CAST(CAST(TIMESTAMP_NTZ'...' AS TIMESTAMP) AS BIGINT)`. + #[test] + fn local_times_in_a_gap_use_the_offset_before_it() { + let cases = [ + // An hour skipped for daylight saving time. + ("America/New_York", "2024-03-10T02:30:00", 1710055800), + // Whole days skipped when the zone crossed the date line. + ("Pacific/Apia", "2011-12-30T12:00:00", 1325282400), + ("Pacific/Kwajalein", "1993-08-21T12:00:00", 745977600), + ]; + for (timezone, local, expected) in cases { + let local: NaiveDateTime = local.parse().unwrap(); + let micros = local.and_utc().timestamp() * 1_000_000; + let utc = TimeZoneRules::parse(timezone) + .unwrap() + .local_to_utc::(micros) + .unwrap(); + assert_eq!(utc, expected * 1_000_000, "{timezone} {local}"); + } + } + #[test] fn spark_offset_spellings_preserve_sign_and_seconds() { let offsets = [ @@ -307,7 +463,7 @@ mod tests { "--01", ] { let timezone = format!("{prefix}{offset}"); - assert!(SparkTimeZone::parse(&timezone).is_err(), "{timezone}"); + assert!(TimeZoneRules::parse(&timezone).is_err(), "{timezone}"); } } for timezone in [ @@ -321,7 +477,7 @@ mod tests { "SystemV/PST9PDT", " UTC", ] { - assert!(SparkTimeZone::parse(timezone).is_err(), "{timezone}"); + assert!(TimeZoneRules::parse(timezone).is_err(), "{timezone}"); } } diff --git a/crates/spark/src/function/mod.rs b/crates/spark/src/function/mod.rs index 483f9b64..f4fd2020 100644 --- a/crates/spark/src/function/mod.rs +++ b/crates/spark/src/function/mod.rs @@ -25,6 +25,7 @@ pub mod predicate; pub mod string; pub mod r#struct; pub mod table; +pub(crate) mod timezone_utils; pub mod url; pub mod utils; pub mod window; diff --git a/crates/spark/src/function/timezone_utils.rs b/crates/spark/src/function/timezone_utils.rs new file mode 100644 index 00000000..1361af38 --- /dev/null +++ b/crates/spark/src/function/timezone_utils.rs @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Conversions between Spark's `TIMESTAMP` and `TIMESTAMP_NTZ` arrays. +//! +//! Spark stores a `TIMESTAMP` as microseconds since the epoch in UTC, and a +//! `TIMESTAMP_NTZ` as a local date-time, in microseconds since the epoch as if +//! it were UTC. Converting between them needs the session time zone. + +use arrow::array::{PrimitiveArray, TimestampMicrosecondArray}; +use arrow::datatypes::TimestampMicrosecondType; +use arrow::temporal_conversions::MICROSECONDS; +use datafusion_common::{Result, exec_datafusion_err}; + +use crate::SparkTimeZone; + +/// Converts UTC timestamps to local date-times in `tz`, like Spark's +/// `convertTz(ts, ZoneOffset.UTC, zoneId)`. The result has no time zone. +pub(crate) fn timestamp_to_local( + array: &TimestampMicrosecondArray, + tz: &SparkTimeZone, +) -> Result { + let rules = tz.rules(); + match rules.fixed_offset_seconds() { + Some(seconds) => add_micros(array, i64::from(seconds) * MICROSECONDS), + None => array + .try_unary(|ts| rules.adjust_to_local_time::(ts)), + } +} + +/// Converts local date-times in `tz` to UTC timestamps, the way Java's +/// `LocalDateTime.atZone` resolves daylight saving gaps and overlaps. The +/// result has no time zone. +pub(crate) fn local_to_timestamp( + array: &TimestampMicrosecondArray, + tz: &SparkTimeZone, +) -> Result { + let rules = tz.rules(); + match rules.fixed_offset_seconds() { + Some(seconds) => add_micros(array, -i64::from(seconds) * MICROSECONDS), + None => array.try_unary(|ts| rules.local_to_utc::(ts)), + } +} + +/// Adds `delta` to every value, dropping the array's time zone. +fn add_micros( + array: &TimestampMicrosecondArray, + delta: i64, +) -> Result { + if delta == 0 { + let (_, values, nulls) = array.clone().into_parts(); + return Ok(PrimitiveArray::new(values, nulls)); + } + array.try_unary(|ts| { + ts.checked_add(delta) + .ok_or_else(|| exec_datafusion_err!("Timestamp {ts} overflows")) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::Array; + use chrono::NaiveDateTime; + + fn micros(datetime: &str) -> i64 { + NaiveDateTime::parse_from_str(datetime, "%Y-%m-%d %H:%M:%S%.f") + .unwrap() + .and_utc() + .timestamp_micros() + } + + fn array(datetime: &str) -> TimestampMicrosecondArray { + TimestampMicrosecondArray::from(vec![Some(micros(datetime)), None]) + } + + fn tz(id: &str) -> SparkTimeZone { + SparkTimeZone::parse(id).unwrap() + } + + #[test] + fn test_local_to_timestamp_non_existent_time() { + // 01:30 does not exist in London on 2024-03-31. Spark moves it forward an + // hour, to 02:30 BST, which is 01:30 UTC. + let result = + local_to_timestamp(&array("2024-03-31 01:30:00"), &tz("Europe/London")) + .unwrap(); + assert_eq!(result.value(0), micros("2024-03-31 01:30:00")); + assert!(result.is_null(1)); + } + + #[test] + fn test_local_to_timestamp_ambiguous_time() { + // 01:30 happens twice in London on 2024-10-27. Spark picks the earlier + // offset, BST, which is 00:30 UTC. + let result = + local_to_timestamp(&array("2024-10-27 01:30:00"), &tz("Europe/London")) + .unwrap(); + assert_eq!(result.value(0), micros("2024-10-27 00:30:00")); + } + + #[test] + fn test_local_to_timestamp_keeps_fraction() { + let result = local_to_timestamp( + &array("2024-01-15 10:30:00.123456"), + &tz("America/New_York"), + ) + .unwrap(); + assert_eq!(result.value(0), micros("2024-01-15 15:30:00.123456")); + } + + #[test] + fn test_system_v_daylight_saving() { + // SystemV/PST8PDT is UTC-7 in summer and UTC-8 in winter. + let pst8pdt = tz("SystemV/PST8PDT"); + let summer = local_to_timestamp(&array("2024-07-04 12:00:00"), &pst8pdt).unwrap(); + assert_eq!(summer.value(0), micros("2024-07-04 19:00:00")); + let winter = local_to_timestamp(&array("2024-01-15 12:00:00"), &pst8pdt).unwrap(); + assert_eq!(winter.value(0), micros("2024-01-15 20:00:00")); + // 01:30 happens twice on 2024-10-27, and resolves to daylight time. + let overlap = + local_to_timestamp(&array("2024-10-27 01:30:00"), &pst8pdt).unwrap(); + assert_eq!(overlap.value(0), micros("2024-10-27 08:30:00")); + // 02:30 does not exist on 2024-04-28, and moves forward to 03:30 PDT. + let gap = local_to_timestamp(&array("2024-04-28 02:30:00"), &pst8pdt).unwrap(); + assert_eq!(gap.value(0), micros("2024-04-28 10:30:00")); + } + + #[test] + fn test_timestamp_to_local() { + // New York is UTC-5 in January and UTC-4 in July. + let new_york = tz("America/New_York"); + let winter = + timestamp_to_local(&array("2024-01-15 15:30:00"), &new_york).unwrap(); + assert_eq!(winter.value(0), micros("2024-01-15 10:30:00")); + assert!(winter.is_null(1)); + let summer = + timestamp_to_local(&array("2024-07-04 16:30:00"), &new_york).unwrap(); + assert_eq!(summer.value(0), micros("2024-07-04 12:30:00")); + // Java's short IDs are accepted, as in Spark. + let pst = timestamp_to_local(&array("2024-01-15 20:00:00"), &tz("PST")).unwrap(); + assert_eq!(pst.value(0), micros("2024-01-15 12:00:00")); + } + + #[test] + fn test_fixed_offsets() { + let input = array("2024-01-15 15:30:00"); + let utc = timestamp_to_local(&input, &tz("UTC")).unwrap(); + assert_eq!(utc.values(), input.values()); + assert_eq!(utc.nulls(), input.nulls()); + + let plus_5_30 = tz("+05:30"); + let local = timestamp_to_local(&input, &plus_5_30).unwrap(); + assert_eq!(local.value(0), micros("2024-01-15 21:00:00")); + let back = local_to_timestamp(&local, &plus_5_30).unwrap(); + assert_eq!(back.value(0), input.value(0)); + } + + #[test] + fn test_out_of_range_timestamp_is_an_error() { + let input = TimestampMicrosecondArray::from(vec![i64::MAX]); + assert!(timestamp_to_local(&input, &tz("Europe/Copenhagen")).is_err()); + assert!(timestamp_to_local(&input, &tz("+01:00")).is_err()); + } +} diff --git a/crates/spark/src/lib.rs b/crates/spark/src/lib.rs index 4930a9e5..462274ae 100644 --- a/crates/spark/src/lib.rs +++ b/crates/spark/src/lib.rs @@ -170,13 +170,15 @@ /// Spark configuration extensions and map-key deduplication policy. pub mod config; +mod error; +mod eval_mode; pub mod function; pub mod planner; pub use config::{MapKeyDedupPolicy, SparkOptions}; -mod eval_mode; - +pub use error::SparkError; pub use eval_mode::EvalMode; +pub use function::datetime::timezone::SparkTimeZone; #[cfg(feature = "core")] mod session_state;