diff --git a/crates/spark/src/function/datetime/dayofweek.rs b/crates/spark/src/function/datetime/dayofweek.rs new file mode 100644 index 0000000..7863c32 --- /dev/null +++ b/crates/spark/src/function/datetime/dayofweek.rs @@ -0,0 +1,382 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use std::sync::Arc; + +use arrow::array::{Array, ArrayRef, AsArray, Int32Array}; +use arrow::compute::cast; +use arrow::datatypes::{DataType, Date32Type, Date64Type, Field, FieldRef}; +use datafusion_common::utils::take_function_args; +use datafusion_common::{Result, ScalarValue, exec_err, internal_err}; +use datafusion_expr::{ + ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, + Signature, +}; +use datafusion_macros::user_doc; + +use super::date_part_name::date_signature; + +const MILLISECONDS_PER_DAY: i64 = 86_400_000; + +/// Spark `DayOfWeek`: Sunday = 1, Monday = 2, ..., Saturday = 7. +/// +/// Spark computes `LocalDate.ofEpochDay(days).getDayOfWeek.plus(1).getValue`, +/// which is defined for every `Int`. Epoch day 0 (1970-01-01) is a Thursday, so +/// `+ 4` moves Sunday to 0 before the shift to 1-based. The day is an `i64` so +/// that a `Date64` outside the `Date32` range still has a day of the week. +#[inline] +pub(super) fn day_of_week(days: i64) -> i32 { + ((days.rem_euclid(7) + 4) % 7 + 1) as i32 +} + +/// Spark `WeekDay`: Monday = 0, Tuesday = 1, ..., Sunday = 6. +/// +/// Spark computes `LocalDate.ofEpochDay(days).getDayOfWeek.ordinal()`. +#[inline] +pub(super) fn week_day(days: i64) -> i32 { + ((days.rem_euclid(7) + 3) % 7) as i32 +} + +/// Applies `kernel` to the epoch day of every date in a `Date32` or `Date64` +/// array, or returns `None` for any other type. +/// +/// `unary` also evaluates the slots under NULLs, which is safe because the +/// kernels are total, and lets the loop vectorize. +fn map_dates i32 + Copy>( + array: &dyn Array, + kernel: F, +) -> Option { + match array.data_type() { + DataType::Date32 => Some( + array + .as_primitive::() + .unary(|days| kernel(i64::from(days))), + ), + // Date64 holds milliseconds since the epoch. Arrow's cast to Date32 + // would turn days outside the i32 range into NULL, and it divides + // toward zero, which moves a time before the epoch to the next day. + DataType::Date64 => Some( + array + .as_primitive::() + .unary(|ms| kernel(ms.div_euclid(MILLISECONDS_PER_DAY))), + ), + _ => None, + } +} + +/// Maps each epoch day in `arg` (a `Date32` or `Date64` scalar or array, +/// possibly dictionary encoded) to an `Int32` with `kernel`. NULL dates give +/// NULL. +/// +/// The value is computed straight from the epoch day rather than through a +/// calendar date, so it is defined for the whole `i32` domain as in Spark. +pub(super) fn epoch_day_field i32 + Copy>( + name: &str, + arg: ColumnarValue, + kernel: F, +) -> Result { + match arg { + ColumnarValue::Scalar(ScalarValue::Date32(days)) => Ok(ColumnarValue::Scalar( + ScalarValue::Int32(days.map(|days| kernel(i64::from(days)))), + )), + ColumnarValue::Scalar(scalar) if scalar.is_null() => { + Ok(ColumnarValue::Scalar(ScalarValue::Int32(None))) + } + ColumnarValue::Scalar(scalar) => { + let array = scalar.to_array_of_size(1)?; + let result = epoch_day_array(name, &array, kernel)?; + Ok(ColumnarValue::Scalar(ScalarValue::try_from_array( + &result, 0, + )?)) + } + ColumnarValue::Array(array) => { + Ok(ColumnarValue::Array(epoch_day_array(name, &array, kernel)?)) + } + } +} + +fn epoch_day_array i32 + Copy>( + name: &str, + array: &ArrayRef, + kernel: F, +) -> Result { + // A dictionary-encoded date column: map the dictionary values, rewrap the + // keys, then unpack to a plain Int32 array, so the work is proportional to + // the cardinality rather than the row count. + if let DataType::Dictionary(_, _) = array.data_type() { + let dict = array.as_any_dictionary(); + let Some(mapped) = map_dates(dict.values().as_ref(), kernel) else { + return exec_err!("`{name}` expects a date, got {}", array.data_type()); + }; + return Ok(cast(&dict.with_values(Arc::new(mapped)), &DataType::Int32)?); + } + let Some(mapped) = map_dates(array.as_ref(), kernel) else { + return exec_err!("`{name}` expects a date, got {}", array.data_type()); + }; + Ok(Arc::new(mapped)) +} + +/// Spark-compatible `dayofweek` expression. +/// +/// +#[user_doc( + doc_section(label = "Time and Date Functions"), + description = "Returns the day of the week as an INT from 1 (Sunday) to 7 (Saturday). \ + This differs from `weekday`, which counts from 0 (Monday) to 6 (Sunday), and from \ + DataFusion's `date_part('dow', ...)`, which counts from 0 (Sunday) to 6 (Saturday). \ + DataFusion has no built-in `dayofweek`. \ + Strings are implicitly cast to DATE. A malformed string is an error, even with ANSI mode off \ + where Spark returns NULL. A string that Spark reads as a partial date, such as '2009' or \ + '2009-07', is also an error, where Spark uses the first day of that year or month. \ + Timestamps are cast to DATE first, so the time of day is ignored. A timestamp with a time zone \ + is converted in its own time zone, not the session time zone that Spark uses. \ + Returns NULL if the input is NULL.", + syntax_example = "dayofweek(date)", + sql_example = r#"```sql +> SELECT dayofweek('2009-07-30'::DATE); +5 +> SELECT dayofweek('2009-07-30'); +5 +> SELECT dayofweek(arrow_cast('2024-01-07 23:59:59', 'Timestamp(Microsecond, None)')); +1 +> SELECT dayofweek(NULL::DATE); +NULL +```"#, + argument( + name = "date", + description = "DATE, TIMESTAMP or string expression. NULL returns NULL." + ) +)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkDayOfWeek { + signature: Signature, +} + +impl Default for SparkDayOfWeek { + fn default() -> Self { + Self::new() + } +} + +impl SparkDayOfWeek { + pub fn new() -> Self { + Self { + signature: date_signature(), + } + } +} + +impl ScalarUDFImpl for SparkDayOfWeek { + fn name(&self) -> &str { + "dayofweek" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + internal_err!("return_field_from_args should be used instead") + } + + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); + Ok(Arc::new(Field::new(self.name(), DataType::Int32, nullable))) + } + + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + let [arg] = take_function_args(self.name(), args.args)?; + epoch_day_field(self.name(), arg, day_of_week) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{Date32Array, Date64Array, DictionaryArray}; + use arrow::datatypes::Int32Type; + use datafusion_common::config::ConfigOptions; + + fn invoke(arg: ColumnarValue, rows: usize) -> Result { + let data_type = arg.data_type(); + SparkDayOfWeek::new().invoke_with_args(ScalarFunctionArgs { + args: vec![arg], + arg_fields: vec![Arc::new(Field::new("d", data_type, true))], + number_rows: rows, + return_field: Arc::new(Field::new("dayofweek", DataType::Int32, true)), + config_options: Arc::new(ConfigOptions::default()), + }) + } + + fn invoke_array(array: ArrayRef) -> Int32Array { + let rows = array.len(); + let result = invoke(ColumnarValue::Array(array), rows) + .unwrap() + .to_array(rows) + .unwrap(); + result.as_primitive::().clone() + } + + #[test] + fn return_field_nullability_matches_input() { + let func = SparkDayOfWeek::new(); + for nullable in [false, true] { + let arg = Arc::new(Field::new("arg", DataType::Date32, nullable)); + let out = func + .return_field_from_args(ReturnFieldArgs { + arg_fields: &[arg], + scalar_arguments: &[None], + }) + .unwrap(); + assert_eq!(out.data_type(), &DataType::Int32); + assert_eq!(out.is_nullable(), nullable); + } + } + + /// 1970-01-01 is a Thursday, so the week either side of the epoch pins + /// both numberings. + #[test] + fn known_weekdays_around_the_epoch() { + // days -4..=3 spans Sunday 1969-12-28 through Sunday 1970-01-04. + let array: ArrayRef = Arc::new(Date32Array::from((-4..=3).collect::>())); + assert_eq!( + invoke_array(array).values(), + // Sun Mon Tue Wed Thu Fri Sat Sun + &[1, 2, 3, 4, 5, 6, 7, 1] + ); + let weekdays: Vec = (-4..=3).map(week_day).collect(); + assert_eq!(weekdays, [6, 0, 1, 2, 3, 4, 5, 6]); + } + + /// Nulls pass through in place, whether they are sparse or dense. + #[test] + fn nulls_are_preserved() { + let sparse: ArrayRef = + Arc::new(Date32Array::from(vec![Some(0), None, Some(1), Some(2)])); + let out = invoke_array(sparse); + assert_eq!( + out.iter().collect::>(), + [Some(5), None, Some(6), Some(7)] + ); + + let dense: ArrayRef = + Arc::new(Date32Array::from(vec![None, None, Some(1), None])); + let out = invoke_array(dense); + assert_eq!(out.iter().collect::>(), [None, None, Some(6), None]); + } + + /// The whole `i32` domain works, as `LocalDate.ofEpochDay` does in Spark. + /// These are outside chrono's representable range. + #[test] + fn full_i32_domain() { + let array: ArrayRef = Arc::new(Date32Array::from(vec![i32::MIN, i32::MAX])); + // Verified against Spark 4.2.0: + // dayofweek(date_from_unix_date(-2147483648)) = 3 (-5877641-06-23) + // dayofweek(date_from_unix_date(2147483647)) = 6 (+5881580-07-11) + // weekday of the same two dates = 1 and 4 + assert_eq!(invoke_array(array).values(), &[3, 6]); + assert_eq!( + [week_day(i32::MIN.into()), week_day(i32::MAX.into())], + [1, 4] + ); + } + + /// The two numberings agree across a full 400-year Gregorian cycle. + #[test] + fn numberings_agree_over_a_full_cycle() { + for days in -146_097..=146_097 { + assert_eq!((day_of_week(days) + 5) % 7, week_day(days), "day {days}"); + } + } + + /// A dictionary-encoded date column unpacks to a plain Int32 result. + #[test] + fn dictionary_input_unpacks() { + let values = + Arc::new(Date32Array::from(vec![Some(0), Some(1), None])) as ArrayRef; + let keys = Int32Array::from(vec![Some(0), Some(1), Some(0), None, Some(2)]); + let dict = DictionaryArray::::new(keys, values); + let out = invoke_array(Arc::new(dict)); + assert_eq!(out.data_type(), &DataType::Int32); + assert_eq!( + out.iter().collect::>(), + [Some(5), Some(6), Some(5), None, None] + ); + } + + /// A dictionary of Date64 values is cast to epoch days before mapping. + /// SQL never reaches this path, because coercion unpacks the dictionary + /// to its value type first, so only a direct call covers it. + #[test] + fn date64_dictionary_input() { + let values = Arc::new(Date64Array::from(vec![Some(0), Some(-86_400_000), None])) + as ArrayRef; + let keys = Int32Array::from(vec![Some(0), Some(1), None, Some(2)]); + let dict = DictionaryArray::::new(keys, values); + let out = invoke_array(Arc::new(dict)); + assert_eq!(out.data_type(), &DataType::Int32); + assert_eq!( + out.iter().collect::>(), + [Some(5), Some(4), None, None] + ); + } + + /// A Date64 is floored to whole days, so the last millisecond before the + /// epoch is on 1969-12-31 (a Wednesday), and days outside the Date32 range + /// still give a weekday rather than NULL. + #[test] + fn date64_outside_the_date32_range() { + let ms = |days: i64| days * MILLISECONDS_PER_DAY; + let first_after = i64::from(i32::MAX) + 1; + let last_before = i64::from(i32::MIN) - 1; + let values = vec![-1, ms(first_after), ms(last_before), i64::MAX, i64::MIN]; + let expected: Vec = [-1, first_after, last_before] + .into_iter() + .chain([i64::MAX, i64::MIN].map(|ms| ms.div_euclid(MILLISECONDS_PER_DAY))) + .map(day_of_week) + .collect(); + // One day after i32::MAX (a Friday) and one day before i32::MIN (a + // Tuesday). + assert_eq!(expected[..3], [4, 7, 2]); + + let plain = invoke_array(Arc::new(Date64Array::from(values.clone()))); + assert_eq!(plain.null_count(), 0); + assert_eq!(plain.values(), expected.as_slice()); + + let keys = Int32Array::from((0..values.len() as i32).collect::>()); + let dict = + DictionaryArray::::new(keys, Arc::new(Date64Array::from(values))); + let out = invoke_array(Arc::new(dict)); + assert_eq!(out.null_count(), 0); + assert_eq!(out.values(), expected.as_slice()); + } + + #[test] + fn rejects_non_date_input() { + let array: ArrayRef = Arc::new(Int32Array::from(vec![1])); + let err = invoke(ColumnarValue::Array(array), 1).unwrap_err(); + assert!(err.to_string().contains("expects a date"), "{err}"); + } + + #[test] + fn scalar_input() -> Result<()> { + match invoke(ColumnarValue::Scalar(ScalarValue::Date32(Some(0))), 1)? { + ColumnarValue::Scalar(ScalarValue::Int32(Some(5))) => {} + other => panic!("unexpected result: {other:?}"), + } + match invoke(ColumnarValue::Scalar(ScalarValue::Date32(None)), 1)? { + ColumnarValue::Scalar(ScalarValue::Int32(None)) => {} + other => panic!("unexpected result: {other:?}"), + } + match invoke(ColumnarValue::Scalar(ScalarValue::Null), 1)? { + ColumnarValue::Scalar(ScalarValue::Int32(None)) => {} + other => panic!("unexpected result: {other:?}"), + } + Ok(()) + } +} diff --git a/crates/spark/src/function/datetime/mod.rs b/crates/spark/src/function/datetime/mod.rs index d5b3433..6e280c9 100644 --- a/crates/spark/src/function/datetime/mod.rs +++ b/crates/spark/src/function/datetime/mod.rs @@ -14,6 +14,7 @@ mod date_part_name; pub mod date_sub; pub mod date_trunc; pub mod dayname; +pub mod dayofweek; pub mod extract; pub mod from_utc_timestamp; pub mod last_day; @@ -44,6 +45,7 @@ make_udf_function!(date_part::SparkDatePart, date_part); make_udf_function!(date_sub::SparkDateSub, date_sub); make_udf_function!(date_trunc::SparkDateTrunc, date_trunc); make_udf_function!(dayname::SparkDayName, dayname); +make_udf_function!(dayofweek::SparkDayOfWeek, dayofweek); make_udf_function!( from_utc_timestamp::SparkFromUtcTimestamp, from_utc_timestamp @@ -122,6 +124,11 @@ pub mod expr_fn { "Returns the three-letter abbreviated day name from a date.", arg1 )); + export_functions!(( + dayofweek, + "Returns the day of the week for date/timestamp as an integer where Sunday = 1, Monday = 2, ..., Saturday = 7.", + arg1 + )); export_functions!(( make_interval, "Make interval from years, months, weeks, days, hours, mins and secs.", @@ -221,6 +228,7 @@ pub fn functions() -> Vec> { date_sub(), date_trunc(), dayname(), + dayofweek(), from_utc_timestamp(), hour(), last_day(), diff --git a/crates/spark/src/function/datetime/weekday.rs b/crates/spark/src/function/datetime/weekday.rs index fe2aa68..4cdbfd3 100644 --- a/crates/spark/src/function/datetime/weekday.rs +++ b/crates/spark/src/function/datetime/weekday.rs @@ -4,18 +4,18 @@ use std::sync::Arc; -use arrow::array::AsArray; -use arrow::compute::{DatePart, date_part}; -use arrow::datatypes::{DataType, Field, FieldRef, Int32Type}; +use arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::types::{NativeType, logical_date}; use datafusion_common::utils::take_function_args; -use datafusion_common::{Result, ScalarValue, internal_err}; +use datafusion_common::{Result, internal_err}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, }; use datafusion_macros::user_doc; +use super::dayofweek::{epoch_day_field, week_day}; + /// Spark-compatible `weekday` expression. /// /// @@ -89,30 +89,16 @@ impl ScalarUDFImpl for SparkWeekDay { fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { let [arg] = take_function_args(self.name(), args.args)?; - match arg { - ColumnarValue::Scalar(scalar) => { - if scalar.is_null() { - return Ok(ColumnarValue::Scalar(ScalarValue::Int32(None))); - } - let arr = scalar.to_array_of_size(1)?; - // `DayOfWeekMonday0` returns 0..=6 with Monday = 0, which - // matches Spark `weekday` semantics exactly. - let weekday_arr = date_part(&arr, DatePart::DayOfWeekMonday0)?; - let value = weekday_arr.as_primitive::().value(0); - Ok(ColumnarValue::Scalar(ScalarValue::Int32(Some(value)))) - } - ColumnarValue::Array(arr) => { - let weekday_arr = date_part(&arr, DatePart::DayOfWeekMonday0)?; - Ok(ColumnarValue::Array(weekday_arr)) - } - } + epoch_day_field(self.name(), arg, week_day) } } #[cfg(test)] mod tests { use super::*; - use arrow::array::{Date32Array, Int32Array}; + use arrow::array::{AsArray, Date32Array, Int32Array}; + use arrow::datatypes::Int32Type; + use datafusion_common::ScalarValue; #[test] fn test_weekday_return_field_nullability_matches_input() { diff --git a/crates/sqllogictest/test_files/datetime/dayofweek.slt b/crates/sqllogictest/test_files/datetime/dayofweek.slt index 300fb54..0a2ef69 100644 --- a/crates/sqllogictest/test_files/datetime/dayofweek.slt +++ b/crates/sqllogictest/test_files/datetime/dayofweek.slt @@ -8,7 +8,219 @@ # For more information, please see: # https://github.com/apache/datafusion/issues/15914 +# Spark `dayofweek` is 1-indexed with Sunday = 1 .. Saturday = 7. +# Expected values verified against Spark 4.2.0. + ## Original Query: SELECT dayofweek('2009-07-30'); ## PySpark 3.5.5 Result: {'dayofweek(2009-07-30)': 5, 'typeof(dayofweek(2009-07-30))': 'int', 'typeof(2009-07-30)': 'string'} -#query -#SELECT dayofweek('2009-07-30'::string); +query I +SELECT dayofweek('2009-07-30'::string); +---- +5 + +query I +SELECT dayofweek('2009-07-30'::DATE); +---- +5 + +# All seven days of one week (2024-01-07 is a Sunday) +query IIIIIII +SELECT dayofweek('2024-01-07'::DATE), dayofweek('2024-01-08'::DATE), dayofweek('2024-01-09'::DATE), + dayofweek('2024-01-10'::DATE), dayofweek('2024-01-11'::DATE), dayofweek('2024-01-12'::DATE), + dayofweek('2024-01-13'::DATE); +---- +1 2 3 4 5 6 7 + +# Epoch, the day before it, and a leap day +query III +SELECT dayofweek('1970-01-01'::DATE), dayofweek('1969-12-31'::DATE), dayofweek('2024-02-29'::DATE); +---- +5 4 5 + +# Calendar boundaries. Spark uses the proleptic Gregorian calendar, so +# 1582-10-04 (the last Julian day before the switch) is a Monday. +query IIII +SELECT dayofweek('0001-01-01'::DATE), dayofweek('9999-12-31'::DATE), + dayofweek('1582-10-04'::DATE), dayofweek('1582-10-15'::DATE); +---- +2 6 2 6 + +# The whole Int32 epoch-day range: -5877641-06-23 and +5881580-07-11 +query II +SELECT dayofweek(date_from_unix_date(-2147483648)), dayofweek(date_from_unix_date(2147483647)); +---- +3 6 + +# The same two dates as a column, so the function sees an array +query I +SELECT dayofweek(d) FROM (VALUES (date_from_unix_date(-2147483648)), (date_from_unix_date(2147483647))) AS t(d); +---- +3 +6 + +# NULL +query I +SELECT dayofweek(NULL::DATE); +---- +NULL + +# Untyped NULL gives a NULL INT +query IT +SELECT dayofweek(NULL), arrow_typeof(dayofweek(NULL)); +---- +NULL Int32 + +# Result type +query T +SELECT arrow_typeof(dayofweek('2024-01-01'::DATE)); +---- +Int32 + +# Column input, including NULL +query I +SELECT dayofweek(d) FROM (VALUES ('2024-01-01'::DATE), (NULL::DATE), ('2024-01-07'::DATE)) AS t(d); +---- +2 +NULL +1 + +# Strings are implicitly cast to dates +query I +SELECT dayofweek('2024-05-20 23:30:00'::STRING); +---- +2 + +query I +SELECT dayofweek(d) FROM (VALUES ('2024-01-01'::STRING), (NULL::STRING), ('2024-01-07'::STRING)) AS t(d); +---- +2 +NULL +1 + +# Spark returns NULL for a malformed string when ANSI mode is off. DataFusion's +# string to date cast errors regardless of ANSI mode. ANSI-aware casting is +# tracked by https://github.com/apache/datafusion-spark/issues/88 +# query I +# SELECT dayofweek('abc'::STRING); +# ---- +# NULL + +# Spark reads '2009' and '2009-07' as the first day of that year or month, in both ANSI +# modes. DataFusion's string to date cast rejects them. Tracked by +# https://github.com/apache/datafusion-spark/issues/88 +# query II +# SELECT dayofweek('2009'::STRING), dayofweek('2009-07'::STRING); +# ---- +# 5 4 + +# Date64 is cast to epoch days +query II +SELECT dayofweek(arrow_cast('2024-01-07'::DATE, 'Date64')), dayofweek(arrow_cast('1969-12-31'::DATE, 'Date64')); +---- +1 4 + +query I +SELECT dayofweek(d) FROM (VALUES (arrow_cast('2024-01-07'::DATE, 'Date64')), (arrow_cast(NULL::DATE, 'Date64'))) AS t(d); +---- +1 +NULL + +# A Date64 outside the Date32 range still has a day of the week, rather than +# NULL: one day after and one day before the Date32 range, then the last +# millisecond before the epoch, which is on 1969-12-31. DataFusion only, since +# Spark has no Date64. +skipif spark +query I +SELECT dayofweek(d) FROM (VALUES + (arrow_cast(185542587187200000, 'Date64')), + (arrow_cast(-185542587273600000, 'Date64')), + (arrow_cast(-1, 'Date64')) +) AS t(d); +---- +7 +2 +4 + +skipif spark +query I +SELECT dayofweek(d) FROM (VALUES + (arrow_cast(arrow_cast(185542587187200000, 'Date64'), 'Dictionary(Int32, Date64)')), + (arrow_cast(arrow_cast(-185542587273600000, 'Date64'), 'Dictionary(Int32, Date64)')), + (arrow_cast(arrow_cast(-1, 'Date64'), 'Dictionary(Int32, Date64)')) +) AS t(d); +---- +7 +2 +4 + +# With ANSI mode on, Spark raises CAST_INVALID_INPUT for a malformed string +statement ok +set datafusion.execution.enable_ansi_mode = true; + +query error Cannot cast string 'abc' to value of Date32 type +SELECT dayofweek('abc'::STRING); + +statement ok +set datafusion.execution.enable_ansi_mode = false; + +# Timestamps are cast to dates, so the time of day is ignored +query I +SELECT dayofweek(arrow_cast('2024-01-07 23:59:59', 'Timestamp(Microsecond, None)')); +---- +1 + +query I +SELECT dayofweek(ts) FROM (VALUES + ('2024-01-01 01:02:03'::TIMESTAMP), + ('2024-01-06 10:20:30'::TIMESTAMP), + ('2024-01-07 23:59:59'::TIMESTAMP), + (NULL::TIMESTAMP) +) AS t(ts); +---- +2 +7 +1 +NULL + +# A timestamp with a time zone is cast to a date in its own time zone. With +# the default UTC session time zone this matches Spark. +query I +SELECT dayofweek(arrow_cast('2024-05-21T02:00:00Z', 'Timestamp(Microsecond, Some("UTC"))')); +---- +3 + +# Spark casts a timestamp to a date in the session time zone, so in +# America/New_York the instant 2024-05-21T02:00:00Z is a Monday (2). DataFusion +# uses the timestamp's own time zone (UTC) and returns 3. Session time zone +# aware timestamp to date casting is tracked by +# https://github.com/apache/datafusion-spark/issues/88 +statement ok +set datafusion.execution.time_zone = 'America/New_York'; + +# query I +# SELECT dayofweek(arrow_cast('2024-05-21T02:00:00Z', 'Timestamp(Microsecond, Some("UTC"))')); +# ---- +# 2 + +statement ok +reset datafusion.execution.time_zone; + +# A dictionary-encoded date column (DataFusion only: Spark has no dictionary type) +skipif spark +query IT +SELECT dayofweek(d), arrow_typeof(dayofweek(d)) FROM (VALUES + (arrow_cast('2024-01-01'::DATE, 'Dictionary(Int32, Date32)')), + (arrow_cast(NULL::DATE, 'Dictionary(Int32, Date32)')), + (arrow_cast('2024-01-07'::DATE, 'Dictionary(Int32, Date32)')) +) AS t(d); +---- +2 Int32 +NULL Int32 +1 Int32 + +# Spark raises DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE for an integer +statement error Function 'dayofweek' requires Date, but received Int32 +SELECT dayofweek(1::INT); + +statement error 'dayofweek' does not support zero arguments +SELECT dayofweek(); diff --git a/crates/sqllogictest/test_files/datetime/weekday.slt b/crates/sqllogictest/test_files/datetime/weekday.slt index 52950b9..94fd7ab 100644 --- a/crates/sqllogictest/test_files/datetime/weekday.slt +++ b/crates/sqllogictest/test_files/datetime/weekday.slt @@ -53,6 +53,20 @@ SELECT weekday('2024-01-07'::DATE); ---- 6 +# The whole Int32 epoch-day range: -5877641-06-23 and +5881580-07-11, which +# are outside chrono's calendar. Verified against Spark 4.2.0. +query II +SELECT weekday(date_from_unix_date(-2147483648)), weekday(date_from_unix_date(2147483647)); +---- +1 4 + +# The same two dates as a column, so the function sees an array +query I +SELECT weekday(d) FROM (VALUES (date_from_unix_date(-2147483648)), (date_from_unix_date(2147483647))) AS t(d); +---- +1 +4 + # NULL handling query I SELECT weekday(NULL::DATE); @@ -105,6 +119,40 @@ SELECT weekday(arrow_cast('2024-01-07 03:00:00', 'Timestamp(Nanosecond, Some("UT ---- 6 +# Date64 is cast to epoch days +query II +SELECT weekday(arrow_cast('2024-01-07'::DATE, 'Date64')), weekday(arrow_cast('1969-12-31'::DATE, 'Date64')); +---- +6 2 + +# A Date64 outside the Date32 range still has a day of the week, rather than +# NULL: one day after and one day before the Date32 range, then the last +# millisecond before the epoch, which is on 1969-12-31. DataFusion only, since +# Spark has no Date64. +skipif spark +query I +SELECT weekday(d) FROM (VALUES + (arrow_cast(185542587187200000, 'Date64')), + (arrow_cast(-185542587273600000, 'Date64')), + (arrow_cast(-1, 'Date64')) +) AS t(d); +---- +5 +0 +2 + +skipif spark +query I +SELECT weekday(d) FROM (VALUES + (arrow_cast(arrow_cast(185542587187200000, 'Date64'), 'Dictionary(Int32, Date64)')), + (arrow_cast(arrow_cast(-185542587273600000, 'Date64'), 'Dictionary(Int32, Date64)')), + (arrow_cast(arrow_cast(-1, 'Date64'), 'Dictionary(Int32, Date64)')) +) AS t(d); +---- +5 +0 +2 + # Error: wrong argument type (string without cast) # xfail spark: Spark implicitly casts a string argument to DATE/TIMESTAMP, invalid strings give NULL (https://github.com/apache/datafusion-spark/issues/41) statement error Function 'weekday' requires Date, but received String