From fa838ed6a0441c34d3252f1767dc6ebb385c172f Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 16 Nov 2026 06:29:03 -0700 Subject: [PATCH 1/4] feat(spark): add Spark-compatible dayofweek Port dayofweek from DataFusion Comet. The value is computed straight from the Date32 epoch day, so it is defined for the whole Int32 range as in Spark. weekday now shares the same kernel, which fixes it returning NULL for dates outside chrono's calendar range. --- .../spark/src/function/datetime/dayofweek.rs | 314 ++++++++++++++++++ crates/spark/src/function/datetime/mod.rs | 8 + crates/spark/src/function/datetime/weekday.rs | 30 +- .../test_files/datetime/dayofweek.slt | 161 ++++++++- .../test_files/datetime/weekday.slt | 7 + 5 files changed, 496 insertions(+), 24 deletions(-) create mode 100644 crates/spark/src/function/datetime/dayofweek.rs diff --git a/crates/spark/src/function/datetime/dayofweek.rs b/crates/spark/src/function/datetime/dayofweek.rs new file mode 100644 index 00000000..12e20788 --- /dev/null +++ b/crates/spark/src/function/datetime/dayofweek.rs @@ -0,0 +1,314 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use std::sync::Arc; + +use arrow::array::{Array, ArrayRef, AsArray, Date32Array, Int32Array}; +use arrow::compute::cast; +use arrow::datatypes::{DataType, Date32Type, Field, FieldRef, Int32Type}; +use datafusion_common::utils::take_function_args; +use datafusion_common::{Result, ScalarValue, exec_err, internal_err}; +use datafusion_expr::{ + ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, + Signature, +}; +use datafusion_macros::user_doc; + +use super::date_part_name::date_signature; + +/// Spark `DayOfWeek`: Sunday = 1, Monday = 2, ..., Saturday = 7. +/// +/// Spark computes `LocalDate.ofEpochDay(days).getDayOfWeek.plus(1).getValue`, +/// which is defined for every `Int`. Epoch day 0 (1970-01-01) is a Thursday, so +/// `+ 4` moves Sunday to 0 before the shift to 1-based. Widening to `i64` first +/// keeps `i32::MAX` from overflowing. +#[inline] +pub(super) fn day_of_week(days: i32) -> i32 { + ((i64::from(days) + 4).rem_euclid(7) + 1) as i32 +} + +/// Spark `WeekDay`: Monday = 0, Tuesday = 1, ..., Sunday = 6. +/// +/// Spark computes `LocalDate.ofEpochDay(days).getDayOfWeek.ordinal()`. +#[inline] +pub(super) fn week_day(days: i32) -> i32 { + (i64::from(days) + 3).rem_euclid(7) as i32 +} + +/// Applies `kernel` to every valid date. +/// +/// `unary` evaluates every slot and vectorizes, `unary_opt` visits only valid +/// slots but costs more per element, so switch on null density. +#[inline] +fn map_dates i32 + Copy>(dates: &Date32Array, kernel: F) -> Int32Array { + if dates.null_count() * 2 <= dates.len() { + dates.unary::<_, Int32Type>(kernel) + } else { + dates.unary_opt::<_, Int32Type>(|d| Some(kernel(d))) + } +} + +/// Maps each epoch day in `arg` (a `Date32` scalar or array, possibly +/// dictionary encoded) to an `Int32` with `kernel`. NULL dates give NULL. +/// +/// The value is computed straight from the epoch day rather than through a +/// calendar date, so it is defined for the whole `i32` domain as in Spark. +pub(super) fn epoch_day_field i32 + Copy>( + name: &str, + arg: ColumnarValue, + kernel: F, +) -> Result { + match arg { + ColumnarValue::Scalar(ScalarValue::Date32(days)) => { + Ok(ColumnarValue::Scalar(ScalarValue::Int32(days.map(kernel)))) + } + ColumnarValue::Scalar(scalar) if scalar.is_null() => { + Ok(ColumnarValue::Scalar(ScalarValue::Int32(None))) + } + ColumnarValue::Scalar(scalar) => { + let array = scalar.to_array_of_size(1)?; + let result = epoch_day_array(name, &array, kernel)?; + Ok(ColumnarValue::Scalar(ScalarValue::try_from_array( + &result, 0, + )?)) + } + ColumnarValue::Array(array) => { + Ok(ColumnarValue::Array(epoch_day_array(name, &array, kernel)?)) + } + } +} + +fn epoch_day_array i32 + Copy>( + name: &str, + array: &ArrayRef, + kernel: F, +) -> Result { + // A dictionary-encoded date column: map the dictionary values, rewrap the + // keys, then unpack to a plain Int32 array, so the work is proportional to + // the cardinality rather than the row count. + if let DataType::Dictionary(_, _) = array.data_type() { + let dict = array.as_any_dictionary(); + let Some(values) = dict.values().as_primitive_opt::() else { + return exec_err!("`{name}` expects a date, got {}", array.data_type()); + }; + let mapped = Arc::new(map_dates(values, kernel)) as ArrayRef; + return Ok(cast(&dict.with_values(mapped), &DataType::Int32)?); + } + let Some(dates) = array.as_primitive_opt::() else { + return exec_err!("`{name}` expects a date, got {}", array.data_type()); + }; + Ok(Arc::new(map_dates(dates, kernel))) +} + +/// Spark-compatible `dayofweek` expression. +/// +/// +#[user_doc( + doc_section(label = "Time and Date Functions"), + description = "Returns the day of the week as an INT from 1 (Sunday) to 7 (Saturday). \ + This differs from `weekday`, which counts from 0 (Monday) to 6 (Sunday), and from \ + DataFusion's `date_part('dow', ...)`, which counts from 0 (Sunday) to 6 (Saturday). \ + DataFusion has no built-in `dayofweek`. \ + Strings are implicitly cast to DATE. A malformed string is an error, even with ANSI mode off \ + where Spark returns NULL. \ + Timestamps are cast to DATE first, so the time of day is ignored. A timestamp with a time zone \ + is converted in its own time zone, not the session time zone that Spark uses. \ + Returns NULL if the input is NULL.", + syntax_example = "dayofweek(date)", + sql_example = r#"```sql +> SELECT dayofweek('2009-07-30'::DATE); +5 +> SELECT dayofweek('2009-07-30'); +5 +> SELECT dayofweek(arrow_cast('2024-01-07 23:59:59', 'Timestamp(Microsecond, None)')); +1 +> SELECT dayofweek(NULL::DATE); +NULL +```"#, + argument( + name = "date", + description = "DATE, TIMESTAMP or string expression. NULL returns NULL." + ) +)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkDayOfWeek { + signature: Signature, +} + +impl Default for SparkDayOfWeek { + fn default() -> Self { + Self::new() + } +} + +impl SparkDayOfWeek { + pub fn new() -> Self { + Self { + signature: date_signature(), + } + } +} + +impl ScalarUDFImpl for SparkDayOfWeek { + fn name(&self) -> &str { + "dayofweek" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + internal_err!("return_field_from_args should be used instead") + } + + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); + Ok(Arc::new(Field::new(self.name(), DataType::Int32, nullable))) + } + + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + let [arg] = take_function_args(self.name(), args.args)?; + epoch_day_field(self.name(), arg, day_of_week) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::DictionaryArray; + use datafusion_common::config::ConfigOptions; + + fn invoke(arg: ColumnarValue, rows: usize) -> Result { + let data_type = arg.data_type(); + SparkDayOfWeek::new().invoke_with_args(ScalarFunctionArgs { + args: vec![arg], + arg_fields: vec![Arc::new(Field::new("d", data_type, true))], + number_rows: rows, + return_field: Arc::new(Field::new("dayofweek", DataType::Int32, true)), + config_options: Arc::new(ConfigOptions::default()), + }) + } + + fn invoke_array(array: ArrayRef) -> Int32Array { + let rows = array.len(); + let result = invoke(ColumnarValue::Array(array), rows) + .unwrap() + .to_array(rows) + .unwrap(); + result.as_primitive::().clone() + } + + #[test] + fn return_field_nullability_matches_input() { + let func = SparkDayOfWeek::new(); + for nullable in [false, true] { + let arg = Arc::new(Field::new("arg", DataType::Date32, nullable)); + let out = func + .return_field_from_args(ReturnFieldArgs { + arg_fields: &[arg], + scalar_arguments: &[None], + }) + .unwrap(); + assert_eq!(out.data_type(), &DataType::Int32); + assert_eq!(out.is_nullable(), nullable); + } + } + + /// 1970-01-01 is a Thursday, so the week either side of the epoch pins + /// both numberings. + #[test] + fn known_weekdays_around_the_epoch() { + // days -4..=3 spans Sunday 1969-12-28 through Sunday 1970-01-04. + let array: ArrayRef = Arc::new(Date32Array::from((-4..=3).collect::>())); + assert_eq!( + invoke_array(array).values(), + // Sun Mon Tue Wed Thu Fri Sat Sun + &[1, 2, 3, 4, 5, 6, 7, 1] + ); + let weekdays: Vec = (-4..=3).map(week_day).collect(); + assert_eq!(weekdays, [6, 0, 1, 2, 3, 4, 5, 6]); + } + + /// Nulls pass through in place, at both null densities `map_dates` handles. + #[test] + fn nulls_are_preserved() { + let sparse: ArrayRef = + Arc::new(Date32Array::from(vec![Some(0), None, Some(1), Some(2)])); + let out = invoke_array(sparse); + assert_eq!( + out.iter().collect::>(), + [Some(5), None, Some(6), Some(7)] + ); + + let dense: ArrayRef = + Arc::new(Date32Array::from(vec![None, None, Some(1), None])); + let out = invoke_array(dense); + assert_eq!(out.iter().collect::>(), [None, None, Some(6), None]); + } + + /// The whole `i32` domain works, as `LocalDate.ofEpochDay` does in Spark. + /// These are outside chrono's representable range. + #[test] + fn full_i32_domain() { + let array: ArrayRef = Arc::new(Date32Array::from(vec![i32::MIN, i32::MAX])); + // Verified against Spark 4.2.0: + // dayofweek(date_from_unix_date(-2147483648)) = 3 (-5877641-06-23) + // dayofweek(date_from_unix_date(2147483647)) = 6 (+5881580-07-11) + // weekday of the same two dates = 1 and 4 + assert_eq!(invoke_array(array).values(), &[3, 6]); + assert_eq!([week_day(i32::MIN), week_day(i32::MAX)], [1, 4]); + } + + /// The two numberings agree across a full 400-year Gregorian cycle. + #[test] + fn numberings_agree_over_a_full_cycle() { + for days in -146_097..=146_097 { + assert_eq!((day_of_week(days) + 5) % 7, week_day(days), "day {days}"); + } + } + + /// A dictionary-encoded date column unpacks to a plain Int32 result. + #[test] + fn dictionary_input_unpacks() { + let values = + Arc::new(Date32Array::from(vec![Some(0), Some(1), None])) as ArrayRef; + let keys = Int32Array::from(vec![Some(0), Some(1), Some(0), None, Some(2)]); + let dict = DictionaryArray::::new(keys, values); + let out = invoke_array(Arc::new(dict)); + assert_eq!(out.data_type(), &DataType::Int32); + assert_eq!( + out.iter().collect::>(), + [Some(5), Some(6), Some(5), None, None] + ); + } + + #[test] + fn rejects_non_date_input() { + let array: ArrayRef = Arc::new(Int32Array::from(vec![1])); + let err = invoke(ColumnarValue::Array(array), 1).unwrap_err(); + assert!(err.to_string().contains("expects a date"), "{err}"); + } + + #[test] + fn scalar_input() -> Result<()> { + match invoke(ColumnarValue::Scalar(ScalarValue::Date32(Some(0))), 1)? { + ColumnarValue::Scalar(ScalarValue::Int32(Some(5))) => {} + other => panic!("unexpected result: {other:?}"), + } + match invoke(ColumnarValue::Scalar(ScalarValue::Date32(None)), 1)? { + ColumnarValue::Scalar(ScalarValue::Int32(None)) => {} + other => panic!("unexpected result: {other:?}"), + } + match invoke(ColumnarValue::Scalar(ScalarValue::Null), 1)? { + ColumnarValue::Scalar(ScalarValue::Int32(None)) => {} + other => panic!("unexpected result: {other:?}"), + } + Ok(()) + } +} diff --git a/crates/spark/src/function/datetime/mod.rs b/crates/spark/src/function/datetime/mod.rs index bb6c8dc8..3143ffdc 100644 --- a/crates/spark/src/function/datetime/mod.rs +++ b/crates/spark/src/function/datetime/mod.rs @@ -11,6 +11,7 @@ mod date_part_name; pub mod date_sub; pub mod date_trunc; pub mod dayname; +pub mod dayofweek; pub mod extract; pub mod from_utc_timestamp; pub mod last_day; @@ -40,6 +41,7 @@ make_udf_function!(date_part::SparkDatePart, date_part); make_udf_function!(date_sub::SparkDateSub, date_sub); make_udf_function!(date_trunc::SparkDateTrunc, date_trunc); make_udf_function!(dayname::SparkDayName, dayname); +make_udf_function!(dayofweek::SparkDayOfWeek, dayofweek); make_udf_function!( from_utc_timestamp::SparkFromUtcTimestamp, from_utc_timestamp @@ -117,6 +119,11 @@ pub mod expr_fn { "Returns the three-letter abbreviated day name from a date.", arg1 )); + export_functions!(( + dayofweek, + "Returns the day of the week for date/timestamp as an integer where Sunday = 1, Monday = 2, ..., Saturday = 7.", + arg1 + )); export_functions!(( make_interval, "Make interval from years, months, weeks, days, hours, mins and secs.", @@ -211,6 +218,7 @@ pub fn functions() -> Vec> { date_sub(), date_trunc(), dayname(), + dayofweek(), from_utc_timestamp(), hour(), last_day(), diff --git a/crates/spark/src/function/datetime/weekday.rs b/crates/spark/src/function/datetime/weekday.rs index fe2aa68c..4cdbfd3b 100644 --- a/crates/spark/src/function/datetime/weekday.rs +++ b/crates/spark/src/function/datetime/weekday.rs @@ -4,18 +4,18 @@ use std::sync::Arc; -use arrow::array::AsArray; -use arrow::compute::{DatePart, date_part}; -use arrow::datatypes::{DataType, Field, FieldRef, Int32Type}; +use arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::types::{NativeType, logical_date}; use datafusion_common::utils::take_function_args; -use datafusion_common::{Result, ScalarValue, internal_err}; +use datafusion_common::{Result, internal_err}; use datafusion_expr::{ Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, }; use datafusion_macros::user_doc; +use super::dayofweek::{epoch_day_field, week_day}; + /// Spark-compatible `weekday` expression. /// /// @@ -89,30 +89,16 @@ impl ScalarUDFImpl for SparkWeekDay { fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { let [arg] = take_function_args(self.name(), args.args)?; - match arg { - ColumnarValue::Scalar(scalar) => { - if scalar.is_null() { - return Ok(ColumnarValue::Scalar(ScalarValue::Int32(None))); - } - let arr = scalar.to_array_of_size(1)?; - // `DayOfWeekMonday0` returns 0..=6 with Monday = 0, which - // matches Spark `weekday` semantics exactly. - let weekday_arr = date_part(&arr, DatePart::DayOfWeekMonday0)?; - let value = weekday_arr.as_primitive::().value(0); - Ok(ColumnarValue::Scalar(ScalarValue::Int32(Some(value)))) - } - ColumnarValue::Array(arr) => { - let weekday_arr = date_part(&arr, DatePart::DayOfWeekMonday0)?; - Ok(ColumnarValue::Array(weekday_arr)) - } - } + epoch_day_field(self.name(), arg, week_day) } } #[cfg(test)] mod tests { use super::*; - use arrow::array::{Date32Array, Int32Array}; + use arrow::array::{AsArray, Date32Array, Int32Array}; + use arrow::datatypes::Int32Type; + use datafusion_common::ScalarValue; #[test] fn test_weekday_return_field_nullability_matches_input() { diff --git a/crates/sqllogictest/test_files/datetime/dayofweek.slt b/crates/sqllogictest/test_files/datetime/dayofweek.slt index 300fb542..b6655375 100644 --- a/crates/sqllogictest/test_files/datetime/dayofweek.slt +++ b/crates/sqllogictest/test_files/datetime/dayofweek.slt @@ -8,7 +8,164 @@ # For more information, please see: # https://github.com/apache/datafusion/issues/15914 +# Spark `dayofweek` is 1-indexed with Sunday = 1 .. Saturday = 7. +# Expected values verified against Spark 4.2.0. + ## Original Query: SELECT dayofweek('2009-07-30'); ## PySpark 3.5.5 Result: {'dayofweek(2009-07-30)': 5, 'typeof(dayofweek(2009-07-30))': 'int', 'typeof(2009-07-30)': 'string'} -#query -#SELECT dayofweek('2009-07-30'::string); +query I +SELECT dayofweek('2009-07-30'::string); +---- +5 + +query I +SELECT dayofweek('2009-07-30'::DATE); +---- +5 + +# All seven days of one week (2024-01-07 is a Sunday) +query IIIIIII +SELECT dayofweek('2024-01-07'::DATE), dayofweek('2024-01-08'::DATE), dayofweek('2024-01-09'::DATE), + dayofweek('2024-01-10'::DATE), dayofweek('2024-01-11'::DATE), dayofweek('2024-01-12'::DATE), + dayofweek('2024-01-13'::DATE); +---- +1 2 3 4 5 6 7 + +# Epoch, the day before it, and a leap day +query III +SELECT dayofweek('1970-01-01'::DATE), dayofweek('1969-12-31'::DATE), dayofweek('2024-02-29'::DATE); +---- +5 4 5 + +# Calendar boundaries. Spark uses the proleptic Gregorian calendar, so +# 1582-10-04 (the last Julian day before the switch) is a Monday. +query IIII +SELECT dayofweek('0001-01-01'::DATE), dayofweek('9999-12-31'::DATE), + dayofweek('1582-10-04'::DATE), dayofweek('1582-10-15'::DATE); +---- +2 6 2 6 + +# The whole Int32 epoch-day range: -5877641-06-23 and +5881580-07-11 +query II +SELECT dayofweek(date_from_unix_date(-2147483648)), dayofweek(date_from_unix_date(2147483647)); +---- +3 6 + +# NULL +query I +SELECT dayofweek(NULL::DATE); +---- +NULL + +# Untyped NULL gives a NULL INT +query IT +SELECT dayofweek(NULL), arrow_typeof(dayofweek(NULL)); +---- +NULL Int32 + +# Result type +query T +SELECT arrow_typeof(dayofweek('2024-01-01'::DATE)); +---- +Int32 + +# Column input, including NULL +query I +SELECT dayofweek(d) FROM (VALUES ('2024-01-01'::DATE), (NULL::DATE), ('2024-01-07'::DATE)) AS t(d); +---- +2 +NULL +1 + +# Strings are implicitly cast to dates +query I +SELECT dayofweek('2024-05-20 23:30:00'::STRING); +---- +2 + +query I +SELECT dayofweek(d) FROM (VALUES ('2024-01-01'::STRING), (NULL::STRING), ('2024-01-07'::STRING)) AS t(d); +---- +2 +NULL +1 + +# Spark returns NULL for a malformed string when ANSI mode is off. DataFusion's +# string to date cast errors regardless of ANSI mode. ANSI-aware casting is +# tracked by https://github.com/apache/datafusion-spark/issues/88 +# query I +# SELECT dayofweek('abc'::STRING); +# ---- +# NULL + +# With ANSI mode on, Spark raises CAST_INVALID_INPUT for a malformed string +statement ok +set datafusion.execution.enable_ansi_mode = true; + +query error Cannot cast string 'abc' to value of Date32 type +SELECT dayofweek('abc'::STRING); + +statement ok +set datafusion.execution.enable_ansi_mode = false; + +# Timestamps are cast to dates, so the time of day is ignored +query I +SELECT dayofweek(arrow_cast('2024-01-07 23:59:59', 'Timestamp(Microsecond, None)')); +---- +1 + +query I +SELECT dayofweek(ts) FROM (VALUES + ('2024-01-01 01:02:03'::TIMESTAMP), + ('2024-01-06 10:20:30'::TIMESTAMP), + ('2024-01-07 23:59:59'::TIMESTAMP), + (NULL::TIMESTAMP) +) AS t(ts); +---- +2 +7 +1 +NULL + +# A timestamp with a time zone is cast to a date in its own time zone. With +# the default UTC session time zone this matches Spark. +query I +SELECT dayofweek(arrow_cast('2024-05-21T02:00:00Z', 'Timestamp(Microsecond, Some("UTC"))')); +---- +3 + +# Spark casts a timestamp to a date in the session time zone, so in +# America/New_York the instant 2024-05-21T02:00:00Z is a Monday (2). DataFusion +# uses the timestamp's own time zone (UTC) and returns 3. Session time zone +# aware timestamp to date casting is tracked by +# https://github.com/apache/datafusion-spark/issues/88 +statement ok +set datafusion.execution.time_zone = 'America/New_York'; + +# query I +# SELECT dayofweek(arrow_cast('2024-05-21T02:00:00Z', 'Timestamp(Microsecond, Some("UTC"))')); +# ---- +# 2 + +statement ok +reset datafusion.execution.time_zone; + +# A dictionary-encoded date column (DataFusion only: Spark has no dictionary type) +skipif spark +query IT +SELECT dayofweek(d), arrow_typeof(dayofweek(d)) FROM (VALUES + (arrow_cast('2024-01-01'::DATE, 'Dictionary(Int32, Date32)')), + (arrow_cast(NULL::DATE, 'Dictionary(Int32, Date32)')), + (arrow_cast('2024-01-07'::DATE, 'Dictionary(Int32, Date32)')) +) AS t(d); +---- +2 Int32 +NULL Int32 +1 Int32 + +# Spark raises DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE for an integer +statement error Function 'dayofweek' requires Date, but received Int32 +SELECT dayofweek(1::INT); + +statement error 'dayofweek' does not support zero arguments +SELECT dayofweek(); diff --git a/crates/sqllogictest/test_files/datetime/weekday.slt b/crates/sqllogictest/test_files/datetime/weekday.slt index f985eed2..3bbc1c4a 100644 --- a/crates/sqllogictest/test_files/datetime/weekday.slt +++ b/crates/sqllogictest/test_files/datetime/weekday.slt @@ -53,6 +53,13 @@ SELECT weekday('2024-01-07'::DATE); ---- 6 +# The whole Int32 epoch-day range: -5877641-06-23 and +5881580-07-11, which +# are outside chrono's calendar. Verified against Spark 4.2.0. +query II +SELECT weekday(date_from_unix_date(-2147483648)), weekday(date_from_unix_date(2147483647)); +---- +1 4 + # NULL handling query I SELECT weekday(NULL::DATE); From be65c39bc4a2defadb120cb3892c61032333eb62 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 10:57:04 -0700 Subject: [PATCH 2/4] fix(spark): accept Date64 in dayofweek and weekday date_signature admits Date64, so cast it to epoch days before computing the day of the week. Note that strings Spark reads as partial dates are an error here, with commented-out queries citing #88. --- .../spark/src/function/datetime/dayofweek.rs | 14 +++++++++-- .../test_files/datetime/dayofweek.slt | 25 +++++++++++++++++++ .../test_files/datetime/weekday.slt | 6 +++++ 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/crates/spark/src/function/datetime/dayofweek.rs b/crates/spark/src/function/datetime/dayofweek.rs index 12e20788..675cf0af 100644 --- a/crates/spark/src/function/datetime/dayofweek.rs +++ b/crates/spark/src/function/datetime/dayofweek.rs @@ -84,12 +84,21 @@ fn epoch_day_array i32 + Copy>( array: &ArrayRef, kernel: F, ) -> Result { + // The date signature also admits Date64, which holds milliseconds since the + // epoch. Arrow's cast turns it into epoch days. + if array.data_type() == &DataType::Date64 { + return epoch_day_array(name, &cast(array, &DataType::Date32)?, kernel); + } // A dictionary-encoded date column: map the dictionary values, rewrap the // keys, then unpack to a plain Int32 array, so the work is proportional to // the cardinality rather than the row count. if let DataType::Dictionary(_, _) = array.data_type() { let dict = array.as_any_dictionary(); - let Some(values) = dict.values().as_primitive_opt::() else { + let values = match dict.values().data_type() { + DataType::Date64 => cast(dict.values(), &DataType::Date32)?, + _ => Arc::clone(dict.values()), + }; + let Some(values) = values.as_primitive_opt::() else { return exec_err!("`{name}` expects a date, got {}", array.data_type()); }; let mapped = Arc::new(map_dates(values, kernel)) as ArrayRef; @@ -111,7 +120,8 @@ fn epoch_day_array i32 + Copy>( DataFusion's `date_part('dow', ...)`, which counts from 0 (Sunday) to 6 (Saturday). \ DataFusion has no built-in `dayofweek`. \ Strings are implicitly cast to DATE. A malformed string is an error, even with ANSI mode off \ - where Spark returns NULL. \ + where Spark returns NULL. A string that Spark reads as a partial date, such as '2009' or \ + '2009-07', is also an error, where Spark uses the first day of that year or month. \ Timestamps are cast to DATE first, so the time of day is ignored. A timestamp with a time zone \ is converted in its own time zone, not the session time zone that Spark uses. \ Returns NULL if the input is NULL.", diff --git a/crates/sqllogictest/test_files/datetime/dayofweek.slt b/crates/sqllogictest/test_files/datetime/dayofweek.slt index b6655375..6f80263a 100644 --- a/crates/sqllogictest/test_files/datetime/dayofweek.slt +++ b/crates/sqllogictest/test_files/datetime/dayofweek.slt @@ -98,6 +98,31 @@ NULL # ---- # NULL +# Spark reads '2009' and '2009-07' as the first day of that year or month, in both ANSI +# modes. DataFusion's string to date cast rejects them. Tracked by +# https://github.com/apache/datafusion-spark/issues/88 +# query II +# SELECT dayofweek('2009'::STRING), dayofweek('2009-07'::STRING); +# ---- +# 5 4 + +# Date64 is cast to epoch days +query II +SELECT dayofweek(arrow_cast('2024-01-07'::DATE, 'Date64')), dayofweek(arrow_cast('1969-12-31'::DATE, 'Date64')); +---- +1 4 + +query I +SELECT dayofweek(d) FROM (VALUES (arrow_cast('2024-01-07'::DATE, 'Date64')), (arrow_cast(NULL::DATE, 'Date64'))) AS t(d); +---- +1 +NULL + +query I +SELECT dayofweek(arrow_cast(arrow_cast('2024-01-07'::DATE, 'Date64'), 'Dictionary(Int32, Date64)')); +---- +1 + # With ANSI mode on, Spark raises CAST_INVALID_INPUT for a malformed string statement ok set datafusion.execution.enable_ansi_mode = true; diff --git a/crates/sqllogictest/test_files/datetime/weekday.slt b/crates/sqllogictest/test_files/datetime/weekday.slt index 3bbc1c4a..2ac76391 100644 --- a/crates/sqllogictest/test_files/datetime/weekday.slt +++ b/crates/sqllogictest/test_files/datetime/weekday.slt @@ -112,6 +112,12 @@ SELECT weekday(arrow_cast('2024-01-07 03:00:00', 'Timestamp(Nanosecond, Some("UT ---- 6 +# Date64 is cast to epoch days +query II +SELECT weekday(arrow_cast('2024-01-07'::DATE, 'Date64')), weekday(arrow_cast('1969-12-31'::DATE, 'Date64')); +---- +6 2 + # Error: wrong argument type (string without cast) statement error Function 'weekday' requires Date, but received String SELECT weekday('not-a-date'); From 3e80f6c55a968cf1e9abfc2d439c5da584f0ed27 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 16:54:08 -0700 Subject: [PATCH 3/4] test(spark): cover the Date64 dictionary arm in dayofweek with a unit test Coercion unpacks a dictionary to its value type before the function runs, so the Date64 dictionary query in dayofweek.slt never reached this arm. Replace it with a unit test that calls invoke_with_args directly. --- .../spark/src/function/datetime/dayofweek.rs | 19 ++++++++++++++++++- .../test_files/datetime/dayofweek.slt | 5 ----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/crates/spark/src/function/datetime/dayofweek.rs b/crates/spark/src/function/datetime/dayofweek.rs index 675cf0af..b190f5eb 100644 --- a/crates/spark/src/function/datetime/dayofweek.rs +++ b/crates/spark/src/function/datetime/dayofweek.rs @@ -191,7 +191,7 @@ impl ScalarUDFImpl for SparkDayOfWeek { #[cfg(test)] mod tests { use super::*; - use arrow::array::DictionaryArray; + use arrow::array::{Date64Array, DictionaryArray}; use datafusion_common::config::ConfigOptions; fn invoke(arg: ColumnarValue, rows: usize) -> Result { @@ -298,6 +298,23 @@ mod tests { ); } + /// A dictionary of Date64 values is cast to epoch days before mapping. + /// SQL never reaches this path, because coercion unpacks the dictionary + /// to its value type first, so only a direct call covers it. + #[test] + fn date64_dictionary_input() { + let values = Arc::new(Date64Array::from(vec![Some(0), Some(-86_400_000), None])) + as ArrayRef; + let keys = Int32Array::from(vec![Some(0), Some(1), None, Some(2)]); + let dict = DictionaryArray::::new(keys, values); + let out = invoke_array(Arc::new(dict)); + assert_eq!(out.data_type(), &DataType::Int32); + assert_eq!( + out.iter().collect::>(), + [Some(5), Some(4), None, None] + ); + } + #[test] fn rejects_non_date_input() { let array: ArrayRef = Arc::new(Int32Array::from(vec![1])); diff --git a/crates/sqllogictest/test_files/datetime/dayofweek.slt b/crates/sqllogictest/test_files/datetime/dayofweek.slt index 6f80263a..43476e91 100644 --- a/crates/sqllogictest/test_files/datetime/dayofweek.slt +++ b/crates/sqllogictest/test_files/datetime/dayofweek.slt @@ -118,11 +118,6 @@ SELECT dayofweek(d) FROM (VALUES (arrow_cast('2024-01-07'::DATE, 'Date64')), (ar 1 NULL -query I -SELECT dayofweek(arrow_cast(arrow_cast('2024-01-07'::DATE, 'Date64'), 'Dictionary(Int32, Date64)')); ----- -1 - # With ANSI mode on, Spark raises CAST_INVALID_INPUT for a malformed string statement ok set datafusion.execution.enable_ansi_mode = true; From ce9eb14b0b2ebf9f85baf522bf71e702c221814e Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 18 Nov 2026 05:53:04 -0700 Subject: [PATCH 4/4] fix(spark): compute Date64 epoch days in i64 in dayofweek and weekday Arrow's Date64 to Date32 cast returns NULL for days outside the i32 range, so a non-null Date64 could give a NULL result from a non-nullable field. The cast also divides toward zero, which put a time before the epoch on the following day. Floor the milliseconds to days in i64 instead, and make the kernels take i64 days. Use `unary` for every batch. The kernels are total, so evaluating the slots under NULLs is safe, and the null-density switch had no benchmark behind it. Add tests for Date64 values outside the Date32 range, plain and dictionary encoded, and for the Date32 range limits as a column. --- .../spark/src/function/datetime/dayofweek.rs | 123 ++++++++++++------ .../test_files/datetime/dayofweek.slt | 35 +++++ .../test_files/datetime/weekday.slt | 35 +++++ 3 files changed, 152 insertions(+), 41 deletions(-) diff --git a/crates/spark/src/function/datetime/dayofweek.rs b/crates/spark/src/function/datetime/dayofweek.rs index b190f5eb..7863c32f 100644 --- a/crates/spark/src/function/datetime/dayofweek.rs +++ b/crates/spark/src/function/datetime/dayofweek.rs @@ -4,9 +4,9 @@ use std::sync::Arc; -use arrow::array::{Array, ArrayRef, AsArray, Date32Array, Int32Array}; +use arrow::array::{Array, ArrayRef, AsArray, Int32Array}; use arrow::compute::cast; -use arrow::datatypes::{DataType, Date32Type, Field, FieldRef, Int32Type}; +use arrow::datatypes::{DataType, Date32Type, Date64Type, Field, FieldRef}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, exec_err, internal_err}; use datafusion_expr::{ @@ -17,52 +17,69 @@ use datafusion_macros::user_doc; use super::date_part_name::date_signature; +const MILLISECONDS_PER_DAY: i64 = 86_400_000; + /// Spark `DayOfWeek`: Sunday = 1, Monday = 2, ..., Saturday = 7. /// /// Spark computes `LocalDate.ofEpochDay(days).getDayOfWeek.plus(1).getValue`, /// which is defined for every `Int`. Epoch day 0 (1970-01-01) is a Thursday, so -/// `+ 4` moves Sunday to 0 before the shift to 1-based. Widening to `i64` first -/// keeps `i32::MAX` from overflowing. +/// `+ 4` moves Sunday to 0 before the shift to 1-based. The day is an `i64` so +/// that a `Date64` outside the `Date32` range still has a day of the week. #[inline] -pub(super) fn day_of_week(days: i32) -> i32 { - ((i64::from(days) + 4).rem_euclid(7) + 1) as i32 +pub(super) fn day_of_week(days: i64) -> i32 { + ((days.rem_euclid(7) + 4) % 7 + 1) as i32 } /// Spark `WeekDay`: Monday = 0, Tuesday = 1, ..., Sunday = 6. /// /// Spark computes `LocalDate.ofEpochDay(days).getDayOfWeek.ordinal()`. #[inline] -pub(super) fn week_day(days: i32) -> i32 { - (i64::from(days) + 3).rem_euclid(7) as i32 +pub(super) fn week_day(days: i64) -> i32 { + ((days.rem_euclid(7) + 3) % 7) as i32 } -/// Applies `kernel` to every valid date. +/// Applies `kernel` to the epoch day of every date in a `Date32` or `Date64` +/// array, or returns `None` for any other type. /// -/// `unary` evaluates every slot and vectorizes, `unary_opt` visits only valid -/// slots but costs more per element, so switch on null density. -#[inline] -fn map_dates i32 + Copy>(dates: &Date32Array, kernel: F) -> Int32Array { - if dates.null_count() * 2 <= dates.len() { - dates.unary::<_, Int32Type>(kernel) - } else { - dates.unary_opt::<_, Int32Type>(|d| Some(kernel(d))) +/// `unary` also evaluates the slots under NULLs, which is safe because the +/// kernels are total, and lets the loop vectorize. +fn map_dates i32 + Copy>( + array: &dyn Array, + kernel: F, +) -> Option { + match array.data_type() { + DataType::Date32 => Some( + array + .as_primitive::() + .unary(|days| kernel(i64::from(days))), + ), + // Date64 holds milliseconds since the epoch. Arrow's cast to Date32 + // would turn days outside the i32 range into NULL, and it divides + // toward zero, which moves a time before the epoch to the next day. + DataType::Date64 => Some( + array + .as_primitive::() + .unary(|ms| kernel(ms.div_euclid(MILLISECONDS_PER_DAY))), + ), + _ => None, } } -/// Maps each epoch day in `arg` (a `Date32` scalar or array, possibly -/// dictionary encoded) to an `Int32` with `kernel`. NULL dates give NULL. +/// Maps each epoch day in `arg` (a `Date32` or `Date64` scalar or array, +/// possibly dictionary encoded) to an `Int32` with `kernel`. NULL dates give +/// NULL. /// /// The value is computed straight from the epoch day rather than through a /// calendar date, so it is defined for the whole `i32` domain as in Spark. -pub(super) fn epoch_day_field i32 + Copy>( +pub(super) fn epoch_day_field i32 + Copy>( name: &str, arg: ColumnarValue, kernel: F, ) -> Result { match arg { - ColumnarValue::Scalar(ScalarValue::Date32(days)) => { - Ok(ColumnarValue::Scalar(ScalarValue::Int32(days.map(kernel)))) - } + ColumnarValue::Scalar(ScalarValue::Date32(days)) => Ok(ColumnarValue::Scalar( + ScalarValue::Int32(days.map(|days| kernel(i64::from(days)))), + )), ColumnarValue::Scalar(scalar) if scalar.is_null() => { Ok(ColumnarValue::Scalar(ScalarValue::Int32(None))) } @@ -79,35 +96,25 @@ pub(super) fn epoch_day_field i32 + Copy>( } } -fn epoch_day_array i32 + Copy>( +fn epoch_day_array i32 + Copy>( name: &str, array: &ArrayRef, kernel: F, ) -> Result { - // The date signature also admits Date64, which holds milliseconds since the - // epoch. Arrow's cast turns it into epoch days. - if array.data_type() == &DataType::Date64 { - return epoch_day_array(name, &cast(array, &DataType::Date32)?, kernel); - } // A dictionary-encoded date column: map the dictionary values, rewrap the // keys, then unpack to a plain Int32 array, so the work is proportional to // the cardinality rather than the row count. if let DataType::Dictionary(_, _) = array.data_type() { let dict = array.as_any_dictionary(); - let values = match dict.values().data_type() { - DataType::Date64 => cast(dict.values(), &DataType::Date32)?, - _ => Arc::clone(dict.values()), - }; - let Some(values) = values.as_primitive_opt::() else { + let Some(mapped) = map_dates(dict.values().as_ref(), kernel) else { return exec_err!("`{name}` expects a date, got {}", array.data_type()); }; - let mapped = Arc::new(map_dates(values, kernel)) as ArrayRef; - return Ok(cast(&dict.with_values(mapped), &DataType::Int32)?); + return Ok(cast(&dict.with_values(Arc::new(mapped)), &DataType::Int32)?); } - let Some(dates) = array.as_primitive_opt::() else { + let Some(mapped) = map_dates(array.as_ref(), kernel) else { return exec_err!("`{name}` expects a date, got {}", array.data_type()); }; - Ok(Arc::new(map_dates(dates, kernel))) + Ok(Arc::new(mapped)) } /// Spark-compatible `dayofweek` expression. @@ -191,7 +198,8 @@ impl ScalarUDFImpl for SparkDayOfWeek { #[cfg(test)] mod tests { use super::*; - use arrow::array::{Date64Array, DictionaryArray}; + use arrow::array::{Date32Array, Date64Array, DictionaryArray}; + use arrow::datatypes::Int32Type; use datafusion_common::config::ConfigOptions; fn invoke(arg: ColumnarValue, rows: usize) -> Result { @@ -245,7 +253,7 @@ mod tests { assert_eq!(weekdays, [6, 0, 1, 2, 3, 4, 5, 6]); } - /// Nulls pass through in place, at both null densities `map_dates` handles. + /// Nulls pass through in place, whether they are sparse or dense. #[test] fn nulls_are_preserved() { let sparse: ArrayRef = @@ -272,7 +280,10 @@ mod tests { // dayofweek(date_from_unix_date(2147483647)) = 6 (+5881580-07-11) // weekday of the same two dates = 1 and 4 assert_eq!(invoke_array(array).values(), &[3, 6]); - assert_eq!([week_day(i32::MIN), week_day(i32::MAX)], [1, 4]); + assert_eq!( + [week_day(i32::MIN.into()), week_day(i32::MAX.into())], + [1, 4] + ); } /// The two numberings agree across a full 400-year Gregorian cycle. @@ -315,6 +326,36 @@ mod tests { ); } + /// A Date64 is floored to whole days, so the last millisecond before the + /// epoch is on 1969-12-31 (a Wednesday), and days outside the Date32 range + /// still give a weekday rather than NULL. + #[test] + fn date64_outside_the_date32_range() { + let ms = |days: i64| days * MILLISECONDS_PER_DAY; + let first_after = i64::from(i32::MAX) + 1; + let last_before = i64::from(i32::MIN) - 1; + let values = vec![-1, ms(first_after), ms(last_before), i64::MAX, i64::MIN]; + let expected: Vec = [-1, first_after, last_before] + .into_iter() + .chain([i64::MAX, i64::MIN].map(|ms| ms.div_euclid(MILLISECONDS_PER_DAY))) + .map(day_of_week) + .collect(); + // One day after i32::MAX (a Friday) and one day before i32::MIN (a + // Tuesday). + assert_eq!(expected[..3], [4, 7, 2]); + + let plain = invoke_array(Arc::new(Date64Array::from(values.clone()))); + assert_eq!(plain.null_count(), 0); + assert_eq!(plain.values(), expected.as_slice()); + + let keys = Int32Array::from((0..values.len() as i32).collect::>()); + let dict = + DictionaryArray::::new(keys, Arc::new(Date64Array::from(values))); + let out = invoke_array(Arc::new(dict)); + assert_eq!(out.null_count(), 0); + assert_eq!(out.values(), expected.as_slice()); + } + #[test] fn rejects_non_date_input() { let array: ArrayRef = Arc::new(Int32Array::from(vec![1])); diff --git a/crates/sqllogictest/test_files/datetime/dayofweek.slt b/crates/sqllogictest/test_files/datetime/dayofweek.slt index 43476e91..0a2ef69d 100644 --- a/crates/sqllogictest/test_files/datetime/dayofweek.slt +++ b/crates/sqllogictest/test_files/datetime/dayofweek.slt @@ -51,6 +51,13 @@ SELECT dayofweek(date_from_unix_date(-2147483648)), dayofweek(date_from_unix_dat ---- 3 6 +# The same two dates as a column, so the function sees an array +query I +SELECT dayofweek(d) FROM (VALUES (date_from_unix_date(-2147483648)), (date_from_unix_date(2147483647))) AS t(d); +---- +3 +6 + # NULL query I SELECT dayofweek(NULL::DATE); @@ -118,6 +125,34 @@ SELECT dayofweek(d) FROM (VALUES (arrow_cast('2024-01-07'::DATE, 'Date64')), (ar 1 NULL +# A Date64 outside the Date32 range still has a day of the week, rather than +# NULL: one day after and one day before the Date32 range, then the last +# millisecond before the epoch, which is on 1969-12-31. DataFusion only, since +# Spark has no Date64. +skipif spark +query I +SELECT dayofweek(d) FROM (VALUES + (arrow_cast(185542587187200000, 'Date64')), + (arrow_cast(-185542587273600000, 'Date64')), + (arrow_cast(-1, 'Date64')) +) AS t(d); +---- +7 +2 +4 + +skipif spark +query I +SELECT dayofweek(d) FROM (VALUES + (arrow_cast(arrow_cast(185542587187200000, 'Date64'), 'Dictionary(Int32, Date64)')), + (arrow_cast(arrow_cast(-185542587273600000, 'Date64'), 'Dictionary(Int32, Date64)')), + (arrow_cast(arrow_cast(-1, 'Date64'), 'Dictionary(Int32, Date64)')) +) AS t(d); +---- +7 +2 +4 + # With ANSI mode on, Spark raises CAST_INVALID_INPUT for a malformed string statement ok set datafusion.execution.enable_ansi_mode = true; diff --git a/crates/sqllogictest/test_files/datetime/weekday.slt b/crates/sqllogictest/test_files/datetime/weekday.slt index 2ac76391..fa9f5335 100644 --- a/crates/sqllogictest/test_files/datetime/weekday.slt +++ b/crates/sqllogictest/test_files/datetime/weekday.slt @@ -60,6 +60,13 @@ SELECT weekday(date_from_unix_date(-2147483648)), weekday(date_from_unix_date(21 ---- 1 4 +# The same two dates as a column, so the function sees an array +query I +SELECT weekday(d) FROM (VALUES (date_from_unix_date(-2147483648)), (date_from_unix_date(2147483647))) AS t(d); +---- +1 +4 + # NULL handling query I SELECT weekday(NULL::DATE); @@ -118,6 +125,34 @@ SELECT weekday(arrow_cast('2024-01-07'::DATE, 'Date64')), weekday(arrow_cast('19 ---- 6 2 +# A Date64 outside the Date32 range still has a day of the week, rather than +# NULL: one day after and one day before the Date32 range, then the last +# millisecond before the epoch, which is on 1969-12-31. DataFusion only, since +# Spark has no Date64. +skipif spark +query I +SELECT weekday(d) FROM (VALUES + (arrow_cast(185542587187200000, 'Date64')), + (arrow_cast(-185542587273600000, 'Date64')), + (arrow_cast(-1, 'Date64')) +) AS t(d); +---- +5 +0 +2 + +skipif spark +query I +SELECT weekday(d) FROM (VALUES + (arrow_cast(arrow_cast(185542587187200000, 'Date64'), 'Dictionary(Int32, Date64)')), + (arrow_cast(arrow_cast(-185542587273600000, 'Date64'), 'Dictionary(Int32, Date64)')), + (arrow_cast(arrow_cast(-1, 'Date64'), 'Dictionary(Int32, Date64)')) +) AS t(d); +---- +5 +0 +2 + # Error: wrong argument type (string without cast) statement error Function 'weekday' requires Date, but received String SELECT weekday('not-a-date');