From faf32ee070f6083ae3b2215ae66a36452dda3066 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 16 Nov 2026 06:30:11 -0700 Subject: [PATCH 1/6] feat(spark): add Spark-compatible make_date Port make_date from DataFusion Comet. Invalid dates return NULL, or raise DATETIME_FIELD_OUT_OF_BOUNDS when ANSI mode is enabled, with the java.time.LocalDate.of messages Spark surfaces. A valid date outside the DATE range raises ARITHMETIC_OVERFLOW in both modes, as in Spark. Closes #149. --- .../spark/src/function/datetime/make_date.rs | 538 ++++++++++++++++++ crates/spark/src/function/datetime/mod.rs | 8 + .../test_files/datetime/make_date.slt | 208 ++++++- 3 files changed, 739 insertions(+), 15 deletions(-) create mode 100644 crates/spark/src/function/datetime/make_date.rs diff --git a/crates/spark/src/function/datetime/make_date.rs b/crates/spark/src/function/datetime/make_date.rs new file mode 100644 index 00000000..98dc1891 --- /dev/null +++ b/crates/spark/src/function/datetime/make_date.rs @@ -0,0 +1,538 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use std::sync::Arc; + +use arrow::array::{Array, Date32Array, Date32Builder, new_empty_array}; +use arrow::datatypes::{DataType, Field, FieldRef}; +use datafusion_common::cast::as_int32_array; +use datafusion_common::types::{NativeType, logical_int32, logical_string}; +use datafusion_common::utils::take_function_args; +use datafusion_common::{Result, ScalarValue, exec_err, internal_err}; +use datafusion_expr::{ + Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, + ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, +}; +use datafusion_macros::user_doc; + +const ARITHMETIC_OVERFLOW_ERROR: &str = r#"[ARITHMETIC_OVERFLOW] overflow. If necessary set "spark.sql.ansi.enabled" to "false" to bypass this error. SQLSTATE: 22003"#; + +/// Spark-compatible `make_date` expression. +/// +/// +#[user_doc( + doc_section(label = "Time and Date Functions"), + description = "Creates a date from `year`, `month` and `day` fields. DataFusion has no built-in \ + equivalent. Years from -999999999 to 999999999 are accepted (year 0 is 1 BC), as long as \ + the date fits the DATE range. When the fields do not form a valid date (for example \ + month 13 or February 29 in a non-leap year), the function returns NULL, or raises \ + DATETIME_FIELD_OUT_OF_BOUNDS when `datafusion.execution.enable_ansi_mode` is true. A \ + valid date outside the DATE range (before -5877641-06-23 or after +5881580-07-11) raises \ + ARITHMETIC_OVERFLOW in both modes, as in Spark. Other numeric and string inputs are cast \ + to INT first, which truncates fractions and raises an error if the value is out of the \ + INT range or is not a valid number. These errors match Spark with ANSI mode enabled. \ + With ANSI mode disabled, Spark wraps an out-of-range BIGINT and returns NULL for a \ + malformed string, while this function still errors (see \ + https://github.com/apache/datafusion-spark/issues/88). Returns NULL if any input is NULL.", + syntax_example = "make_date(year, month, day)", + sql_example = r#"```sql +> SELECT make_date(2013, 7, 15); +2013-07-15 +> SELECT make_date(2019, 7, NULL); +NULL +> SELECT make_date(2023, 2, 29); +NULL +```"#, + argument( + name = "year", + description = "The year, from -999999999 to 999999999, as an INT. Other integer, floating-point, decimal and string inputs are cast to INT. NULL returns NULL." + ), + argument( + name = "month", + description = "The month of the year, from 1 (January) to 12 (December), as an INT. NULL returns NULL." + ), + argument( + name = "day", + description = "The day of the month, from 1 to 31, as an INT. NULL returns NULL." + ) +)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkMakeDate { + signature: Signature, +} + +impl Default for SparkMakeDate { + fn default() -> Self { + Self::new() + } +} + +impl SparkMakeDate { + pub fn new() -> Self { + // Spark declares the arguments as INT and implicitly casts other numeric + // and string types to it. DataFusion types bare integer literals as BIGINT. + let int32 = || { + Coercion::new_implicit( + TypeSignatureClass::Native(logical_int32()), + vec![ + TypeSignatureClass::Integer, + TypeSignatureClass::Float, + TypeSignatureClass::Decimal, + TypeSignatureClass::Native(logical_string()), + ], + NativeType::Int32, + ) + }; + Self { + signature: Signature::coercible( + vec![int32(), int32(), int32()], + Volatility::Immutable, + ), + } + } +} + +impl ScalarUDFImpl for SparkMakeDate { + fn name(&self) -> &str { + "make_date" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + internal_err!("return_field_from_args should be used instead") + } + + fn return_field_from_args(&self, _args: ReturnFieldArgs) -> Result { + // An invalid date returns NULL when ANSI mode is off, which is not known + // at planning time, so the result is always nullable. + Ok(Arc::new(Field::new(self.name(), DataType::Date32, true))) + } + + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + let fail_on_error = args.config_options.execution.enable_ansi_mode; + let number_rows = args.number_rows; + let [year, month, day] = take_function_args(self.name(), args.args)?; + + if let ( + ColumnarValue::Scalar(year), + ColumnarValue::Scalar(month), + ColumnarValue::Scalar(day), + ) = (&year, &month, &day) + { + // Spark does not evaluate the expression when there are no rows, so an + // invalid constant date must not raise an error on an empty batch. + if number_rows == 0 { + return Ok(ColumnarValue::Array(new_empty_array(&DataType::Date32))); + } + let days = match (year, month, day) { + ( + ScalarValue::Int32(Some(y)), + ScalarValue::Int32(Some(m)), + ScalarValue::Int32(Some(d)), + ) => make_date(*y, *m, *d, fail_on_error)?, + ( + ScalarValue::Int32(_) | ScalarValue::Null, + ScalarValue::Int32(_) | ScalarValue::Null, + ScalarValue::Int32(_) | ScalarValue::Null, + ) => None, + _ => { + return internal_err!( + "make_date expects INT arguments, got {}, {}, {}", + year.data_type(), + month.data_type(), + day.data_type() + ); + } + }; + return Ok(ColumnarValue::Scalar(ScalarValue::Date32(days))); + } + + let year = year.into_array(number_rows)?; + let month = month.into_array(number_rows)?; + let day = day.into_array(number_rows)?; + let years = as_int32_array(&year)?; + let months = as_int32_array(&month)?; + let days = as_int32_array(&day)?; + + let mut builder = Date32Builder::with_capacity(number_rows); + for i in 0..number_rows { + if years.is_null(i) || months.is_null(i) || days.is_null(i) { + builder.append_null(); + continue; + } + builder.append_option(make_date( + years.value(i), + months.value(i), + days.value(i), + fail_on_error, + )?); + } + let result: Date32Array = builder.finish(); + Ok(ColumnarValue::Array(Arc::new(result))) + } +} + +/// Computes the DATE (days since 1970-01-01) for `year`, `month` and `day`, as +/// Spark does with `localDateToDays(LocalDate.of(year, month, day))`. +/// +/// Returns `Ok(None)` for an invalid date when `fail_on_error` is false, and an +/// error otherwise. A valid date whose day count does not fit in an INT raises +/// `ARITHMETIC_OVERFLOW` regardless of `fail_on_error`, because Spark only +/// catches the `DateTimeException` from `LocalDate.of` and not the overflow of +/// `Math.toIntExact`. +fn make_date( + year: i32, + month: i32, + day: i32, + fail_on_error: bool, +) -> Result> { + if let Some(message) = invalid_date_message(year, month, day) { + if fail_on_error { + return exec_err!( + "[DATETIME_FIELD_OUT_OF_BOUNDS.WITH_SUGGESTION] {message}. If necessary set \"spark.sql.ansi.enabled\" to \"false\" to bypass this error. SQLSTATE: 22023" + ); + } + return Ok(None); + } + match i32::try_from(days_from_civil(year, month, day)) { + Ok(days) => Ok(Some(days)), + Err(_) => exec_err!("{ARITHMETIC_OVERFLOW_ERROR}"), + } +} + +/// Returns the message `java.time.LocalDate.of` raises for an invalid date, or +/// `None` if the date is valid. The checks run in the same order as in Java: +/// year range, month range, day range, then the day against the month length. +fn invalid_date_message(year: i32, month: i32, day: i32) -> Option { + const MONTH_NAMES: [&str; 12] = [ + "JANUARY", + "FEBRUARY", + "MARCH", + "APRIL", + "MAY", + "JUNE", + "JULY", + "AUGUST", + "SEPTEMBER", + "OCTOBER", + "NOVEMBER", + "DECEMBER", + ]; + if !(-999_999_999..=999_999_999).contains(&year) { + return Some(format!( + "Invalid value for Year (valid values -999999999 - 999999999): {year}" + )); + } + if !(1..=12).contains(&month) { + return Some(format!( + "Invalid value for MonthOfYear (valid values 1 - 12): {month}" + )); + } + if !(1..=31).contains(&day) { + return Some(format!( + "Invalid value for DayOfMonth (valid values 1 - 28/31): {day}" + )); + } + if day > days_in_month(year, month) { + if month == 2 && day == 29 { + return Some(format!( + "Invalid date 'February 29' as '{year}' is not a leap year" + )); + } + return Some(format!( + "Invalid date '{} {day}'", + MONTH_NAMES[(month - 1) as usize] + )); + } + None +} + +/// Proleptic Gregorian leap year rule, as used by `java.time`. +fn is_leap_year(year: i32) -> bool { + year % 4 == 0 && (year % 100 != 0 || year % 400 == 0) +} + +fn days_in_month(year: i32, month: i32) -> i32 { + match month { + 2 if is_leap_year(year) => 29, + 2 => 28, + 4 | 6 | 9 | 11 => 30, + _ => 31, + } +} + +/// Days since 1970-01-01 for a valid proleptic Gregorian date, computed in +/// `i64` so that the full `java.time` year range does not overflow. +/// Based on Howard Hinnant's `days_from_civil` algorithm. +fn days_from_civil(year: i32, month: i32, day: i32) -> i64 { + let (year, month, day) = (i64::from(year), i64::from(month), i64::from(day)); + let year = if month <= 2 { year - 1 } else { year }; + let era = year.div_euclid(400); + let year_of_era = year.rem_euclid(400); + let shifted_month = (month + 9) % 12; // March = 0, ..., February = 11 + let day_of_year = (153 * shifted_month + 2) / 5 + day - 1; + let day_of_era = + year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year; + era * 146_097 + day_of_era - 719_468 +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::Int32Array; + use datafusion_common::config::ConfigOptions; + + fn invoke( + args: Vec, + number_rows: usize, + ansi: bool, + ) -> Result { + let mut config = ConfigOptions::default(); + config.execution.enable_ansi_mode = ansi; + let arg_fields = args + .iter() + .enumerate() + .map(|(i, a)| Arc::new(Field::new(format!("a{i}"), a.data_type(), true))) + .collect(); + SparkMakeDate::new().invoke_with_args(ScalarFunctionArgs { + args, + arg_fields, + number_rows, + return_field: Arc::new(Field::new("make_date", DataType::Date32, true)), + config_options: Arc::new(config), + }) + } + + fn scalar(v: Option) -> ColumnarValue { + ColumnarValue::Scalar(ScalarValue::Int32(v)) + } + + fn array(v: Vec>) -> ColumnarValue { + ColumnarValue::Array(Arc::new(Int32Array::from(v))) + } + + fn ok(y: i32, m: i32, d: i32) -> Option { + make_date(y, m, d, false).unwrap() + } + + #[test] + fn test_make_date_valid() { + assert_eq!(ok(1970, 1, 1), Some(0)); + assert_eq!(ok(1970, 1, 2), Some(1)); + assert_eq!(ok(1969, 12, 31), Some(-1)); + assert_eq!(ok(2000, 2, 29), Some(11_016)); + assert_eq!(ok(2004, 2, 29), Some(12_477)); + assert_eq!(ok(2013, 7, 15), Some(15_901)); + } + + #[test] + fn test_make_date_invalid_month() { + assert_eq!(ok(2023, 0, 15), None); + assert_eq!(ok(2023, 13, 15), None); + assert_eq!(ok(2023, -1, 15), None); + } + + #[test] + fn test_make_date_invalid_day() { + assert_eq!(ok(2023, 6, 0), None); + assert_eq!(ok(2023, 6, 32), None); + assert_eq!(ok(2023, 6, -1), None); + } + + #[test] + fn test_make_date_invalid_dates() { + assert_eq!(ok(2023, 2, 30), None); + assert_eq!(ok(2023, 2, 29), None); + // 1900 is not a leap year (divisible by 100 but not 400) + assert_eq!(ok(1900, 2, 29), None); + assert_eq!(ok(2100, 2, 29), None); + assert_eq!(ok(2023, 4, 31), None); + assert_eq!(ok(1_000_000_000, 1, 1), None); + assert_eq!(ok(-1_000_000_000, 1, 1), None); + } + + #[test] + fn test_make_date_wide_year_range() { + assert_eq!(ok(0, 1, 1), Some(-719_528)); + assert_eq!(ok(-1, 1, 1), Some(-719_893)); + assert_eq!(ok(300_000, 6, 15), Some(108_853_388)); + assert_eq!(ok(300_000, 2, 29), Some(108_853_281)); + assert_eq!(ok(5_881_580, 7, 11), Some(i32::MAX)); + assert_eq!(ok(-5_877_641, 6, 23), Some(i32::MIN)); + } + + #[test] + fn test_make_date_overflow_errors_in_both_modes() { + for ansi in [false, true] { + for (y, m, d) in [ + (5_881_580, 7, 12), + (-5_877_641, 6, 22), + (999_999_999, 12, 31), + ] { + let err = make_date(y, m, d, ansi).unwrap_err().to_string(); + assert!(err.contains("[ARITHMETIC_OVERFLOW]"), "{err}"); + } + } + } + + #[test] + fn test_make_date_ansi_messages() { + let cases = [ + ( + 2023, + 2, + 29, + "Invalid date 'February 29' as '2023' is not a leap year", + ), + (2023, 4, 31, "Invalid date 'APRIL 31'"), + ( + 2023, + 13, + 40, + "Invalid value for MonthOfYear (valid values 1 - 12): 13", + ), + ( + 2023, + 6, + 0, + "Invalid value for DayOfMonth (valid values 1 - 28/31): 0", + ), + ( + 1_000_000_000, + 13, + 40, + "Invalid value for Year (valid values -999999999 - 999999999): 1000000000", + ), + ]; + for (y, m, d, message) in cases { + let err = make_date(y, m, d, true).unwrap_err().to_string(); + assert!( + err.contains(&format!( + "[DATETIME_FIELD_OUT_OF_BOUNDS.WITH_SUGGESTION] {message}. If necessary" + )), + "{err}" + ); + assert!(err.ends_with("SQLSTATE: 22023"), "{err}"); + } + } + + #[test] + fn test_make_date_scalar() -> Result<()> { + let result = invoke( + vec![scalar(Some(2013)), scalar(Some(7)), scalar(Some(15))], + 1, + false, + )?; + let ColumnarValue::Scalar(ScalarValue::Date32(Some(15_901))) = result else { + panic!("unexpected {result:?}"); + }; + let result = invoke( + vec![scalar(Some(2023)), scalar(Some(2)), scalar(Some(29))], + 1, + false, + )?; + let ColumnarValue::Scalar(ScalarValue::Date32(None)) = result else { + panic!("unexpected {result:?}"); + }; + let result = invoke( + vec![ + scalar(Some(2019)), + scalar(Some(7)), + ColumnarValue::Scalar(ScalarValue::Null), + ], + 1, + true, + )?; + let ColumnarValue::Scalar(ScalarValue::Date32(None)) = result else { + panic!("unexpected {result:?}"); + }; + assert!( + invoke( + vec![scalar(Some(2023)), scalar(Some(2)), scalar(Some(29))], + 1, + true + ) + .is_err() + ); + Ok(()) + } + + #[test] + fn test_make_date_scalar_empty_batch_does_not_validate() -> Result<()> { + let result = invoke( + vec![scalar(Some(2024)), scalar(Some(13)), scalar(Some(1))], + 0, + true, + )?; + let ColumnarValue::Array(array) = result else { + panic!("expected an empty array"); + }; + assert_eq!(array.data_type(), &DataType::Date32); + assert!(array.is_empty()); + Ok(()) + } + + #[test] + fn test_make_date_array() -> Result<()> { + let years = array(vec![Some(2024), Some(2023), None, Some(2020), Some(2023)]); + let result = invoke( + vec![ + years, + scalar(Some(2)), + array(vec![Some(29), Some(29), Some(1), Some(29), None]), + ], + 5, + false, + )? + .into_array(5)?; + let result = result.as_any().downcast_ref::().unwrap(); + assert_eq!( + result.iter().collect::>(), + vec![Some(19_782), None, None, Some(18_321), None] + ); + Ok(()) + } + + #[test] + fn test_make_date_array_ansi() -> Result<()> { + // NULL inputs are skipped before validation, so an invalid month next to a + // NULL year does not raise an error. + let result = invoke( + vec![ + array(vec![Some(2024), None]), + array(vec![Some(1), Some(13)]), + scalar(Some(31)), + ], + 2, + true, + )? + .into_array(2)?; + let result = result.as_any().downcast_ref::().unwrap(); + assert_eq!(result.iter().collect::>(), vec![Some(19_753), None]); + + let err = invoke( + vec![ + array(vec![Some(2024), Some(2023)]), + scalar(Some(2)), + scalar(Some(29)), + ], + 2, + true, + ) + .unwrap_err() + .to_string(); + assert!( + err.contains("Invalid date 'February 29' as '2023'"), + "{err}" + ); + Ok(()) + } +} diff --git a/crates/spark/src/function/datetime/mod.rs b/crates/spark/src/function/datetime/mod.rs index bf6a77a8..c3727215 100644 --- a/crates/spark/src/function/datetime/mod.rs +++ b/crates/spark/src/function/datetime/mod.rs @@ -17,6 +17,7 @@ pub mod dayname; pub mod extract; pub mod from_utc_timestamp; pub mod last_day; +pub mod make_date; pub mod make_dt_interval; pub mod make_interval; pub mod monthname; @@ -51,6 +52,7 @@ make_udf_function!(extract::SparkHour, hour); make_udf_function!(extract::SparkMinute, minute); make_udf_function!(extract::SparkSecond, second); make_udf_function!(last_day::SparkLastDay, last_day); +make_udf_function!(make_date::SparkMakeDate, make_date); make_udf_function!(make_dt_interval::SparkMakeDtInterval, make_dt_interval); make_udf_function!(make_interval::SparkMakeInterval, make_interval); make_udf_function!(monthname::SparkMonthName, monthname); @@ -110,6 +112,11 @@ pub mod expr_fn { "Returns the last day of the month which the date belongs to.", arg1 )); + export_functions!(( + make_date, + "Creates a date from `year`, `month` and `day` fields.", + year month day + )); export_functions!(( make_dt_interval, "Make a day time interval from given days, hours, mins and secs (return type is actually a Duration(Microsecond))", @@ -217,6 +224,7 @@ pub fn functions() -> Vec> { from_utc_timestamp(), hour(), last_day(), + make_date(), make_dt_interval(), make_interval(), minute(), diff --git a/crates/sqllogictest/test_files/datetime/make_date.slt b/crates/sqllogictest/test_files/datetime/make_date.slt index 683e9fa7..c20d90e1 100644 --- a/crates/sqllogictest/test_files/datetime/make_date.slt +++ b/crates/sqllogictest/test_files/datetime/make_date.slt @@ -2,18 +2,196 @@ # SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership # SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -# This file was originally created by a porting script from: -# https://github.com/lakehq/sail/tree/43b6ed8221de5c4c4adbedbb267ae1351158b43c/crates/sail-spark-connect/tests/gold_data/function -# This file is part of the implementation of the datafusion-spark function library. -# For more information, please see: -# https://github.com/apache/datafusion/issues/15914 - -## Original Query: SELECT make_date(2013, 7, 15); -## PySpark 3.5.5 Result: {'make_date(2013, 7, 15)': datetime.date(2013, 7, 15), 'typeof(make_date(2013, 7, 15))': 'date', 'typeof(2013)': 'int', 'typeof(7)': 'int', 'typeof(15)': 'int'} -#query -#SELECT make_date(2013::int, 7::int, 15::int); - -## Original Query: SELECT make_date(2019, 7, NULL); -## PySpark 3.5.5 Result: {'make_date(2019, 7, NULL)': None, 'typeof(make_date(2019, 7, NULL))': 'date', 'typeof(2019)': 'int', 'typeof(7)': 'int', 'typeof(NULL)': 'void'} -#query -#SELECT make_date(2019::int, 7::int, NULL::void); +# Tests for Spark-compatible make_date +# Reference: https://spark.apache.org/docs/latest/api/sql/index.html#make_date +# Expected values verified with pyspark==4.2.0, with spark.sql.ansi.enabled both +# false and true. +# Known divergences are captured as commented-out queries below. + +query D +SELECT make_date(2013::int, 7::int, 15::int); +---- +2013-07-15 + +query T +SELECT arrow_typeof(make_date(2013::int, 7::int, 15::int)); +---- +Date32 + +# NULL in any argument returns NULL +query DDDD +SELECT make_date(2019::int, 7::int, NULL), make_date(NULL, 7::int, 1::int), + make_date(2019::int, NULL::int, 1::int), make_date(2024::int, 13::int, NULL); +---- +NULL NULL NULL NULL + +# Leap years follow the proleptic Gregorian rules +query DDD +SELECT make_date(2024, 2, 29), make_date(2000, 2, 29), make_date(1970, 1, 1); +---- +2024-02-29 2000-02-29 1970-01-01 + +# Invalid dates return NULL when ANSI mode is off +query DDDDDDDDDD +SELECT make_date(2023, 2, 29), make_date(1900, 2, 29), make_date(2023, 2, 30), + make_date(2023, 4, 31), make_date(2023, 13, 1), make_date(2023, 0, 1), + make_date(2023, 6, 0), make_date(2023, 6, 32), make_date(1000000000, 1, 1), + make_date(-1000000000, 1, 1); +---- +NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL + +# Years outside 0001-9999 are accepted; year 0 is 1 BC +query IIIIII +SELECT unix_date(make_date(0, 1, 1)), unix_date(make_date(-1, 1, 1)), + unix_date(make_date(300000, 6, 15)), unix_date(make_date(300000, 2, 29)), + unix_date(make_date(5881580, 7, 11)), unix_date(make_date(-5877641, 6, 23)); +---- +-719528 -719893 108853388 108853281 2147483647 -2147483648 + +# A valid date outside the DATE range raises ARITHMETIC_OVERFLOW in both modes +query error \[ARITHMETIC_OVERFLOW\] overflow +SELECT make_date(5881580, 7, 12); + +query error \[ARITHMETIC_OVERFLOW\] overflow +SELECT make_date(-5877641, 6, 22); + +query error \[ARITHMETIC_OVERFLOW\] overflow +SELECT make_date(999999999, 12, 31); + +# Array inputs +statement ok +CREATE TABLE ymd AS VALUES + (2024, 1, 31), + (2023, 2, 29), + (NULL, 1, 1), + (2020, 2, 29), + (2024, NULL, 1), + (2024, 13, 1); + +query D +SELECT make_date(column1::int, column2::int, column3::int) FROM ymd; +---- +2024-01-31 +NULL +NULL +2020-02-29 +NULL +NULL + +# Mixed array and scalar inputs +query D +SELECT make_date(column1::int, 2, 29) FROM ymd; +---- +2024-02-29 +NULL +NULL +2020-02-29 +2024-02-29 +2024-02-29 + +# Other integer types are cast to INT +query D +SELECT make_date(arrow_cast(24, 'Int8'), arrow_cast(1, 'Int16'), arrow_cast(1, 'Int64')); +---- +0024-01-01 + +# Floating-point and decimal inputs are truncated toward zero +query DD +SELECT make_date(2024.7::DOUBLE, 1.9::DOUBLE, 1) AS d1, make_date(2024.7::DECIMAL(5, 1), 1.9::DECIMAL(2, 1), 1) AS d2; +---- +2024-01-01 2024-01-01 + +# Strings are cast to INT +query D +SELECT make_date('2024', '2', '3'); +---- +2024-02-03 + +# Spark 4.2.0 returns NULL for a malformed string when ANSI mode is off; DataFusion errors +# https://github.com/apache/datafusion-spark/issues/88 +# query D +# SELECT make_date('abc', '2', '3'); +# ---- +# NULL + +# Spark 4.2.0 wraps an out-of-range BIGINT to INT when ANSI mode is off (giving an +# invalid year and so NULL); DataFusion errors +# https://github.com/apache/datafusion-spark/issues/88 +# query D +# SELECT make_date(3000000000, 1, 1); +# ---- +# NULL + +# BOOLEAN is not accepted +query error +SELECT make_date(true, 1, 1); + +# A constant invalid date is not evaluated when there are no rows +statement ok +set datafusion.execution.enable_ansi_mode = true; + +query D +SELECT make_date(2024, 13, 1) FROM ymd WHERE column1 < 0; +---- + +statement ok +set datafusion.execution.enable_ansi_mode = false; + +# Invalid dates raise DATETIME_FIELD_OUT_OF_BOUNDS under ANSI mode, with the +# message from java.time.LocalDate.of +statement ok +set datafusion.execution.enable_ansi_mode = true; + +query D +SELECT make_date(2024, 2, 29); +---- +2024-02-29 + +query D +SELECT make_date(2019, 7, NULL); +---- +NULL + +query error \[DATETIME_FIELD_OUT_OF_BOUNDS\.WITH_SUGGESTION\] Invalid date 'February 29' as '2023' is not a leap year\. If necessary set "spark\.sql\.ansi\.enabled" to "false" to bypass this error\. SQLSTATE: 22023 +SELECT make_date(2023, 2, 29); + +query error \[DATETIME_FIELD_OUT_OF_BOUNDS\.WITH_SUGGESTION\] Invalid date 'APRIL 31'\. +SELECT make_date(2023, 4, 31); + +query error \[DATETIME_FIELD_OUT_OF_BOUNDS\.WITH_SUGGESTION\] Invalid date 'FEBRUARY 30'\. +SELECT make_date(2023, 2, 30); + +query error \[DATETIME_FIELD_OUT_OF_BOUNDS\.WITH_SUGGESTION\] Invalid value for MonthOfYear \(valid values 1 - 12\): 13\. +SELECT make_date(2023, 13, 40); + +query error \[DATETIME_FIELD_OUT_OF_BOUNDS\.WITH_SUGGESTION\] Invalid value for DayOfMonth \(valid values 1 - 28/31\): 0\. +SELECT make_date(2023, 6, 0); + +query error \[DATETIME_FIELD_OUT_OF_BOUNDS\.WITH_SUGGESTION\] Invalid value for Year \(valid values -999999999 - 999999999\): 1000000000\. +SELECT make_date(1000000000, 13, 40); + +query error \[DATETIME_FIELD_OUT_OF_BOUNDS\.WITH_SUGGESTION\] Invalid value for Year \(valid values -999999999 - 999999999\): -1000000000\. +SELECT make_date(-1000000000, 1, 1); + +query error \[DATETIME_FIELD_OUT_OF_BOUNDS\.WITH_SUGGESTION\] Invalid date 'February 29' as '2023' is not a leap year +SELECT make_date(column1::int, column2::int, column3::int) FROM ymd; + +query error \[ARITHMETIC_OVERFLOW\] overflow +SELECT make_date(5881580, 7, 12); + +# Rows with a NULL input are not validated +query D +SELECT make_date(column1::int, column2::int, column3::int) FROM ymd WHERE column1 IS NULL OR column2 IS NULL; +---- +NULL +NULL + +# A BIGINT outside the INT range raises an error, as Spark's CAST_OVERFLOW does +query error Can't cast value 3000000000 to type Int32 +SELECT make_date(3000000000, 1, 1); + +statement ok +set datafusion.execution.enable_ansi_mode = false; + +statement ok +DROP TABLE ymd; From 777724a364484d802583c28cb1d9b12bba6da401 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 10:47:14 -0700 Subject: [PATCH 2/6] fix(spark): document make_date's differences from the DataFusion built-in Say how it differs from DataFusion's make_date, which it replaces, and which ANSI-off casts still differ from Spark, with commented-out queries citing #88. Share the INT coercion with date_from_unix_date. Test the literal NULL short-circuit in ANSI mode, and drop an empty-batch query that the unit test already covers. --- .../function/datetime/date_from_unix_date.rs | 25 +++++-------- .../src/function/datetime/date_part_name.rs | 23 ++++++++++-- .../spark/src/function/datetime/make_date.rs | 37 +++++++------------ .../test_files/datetime/make_date.slt | 35 ++++++++++++------ 4 files changed, 67 insertions(+), 53 deletions(-) diff --git a/crates/spark/src/function/datetime/date_from_unix_date.rs b/crates/spark/src/function/datetime/date_from_unix_date.rs index ed29a7da..deed4a21 100644 --- a/crates/spark/src/function/datetime/date_from_unix_date.rs +++ b/crates/spark/src/function/datetime/date_from_unix_date.rs @@ -7,15 +7,16 @@ use std::sync::Arc; use arrow::array::{Array, Date32Array}; use arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::cast::as_int32_array; -use datafusion_common::types::{NativeType, logical_int32, logical_string}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, internal_err}; use datafusion_expr::{ - Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, - ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, + ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, + Signature, Volatility, }; use datafusion_macros::user_doc; +use super::date_part_name::int32_coercion; + /// Spark-compatible `date_from_unix_date` expression. /// /// @@ -56,20 +57,12 @@ impl Default for SparkDateFromUnixDate { impl SparkDateFromUnixDate { pub fn new() -> Self { - // Spark declares the argument as INT and implicitly casts other numeric - // and string types to it. DataFusion types bare integer literals as BIGINT. - let int32 = Coercion::new_implicit( - TypeSignatureClass::Native(logical_int32()), - vec![ - TypeSignatureClass::Integer, - TypeSignatureClass::Float, - TypeSignatureClass::Decimal, - TypeSignatureClass::Native(logical_string()), - ], - NativeType::Int32, - ); + // Spark declares the argument as INT. Self { - signature: Signature::coercible(vec![int32], Volatility::Immutable), + signature: Signature::coercible( + vec![int32_coercion()], + Volatility::Immutable, + ), } } } diff --git a/crates/spark/src/function/datetime/date_part_name.rs b/crates/spark/src/function/datetime/date_part_name.rs index 9d340252..2a008dbc 100644 --- a/crates/spark/src/function/datetime/date_part_name.rs +++ b/crates/spark/src/function/datetime/date_part_name.rs @@ -2,15 +2,15 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -//! Shared code for `dayname` and `monthname`, which map one [`DatePart`] of a -//! date to a name. +//! Shared code for the date functions: the argument coercions, and the +//! [`DatePart`] to name mapping that `dayname` and `monthname` use. use std::sync::Arc; use arrow::array::{AsArray, StringArray}; use arrow::compute::{DatePart, date_part}; use arrow::datatypes::Int32Type; -use datafusion_common::types::{NativeType, logical_date, logical_string}; +use datafusion_common::types::{NativeType, logical_date, logical_int32, logical_string}; use datafusion_common::{Result, ScalarValue}; use datafusion_expr::{ Coercion, ColumnarValue, Signature, TypeSignatureClass, Volatility, @@ -34,6 +34,23 @@ pub(super) fn date_signature() -> Signature { ) } +/// One INT argument. Spark implicitly casts other numeric and string types to +/// INT, and DataFusion types bare integer literals as BIGINT. The cast is +/// DataFusion's, which errors where Spark with ANSI mode off returns NULL or +/// wraps (see ). +pub(super) fn int32_coercion() -> Coercion { + Coercion::new_implicit( + TypeSignatureClass::Native(logical_int32()), + vec![ + TypeSignatureClass::Integer, + TypeSignatureClass::Float, + TypeSignatureClass::Decimal, + TypeSignatureClass::Native(logical_string()), + ], + NativeType::Int32, + ) +} + /// Extracts `part` from each date in `arg` and maps it to a name with `name`. /// NULL dates, and values `name` returns `None` for, give NULL. pub(super) fn date_part_name( diff --git a/crates/spark/src/function/datetime/make_date.rs b/crates/spark/src/function/datetime/make_date.rs index 98dc1891..9f8ad444 100644 --- a/crates/spark/src/function/datetime/make_date.rs +++ b/crates/spark/src/function/datetime/make_date.rs @@ -7,15 +7,16 @@ use std::sync::Arc; use arrow::array::{Array, Date32Array, Date32Builder, new_empty_array}; use arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::cast::as_int32_array; -use datafusion_common::types::{NativeType, logical_int32, logical_string}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, exec_err, internal_err}; use datafusion_expr::{ - Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, - ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, + ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, + Signature, Volatility, }; use datafusion_macros::user_doc; +use super::date_part_name::int32_coercion; + const ARITHMETIC_OVERFLOW_ERROR: &str = r#"[ARITHMETIC_OVERFLOW] overflow. If necessary set "spark.sql.ansi.enabled" to "false" to bypass this error. SQLSTATE: 22003"#; /// Spark-compatible `make_date` expression. @@ -23,8 +24,7 @@ const ARITHMETIC_OVERFLOW_ERROR: &str = r#"[ARITHMETIC_OVERFLOW] overflow. If ne /// #[user_doc( doc_section(label = "Time and Date Functions"), - description = "Creates a date from `year`, `month` and `day` fields. DataFusion has no built-in \ - equivalent. Years from -999999999 to 999999999 are accepted (year 0 is 1 BC), as long as \ + description = "Creates a date from `year`, `month` and `day` fields. Years from -999999999 to 999999999 are accepted (year 0 is 1 BC), as long as \ the date fits the DATE range. When the fields do not form a valid date (for example \ month 13 or February 29 in a non-leap year), the function returns NULL, or raises \ DATETIME_FIELD_OUT_OF_BOUNDS when `datafusion.execution.enable_ansi_mode` is true. A \ @@ -32,9 +32,13 @@ const ARITHMETIC_OVERFLOW_ERROR: &str = r#"[ARITHMETIC_OVERFLOW] overflow. If ne ARITHMETIC_OVERFLOW in both modes, as in Spark. Other numeric and string inputs are cast \ to INT first, which truncates fractions and raises an error if the value is out of the \ INT range or is not a valid number. These errors match Spark with ANSI mode enabled. \ - With ANSI mode disabled, Spark wraps an out-of-range BIGINT and returns NULL for a \ - malformed string, while this function still errors (see \ - https://github.com/apache/datafusion-spark/issues/88). Returns NULL if any input is NULL.", + With ANSI mode disabled, Spark wraps an out-of-range BIGINT, returns NULL for a \ + malformed string, an infinite DOUBLE or a DECIMAL out of the INT range, and casts NaN \ + to 0, while this function still errors (see \ + https://github.com/apache/datafusion-spark/issues/88). Returns NULL if any input is \ + NULL. DataFusion's built-in `make_date` differs: it raises an error for an invalid \ + date regardless of ANSI mode, accepts only integer and string arguments, and is \ + limited to years -262143 to 262142.", syntax_example = "make_date(year, month, day)", sql_example = r#"```sql > SELECT make_date(2013, 7, 15); @@ -70,23 +74,10 @@ impl Default for SparkMakeDate { impl SparkMakeDate { pub fn new() -> Self { - // Spark declares the arguments as INT and implicitly casts other numeric - // and string types to it. DataFusion types bare integer literals as BIGINT. - let int32 = || { - Coercion::new_implicit( - TypeSignatureClass::Native(logical_int32()), - vec![ - TypeSignatureClass::Integer, - TypeSignatureClass::Float, - TypeSignatureClass::Decimal, - TypeSignatureClass::Native(logical_string()), - ], - NativeType::Int32, - ) - }; + // Spark declares the arguments as INT. Self { signature: Signature::coercible( - vec![int32(), int32(), int32()], + vec![int32_coercion(), int32_coercion(), int32_coercion()], Volatility::Immutable, ), } diff --git a/crates/sqllogictest/test_files/datetime/make_date.slt b/crates/sqllogictest/test_files/datetime/make_date.slt index c20d90e1..0aa85018 100644 --- a/crates/sqllogictest/test_files/datetime/make_date.slt +++ b/crates/sqllogictest/test_files/datetime/make_date.slt @@ -122,21 +122,28 @@ SELECT make_date('2024', '2', '3'); # ---- # NULL +# Spark 4.2.0 casts NaN to 0 when ANSI mode is off, and returns NULL for an infinite +# DOUBLE or a DECIMAL out of the INT range; DataFusion errors +# https://github.com/apache/datafusion-spark/issues/88 +# query D +# SELECT make_date(CAST('NaN' AS DOUBLE), 1, 1); +# ---- +# 0000-01-01 +# +# query D +# SELECT make_date(CAST('Infinity' AS DOUBLE), 1, 1); +# ---- +# NULL +# +# query D +# SELECT make_date(CAST(99999999999 AS DECIMAL(20,0)), 1, 1); +# ---- +# NULL + # BOOLEAN is not accepted query error SELECT make_date(true, 1, 1); -# A constant invalid date is not evaluated when there are no rows -statement ok -set datafusion.execution.enable_ansi_mode = true; - -query D -SELECT make_date(2024, 13, 1) FROM ymd WHERE column1 < 0; ----- - -statement ok -set datafusion.execution.enable_ansi_mode = false; - # Invalid dates raise DATETIME_FIELD_OUT_OF_BOUNDS under ANSI mode, with the # message from java.time.LocalDate.of statement ok @@ -152,6 +159,12 @@ SELECT make_date(2019, 7, NULL); ---- NULL +# A NULL argument returns NULL before the other fields are checked +query DD +SELECT make_date(2024, 13, NULL), make_date(NULL, 13, 40); +---- +NULL NULL + query error \[DATETIME_FIELD_OUT_OF_BOUNDS\.WITH_SUGGESTION\] Invalid date 'February 29' as '2023' is not a leap year\. If necessary set "spark\.sql\.ansi\.enabled" to "false" to bypass this error\. SQLSTATE: 22023 SELECT make_date(2023, 2, 29); From 0f6d284f39459e5d2e250b093f8f7e2246f57d6c Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 10:54:31 -0700 Subject: [PATCH 3/6] docs(spark): link the shared input-type coercion issue from int32_coercion --- crates/spark/src/function/datetime/date_part_name.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/spark/src/function/datetime/date_part_name.rs b/crates/spark/src/function/datetime/date_part_name.rs index 2a008dbc..a593b3d1 100644 --- a/crates/spark/src/function/datetime/date_part_name.rs +++ b/crates/spark/src/function/datetime/date_part_name.rs @@ -37,7 +37,9 @@ pub(super) fn date_signature() -> Signature { /// One INT argument. Spark implicitly casts other numeric and string types to /// INT, and DataFusion types bare integer literals as BIGINT. The cast is /// DataFusion's, which errors where Spark with ANSI mode off returns NULL or -/// wraps (see ). +/// wraps (see ). A +/// stopgap until input types are declared and coerced in one place +/// (). pub(super) fn int32_coercion() -> Coercion { Coercion::new_implicit( TypeSignatureClass::Native(logical_int32()), From 98e1f480e5444c9ddf550bf8644e073fe2fb74d2 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 16:53:30 -0700 Subject: [PATCH 4/6] docs(spark): correct make_date's ANSI-off DECIMAL and DOUBLE cast divergences --- .../src/function/datetime/date_part_name.rs | 4 ++-- .../spark/src/function/datetime/make_date.rs | 4 ++-- .../test_files/datetime/make_date.slt | 18 ++++++++++++++++-- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/crates/spark/src/function/datetime/date_part_name.rs b/crates/spark/src/function/datetime/date_part_name.rs index a593b3d1..3bc4c085 100644 --- a/crates/spark/src/function/datetime/date_part_name.rs +++ b/crates/spark/src/function/datetime/date_part_name.rs @@ -36,8 +36,8 @@ pub(super) fn date_signature() -> Signature { /// One INT argument. Spark implicitly casts other numeric and string types to /// INT, and DataFusion types bare integer literals as BIGINT. The cast is -/// DataFusion's, which errors where Spark with ANSI mode off returns NULL or -/// wraps (see ). A +/// DataFusion's, which errors where Spark with ANSI mode off returns NULL, +/// wraps, or saturates (see ). A /// stopgap until input types are declared and coerced in one place /// (). pub(super) fn int32_coercion() -> Coercion { diff --git a/crates/spark/src/function/datetime/make_date.rs b/crates/spark/src/function/datetime/make_date.rs index 9f8ad444..695fedc4 100644 --- a/crates/spark/src/function/datetime/make_date.rs +++ b/crates/spark/src/function/datetime/make_date.rs @@ -32,8 +32,8 @@ const ARITHMETIC_OVERFLOW_ERROR: &str = r#"[ARITHMETIC_OVERFLOW] overflow. If ne ARITHMETIC_OVERFLOW in both modes, as in Spark. Other numeric and string inputs are cast \ to INT first, which truncates fractions and raises an error if the value is out of the \ INT range or is not a valid number. These errors match Spark with ANSI mode enabled. \ - With ANSI mode disabled, Spark wraps an out-of-range BIGINT, returns NULL for a \ - malformed string, an infinite DOUBLE or a DECIMAL out of the INT range, and casts NaN \ + With ANSI mode disabled, Spark wraps an out-of-range BIGINT or DECIMAL, returns NULL \ + for a malformed string or a floating-point value out of the INT range, and casts NaN \ to 0, while this function still errors (see \ https://github.com/apache/datafusion-spark/issues/88). Returns NULL if any input is \ NULL. DataFusion's built-in `make_date` differs: it raises an error for an invalid \ diff --git a/crates/sqllogictest/test_files/datetime/make_date.slt b/crates/sqllogictest/test_files/datetime/make_date.slt index 0aa85018..4931fb48 100644 --- a/crates/sqllogictest/test_files/datetime/make_date.slt +++ b/crates/sqllogictest/test_files/datetime/make_date.slt @@ -122,8 +122,8 @@ SELECT make_date('2024', '2', '3'); # ---- # NULL -# Spark 4.2.0 casts NaN to 0 when ANSI mode is off, and returns NULL for an infinite -# DOUBLE or a DECIMAL out of the INT range; DataFusion errors +# When ANSI mode is off, Spark 4.2.0 casts NaN to 0, and saturates a DOUBLE out of the +# INT range to 2147483647 (never a valid year, so NULL); DataFusion errors # https://github.com/apache/datafusion-spark/issues/88 # query D # SELECT make_date(CAST('NaN' AS DOUBLE), 1, 1); @@ -136,6 +136,20 @@ SELECT make_date('2024', '2', '3'); # NULL # # query D +# SELECT make_date(CAST(1e10 AS DOUBLE), 1, 1); +# ---- +# NULL + +# When ANSI mode is off, Spark 4.2.0 wraps an out-of-range DECIMAL to INT like a BIGINT +# (toLong.toInt): 4294969320 wraps to 2024, while 99999999999 wraps to 1215752191, an +# invalid year; DataFusion errors +# https://github.com/apache/datafusion-spark/issues/88 +# query D +# SELECT make_date(CAST(4294969320 AS DECIMAL(20,0)), 1, 1); +# ---- +# 2024-01-01 +# +# query D # SELECT make_date(CAST(99999999999 AS DECIMAL(20,0)), 1, 1); # ---- # NULL From 04199a2c3b2b6c9b9e1f79600ae570619a9210ef Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 19 Nov 2026 05:36:42 -0700 Subject: [PATCH 5/6] docs(spark): note make_date truncates fractional strings when ANSI is off Spark with ANSI mode off truncates a string with a fractional part when casting it to INT, while this function errors. Record it in the make_date docs and slt, and make the int32_coercion comment general instead of listing each case. --- crates/spark/src/function/datetime/date_part_name.rs | 4 ++-- crates/spark/src/function/datetime/make_date.rs | 6 +++--- crates/sqllogictest/test_files/datetime/make_date.slt | 7 +++++++ 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/crates/spark/src/function/datetime/date_part_name.rs b/crates/spark/src/function/datetime/date_part_name.rs index 3bc4c085..3032336f 100644 --- a/crates/spark/src/function/datetime/date_part_name.rs +++ b/crates/spark/src/function/datetime/date_part_name.rs @@ -36,8 +36,8 @@ pub(super) fn date_signature() -> Signature { /// One INT argument. Spark implicitly casts other numeric and string types to /// INT, and DataFusion types bare integer literals as BIGINT. The cast is -/// DataFusion's, which errors where Spark with ANSI mode off returns NULL, -/// wraps, or saturates (see ). A +/// DataFusion's, which errors in cases where Spark with ANSI mode off returns a +/// value or NULL (see ). A /// stopgap until input types are declared and coerced in one place /// (). pub(super) fn int32_coercion() -> Coercion { diff --git a/crates/spark/src/function/datetime/make_date.rs b/crates/spark/src/function/datetime/make_date.rs index 695fedc4..d8e020cb 100644 --- a/crates/spark/src/function/datetime/make_date.rs +++ b/crates/spark/src/function/datetime/make_date.rs @@ -32,9 +32,9 @@ const ARITHMETIC_OVERFLOW_ERROR: &str = r#"[ARITHMETIC_OVERFLOW] overflow. If ne ARITHMETIC_OVERFLOW in both modes, as in Spark. Other numeric and string inputs are cast \ to INT first, which truncates fractions and raises an error if the value is out of the \ INT range or is not a valid number. These errors match Spark with ANSI mode enabled. \ - With ANSI mode disabled, Spark wraps an out-of-range BIGINT or DECIMAL, returns NULL \ - for a malformed string or a floating-point value out of the INT range, and casts NaN \ - to 0, while this function still errors (see \ + With ANSI mode disabled, Spark wraps an out-of-range BIGINT or DECIMAL, truncates a \ + string with a fractional part, returns NULL for a malformed string or a floating-point \ + value out of the INT range, and casts NaN to 0, while this function still errors (see \ https://github.com/apache/datafusion-spark/issues/88). Returns NULL if any input is \ NULL. DataFusion's built-in `make_date` differs: it raises an error for an invalid \ date regardless of ANSI mode, accepts only integer and string arguments, and is \ diff --git a/crates/sqllogictest/test_files/datetime/make_date.slt b/crates/sqllogictest/test_files/datetime/make_date.slt index 4931fb48..9dfa3a78 100644 --- a/crates/sqllogictest/test_files/datetime/make_date.slt +++ b/crates/sqllogictest/test_files/datetime/make_date.slt @@ -114,6 +114,13 @@ SELECT make_date('2024', '2', '3'); # ---- # NULL +# Spark 4.2.0 truncates a string with a fractional part when ANSI mode is off; DataFusion errors +# https://github.com/apache/datafusion-spark/issues/88 +# query D +# SELECT make_date('2024.7', '1', '1'); +# ---- +# 2024-01-01 + # Spark 4.2.0 wraps an out-of-range BIGINT to INT when ANSI mode is off (giving an # invalid year and so NULL); DataFusion errors # https://github.com/apache/datafusion-spark/issues/88 From 1815b8b32126f976cf67c1888cf2862cba7001bc Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 19 Nov 2026 05:39:42 -0700 Subject: [PATCH 6/6] test(spark): mark make_date's Spark differences with xfail comments Turn the commented-out ANSI-off cast records into live records that assert DataFusion's current error, each with a # xfail spark: marker that links #88, so the PySpark runner reports them and flags when they start matching. Skip the unix_date record in evaluate mode, as unix.slt does, until #306 is fixed. --- .../test_files/datetime/make_date.slt | 89 ++++++++----------- 1 file changed, 36 insertions(+), 53 deletions(-) diff --git a/crates/sqllogictest/test_files/datetime/make_date.slt b/crates/sqllogictest/test_files/datetime/make_date.slt index 9dfa3a78..c8a4fe34 100644 --- a/crates/sqllogictest/test_files/datetime/make_date.slt +++ b/crates/sqllogictest/test_files/datetime/make_date.slt @@ -41,6 +41,9 @@ SELECT make_date(2023, 2, 29), make_date(1900, 2, 29), make_date(2023, 2, 30), NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL # Years outside 0001-9999 are accepted; year 0 is 1 BC +# `unix_date` has no `invoke_with_args` yet, so this is skipped in evaluate mode +# (https://github.com/apache/datafusion-spark/issues/306). +skipif evaluate query IIIIII SELECT unix_date(make_date(0, 1, 1)), unix_date(make_date(-1, 1, 1)), unix_date(make_date(300000, 6, 15)), unix_date(make_date(300000, 2, 29)), @@ -107,59 +110,39 @@ SELECT make_date('2024', '2', '3'); ---- 2024-02-03 -# Spark 4.2.0 returns NULL for a malformed string when ANSI mode is off; DataFusion errors -# https://github.com/apache/datafusion-spark/issues/88 -# query D -# SELECT make_date('abc', '2', '3'); -# ---- -# NULL - -# Spark 4.2.0 truncates a string with a fractional part when ANSI mode is off; DataFusion errors -# https://github.com/apache/datafusion-spark/issues/88 -# query D -# SELECT make_date('2024.7', '1', '1'); -# ---- -# 2024-01-01 - -# Spark 4.2.0 wraps an out-of-range BIGINT to INT when ANSI mode is off (giving an -# invalid year and so NULL); DataFusion errors -# https://github.com/apache/datafusion-spark/issues/88 -# query D -# SELECT make_date(3000000000, 1, 1); -# ---- -# NULL - -# When ANSI mode is off, Spark 4.2.0 casts NaN to 0, and saturates a DOUBLE out of the -# INT range to 2147483647 (never a valid year, so NULL); DataFusion errors -# https://github.com/apache/datafusion-spark/issues/88 -# query D -# SELECT make_date(CAST('NaN' AS DOUBLE), 1, 1); -# ---- -# 0000-01-01 -# -# query D -# SELECT make_date(CAST('Infinity' AS DOUBLE), 1, 1); -# ---- -# NULL -# -# query D -# SELECT make_date(CAST(1e10 AS DOUBLE), 1, 1); -# ---- -# NULL - -# When ANSI mode is off, Spark 4.2.0 wraps an out-of-range DECIMAL to INT like a BIGINT -# (toLong.toInt): 4294969320 wraps to 2024, while 99999999999 wraps to 1215752191, an -# invalid year; DataFusion errors -# https://github.com/apache/datafusion-spark/issues/88 -# query D -# SELECT make_date(CAST(4294969320 AS DECIMAL(20,0)), 1, 1); -# ---- -# 2024-01-01 -# -# query D -# SELECT make_date(CAST(99999999999 AS DECIMAL(20,0)), 1, 1); -# ---- -# NULL +# With ANSI mode off, Spark's cast to INT returns a value or NULL where DataFusion's errors + +# xfail spark: Spark returns NULL for a malformed string, DataFusion's cast errors (https://github.com/apache/datafusion-spark/issues/88) +query error Cannot cast string 'abc' to value of Int32 type +SELECT make_date('abc', '2', '3'); + +# xfail spark: Spark truncates a string with a fractional part, giving 2024-01-01, DataFusion's cast errors (https://github.com/apache/datafusion-spark/issues/88) +query error Cannot cast string '2024\.7' to value of Int32 type +SELECT make_date('2024.7', '1', '1'); + +# xfail spark: Spark wraps an out-of-range BIGINT to INT, an invalid year, so NULL, DataFusion's cast errors (https://github.com/apache/datafusion-spark/issues/88) +query error Can't cast value 3000000000 to type Int32 +SELECT make_date(3000000000, 1, 1); + +# xfail spark: Spark casts NaN to 0, giving 0000-01-01, DataFusion's cast errors (https://github.com/apache/datafusion-spark/issues/88) +query error Can't cast value NaN to type Int32 +SELECT CAST(make_date(CAST('NaN' AS DOUBLE), 1, 1) AS STRING); + +# xfail spark: Spark saturates an out-of-range DOUBLE to 2147483647, an invalid year, so NULL, DataFusion's cast errors (https://github.com/apache/datafusion-spark/issues/88) +query error Can't cast value inf to type Int32 +SELECT make_date(CAST('Infinity' AS DOUBLE), 1, 1); + +# xfail spark: Spark saturates an out-of-range DOUBLE to 2147483647, an invalid year, so NULL, DataFusion's cast errors (https://github.com/apache/datafusion-spark/issues/88) +query error Can't cast value 10000000000\.0 to type Int32 +SELECT make_date(CAST(1e10 AS DOUBLE), 1, 1); + +# xfail spark: Spark wraps an out-of-range DECIMAL like a BIGINT (toLong.toInt), so 4294969320 gives 2024-01-01, DataFusion's cast errors (https://github.com/apache/datafusion-spark/issues/88) +query error value of 4294969320 is out of range Int32 +SELECT make_date(CAST(4294969320 AS DECIMAL(20,0)), 1, 1); + +# xfail spark: Spark wraps an out-of-range DECIMAL like a BIGINT (toLong.toInt), to 1215752191, an invalid year, so NULL, DataFusion's cast errors (https://github.com/apache/datafusion-spark/issues/88) +query error value of 99999999999 is out of range Int32 +SELECT make_date(CAST(99999999999 AS DECIMAL(20,0)), 1, 1); # BOOLEAN is not accepted query error