diff --git a/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md b/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md index 323de0dcd30..c2f550315b5 100644 --- a/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md +++ b/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md @@ -148,7 +148,7 @@ does. The following forms are supported: - `Z`, and `UTC`, `GMT` or `UT` on their own or followed by an offset (e.g. `UTC`, `GMT-01:00`, `UTC+8`). - Offsets in the forms `+H`, `+HH`, `+HHMM`, `+HHMMSS`, `+H:M`, `+HH:MM` and `+H:MM:SS` - (e.g. `+07:30`, `-0800`, `+080000`, `+08:00:00`). + (e.g. `+07:30`, `-0800`, `+080000`, `+08:00:00`, `+08:00:30`, `+080030`). - The Java `ZoneId.SHORT_IDS` aliases `ACT`, `AET`, `AGT`, `ART`, `AST`, `BET`, `BST`, `CAT`, `CNT`, `CST`, `CTT`, `EAT`, `ECT`, `IET`, `IST`, `JST`, `MIT`, `NET`, `NST`, `PLT`, `PNT`, `PRT`, `PST`, `SST` and `VST`, which map to a region and follow its daylight saving time, and @@ -156,9 +156,7 @@ does. The following forms are supported: - Time zone database region ids, with or without a `/` and with or without a space before them (e.g. `Europe/Moscow`, `America/Los_Angeles`, `CET`, `Zulu`, `Etc/GMT+8`). -Zone ids are case sensitive, as in Spark. Offsets with a non-zero seconds part (e.g. `+08:00:30`) -are not supported: Comet returns `NULL`, or raises `CAST_INVALID_INPUT` in ANSI mode, where Spark -returns a value. See [#7224](https://github.com/apache/datafusion-comet/issues/7224). +Zone ids are case sensitive, as in Spark. ## String to TimestampNTZ diff --git a/native/spark-expr/src/conversion_funcs/string.rs b/native/spark-expr/src/conversion_funcs/string.rs index 18ad2e6d5d5..ebed1082d17 100644 --- a/native/spark-expr/src/conversion_funcs/string.rs +++ b/native/spark-expr/src/conversion_funcs/string.rs @@ -1624,15 +1624,7 @@ fn timestamp_parser( // padding is accepted there. // Check the prefix, not the base patterns: a zone suffix can hide a time-only match. if trimmed.is_empty() || (is_spark4_plus && start > 0 && trimmed.starts_with('T')) { - return if eval_mode == EvalMode::Ansi { - Err(SparkError::InvalidInputInCastToDatetime { - value: value.to_string(), - from_type: "STRING".to_string(), - to_type: "TIMESTAMP".to_string(), - }) - } else { - Ok(None) - }; + return invalid_timestamp(value, eval_mode); } let value = trimmed; @@ -1644,7 +1636,7 @@ fn timestamp_parser( // base pattern. This prevents the '-' in plain date strings like "2015-03-18" // from being misidentified as a negative-offset sign. if pattern.is_none() { - if let Some((stripped, suffix_tz)) = extract_offset_suffix(value) { + if let Some((stripped, suffix_tz, residual_secs)) = extract_offset_suffix(value) { // Spark applies Java String.trim to the zone, not Unicode whitespace trimming. let stripped = stripped.trim_end_matches(|c: char| c <= '\u{20}'); // A zone suffix is only meaningful after the seconds segment. Otherwise fall @@ -1652,7 +1644,18 @@ fn timestamp_parser( // reported as malformed (null, or CAST_INVALID_INPUT under ANSI) like Spark does. let stripped_pattern = classify_timestamp_pattern(stripped); if stripped_pattern.is_some_and(TimestampPattern::ends_with_seconds_segment) { - return parse_timestamp_pattern(stripped, stripped_pattern, eval_mode, &suffix_tz); + let parsed = + parse_timestamp_pattern(stripped, stripped_pattern, eval_mode, &suffix_tz)?; + // Arrow's `Tz` holds whole minutes, so an offset such as `+08:00:30` arrives as + // the `+08:00` zone plus a 30-second residual. Subtract the residual to land on + // the exact instant Spark computes. An instant pushed below the representable + // range is out of range, which Spark reports like any other invalid value. + let adjusted = parsed + .and_then(|micros| micros.checked_sub(i64::from(residual_secs) * 1_000_000)); + return match adjusted { + Some(micros) => Ok(Some(micros)), + None => invalid_timestamp(value, eval_mode), + }; } } } @@ -1741,12 +1744,6 @@ fn parse_sign_offset(s: &str) -> Option { /// Constructs a [`Tz`] from an offset measured in seconds. /// E.g. `+7*3600 + 30*60` -> `"+07:30"`. fn tz_from_offset_secs(secs: i32) -> Option { - // `Tz` offsets have minute precision, so an offset with a non-zero seconds part such as - // "+08:00:30" cannot be represented. Returning `None` leaves the value unparsed, while - // Spark accepts it: https://github.com/apache/datafusion-comet/issues/7224 - if secs % 60 != 0 { - return None; - } let abs = secs.abs(); let h = abs / 3600; let m = (abs % 3600) / 60; @@ -1754,6 +1751,14 @@ fn tz_from_offset_secs(secs: i32) -> Option { Tz::from_str(&format!("{}{:02}:{:02}", sign, h, m)).ok() } +/// Splits an offset into the minute-precision [`Tz`] arrow can represent and the residual +/// seconds it cannot carry. `tz_from_offset_secs` truncates toward zero, so the residual keeps +/// the sign of the offset and applying both parts in turn reproduces the exact offset. +fn tz_and_residual_secs(secs: i32) -> Option<(Tz, i32)> { + let tz = tz_from_offset_secs(secs)?; + Some((tz, secs - (secs / 60) * 60)) +} + /// `java.time.ZoneId.SHORT_IDS`, which Spark passes to `ZoneId.of` when it parses the zone /// of a timestamp string. const JAVA_SHORT_IDS: &[(&str, &str)] = &[ @@ -1832,9 +1837,13 @@ fn extract_zone_id_suffix(value: &str) -> Option<(&str, Tz)> { }) } -/// If `value` ends with a recognised timezone suffix, returns `(datetime_prefix, Tz)`. +/// If `value` ends with a recognised timezone suffix, returns +/// `(datetime_prefix, Tz, residual_seconds)`. /// Returns `None` when no suffix is found. /// +/// The [`Tz`] carries only whole minutes, so a suffix such as `+08:00:30` is returned as the +/// `+08:00` zone plus a 30-second residual the caller subtracts after parsing. +/// /// Recognised forms (in matching priority order): /// Z -> UTC+0 /// UTC / " UTC" -> UTC+0 (or UTC +/- offset, e.g. "UTC+0", " UTC+07:30") @@ -1847,10 +1856,10 @@ fn extract_zone_id_suffix(value: &str) -> Option<(&str, Tz)> { /// /// **The caller must ensure the value does not already match a base timestamp pattern.** /// Without that guard a bare '-' in "2015-03-18" would be misread as a -18:00 offset. -fn extract_offset_suffix(value: &str) -> Option<(&str, Tz)> { +fn extract_offset_suffix(value: &str) -> Option<(&str, Tz, i32)> { // 1. Z suffix if let Some(stripped) = value.strip_suffix('Z') { - return Some((stripped, tz_from_offset_secs(0)?)); + return Some((stripped, tz_from_offset_secs(0)?, 0)); } // 2. Named text-prefix forms: "UTC", "GMT", "UT" (optionally space-prefixed), @@ -1871,7 +1880,8 @@ fn extract_offset_suffix(value: &str) -> Option<(&str, Tz)> { if !before.ends_with('/') { if let Some(secs) = parse_sign_offset(offset_str) { let before = before.strip_suffix(' ').unwrap_or(before); - return Some((before, tz_from_offset_secs(secs)?)); + let (tz, residual_secs) = tz_and_residual_secs(secs)?; + return Some((before, tz, residual_secs)); } } } @@ -1879,7 +1889,7 @@ fn extract_offset_suffix(value: &str) -> Option<(&str, Tz)> { // 3. Region ids and Java `ZoneId.SHORT_IDS` aliases, with or without a leading space: // " Europe/Moscow", "America/Los_Angeles", " CET", " Zulu", " Etc/GMT+8", " PST". if let Some((prefix, tz)) = extract_zone_id_suffix(value) { - return Some((prefix, tz)); + return Some((prefix, tz, 0)); } // 4. Bare +/-offset: find the rightmost '+' or '-' and try to parse everything @@ -1895,7 +1905,8 @@ fn extract_offset_suffix(value: &str) -> Option<(&str, Tz)> { if let Some(pos) = last_sign { let offset_str = &value[pos..]; if let Some(secs) = parse_sign_offset(offset_str) { - return Some((&value[..pos], tz_from_offset_secs(secs)?)); + let (tz, residual_secs) = tz_and_residual_secs(secs)?; + return Some((&value[..pos], tz, residual_secs)); } } @@ -2167,15 +2178,7 @@ fn parse_timestamp_pattern( }; if timestamp.is_none() { - return if eval_mode == EvalMode::Ansi { - Err(SparkError::InvalidInputInCastToDatetime { - value: value.to_string(), - from_type: "STRING".to_string(), - to_type: "TIMESTAMP".to_string(), - }) - } else { - Ok(None) - }; + return invalid_timestamp(value, eval_mode); } Ok(timestamp) @@ -2206,7 +2209,7 @@ fn timestamp_ntz_parser( // pattern matches, so the inner parser reports it as malformed. let (value_to_parse, pattern) = if pattern.is_none() { match extract_offset_suffix(value) { - Some((stripped, _tz)) => { + Some((stripped, _, _residual_secs)) => { // Spark applies Java String.trim to the zone, not Unicode whitespace trimming. let stripped = stripped.trim_end_matches(|c: char| c <= '\u{20}'); let stripped_pattern = classify_timestamp_pattern(stripped); @@ -2227,6 +2230,19 @@ fn timestamp_ntz_parser( timestamp_ntz_parser_inner(value_to_parse, pattern, eval_mode) } +/// ANSI mode raises on an unparseable TIMESTAMP value; the other modes return null. +fn invalid_timestamp(value: &str, eval_mode: EvalMode) -> SparkResult> { + if eval_mode == EvalMode::Ansi { + Err(SparkError::InvalidInputInCastToDatetime { + value: value.to_string(), + from_type: "STRING".to_string(), + to_type: "TIMESTAMP".to_string(), + }) + } else { + Ok(None) + } +} + /// ANSI mode raises on an unparseable TIMESTAMP_NTZ value; the other modes return null. fn invalid_ntz_timestamp(value: &str, eval_mode: EvalMode) -> SparkResult> { if eval_mode == EvalMode::Ansi { @@ -4103,11 +4119,20 @@ mod tests { "2020-01-01 12:34:56 GMT+053000", Some(utc - 5 * hour - 30 * 60 * 1_000_000), ), - ("2020-01-01 12:34:56 +080030", None), + // A non-zero seconds part is applied as a residual after the minute-precision zone + ( + "2020-01-01 12:34:56 +080030", + Some(utc - 8 * hour - 30 * 1_000_000), + ), + ( + "2020-01-01 12:34:56 +08:00:30", + Some(utc - 8 * hour - 30 * 1_000_000), + ), + ( + "2020-01-01 12:34:56-01:30:15", + Some(utc + hour + 30 * 60 * 1_000_000 + 15 * 1_000_000), + ), ("2020-01-01 12:34:56 +08000", None), - // A non-zero seconds part is not supported yet (`Tz` has minute precision), - // see https://github.com/apache/datafusion-comet/issues/7224 - ("2020-01-01 12:34:56 +08:00:30", None), // UTC/GMT/UT names, and region ids with those names after a '/' ("2020-01-01 12:34:56 UTC", Some(utc)), ("2020-01-01 12:34:56UTC", Some(utc)), @@ -4878,6 +4903,12 @@ mod tests { ("-18:00", -64_800), ("+1800", 64_800), ("-1800", -64_800), + ("+08:00:30", 28_830), + ("+8:00:30", 28_830), + ("-01:30:15", -5_415), + ("+080030", 28_830), + ("+180000", 64_800), + ("-180000", -64_800), ] { assert_eq!(parse_sign_offset(input), Some(seconds), "{input:?}"); } @@ -4890,6 +4921,13 @@ mod tests { "+19", "-1900", "+8:", + "+8:0:3", + "+8:0:30", + "+08:0:30", + "+08:00:3", + "+08:00:60", + "+18:00:01", + "+0800 30", "+1:+1", "-1:-1", "+1\u{967}", @@ -4901,6 +4939,27 @@ mod tests { } } + #[test] + #[cfg_attr(miri, ignore)] + fn timestamp_parser_rejects_residual_offset_underflow() { + let tz = &Tz::from_str("UTC").unwrap(); + // `i64::MIN` microseconds is the earliest representable instant, so a positive residual + // pushes it out of range. Spark returns null there instead of wrapping. + for mode in [EvalMode::Legacy, EvalMode::Try] { + assert_eq!( + timestamp_parser("-290308-12-21T19:59:05.224192+00:00:30", mode, tz, true).unwrap(), + None + ); + } + assert!(timestamp_parser( + "-290308-12-21T19:59:05.224192+00:00:30", + EvalMode::Ansi, + tz, + true + ) + .is_err()); + } + #[test] #[cfg_attr(miri, ignore)] fn timestamp_parser_spark_segment_rules_test() { diff --git a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala index 9d37f51fcaa..1bc1ec24c79 100644 --- a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala @@ -1918,6 +1918,12 @@ class CometNativeCastSuite "2020-01-01T12:34:56-08:00", // Single-digit hour offset (extract_offset_suffix supports ±H:MM) "2020-01-01T12:34:56+5:30", + // Second-precision offsets, the forms Java's ZoneOffset.of accepts + "2020-01-01T12:34:56+08:00:30", + "2020-01-01T12:34:56-01:30:15", + "2020-01-01T12:34:56+080030", + // Spark rejects a short minute or second segment once seconds are present + "2020-01-01T12:34:56+8:0:3", // T-prefixed time-only with colon "T12:34", "T12:34:56", @@ -2011,6 +2017,9 @@ class CometNativeCastSuite "2020-06-15T12:30:00+05:30", "2020-06-15T12:30:00-08:00", "2020-06-15T12:30:00 UTC", + "2020-06-15T12:30:00+08:00:30", + "2020-06-15T12:30:00-01:30:15", + "2020-06-15T12:30:00+8:0:3", "2024-03-10 02:30:00", "2024-11-03 01:30:00", "not a timestamp",