diff --git a/native/common/src/error.rs b/native/common/src/error.rs index a33613f3099..113b6ff8b9a 100644 --- a/native/common/src/error.rs +++ b/native/common/src/error.rs @@ -38,7 +38,8 @@ pub enum SparkError { }, /// Like CastInvalidValue but maps to SparkDateTimeException instead of SparkNumberFormatException. - /// Used for string → timestamp/date cast failures. + /// Used for string → timestamp/date cast failures and for NaN or an infinity cast to + /// timestamp. #[error("[CAST_INVALID_INPUT] The value '{value}' of the type \"{from_type}\" cannot be cast to \"{to_type}\" \ because it is malformed. Correct the value as per the syntax, or change its target type. \ Use `try_cast` to tolerate malformed input and return NULL instead. If necessary \ diff --git a/native/spark-expr/src/conversion_funcs/numeric.rs b/native/spark-expr/src/conversion_funcs/numeric.rs index 9a9a7bf915e..031af82e0ba 100644 --- a/native/spark-expr/src/conversion_funcs/numeric.rs +++ b/native/spark-expr/src/conversion_funcs/numeric.rs @@ -109,37 +109,41 @@ fn float_timestamp_micros(val: f64) -> Option { } /// ANSI error for a value [`float_timestamp_micros`] rejected. NaN and infinity -/// are `CAST_INVALID_INPUT` into TIMESTAMP. A finite value whose product does -/// not fit is `CAST_OVERFLOW` into BIGINT. The source type is DOUBLE even when -/// the array was float32, because the check runs after the widening. +/// are `CAST_INVALID_INPUT` into TIMESTAMP, raised as `SparkDateTimeException` +/// and spelled the way Spark prints a double (`NaN`, `Infinity`). A finite value +/// whose product does not fit is `CAST_OVERFLOW` into BIGINT. The source type is +/// DOUBLE even when the array was float32, because the check runs after the +/// widening. fn float_timestamp_error(val: f64) -> SparkError { if val.is_nan() || val.is_infinite() { - SparkError::CastInvalidValue { - value: val.to_string(), + SparkError::InvalidInputInCastToDatetime { + value: double_sql_text(val), from_type: "DOUBLE".to_string(), to_type: "TIMESTAMP".to_string(), } } else { let micros = val * MICROS_PER_SECOND as f64; SparkError::CastOverFlow { - value: float_timestamp_overflow_text(micros), + value: double_sql_text(micros), from_type: "DOUBLE".to_string(), to_type: "BIGINT".to_string(), } } } -fn float_timestamp_overflow_text(micros: f64) -> String { - if micros.is_infinite() { - if micros.is_sign_positive() { +/// Text for a double in an error, which the JVM side parses back to a double. NaN and the +/// infinities use Spark's spelling. +fn double_sql_text(value: f64) -> String { + if value.is_infinite() { + if value.is_sign_positive() { "Infinity".to_string() } else { "-Infinity".to_string() } - } else if micros.is_nan() { + } else if value.is_nan() { "NaN".to_string() } else { - format!("{:e}", micros).to_uppercase() + "D" + format!("{:e}", value).to_uppercase() + "D" } } @@ -2946,17 +2950,36 @@ mod tests { } #[test] - fn test_cast_float_invalid_from_type_is_double() { + fn test_cast_float_invalid_is_datetime_error() { let tz = Some(Arc::from("UTC")); - for value in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { - let array: ArrayRef = Arc::new(Float64Array::from(vec![Some(value)])); - let err = cast_float_to_timestamp(&array, &tz, EvalMode::Ansi).unwrap_err(); - let from_type = match err { - SparkError::CastInvalidValue { from_type, .. } - | SparkError::InvalidInputInCastToDatetime { from_type, .. } => from_type, - other => panic!("expected an invalid-input error, got {other:?}"), - }; - assert_eq!(from_type, "DOUBLE"); + let cases: [(ArrayRef, &str); 4] = [ + (Arc::new(Float64Array::from(vec![f64::NAN])), "NaN"), + ( + Arc::new(Float64Array::from(vec![f64::INFINITY])), + "Infinity", + ), + ( + Arc::new(Float64Array::from(vec![f64::NEG_INFINITY])), + "-Infinity", + ), + ( + Arc::new(Float32Array::from(vec![f32::INFINITY])), + "Infinity", + ), + ]; + for (array, expected) in cases { + match cast_float_to_timestamp(&array, &tz, EvalMode::Ansi).unwrap_err() { + SparkError::InvalidInputInCastToDatetime { + value, + from_type, + to_type, + } => { + assert_eq!(value, expected); + assert_eq!(from_type, "DOUBLE"); + assert_eq!(to_type, "TIMESTAMP"); + } + other => panic!("expected InvalidInputInCastToDatetime, got {other:?}"), + } } } diff --git a/spark/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimSparkErrorConverter.scala b/spark/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimSparkErrorConverter.scala index c8a7dc78c8f..326b6da3380 100644 --- a/spark/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimSparkErrorConverter.scala +++ b/spark/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimSparkErrorConverter.scala @@ -220,6 +220,14 @@ trait ShimSparkErrorConverter { QueryExecutionErrors .invalidInputInCastToNumberError(targetType, str, sqlCtx(context))) + case "InvalidInputInCastToDatetime" if params("fromType").toString == "DOUBLE" => + val targetType = getDataType(params("toType").toString) + Some( + QueryExecutionErrors.invalidInputInCastToDatetimeError( + parseDoubleLiteral(params("value").toString), + targetType, + sqlCtx(context))) + case "InvalidInputInCastToDatetime" => val expression = s"'${params("value").toString.replace("\\", "\\\\").replace("'", "\\'")}'" diff --git a/spark/src/main/spark-4.x/org/apache/spark/sql/comet/shims/ShimSparkErrorConverter.scala b/spark/src/main/spark-4.x/org/apache/spark/sql/comet/shims/ShimSparkErrorConverter.scala index d2fed8a2dc3..53cd3f671ef 100644 --- a/spark/src/main/spark-4.x/org/apache/spark/sql/comet/shims/ShimSparkErrorConverter.scala +++ b/spark/src/main/spark-4.x/org/apache/spark/sql/comet/shims/ShimSparkErrorConverter.scala @@ -252,6 +252,14 @@ trait ShimSparkErrorConverter { QueryExecutionErrors .invalidInputInCastToNumberError(targetType, str, context.headOption.orNull)) + case "InvalidInputInCastToDatetime" if params("fromType").toString == "DOUBLE" => + val targetType = getDataType(params("toType").toString) + Some( + QueryExecutionErrors.invalidInputInCastToDatetimeError( + parseDoubleLiteral(params("value").toString), + targetType, + context.headOption.orNull)) + case "InvalidInputInCastToDatetime" => val str = UTF8String.fromString(params("value").toString) val targetType = getDataType(params("toType").toString) diff --git a/spark/src/test/resources/sql-tests/expressions/cast/cast_float_to_timestamp_ansi.sql b/spark/src/test/resources/sql-tests/expressions/cast/cast_float_to_timestamp_ansi.sql new file mode 100644 index 00000000000..f093262b879 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/cast/cast_float_to_timestamp_ansi.sql @@ -0,0 +1,55 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ANSI casts of NaN and infinities to TIMESTAMP (#7001). Spark raises SparkDateTimeException +-- with CAST_INVALID_INPUT and prints the value as a DOUBLE, also for FLOAT input. + +-- Config: spark.sql.ansi.enabled=true + +statement +CREATE TABLE test_cast_float_ts_ansi(id int, d double, f float) USING parquet + +statement +INSERT INTO test_cast_float_ts_ansi VALUES (1, 1.5, 1.5), (2, double('NaN'), float('NaN')), (3, double('Infinity'), float('Infinity')), (4, double('-Infinity'), float('-Infinity')) + +-- sentinel: a finite value casts natively +query +SELECT cast(d AS timestamp), cast(f AS timestamp) FROM test_cast_float_ts_ansi WHERE id = 1 + +-- try_cast returns NULL for every row +query +SELECT id, try_cast(d AS timestamp), try_cast(f AS timestamp) FROM test_cast_float_ts_ansi ORDER BY id + +-- same exception class, error class and SQLSTATE as Spark +query expect_error_class(CAST_INVALID_INPUT) +SELECT cast(d AS timestamp) FROM test_cast_float_ts_ansi WHERE id = 2 + +query expect_error_class(CAST_INVALID_INPUT) +SELECT cast(f AS timestamp) FROM test_cast_float_ts_ansi WHERE id = 4 + +-- same message text as Spark +query expect_error(The value Infinity of the type "DOUBLE" cannot be cast to "TIMESTAMP") +SELECT cast(d AS timestamp) FROM test_cast_float_ts_ansi WHERE id = 3 + +query expect_error(The value NaN of the type "DOUBLE" cannot be cast to "TIMESTAMP") +SELECT cast(f AS timestamp) FROM test_cast_float_ts_ansi WHERE id = 2 + +query expect_error(The value -Infinity of the type "DOUBLE" cannot be cast to "TIMESTAMP") +SELECT cast(d AS timestamp) FROM test_cast_float_ts_ansi WHERE id = 4 + +query expect_error(The value Infinity of the type "DOUBLE" cannot be cast to "TIMESTAMP") +SELECT cast(f AS timestamp) FROM test_cast_float_ts_ansi WHERE id = 3 diff --git a/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala b/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala index c36d5d69f33..bd8971f59a8 100644 --- a/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala +++ b/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala @@ -19,6 +19,8 @@ package org.apache.comet +import java.time.DateTimeException + import org.scalatest.funsuite.AnyFunSuite import org.apache.spark.{SparkException, SparkThrowable} @@ -45,6 +47,37 @@ class SparkErrorConverterSuite extends AnyFunSuite { } } + private def castToDatetimeError(fromType: String, value: String): Throwable = + SparkErrorConverter + .convertErrorType( + "InvalidInputInCastToDatetime", + "CAST_INVALID_INPUT", + Map("fromType" -> fromType, "toType" -> "TIMESTAMP", "value" -> value), + Array.empty, + "") + .getOrElse(fail("Expected InvalidInputInCastToDatetime to be converted")) + + test("InvalidInputInCastToDatetime prints a double value as Spark does") { + for (value <- Seq("NaN", "Infinity", "-Infinity")) { + val error = castToDatetimeError("DOUBLE", value) + // SparkDateTimeException is private[spark], so check its public parent. + assert(error.isInstanceOf[DateTimeException]) + assert(error.asInstanceOf[SparkThrowable].getErrorClass == "CAST_INVALID_INPUT") + assert( + error.getMessage.contains( + s"""The value $value of the type "DOUBLE" cannot be cast to "TIMESTAMP""""), + error.getMessage) + } + } + + test("InvalidInputInCastToDatetime still quotes a string value") { + val error = castToDatetimeError("STRING", "abc") + assert(error.isInstanceOf[DateTimeException]) + assert( + error.getMessage.contains("""The value 'abc' of the type "STRING" cannot be cast"""), + error.getMessage) + } + test("ArithmeticOverflow without a suggested function remains compatible") { // Accept both older payloads without the key and unary errors with an empty function. for (params <- Seq(