Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion native/common/src/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ pub enum SparkError {
},

/// Like CastInvalidValue but maps to SparkDateTimeException instead of SparkNumberFormatException.
/// Used for string → timestamp/date cast failures.
/// Used for string → timestamp/date cast failures and for NaN or an infinity cast to
/// timestamp.
#[error("[CAST_INVALID_INPUT] The value '{value}' of the type \"{from_type}\" cannot be cast to \"{to_type}\" \
because it is malformed. Correct the value as per the syntax, or change its target type. \
Use `try_cast` to tolerate malformed input and return NULL instead. If necessary \
Expand Down
65 changes: 44 additions & 21 deletions native/spark-expr/src/conversion_funcs/numeric.rs
Original file line number Diff line number Diff line change
Expand Up @@ -109,37 +109,41 @@ fn float_timestamp_micros(val: f64) -> Option<i64> {
}

/// ANSI error for a value [`float_timestamp_micros`] rejected. NaN and infinity
/// are `CAST_INVALID_INPUT` into TIMESTAMP. A finite value whose product does
/// not fit is `CAST_OVERFLOW` into BIGINT. The source type is DOUBLE even when
/// the array was float32, because the check runs after the widening.
/// are `CAST_INVALID_INPUT` into TIMESTAMP, raised as `SparkDateTimeException`
/// and spelled the way Spark prints a double (`NaN`, `Infinity`). A finite value
/// whose product does not fit is `CAST_OVERFLOW` into BIGINT. The source type is
/// DOUBLE even when the array was float32, because the check runs after the
/// widening.
fn float_timestamp_error(val: f64) -> SparkError {
if val.is_nan() || val.is_infinite() {
SparkError::CastInvalidValue {
value: val.to_string(),
SparkError::InvalidInputInCastToDatetime {
value: double_sql_text(val),
from_type: "DOUBLE".to_string(),
to_type: "TIMESTAMP".to_string(),
}
} else {
let micros = val * MICROS_PER_SECOND as f64;
SparkError::CastOverFlow {
value: float_timestamp_overflow_text(micros),
value: double_sql_text(micros),
from_type: "DOUBLE".to_string(),
to_type: "BIGINT".to_string(),
}
}
}

fn float_timestamp_overflow_text(micros: f64) -> String {
if micros.is_infinite() {
if micros.is_sign_positive() {
/// Text for a double in an error, which the JVM side parses back to a double. NaN and the
/// infinities use Spark's spelling.
fn double_sql_text(value: f64) -> String {
if value.is_infinite() {
if value.is_sign_positive() {
"Infinity".to_string()
} else {
"-Infinity".to_string()
}
} else if micros.is_nan() {
} else if value.is_nan() {
"NaN".to_string()
} else {
format!("{:e}", micros).to_uppercase() + "D"
format!("{:e}", value).to_uppercase() + "D"
}
}

Expand Down Expand Up @@ -2946,17 +2950,36 @@ mod tests {
}

#[test]
fn test_cast_float_invalid_from_type_is_double() {
fn test_cast_float_invalid_is_datetime_error() {
let tz = Some(Arc::from("UTC"));
for value in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] {
let array: ArrayRef = Arc::new(Float64Array::from(vec![Some(value)]));
let err = cast_float_to_timestamp(&array, &tz, EvalMode::Ansi).unwrap_err();
let from_type = match err {
SparkError::CastInvalidValue { from_type, .. }
| SparkError::InvalidInputInCastToDatetime { from_type, .. } => from_type,
other => panic!("expected an invalid-input error, got {other:?}"),
};
assert_eq!(from_type, "DOUBLE");
let cases: [(ArrayRef, &str); 4] = [
(Arc::new(Float64Array::from(vec![f64::NAN])), "NaN"),
(
Arc::new(Float64Array::from(vec![f64::INFINITY])),
"Infinity",
),
(
Arc::new(Float64Array::from(vec![f64::NEG_INFINITY])),
"-Infinity",
),
(
Arc::new(Float32Array::from(vec![f32::INFINITY])),
"Infinity",
),
];
for (array, expected) in cases {
match cast_float_to_timestamp(&array, &tz, EvalMode::Ansi).unwrap_err() {
SparkError::InvalidInputInCastToDatetime {
value,
from_type,
to_type,
} => {
assert_eq!(value, expected);
assert_eq!(from_type, "DOUBLE");
assert_eq!(to_type, "TIMESTAMP");
}
other => panic!("expected InvalidInputInCastToDatetime, got {other:?}"),
}
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,14 @@ trait ShimSparkErrorConverter {
QueryExecutionErrors
.invalidInputInCastToNumberError(targetType, str, sqlCtx(context)))

case "InvalidInputInCastToDatetime" if params("fromType").toString == "DOUBLE" =>
val targetType = getDataType(params("toType").toString)
Some(
QueryExecutionErrors.invalidInputInCastToDatetimeError(
parseDoubleLiteral(params("value").toString),
targetType,
sqlCtx(context)))

case "InvalidInputInCastToDatetime" =>
val expression =
s"'${params("value").toString.replace("\\", "\\\\").replace("'", "\\'")}'"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,14 @@ trait ShimSparkErrorConverter {
QueryExecutionErrors
.invalidInputInCastToNumberError(targetType, str, context.headOption.orNull))

case "InvalidInputInCastToDatetime" if params("fromType").toString == "DOUBLE" =>
val targetType = getDataType(params("toType").toString)
Some(
QueryExecutionErrors.invalidInputInCastToDatetimeError(
parseDoubleLiteral(params("value").toString),
targetType,
context.headOption.orNull))

case "InvalidInputInCastToDatetime" =>
val str = UTF8String.fromString(params("value").toString)
val targetType = getDataType(params("toType").toString)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
-- Licensed to the Apache Software Foundation (ASF) under one
-- or more contributor license agreements. See the NOTICE file
-- distributed with this work for additional information
-- regarding copyright ownership. The ASF licenses this file
-- to you under the Apache License, Version 2.0 (the
-- "License"); you may not use this file except in compliance
-- with the License. You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing,
-- software distributed under the License is distributed on an
-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-- KIND, either express or implied. See the License for the
-- specific language governing permissions and limitations
-- under the License.

-- ANSI casts of NaN and infinities to TIMESTAMP (#7001). Spark raises SparkDateTimeException
-- with CAST_INVALID_INPUT and prints the value as a DOUBLE, also for FLOAT input.

-- Config: spark.sql.ansi.enabled=true

statement
CREATE TABLE test_cast_float_ts_ansi(id int, d double, f float) USING parquet

statement
INSERT INTO test_cast_float_ts_ansi VALUES (1, 1.5, 1.5), (2, double('NaN'), float('NaN')), (3, double('Infinity'), float('Infinity')), (4, double('-Infinity'), float('-Infinity'))

-- sentinel: a finite value casts natively
query
SELECT cast(d AS timestamp), cast(f AS timestamp) FROM test_cast_float_ts_ansi WHERE id = 1

-- try_cast returns NULL for every row
query
SELECT id, try_cast(d AS timestamp), try_cast(f AS timestamp) FROM test_cast_float_ts_ansi ORDER BY id

-- same exception class, error class and SQLSTATE as Spark
query expect_error_class(CAST_INVALID_INPUT)
SELECT cast(d AS timestamp) FROM test_cast_float_ts_ansi WHERE id = 2

query expect_error_class(CAST_INVALID_INPUT)
SELECT cast(f AS timestamp) FROM test_cast_float_ts_ansi WHERE id = 4

-- same message text as Spark
query expect_error(The value Infinity of the type "DOUBLE" cannot be cast to "TIMESTAMP")
SELECT cast(d AS timestamp) FROM test_cast_float_ts_ansi WHERE id = 3

query expect_error(The value NaN of the type "DOUBLE" cannot be cast to "TIMESTAMP")
SELECT cast(f AS timestamp) FROM test_cast_float_ts_ansi WHERE id = 2

query expect_error(The value -Infinity of the type "DOUBLE" cannot be cast to "TIMESTAMP")
SELECT cast(d AS timestamp) FROM test_cast_float_ts_ansi WHERE id = 4

query expect_error(The value Infinity of the type "DOUBLE" cannot be cast to "TIMESTAMP")
SELECT cast(f AS timestamp) FROM test_cast_float_ts_ansi WHERE id = 3
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@

package org.apache.comet

import java.time.DateTimeException

import org.scalatest.funsuite.AnyFunSuite

import org.apache.spark.{SparkException, SparkThrowable}
Expand All @@ -45,6 +47,37 @@ class SparkErrorConverterSuite extends AnyFunSuite {
}
}

private def castToDatetimeError(fromType: String, value: String): Throwable =
SparkErrorConverter
.convertErrorType(
"InvalidInputInCastToDatetime",
"CAST_INVALID_INPUT",
Map("fromType" -> fromType, "toType" -> "TIMESTAMP", "value" -> value),
Array.empty,
"")
.getOrElse(fail("Expected InvalidInputInCastToDatetime to be converted"))

test("InvalidInputInCastToDatetime prints a double value as Spark does") {
for (value <- Seq("NaN", "Infinity", "-Infinity")) {
val error = castToDatetimeError("DOUBLE", value)
// SparkDateTimeException is private[spark], so check its public parent.
assert(error.isInstanceOf[DateTimeException])
assert(error.asInstanceOf[SparkThrowable].getErrorClass == "CAST_INVALID_INPUT")
assert(
error.getMessage.contains(
s"""The value $value of the type "DOUBLE" cannot be cast to "TIMESTAMP""""),
error.getMessage)
}
}

test("InvalidInputInCastToDatetime still quotes a string value") {
val error = castToDatetimeError("STRING", "abc")
assert(error.isInstanceOf[DateTimeException])
assert(
error.getMessage.contains("""The value 'abc' of the type "STRING" cannot be cast"""),
error.getMessage)
}

test("ArithmeticOverflow without a suggested function remains compatible") {
// Accept both older payloads without the key and unary errors with an empty function.
for (params <- Seq(
Expand Down