diff --git a/dev/diffs/3.5.9.diff b/dev/diffs/3.5.9.diff index c41291d25d5..7dc2484e9a8 100644 --- a/dev/diffs/3.5.9.diff +++ b/dev/diffs/3.5.9.diff @@ -256,6 +256,20 @@ index fac23b4a26f..2b73732c33f 100644 -- -- INT8 -- Test int8 64-bit integers. +diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql +index cbc9c2a26f6..e690156a325 100644 +--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql ++++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql +@@ -6,6 +6,9 @@ + -- https://github.com/postgres/postgres/blob/REL_12_BETA2/src/test/regress/sql/numeric.sql + -- + ++-- TODO: https://github.com/apache/datafusion-comet/issues/6797 ++--SET spark.comet.enabled = false ++ + -- [SPARK-28318] Decimal can only support precision up to 38. We rewrite numeric(210,10) to decimal(38,10). + CREATE TABLE num_data (id int, val decimal(38,10)) USING parquet; + CREATE TABLE num_exp_add (id1 int, id2 int, expected decimal(38,10)) USING parquet; diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql index 0efe0877e9b..423d3b3d76d 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql diff --git a/dev/diffs/4.0.4.diff b/dev/diffs/4.0.4.diff index 772ffbcdadb..bc79537db13 100644 --- a/dev/diffs/4.0.4.diff +++ b/dev/diffs/4.0.4.diff @@ -308,6 +308,20 @@ index fac23b4a26f..98b12ae5ccc 100644 CREATE TABLE INT8_TBL(q1 bigint, q2 bigint) USING parquet; -- PostgreSQL implicitly casts string literals to data with integral types, but +diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql +index cbc9c2a26f6..e690156a325 100644 +--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql ++++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql +@@ -6,6 +6,9 @@ + -- https://github.com/postgres/postgres/blob/REL_12_BETA2/src/test/regress/sql/numeric.sql + -- + ++-- TODO: https://github.com/apache/datafusion-comet/issues/6797 ++--SET spark.comet.enabled = false ++ + -- [SPARK-28318] Decimal can only support precision up to 38. We rewrite numeric(210,10) to decimal(38,10). + CREATE TABLE num_data (id int, val decimal(38,10)) USING parquet; + CREATE TABLE num_exp_add (id1 int, id2 int, expected decimal(38,10)) USING parquet; diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql index 0efe0877e9b..f9df0400c99 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql diff --git a/dev/diffs/4.1.3.diff b/dev/diffs/4.1.3.diff index daf53c6773c..4b4fece2c2a 100644 --- a/dev/diffs/4.1.3.diff +++ b/dev/diffs/4.1.3.diff @@ -310,6 +310,20 @@ index fac23b4a26f..98b12ae5ccc 100644 CREATE TABLE INT8_TBL(q1 bigint, q2 bigint) USING parquet; -- PostgreSQL implicitly casts string literals to data with integral types, but +diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql +index cbc9c2a26f6..e690156a325 100644 +--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql ++++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql +@@ -6,6 +6,9 @@ + -- https://github.com/postgres/postgres/blob/REL_12_BETA2/src/test/regress/sql/numeric.sql + -- + ++-- TODO: https://github.com/apache/datafusion-comet/issues/6797 ++--SET spark.comet.enabled = false ++ + -- [SPARK-28318] Decimal can only support precision up to 38. We rewrite numeric(210,10) to decimal(38,10). + CREATE TABLE num_data (id int, val decimal(38,10)) USING parquet; + CREATE TABLE num_exp_add (id1 int, id2 int, expected decimal(38,10)) USING parquet; diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql index 0efe0877e9b..f9df0400c99 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql diff --git a/dev/diffs/4.2.0.diff b/dev/diffs/4.2.0.diff index 40aad4b1d59..0bdb7579c9f 100644 --- a/dev/diffs/4.2.0.diff +++ b/dev/diffs/4.2.0.diff @@ -312,6 +312,20 @@ index fac23b4a26f..98b12ae5ccc 100644 CREATE TABLE INT8_TBL(q1 bigint, q2 bigint) USING parquet; -- PostgreSQL implicitly casts string literals to data with integral types, but +diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql +index cbc9c2a26f6..e690156a325 100644 +--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql ++++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql +@@ -6,6 +6,9 @@ + -- https://github.com/postgres/postgres/blob/REL_12_BETA2/src/test/regress/sql/numeric.sql + -- + ++-- TODO: https://github.com/apache/datafusion-comet/issues/6797 ++--SET spark.comet.enabled = false ++ + -- [SPARK-28318] Decimal can only support precision up to 38. We rewrite numeric(210,10) to decimal(38,10). + CREATE TABLE num_data (id int, val decimal(38,10)) USING parquet; + CREATE TABLE num_exp_add (id1 int, id2 int, expected decimal(38,10)) USING parquet; diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql index 0efe0877e9b..f9df0400c99 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql diff --git a/docs/source/user-guide/latest/compatibility/index.md b/docs/source/user-guide/latest/compatibility/index.md index dca663d9228..9e88e4dab7e 100644 --- a/docs/source/user-guide/latest/compatibility/index.md +++ b/docs/source/user-guide/latest/compatibility/index.md @@ -181,6 +181,11 @@ divergence: - Spark 4.2 introduced additional ANSI arithmetic overflow behavior differences that Comet does not yet track ([#4967](https://github.com/apache/datafusion-comet/issues/4967)). +- An `INSERT` that narrows a numeric column reports an overflow as `CAST_OVERFLOW_IN_TABLE_INSERT`, + whatever `spark.sql.ansi.enabled` is. When an expression under that cast occurs more than once in + the projection, Spark's subexpression elimination evaluates it outside the check and reports the + original error, such as `CAST_OVERFLOW`. Comet still reports `CAST_OVERFLOW_IN_TABLE_INSERT`, as + Spark does with `spark.sql.subexpressionElimination.enabled=false`. ## Known result-value divergences diff --git a/docs/source/user-guide/latest/expressions.md b/docs/source/user-guide/latest/expressions.md index b73fef7aa2b..aa0396e6657 100644 --- a/docs/source/user-guide/latest/expressions.md +++ b/docs/source/user-guide/latest/expressions.md @@ -704,6 +704,7 @@ Comet also accelerates a number of Catalyst expressions that have no Spark SQL f - **Operator and optimizer-injected expressions:** runtime bloom-filter join probes (`BloomFilterMightContain`, `BloomFilterAggregate`), optimized `IN` sets (`InSet`), scalar subqueries (`ScalarSubquery`), and floating-point normalization (`KnownFloatingPointNormalized`). - **Accessor expressions (subscript and field access, not functions):** struct field access (`col.field`), array element access (`arr[i]`), and map value access (`map[key]`). - **Internal decimal arithmetic:** `CheckOverflow`, `MakeDecimal`, and `UnscaledValue`, which the analyzer inserts around decimal operations. +- **Table insert casts:** `CheckOverflowInTableInsert`, which the analyzer wraps around the cast an `INSERT` adds to narrow a numeric column. - **Missing-data predicates:** `AtLeastNNonNulls`, used by `DataFrame.na.drop`, counts non-null and non-NaN values natively. - **User-defined functions:** Scala UDFs registered through the DataFrame or SQL API. - **DataSource V2 catalog functions:** Iceberg's system functions `bucket`, `truncate`, `years`, `months`, `days`, and `hours` (for example `system.bucket(16, id)`) run natively; see [Iceberg system functions](iceberg.md#iceberg-system-functions). Other DataSource V2 catalog functions run in Spark. diff --git a/native/common/src/error.rs b/native/common/src/error.rs index c227c3f1b8e..a9f0cde687e 100644 --- a/native/common/src/error.rs +++ b/native/common/src/error.rs @@ -69,6 +69,15 @@ pub enum SparkError { to_type: String, }, + #[error("[CAST_OVERFLOW_IN_TABLE_INSERT] Fail to assign a value of \"{from_type}\" type to the \ + \"{to_type}\" type column or variable `{column_name}` due to an overflow. Use `try_cast` on \ + the input value to tolerate overflow and return NULL instead.")] + CastOverflowInTableInsert { + from_type: String, + to_type: String, + column_name: String, + }, + #[error("[CANNOT_PARSE_DECIMAL] Cannot parse decimal.")] CannotParseDecimal, @@ -358,6 +367,7 @@ impl SparkError { SparkError::NumericValueOutOfRange { .. } => "NumericValueOutOfRange", SparkError::NumericOutOfRange { .. } => "NumericOutOfRange", SparkError::CastOverFlow { .. } => "CastOverFlow", + SparkError::CastOverflowInTableInsert { .. } => "CastOverflowInTableInsert", SparkError::CannotParseDecimal => "CannotParseDecimal", SparkError::ArithmeticOverflow { .. } => "ArithmeticOverflow", SparkError::IntegralDivideOverflow => "IntegralDivideOverflow", @@ -464,6 +474,17 @@ impl SparkError { "toType": to_type, }) } + SparkError::CastOverflowInTableInsert { + from_type, + to_type, + column_name, + } => { + serde_json::json!({ + "fromType": from_type, + "toType": to_type, + "columnName": column_name, + }) + } SparkError::ArithmeticOverflow { from_type, function_name, @@ -717,6 +738,15 @@ impl SparkError { } } + /// Whether Spark raises this error as a `SparkArithmeticException`. + pub fn is_arithmetic(&self) -> bool { + matches!( + self.exception_class(), + "org/apache/spark/SparkArithmeticException" + | "org/apache/spark/sql/comet/CastOverflowException" + ) + } + /// Returns the appropriate Spark exception class for this error pub fn exception_class(&self) -> &'static str { match self { @@ -732,7 +762,10 @@ impl SparkError { | SparkError::BinaryArithmeticOverflow { .. } | SparkError::IntervalArithmeticOverflowWithSuggestion { .. } | SparkError::IntervalArithmeticOverflowWithoutSuggestion - | SparkError::DatetimeOverflow => "org/apache/spark/SparkArithmeticException", + | SparkError::DatetimeOverflow + | SparkError::CastOverflowInTableInsert { .. } => { + "org/apache/spark/SparkArithmeticException" + } // CastOverflow gets special handling with CastOverflowException SparkError::CastOverFlow { .. } => "org/apache/spark/sql/comet/CastOverflowException", @@ -829,6 +862,7 @@ impl SparkError { SparkError::CastInvalidValue { .. } => Some("CAST_INVALID_INPUT"), SparkError::InvalidInputInCastToDatetime { .. } => Some("CAST_INVALID_INPUT"), SparkError::CastOverFlow { .. } => Some("CAST_OVERFLOW"), + SparkError::CastOverflowInTableInsert { .. } => Some("CAST_OVERFLOW_IN_TABLE_INSERT"), SparkError::NumericValueOutOfRange { .. } => { Some("NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION") } diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index 72071d9dd19..445350daefa 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -121,10 +121,10 @@ use datafusion_comet_proto::{ use datafusion_comet_spark_expr::{ coerce_to_common_type, create_case_when, create_if_expr, decimal128_add_sub_width, jvm_udf::JvmScalarUdfExpr, spark_in_list, ArrayInsert, AtLeastNNonNulls, Cast, CheckOverflow, - CreateNamedStruct, DecimalRescaleCheckOverflow, FloatOperands, GetArrayStructFields, - GetStructField, IfExpr, ListExtract, MakeDateExpr, NormalizeNaNAndZero, NormalizeNestedFloats, - NullShortCircuit, SparkCastOptions, ToJson, UnboundColumn, WideDecimalBinaryExpr, - WideDecimalOp, + CheckOverflowInTableInsert, CreateNamedStruct, DecimalRescaleCheckOverflow, FloatOperands, + GetArrayStructFields, GetStructField, IfExpr, ListExtract, MakeDateExpr, NormalizeNaNAndZero, + NormalizeNestedFloats, NullShortCircuit, SparkCastOptions, ToJson, UnboundColumn, + WideDecimalBinaryExpr, WideDecimalOp, }; use itertools::Itertools; use jni::objects::{Global, JObject}; @@ -932,6 +932,15 @@ impl PhysicalPlanner { query_context, ))) } + ExprStruct::CheckOverflowInTableInsert(expr) => { + let child = self.create_expr(expr.child.as_ref().unwrap(), input_schema)?; + Ok(Arc::new(CheckOverflowInTableInsert::new( + child, + expr.from_type.clone(), + expr.to_type.clone(), + expr.column_name.clone(), + ))) + } ExprStruct::ScalarFunc(expr) => { let func = self.create_scalar_function_expr(expr, input_schema); match expr.func.as_ref() { diff --git a/native/proto/src/proto/expr.proto b/native/proto/src/proto/expr.proto index bb39564dce2..71fb3e8f40d 100644 --- a/native/proto/src/proto/expr.proto +++ b/native/proto/src/proto/expr.proto @@ -96,6 +96,7 @@ message Expr { Uuid uuid = 74; NativeScalarUdf native_scalar_udf = 75; AtLeastNNonNulls at_least_n_non_nulls = 76; + CheckOverflowInTableInsert check_overflow_in_table_insert = 77; } reserved 20; @@ -593,6 +594,14 @@ message CheckOverflow { bool fail_on_error = 3; } +// The types are Spark SQL type names, which the JVM parses back to build Spark's error. +message CheckOverflowInTableInsert { + Expr child = 1; + string from_type = 2; + string to_type = 3; + string column_name = 4; +} + message ScalarFunc { string func = 1; repeated Expr args = 2; diff --git a/native/spark-expr/src/conversion_funcs/check_overflow_in_table_insert.rs b/native/spark-expr/src/conversion_funcs/check_overflow_in_table_insert.rs new file mode 100644 index 00000000000..6a7d8661a7e --- /dev/null +++ b/native/spark-expr/src/conversion_funcs/check_overflow_in_table_insert.rs @@ -0,0 +1,347 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt::{Display, Formatter}; +use std::hash::{Hash, Hasher}; +use std::sync::Arc; + +use arrow::datatypes::{DataType, Schema}; +use arrow::record_batch::RecordBatch; +use datafusion::common::{DataFusionError, Result}; +use datafusion::logical_expr::ColumnarValue; +use datafusion::physical_expr::PhysicalExpr; +use datafusion_comet_common::error_chain; +use datafusion_comet_jni_bridge::errors::CometError; + +use crate::{SparkError, SparkErrorWithContext}; + +/// Spark's `CheckOverflowInTableInsert`: evaluates the cast that a table insert added and turns +/// any `SparkArithmeticException` from it into `CAST_OVERFLOW_IN_TABLE_INSERT`, which names the +/// column instead of the value and carries no query context. +#[derive(Debug)] +pub struct CheckOverflowInTableInsert { + child: Arc, + from_type: String, + to_type: String, + column_name: String, +} + +impl CheckOverflowInTableInsert { + pub fn new( + child: Arc, + from_type: String, + to_type: String, + column_name: String, + ) -> Self { + Self { + child, + from_type, + to_type, + column_name, + } + } + + fn table_insert_error(&self) -> DataFusionError { + SparkError::CastOverflowInTableInsert { + from_type: self.from_type.clone(), + to_type: self.to_type.clone(), + column_name: self.column_name.clone(), + } + .into() + } +} + +impl PartialEq for CheckOverflowInTableInsert { + fn eq(&self, other: &Self) -> bool { + self.child.eq(&other.child) + && self.from_type == other.from_type + && self.to_type == other.to_type + && self.column_name == other.column_name + } +} + +impl Eq for CheckOverflowInTableInsert {} + +impl Hash for CheckOverflowInTableInsert { + fn hash(&self, state: &mut H) { + self.child.hash(state); + self.from_type.hash(state); + self.to_type.hash(state); + self.column_name.hash(state); + } +} + +impl Display for CheckOverflowInTableInsert { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "CheckOverflowInTableInsert({}, column: {})", + self.child, self.column_name + ) + } +} + +impl PhysicalExpr for CheckOverflowInTableInsert { + fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + self.child.fmt_sql(f) + } + + fn data_type(&self, input_schema: &Schema) -> Result { + self.child.data_type(input_schema) + } + + fn nullable(&self, input_schema: &Schema) -> Result { + self.child.nullable(input_schema) + } + + fn evaluate(&self, batch: &RecordBatch) -> Result { + match self.child.evaluate(batch) { + Err(e) if is_arithmetic_error(&e) => Err(self.table_insert_error()), + other => other, + } + } + + fn children(&self) -> Vec<&Arc> { + vec![&self.child] + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + match children.as_slice() { + [child] => Ok(Arc::new(Self::new( + Arc::clone(child), + self.from_type.clone(), + self.to_type.clone(), + self.column_name.clone(), + ))), + _ => Err(DataFusionError::Internal( + "CheckOverflowInTableInsert should have exactly one child".to_string(), + )), + } + } +} + +/// Whether Spark raises this error as a `SparkArithmeticException`: a native `SparkError`, or an +/// exception thrown by a child that runs Spark's own code through the JVM codegen dispatcher. +/// Like the JNI layer, it takes the first Spark error in the `source()` chain. +fn is_arithmetic_error(error: &DataFusionError) -> bool { + error_chain(error) + .find_map(|e| { + if let Some(CometError::JavaException { class, .. }) = e.downcast_ref::() { + Some(class == "org.apache.spark.SparkArithmeticException") + } else if let Some(e) = e.downcast_ref::() { + Some(e.error.is_arithmetic()) + } else { + e.downcast_ref::() + .map(SparkError::is_arithmetic) + } + }) + .unwrap_or(false) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::QueryContext; + use arrow::array::{Array, Int64Array}; + use arrow::datatypes::Field; + use datafusion::physical_expr::expressions::Column; + + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] + enum Failure { + CastOverflow, + DecimalOutOfRange, + DivideByZero, + InvalidInput, + } + + impl Failure { + fn error(self) -> SparkError { + match self { + Failure::CastOverflow => SparkError::CastOverFlow { + value: "3000000000L".to_string(), + from_type: "BIGINT".to_string(), + to_type: "INT".to_string(), + }, + Failure::DecimalOutOfRange => SparkError::NumericValueOutOfRange { + value: "1234567.89".to_string(), + precision: 10, + scale: 2, + }, + Failure::DivideByZero => SparkError::DivideByZero, + Failure::InvalidInput => SparkError::CastInvalidValue { + value: "'abc'".to_string(), + from_type: "STRING".to_string(), + to_type: "INT".to_string(), + }, + } + } + } + + /// A child that returns column `c`, or fails. A wrapped error comes inside + /// `DataFusionError::Context`, as DataFusion adds it on the way up. + #[derive(Debug, PartialEq, Eq, Hash)] + struct Child { + failure: Option, + with_context: bool, + wrapped: bool, + } + + impl Display for Child { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "child") + } + } + + impl PhysicalExpr for Child { + fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "child") + } + + fn data_type(&self, _input_schema: &Schema) -> Result { + Ok(DataType::Int64) + } + + fn nullable(&self, _input_schema: &Schema) -> Result { + Ok(true) + } + + fn evaluate(&self, batch: &RecordBatch) -> Result { + let Some(failure) = self.failure else { + return Column::new("c", 0).evaluate(batch); + }; + let error = failure.error(); + let error = if self.with_context { + let context = QueryContext::new("SELECT c".to_string(), 7, 7, None, None, 1, 7); + SparkErrorWithContext::with_context(error, Arc::new(context)).into() + } else { + error.into() + }; + if self.wrapped { + Err(DataFusionError::Context( + "child".to_string(), + Box::new(error), + )) + } else { + Err(error) + } + } + + fn children(&self) -> Vec<&Arc> { + vec![] + } + + fn with_new_children( + self: Arc, + _children: Vec>, + ) -> Result> { + Ok(self) + } + } + + fn evaluate( + failure: Option, + with_context: bool, + wrapped: bool, + ) -> Result { + let schema = Arc::new(Schema::new(vec![Field::new("c", DataType::Int64, true)])); + let batch = RecordBatch::try_new( + schema, + vec![Arc::new(Int64Array::from(vec![Some(1), None, Some(-2)]))], + ) + .unwrap(); + let child = Child { + failure, + with_context, + wrapped, + }; + CheckOverflowInTableInsert::new( + Arc::new(child), + "BIGINT".to_string(), + "INT".to_string(), + "c".to_string(), + ) + .evaluate(&batch) + } + + /// The first Spark error's message, and whether it carries a query context. + fn spark_error(result: Result) -> (String, bool) { + let Err(error) = result else { + panic!("expected a SparkError, got {result:?}"); + }; + let found = error_chain(&error) + .find_map(|e| { + if let Some(e) = e.downcast_ref::() { + Some((e.error.to_string(), e.context.is_some())) + } else { + e.downcast_ref::() + .map(|e| (e.to_string(), false)) + } + }) + .expect("a SparkError"); + found + } + + #[test] + fn values_pass_through() { + let ColumnarValue::Array(array) = evaluate(None, false, false).unwrap() else { + panic!("expected an array"); + }; + let array = array.as_any().downcast_ref::().unwrap(); + assert_eq!(array, &Int64Array::from(vec![Some(1), None, Some(-2)])); + } + + #[test] + fn arithmetic_errors_become_table_insert_errors() { + let failures = [ + Failure::CastOverflow, + Failure::DecimalOutOfRange, + Failure::DivideByZero, + ]; + for failure in failures { + for (with_context, wrapped) in + [(false, false), (true, false), (false, true), (true, true)] + { + let clue = format!("{failure:?} with context {with_context}, wrapped {wrapped}"); + let result = evaluate(Some(failure), with_context, wrapped); + let (message, has_context) = spark_error(result); + assert_eq!( + message, + "[CAST_OVERFLOW_IN_TABLE_INSERT] Fail to assign a value of \"BIGINT\" type \ + to the \"INT\" type column or variable `c` due to an overflow. Use \ + `try_cast` on the input value to tolerate overflow and return NULL instead.", + "{clue}" + ); + assert!(!has_context, "{clue}"); + } + } + } + + #[test] + fn other_errors_are_unchanged() { + for (with_context, wrapped) in [(false, false), (true, false), (false, true), (true, true)] + { + let result = evaluate(Some(Failure::InvalidInput), with_context, wrapped); + assert_eq!(matches!(result, Err(DataFusionError::Context(..))), wrapped); + let (message, has_context) = spark_error(result); + assert_eq!(message, Failure::InvalidInput.error().to_string()); + assert_eq!(has_context, with_context); + } + } +} diff --git a/native/spark-expr/src/conversion_funcs/mod.rs b/native/spark-expr/src/conversion_funcs/mod.rs index 098cb74ef90..beb4ec54fac 100644 --- a/native/spark-expr/src/conversion_funcs/mod.rs +++ b/native/spark-expr/src/conversion_funcs/mod.rs @@ -17,11 +17,13 @@ mod boolean; pub mod cast; +mod check_overflow_in_table_insert; mod numeric; mod string; mod temporal; pub(crate) mod trim; mod utils; +pub use check_overflow_in_table_insert::CheckOverflowInTableInsert; pub use numeric::{java_float_string, write_java_float_string, JavaFloatString}; pub(crate) use string::ymd_to_epoch_day; diff --git a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala index 72c9c8eb546..a0b518134d3 100644 --- a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala +++ b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala @@ -22,7 +22,7 @@ package org.apache.comet.expressions import scala.util.Try import org.apache.spark.SPARK_VERSION -import org.apache.spark.sql.catalyst.expressions.{Attribute, Cast, Expression, Literal} +import org.apache.spark.sql.catalyst.expressions.{Attribute, Cast, CheckOverflowInTableInsert, Expression, Literal} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{ArrayType, DataType, DataTypes, DecimalType, MapType, NullType, NumericType, StructType, TimestampNTZType, TimestampType} @@ -645,3 +645,30 @@ object CometCast .getOrElse(Compatible()) } } + +/** + * Spark rethrows the child's error unchanged when the optimizer has replaced the cast + * (SPARK-42286), so only a `Cast` child is wrapped. + */ +object CometCheckOverflowInTableInsert extends CometExpressionSerde[CheckOverflowInTableInsert] { + + override def convert( + expr: CheckOverflowInTableInsert, + inputs: Seq[Attribute], + binding: Boolean): Option[ExprOuterClass.Expr] = { + expr.child match { + case cast: Cast => + exprToProtoInternal(cast, inputs, binding).map { child => + val builder = ExprOuterClass.CheckOverflowInTableInsert + .newBuilder() + .setChild(child) + .setFromType(cast.child.dataType.sql) + .setToType(cast.dataType.sql) + .setColumnName(expr.columnName) + ExprOuterClass.Expr.newBuilder().setCheckOverflowInTableInsert(builder).build() + } + case child => + exprToProtoInternal(child, inputs, binding) + } + } +} diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 2d1d355fc75..fbf82dc6320 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -336,7 +336,8 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { classOf[ParseUrl] -> CometParseUrl) private val conversionExpressions: Map[Class[_ <: Expression], CometExpressionSerde[_]] = Map( - classOf[Cast] -> CometCast) + classOf[Cast] -> CometCast, + classOf[CheckOverflowInTableInsert] -> CometCheckOverflowInTableInsert) private val jsonExpressions: Map[Class[_ <: Expression], CometExpressionSerde[_]] = Map( classOf[LengthOfJsonArray] -> CometLengthOfJsonArray, diff --git a/spark/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimSparkErrorConverter.scala b/spark/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimSparkErrorConverter.scala index 326b6da3380..3dcdd1bdaf0 100644 --- a/spark/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimSparkErrorConverter.scala +++ b/spark/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimSparkErrorConverter.scala @@ -271,6 +271,13 @@ trait ShimSparkErrorConverter { Some(QueryExecutionErrors.castingCauseOverflowError(typedValue, fromType, toType)) + case "CastOverflowInTableInsert" => + Some( + QueryExecutionErrors.castingCauseOverflowErrorInTableInsert( + getDataType(params("fromType").toString), + getDataType(params("toType").toString), + params("columnName").toString)) + case "CannotParseDecimal" => Some(QueryExecutionErrors.cannotParseDecimalError()) diff --git a/spark/src/main/spark-4.x/org/apache/spark/sql/comet/shims/ShimSparkErrorConverter.scala b/spark/src/main/spark-4.x/org/apache/spark/sql/comet/shims/ShimSparkErrorConverter.scala index 53cd3f671ef..fda2d760826 100644 --- a/spark/src/main/spark-4.x/org/apache/spark/sql/comet/shims/ShimSparkErrorConverter.scala +++ b/spark/src/main/spark-4.x/org/apache/spark/sql/comet/shims/ShimSparkErrorConverter.scala @@ -299,6 +299,13 @@ trait ShimSparkErrorConverter { Some(QueryExecutionErrors.castingCauseOverflowError(typedValue, fromType, toType)) + case "CastOverflowInTableInsert" => + Some( + QueryExecutionErrors.castingCauseOverflowErrorInTableInsert( + getDataType(params("fromType").toString), + getDataType(params("toType").toString), + params("columnName").toString)) + case "CannotParseDecimal" => Some(QueryExecutionErrors.cannotParseDecimalError()) diff --git a/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala b/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala index e44978c1d56..2bbfeddc789 100644 --- a/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala +++ b/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala @@ -78,6 +78,23 @@ class SparkErrorConverterSuite extends AnyFunSuite { error.getMessage) } + test("CastOverflowInTableInsert names the column as Spark does") { + val error = SparkErrorConverter + .convertErrorType( + "CastOverflowInTableInsert", + "CAST_OVERFLOW_IN_TABLE_INSERT", + Map("fromType" -> "DECIMAL(20,2)", "toType" -> "DECIMAL(10,2)", "columnName" -> "s.x"), + Array.empty, + "") + .getOrElse(fail("Expected CastOverflowInTableInsert to be converted")) + assert(error.isInstanceOf[ArithmeticException]) + assert(error.asInstanceOf[SparkThrowable].getErrorClass == "CAST_OVERFLOW_IN_TABLE_INSERT") + assert(error.asInstanceOf[SparkThrowable].getQueryContext.isEmpty) + assert(error.getMessage.contains("\"DECIMAL(20,2)\""), error.getMessage) + assert(error.getMessage.contains("\"DECIMAL(10,2)\""), error.getMessage) + assert(error.getMessage.contains("`s`.`x`"), error.getMessage) + } + test("ArithmeticOverflow without a suggested function remains compatible") { // Accept both older payloads without the key and unary errors with an empty function. for (params <- Seq( diff --git a/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala b/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala index 5f4efe8ce57..468157e1d3f 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala @@ -36,13 +36,14 @@ import org.apache.parquet.hadoop.ParquetFileReader import org.apache.parquet.hadoop.metadata.CompressionCodecName import org.apache.parquet.hadoop.util.HadoopInputFile import org.apache.parquet.schema.{MessageType, Type} -import org.apache.spark.SPARK_VERSION_SHORT +import org.apache.spark.{SPARK_VERSION_SHORT, SparkThrowable} import org.apache.spark.internal.io.{FileCommitProtocol, FileNameSpec} import org.apache.spark.internal.io.cloud.{BindingParquetOutputCommitter, PathOutputCommitProtocol} import org.apache.spark.memory.MemoryMode import org.apache.spark.sql.{AnalysisException, DataFrame, Row, SaveMode} import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow, TableIdentifier} -import org.apache.spark.sql.comet.{CometBatchScanExec, CometColumnarToRowExec, CometNativeScanExec, CometNativeWriteExec, CometScanExec, CometSparkToColumnarExec, CometWriteFilesExec} +import org.apache.spark.sql.catalyst.optimizer.ConstantFolding +import org.apache.spark.sql.comet.{CometBatchScanExec, CometColumnarToRowExec, CometNativeScanExec, CometNativeWriteExec, CometProjectExec, CometScanExec, CometSparkToColumnarExec, CometWriteFilesExec} import org.apache.spark.sql.execution.{ColumnarToRowTransition, FileSourceScanExec, SparkPlan, SQLExecution} import org.apache.spark.sql.execution.command.DataWritingCommandExec import org.apache.spark.sql.execution.datasources.{BasicWriteTaskStats, BasicWriteTaskStatsTracker, FileFormatWriter, SQLHadoopMapReduceCommitProtocol, WriteFilesExec, WriteJobStatsTracker, WriteTaskStats, WriteTaskStatsTracker} @@ -53,7 +54,7 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{ArrayType, IntegerType, LongType, MapType, Metadata, MetadataBuilder, StringType, StructField, StructType} import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} -import org.apache.comet.CometConf +import org.apache.comet.{CometConf, CometNativeException} import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus import org.apache.comet.hadoop.fs.RecordingFileSystem import org.apache.comet.rules.RevertNativeForTransitionHeavyStages @@ -1694,6 +1695,153 @@ class CometParquetWriterSuite extends CometParquetWriterTestBase { } } + private def withInsertWriter(nativeWriter: Boolean)(f: => Unit): Unit = + if (nativeWriter) withNativeWriter(f) else f + + private def assertCometInsert(plan: SparkPlan, nativeWriter: Boolean): Unit = + if (nativeWriter) { + assertHasCometNativeWriteExec(plan) + } else { + assert(plan.exists(_.isInstanceOf[CometProjectExec]), plan.treeString) + } + + private def withNarrowingTables(sourceType: String, targetType: String, values: Seq[String])( + f: => Unit): Unit = + withTable("comet_narrow_source", "comet_narrow_target") { + sql(s"CREATE TABLE comet_narrow_source(c $sourceType) USING parquet") + sql(s"CREATE TABLE comet_narrow_target(c $targetType) USING parquet") + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { + sql(s"INSERT INTO comet_narrow_source VALUES ${values.map(v => s"($v)").mkString(", ")}") + } + withClue(s"$sourceType -> $targetType: ")(f) + } + + /** Runs `select` into the target in Spark and in Comet, and returns Spark's error. */ + private def assertInsertFailsAsSpark( + select: String, + nativeWriter: Boolean): SparkThrowable with Throwable = { + val insert = s"INSERT INTO comet_narrow_target SELECT $select FROM comet_narrow_source" + def sparkError(e: Throwable): SparkThrowable with Throwable = + causeChain(e) + .collect { case t: SparkThrowable with Throwable => t } + .lastOption + .getOrElse(fail(s"$insert did not raise a SparkThrowable", e)) + var expected: SparkThrowable with Throwable = null + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { + expected = sparkError(intercept[Exception](sql(insert))) + } + withInsertWriter(nativeWriter) { + val (plan, e) = captureFailedWritePlan(sql(insert)) + assertCometInsert(plan, nativeWriter) + assert(!causeChain(e).exists(_.isInstanceOf[CometNativeException]), e) + val actual = sparkError(e) + assert(actual.getClass == expected.getClass) + assert(actual.getErrorClass == expected.getErrorClass) + assert(actual.getSqlState == expected.getSqlState) + assert(actual.getMessage == expected.getMessage) + } + expected + } + + for (nativeWriter <- Seq(false, true)) { + test(s"a narrowing INSERT stays in Comet (native writer: $nativeWriter)") { + val inserts = Seq( + ("bigint", "int", Seq("1", "NULL", "2147483647", "-2147483648")), + ("int", "tinyint", Seq("127", "-128")), + ("bigint", "smallint", Seq("32767", "-32768")), + ("double", "bigint", Seq("1.5", "-2.9", "NULL")), + ("decimal(20,2)", "decimal(10,2)", Seq("1.25", "-99999999.99", "NULL")), + ("decimal(20,2)", "int", Seq("1.99", "-7.5")), + ("struct", "struct", Seq("named_struct('x', 1)", "NULL"))) + inserts.foreach { case (sourceType, targetType, values) => + withNarrowingTables(sourceType, targetType, values) { + withTable("comet_narrow_expected") { + sql(s"CREATE TABLE comet_narrow_expected(c $targetType) USING parquet") + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { + sql("INSERT INTO comet_narrow_expected SELECT c FROM comet_narrow_source") + } + withInsertWriter(nativeWriter) { + val plan = captureWritePlan( + sql("INSERT INTO comet_narrow_target SELECT c FROM comet_narrow_source")) + assertCometInsert(plan, nativeWriter) + } + checkAnswer(spark.table("comet_narrow_target"), spark.table("comet_narrow_expected")) + } + } + } + } + + test( + s"a narrowing INSERT that overflows fails as Spark does (native writer: $nativeWriter)") { + val overflows = Seq( + ("bigint", "int", "3000000000"), + ("bigint", "int", "-2147483649"), + ("int", "tinyint", "128"), + ("bigint", "smallint", "-32769"), + ("double", "int", "double('NaN')"), + ("double", "bigint", "double('Infinity')"), + ("double", "bigint", "1e20"), + ("double", "decimal(10,2)", "1e12"), + ("decimal(20,2)", "decimal(10,2)", "123456789012.34"), + ("decimal(20,2)", "int", "3000000000.5"), + ("struct", "struct", "named_struct('x', 3000000000)")) + overflows.foreach { case (sourceType, targetType, value) => + withNarrowingTables(sourceType, targetType, Seq(value)) { + val error = assertInsertFailsAsSpark("c", nativeWriter) + assert(error.getErrorClass == "CAST_OVERFLOW_IN_TABLE_INSERT", error) + } + } + + // Spark reports any arithmetic error under the cast as an overflow of the column. + withNarrowingTables("bigint", "int", Seq("1")) { + withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { + val error = assertInsertFailsAsSpark("c div 0", nativeWriter) + assert(error.getErrorClass == "CAST_OVERFLOW_IN_TABLE_INSERT", error) + } + } + + // Other errors under the cast keep their class and query context. + withNarrowingTables("string", "int", Seq("'abc'")) { + withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { + val error = assertInsertFailsAsSpark("CAST(c AS BIGINT)", nativeWriter) + assert(error.getErrorClass == "CAST_INVALID_INPUT", error) + assert(error.getQueryContext.nonEmpty, error) + } + } + } + + test( + s"a narrowing INSERT with a rewritten cast keeps its error (native writer: $nativeWriter)") { + // SPARK-42286: Spark rethrows the child's error as it is when the optimizer has replaced the + // cast under CheckOverflowInTableInsert, here by pushing it into the CASE WHEN branches. + val inserts = Seq( + "tinyint" -> ("0 - (CASE WHEN c = 1.2345678901234567E19D THEN 1.2345678901234567E19D " + + "ELSE c END)"), + "decimal(9,0)" -> "0 - (CASE WHEN c = 1 THEN 1 ELSE c END)") + inserts.foreach { case (targetType, select) => + withNarrowingTables("double", targetType, Seq("1.2345678901234567E19D")) { + val error = assertInsertFailsAsSpark(select, nativeWriter) + assert(error.getErrorClass != "CAST_OVERFLOW_IN_TABLE_INSERT", error) + } + } + } + } + + test("a narrowing INSERT of an overflowing literal fails as Spark does") { + withNarrowingTables("bigint", "int", Seq("1")) { + withSQLConf(SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> ConstantFolding.ruleName) { + val error = assertInsertFailsAsSpark("3000000000L", nativeWriter = false) + assert(error.getErrorClass == "CAST_OVERFLOW_IN_TABLE_INSERT", error) + val (plan, _) = captureFailedWritePlan( + sql("INSERT INTO comet_narrow_target SELECT 3000000000L FROM comet_narrow_source")) + assertExpressionImpl( + plan, + native = Seq("checkoverflowintableinsert"), + dispatched = Seq("cast")) + } + } + } + test("an empty partition writes no file and still commits") { // The write task's `partitionId != 0 && !batches.hasNext` branch must skip newTaskTempFile // altogether and still commit the task, matching FileFormatWriter's EmptyDirectoryDataWriter.