From 6411b9bb17e4cfdca29999e8f1f2f59090522427 Mon Sep 17 00:00:00 2001 From: FreakingMind <55959207+FreakingMind@users.noreply.github.com> Date: Tue, 17 Nov 2026 01:35:30 +0300 Subject: [PATCH 1/4] feat: run CheckOverflowInTableInsert natively Spark wraps the cast of an INSERT that narrows a numeric column in CheckOverflowInTableInsert, which Comet did not support, so the Project that feeds the write fell back to Spark along with the native write. The native wrapper keeps the cast native and turns an arithmetic error from it into CAST_OVERFLOW_IN_TABLE_INSERT, built on the JVM by Spark's own error factory so each version keeps its message. Closes #7656 --- docs/source/user-guide/latest/expressions.md | 1 + native/common/src/error.rs | 36 +- native/core/src/execution/planner.rs | 17 +- native/proto/src/proto/expr.proto | 9 + .../check_overflow_in_table_insert.rs | 318 ++++++++++++++++++ native/spark-expr/src/conversion_funcs/mod.rs | 2 + .../apache/comet/expressions/CometCast.scala | 29 +- .../apache/comet/serde/QueryPlanSerde.scala | 3 +- .../comet/shims/ShimSparkErrorConverter.scala | 7 + .../comet/shims/ShimSparkErrorConverter.scala | 7 + .../comet/SparkErrorConverterSuite.scala | 17 + .../parquet/CometParquetWriterSuite.scala | 154 ++++++++- 12 files changed, 590 insertions(+), 10 deletions(-) create mode 100644 native/spark-expr/src/conversion_funcs/check_overflow_in_table_insert.rs diff --git a/docs/source/user-guide/latest/expressions.md b/docs/source/user-guide/latest/expressions.md index b73fef7aa2b..0831ced7a66 100644 --- a/docs/source/user-guide/latest/expressions.md +++ b/docs/source/user-guide/latest/expressions.md @@ -704,6 +704,7 @@ Comet also accelerates a number of Catalyst expressions that have no Spark SQL f - **Operator and optimizer-injected expressions:** runtime bloom-filter join probes (`BloomFilterMightContain`, `BloomFilterAggregate`), optimized `IN` sets (`InSet`), scalar subqueries (`ScalarSubquery`), and floating-point normalization (`KnownFloatingPointNormalized`). - **Accessor expressions (subscript and field access, not functions):** struct field access (`col.field`), array element access (`arr[i]`), and map value access (`map[key]`). - **Internal decimal arithmetic:** `CheckOverflow`, `MakeDecimal`, and `UnscaledValue`, which the analyzer inserts around decimal operations. +- **Table insert casts:** `CheckOverflowInTableInsert`, which Spark wraps around the cast an `INSERT` adds under the default `spark.sql.storeAssignmentPolicy=ANSI`, so that an arithmetic error from the cast is reported as `CAST_OVERFLOW_IN_TABLE_INSERT`. When an expression under the cast occurs more than once in the projection, Spark's subexpression elimination evaluates it outside the wrapper and Spark reports the original error, such as `CAST_OVERFLOW`. Comet reports `CAST_OVERFLOW_IN_TABLE_INSERT` there, as Spark does with `spark.sql.subexpressionElimination.enabled=false`. - **Missing-data predicates:** `AtLeastNNonNulls`, used by `DataFrame.na.drop`, counts non-null and non-NaN values natively. - **User-defined functions:** Scala UDFs registered through the DataFrame or SQL API. - **DataSource V2 catalog functions:** Iceberg's system functions `bucket`, `truncate`, `years`, `months`, `days`, and `hours` (for example `system.bucket(16, id)`) run natively; see [Iceberg system functions](iceberg.md#iceberg-system-functions). Other DataSource V2 catalog functions run in Spark. diff --git a/native/common/src/error.rs b/native/common/src/error.rs index c227c3f1b8e..a9f0cde687e 100644 --- a/native/common/src/error.rs +++ b/native/common/src/error.rs @@ -69,6 +69,15 @@ pub enum SparkError { to_type: String, }, + #[error("[CAST_OVERFLOW_IN_TABLE_INSERT] Fail to assign a value of \"{from_type}\" type to the \ + \"{to_type}\" type column or variable `{column_name}` due to an overflow. Use `try_cast` on \ + the input value to tolerate overflow and return NULL instead.")] + CastOverflowInTableInsert { + from_type: String, + to_type: String, + column_name: String, + }, + #[error("[CANNOT_PARSE_DECIMAL] Cannot parse decimal.")] CannotParseDecimal, @@ -358,6 +367,7 @@ impl SparkError { SparkError::NumericValueOutOfRange { .. } => "NumericValueOutOfRange", SparkError::NumericOutOfRange { .. } => "NumericOutOfRange", SparkError::CastOverFlow { .. } => "CastOverFlow", + SparkError::CastOverflowInTableInsert { .. } => "CastOverflowInTableInsert", SparkError::CannotParseDecimal => "CannotParseDecimal", SparkError::ArithmeticOverflow { .. } => "ArithmeticOverflow", SparkError::IntegralDivideOverflow => "IntegralDivideOverflow", @@ -464,6 +474,17 @@ impl SparkError { "toType": to_type, }) } + SparkError::CastOverflowInTableInsert { + from_type, + to_type, + column_name, + } => { + serde_json::json!({ + "fromType": from_type, + "toType": to_type, + "columnName": column_name, + }) + } SparkError::ArithmeticOverflow { from_type, function_name, @@ -717,6 +738,15 @@ impl SparkError { } } + /// Whether Spark raises this error as a `SparkArithmeticException`. + pub fn is_arithmetic(&self) -> bool { + matches!( + self.exception_class(), + "org/apache/spark/SparkArithmeticException" + | "org/apache/spark/sql/comet/CastOverflowException" + ) + } + /// Returns the appropriate Spark exception class for this error pub fn exception_class(&self) -> &'static str { match self { @@ -732,7 +762,10 @@ impl SparkError { | SparkError::BinaryArithmeticOverflow { .. } | SparkError::IntervalArithmeticOverflowWithSuggestion { .. } | SparkError::IntervalArithmeticOverflowWithoutSuggestion - | SparkError::DatetimeOverflow => "org/apache/spark/SparkArithmeticException", + | SparkError::DatetimeOverflow + | SparkError::CastOverflowInTableInsert { .. } => { + "org/apache/spark/SparkArithmeticException" + } // CastOverflow gets special handling with CastOverflowException SparkError::CastOverFlow { .. } => "org/apache/spark/sql/comet/CastOverflowException", @@ -829,6 +862,7 @@ impl SparkError { SparkError::CastInvalidValue { .. } => Some("CAST_INVALID_INPUT"), SparkError::InvalidInputInCastToDatetime { .. } => Some("CAST_INVALID_INPUT"), SparkError::CastOverFlow { .. } => Some("CAST_OVERFLOW"), + SparkError::CastOverflowInTableInsert { .. } => Some("CAST_OVERFLOW_IN_TABLE_INSERT"), SparkError::NumericValueOutOfRange { .. } => { Some("NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION") } diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index 72071d9dd19..445350daefa 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -121,10 +121,10 @@ use datafusion_comet_proto::{ use datafusion_comet_spark_expr::{ coerce_to_common_type, create_case_when, create_if_expr, decimal128_add_sub_width, jvm_udf::JvmScalarUdfExpr, spark_in_list, ArrayInsert, AtLeastNNonNulls, Cast, CheckOverflow, - CreateNamedStruct, DecimalRescaleCheckOverflow, FloatOperands, GetArrayStructFields, - GetStructField, IfExpr, ListExtract, MakeDateExpr, NormalizeNaNAndZero, NormalizeNestedFloats, - NullShortCircuit, SparkCastOptions, ToJson, UnboundColumn, WideDecimalBinaryExpr, - WideDecimalOp, + CheckOverflowInTableInsert, CreateNamedStruct, DecimalRescaleCheckOverflow, FloatOperands, + GetArrayStructFields, GetStructField, IfExpr, ListExtract, MakeDateExpr, NormalizeNaNAndZero, + NormalizeNestedFloats, NullShortCircuit, SparkCastOptions, ToJson, UnboundColumn, + WideDecimalBinaryExpr, WideDecimalOp, }; use itertools::Itertools; use jni::objects::{Global, JObject}; @@ -932,6 +932,15 @@ impl PhysicalPlanner { query_context, ))) } + ExprStruct::CheckOverflowInTableInsert(expr) => { + let child = self.create_expr(expr.child.as_ref().unwrap(), input_schema)?; + Ok(Arc::new(CheckOverflowInTableInsert::new( + child, + expr.from_type.clone(), + expr.to_type.clone(), + expr.column_name.clone(), + ))) + } ExprStruct::ScalarFunc(expr) => { let func = self.create_scalar_function_expr(expr, input_schema); match expr.func.as_ref() { diff --git a/native/proto/src/proto/expr.proto b/native/proto/src/proto/expr.proto index bb39564dce2..71fb3e8f40d 100644 --- a/native/proto/src/proto/expr.proto +++ b/native/proto/src/proto/expr.proto @@ -96,6 +96,7 @@ message Expr { Uuid uuid = 74; NativeScalarUdf native_scalar_udf = 75; AtLeastNNonNulls at_least_n_non_nulls = 76; + CheckOverflowInTableInsert check_overflow_in_table_insert = 77; } reserved 20; @@ -593,6 +594,14 @@ message CheckOverflow { bool fail_on_error = 3; } +// The types are Spark SQL type names, which the JVM parses back to build Spark's error. +message CheckOverflowInTableInsert { + Expr child = 1; + string from_type = 2; + string to_type = 3; + string column_name = 4; +} + message ScalarFunc { string func = 1; repeated Expr args = 2; diff --git a/native/spark-expr/src/conversion_funcs/check_overflow_in_table_insert.rs b/native/spark-expr/src/conversion_funcs/check_overflow_in_table_insert.rs new file mode 100644 index 00000000000..0994942814a --- /dev/null +++ b/native/spark-expr/src/conversion_funcs/check_overflow_in_table_insert.rs @@ -0,0 +1,318 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt::{Display, Formatter}; +use std::hash::{Hash, Hasher}; +use std::sync::Arc; + +use arrow::datatypes::{DataType, Schema}; +use arrow::record_batch::RecordBatch; +use datafusion::common::{DataFusionError, Result}; +use datafusion::logical_expr::ColumnarValue; +use datafusion::physical_expr::PhysicalExpr; +use datafusion_comet_jni_bridge::errors::CometError; + +use crate::{SparkError, SparkErrorWithContext}; + +/// Spark's `CheckOverflowInTableInsert`: evaluates the cast that a table insert added and turns +/// any `SparkArithmeticException` from it into `CAST_OVERFLOW_IN_TABLE_INSERT`, which names the +/// column instead of the value and carries no query context. +#[derive(Debug)] +pub struct CheckOverflowInTableInsert { + child: Arc, + from_type: String, + to_type: String, + column_name: String, +} + +impl CheckOverflowInTableInsert { + pub fn new( + child: Arc, + from_type: String, + to_type: String, + column_name: String, + ) -> Self { + Self { + child, + from_type, + to_type, + column_name, + } + } + + fn table_insert_error(&self) -> DataFusionError { + SparkError::CastOverflowInTableInsert { + from_type: self.from_type.clone(), + to_type: self.to_type.clone(), + column_name: self.column_name.clone(), + } + .into() + } +} + +impl PartialEq for CheckOverflowInTableInsert { + fn eq(&self, other: &Self) -> bool { + self.child.eq(&other.child) + && self.from_type == other.from_type + && self.to_type == other.to_type + && self.column_name == other.column_name + } +} + +impl Eq for CheckOverflowInTableInsert {} + +impl Hash for CheckOverflowInTableInsert { + fn hash(&self, state: &mut H) { + self.child.hash(state); + self.from_type.hash(state); + self.to_type.hash(state); + self.column_name.hash(state); + } +} + +impl Display for CheckOverflowInTableInsert { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "CheckOverflowInTableInsert({}, column: {})", + self.child, self.column_name + ) + } +} + +impl PhysicalExpr for CheckOverflowInTableInsert { + fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + self.child.fmt_sql(f) + } + + fn data_type(&self, input_schema: &Schema) -> Result { + self.child.data_type(input_schema) + } + + fn nullable(&self, input_schema: &Schema) -> Result { + self.child.nullable(input_schema) + } + + fn evaluate(&self, batch: &RecordBatch) -> Result { + match self.child.evaluate(batch) { + Err(e) if is_arithmetic_error(&e) => Err(self.table_insert_error()), + other => other, + } + } + + fn children(&self) -> Vec<&Arc> { + vec![&self.child] + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + match children.as_slice() { + [child] => Ok(Arc::new(Self::new( + Arc::clone(child), + self.from_type.clone(), + self.to_type.clone(), + self.column_name.clone(), + ))), + _ => Err(DataFusionError::Internal( + "CheckOverflowInTableInsert should have exactly one child".to_string(), + )), + } + } +} + +/// Whether Spark raises this error as a `SparkArithmeticException`: a native `SparkError`, or an +/// exception thrown by a child that runs Spark's own code through the JVM codegen dispatcher. +/// Only the top of a `DataFusionError::External` is checked, because that is where the JNI layer +/// reads a Spark error from. +fn is_arithmetic_error(error: &DataFusionError) -> bool { + let DataFusionError::External(e) = error else { + return false; + }; + if let Some(CometError::JavaException { class, .. }) = e.downcast_ref::() { + return class == "org.apache.spark.SparkArithmeticException"; + } + e.downcast_ref::() + .or_else(|| e.downcast_ref::().map(|e| &e.error)) + .is_some_and(SparkError::is_arithmetic) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::QueryContext; + use arrow::array::{Array, Int64Array}; + use arrow::datatypes::Field; + use datafusion::physical_expr::expressions::Column; + + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] + enum Failure { + CastOverflow, + DecimalOutOfRange, + DivideByZero, + InvalidInput, + } + + impl Failure { + fn error(self) -> SparkError { + match self { + Failure::CastOverflow => SparkError::CastOverFlow { + value: "3000000000L".to_string(), + from_type: "BIGINT".to_string(), + to_type: "INT".to_string(), + }, + Failure::DecimalOutOfRange => SparkError::NumericValueOutOfRange { + value: "1234567.89".to_string(), + precision: 10, + scale: 2, + }, + Failure::DivideByZero => SparkError::DivideByZero, + Failure::InvalidInput => SparkError::CastInvalidValue { + value: "'abc'".to_string(), + from_type: "STRING".to_string(), + to_type: "INT".to_string(), + }, + } + } + } + + /// A child that returns column `c`, or fails. + #[derive(Debug, PartialEq, Eq, Hash)] + struct Child { + failure: Option, + with_context: bool, + } + + impl Display for Child { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "child") + } + } + + impl PhysicalExpr for Child { + fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "child") + } + + fn data_type(&self, _input_schema: &Schema) -> Result { + Ok(DataType::Int64) + } + + fn nullable(&self, _input_schema: &Schema) -> Result { + Ok(true) + } + + fn evaluate(&self, batch: &RecordBatch) -> Result { + let Some(failure) = self.failure else { + return Column::new("c", 0).evaluate(batch); + }; + let error = failure.error(); + if self.with_context { + let context = QueryContext::new("SELECT c".to_string(), 7, 7, None, None, 1, 7); + Err(SparkErrorWithContext::with_context(error, Arc::new(context)).into()) + } else { + Err(error.into()) + } + } + + fn children(&self) -> Vec<&Arc> { + vec![] + } + + fn with_new_children( + self: Arc, + _children: Vec>, + ) -> Result> { + Ok(self) + } + } + + fn evaluate(failure: Option, with_context: bool) -> Result { + let schema = Arc::new(Schema::new(vec![Field::new("c", DataType::Int64, true)])); + let batch = RecordBatch::try_new( + schema, + vec![Arc::new(Int64Array::from(vec![Some(1), None, Some(-2)]))], + ) + .unwrap(); + let child = Child { + failure, + with_context, + }; + CheckOverflowInTableInsert::new( + Arc::new(child), + "BIGINT".to_string(), + "INT".to_string(), + "c".to_string(), + ) + .evaluate(&batch) + } + + /// The Spark error's message, and whether it carries a query context. + fn spark_error(result: Result) -> (String, bool) { + let Err(DataFusionError::External(e)) = result else { + panic!("expected a SparkError, got {result:?}"); + }; + if let Some(e) = e.downcast_ref::() { + (e.error.to_string(), e.context.is_some()) + } else { + let e = e.downcast_ref::().expect("a SparkError"); + (e.to_string(), false) + } + } + + #[test] + fn values_pass_through() { + let ColumnarValue::Array(array) = evaluate(None, false).unwrap() else { + panic!("expected an array"); + }; + let array = array.as_any().downcast_ref::().unwrap(); + assert_eq!(array, &Int64Array::from(vec![Some(1), None, Some(-2)])); + } + + #[test] + fn arithmetic_errors_become_table_insert_errors() { + let failures = [ + Failure::CastOverflow, + Failure::DecimalOutOfRange, + Failure::DivideByZero, + ]; + for failure in failures { + for with_context in [false, true] { + let (message, has_context) = spark_error(evaluate(Some(failure), with_context)); + assert_eq!( + message, + "[CAST_OVERFLOW_IN_TABLE_INSERT] Fail to assign a value of \"BIGINT\" type \ + to the \"INT\" type column or variable `c` due to an overflow. Use \ + `try_cast` on the input value to tolerate overflow and return NULL instead.", + "{failure:?} with context {with_context}" + ); + assert!(!has_context, "{failure:?} with context {with_context}"); + } + } + } + + #[test] + fn other_errors_are_unchanged() { + for with_context in [false, true] { + let failure = Some(Failure::InvalidInput); + let (message, has_context) = spark_error(evaluate(failure, with_context)); + assert_eq!(message, Failure::InvalidInput.error().to_string()); + assert_eq!(has_context, with_context); + } + } +} diff --git a/native/spark-expr/src/conversion_funcs/mod.rs b/native/spark-expr/src/conversion_funcs/mod.rs index 098cb74ef90..beb4ec54fac 100644 --- a/native/spark-expr/src/conversion_funcs/mod.rs +++ b/native/spark-expr/src/conversion_funcs/mod.rs @@ -17,11 +17,13 @@ mod boolean; pub mod cast; +mod check_overflow_in_table_insert; mod numeric; mod string; mod temporal; pub(crate) mod trim; mod utils; +pub use check_overflow_in_table_insert::CheckOverflowInTableInsert; pub use numeric::{java_float_string, write_java_float_string, JavaFloatString}; pub(crate) use string::ymd_to_epoch_day; diff --git a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala index 72c9c8eb546..a0b518134d3 100644 --- a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala +++ b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala @@ -22,7 +22,7 @@ package org.apache.comet.expressions import scala.util.Try import org.apache.spark.SPARK_VERSION -import org.apache.spark.sql.catalyst.expressions.{Attribute, Cast, Expression, Literal} +import org.apache.spark.sql.catalyst.expressions.{Attribute, Cast, CheckOverflowInTableInsert, Expression, Literal} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{ArrayType, DataType, DataTypes, DecimalType, MapType, NullType, NumericType, StructType, TimestampNTZType, TimestampType} @@ -645,3 +645,30 @@ object CometCast .getOrElse(Compatible()) } } + +/** + * Spark rethrows the child's error unchanged when the optimizer has replaced the cast + * (SPARK-42286), so only a `Cast` child is wrapped. + */ +object CometCheckOverflowInTableInsert extends CometExpressionSerde[CheckOverflowInTableInsert] { + + override def convert( + expr: CheckOverflowInTableInsert, + inputs: Seq[Attribute], + binding: Boolean): Option[ExprOuterClass.Expr] = { + expr.child match { + case cast: Cast => + exprToProtoInternal(cast, inputs, binding).map { child => + val builder = ExprOuterClass.CheckOverflowInTableInsert + .newBuilder() + .setChild(child) + .setFromType(cast.child.dataType.sql) + .setToType(cast.dataType.sql) + .setColumnName(expr.columnName) + ExprOuterClass.Expr.newBuilder().setCheckOverflowInTableInsert(builder).build() + } + case child => + exprToProtoInternal(child, inputs, binding) + } + } +} diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 2d1d355fc75..fbf82dc6320 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -336,7 +336,8 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { classOf[ParseUrl] -> CometParseUrl) private val conversionExpressions: Map[Class[_ <: Expression], CometExpressionSerde[_]] = Map( - classOf[Cast] -> CometCast) + classOf[Cast] -> CometCast, + classOf[CheckOverflowInTableInsert] -> CometCheckOverflowInTableInsert) private val jsonExpressions: Map[Class[_ <: Expression], CometExpressionSerde[_]] = Map( classOf[LengthOfJsonArray] -> CometLengthOfJsonArray, diff --git a/spark/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimSparkErrorConverter.scala b/spark/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimSparkErrorConverter.scala index 326b6da3380..3dcdd1bdaf0 100644 --- a/spark/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimSparkErrorConverter.scala +++ b/spark/src/main/spark-3.5/org/apache/spark/sql/comet/shims/ShimSparkErrorConverter.scala @@ -271,6 +271,13 @@ trait ShimSparkErrorConverter { Some(QueryExecutionErrors.castingCauseOverflowError(typedValue, fromType, toType)) + case "CastOverflowInTableInsert" => + Some( + QueryExecutionErrors.castingCauseOverflowErrorInTableInsert( + getDataType(params("fromType").toString), + getDataType(params("toType").toString), + params("columnName").toString)) + case "CannotParseDecimal" => Some(QueryExecutionErrors.cannotParseDecimalError()) diff --git a/spark/src/main/spark-4.x/org/apache/spark/sql/comet/shims/ShimSparkErrorConverter.scala b/spark/src/main/spark-4.x/org/apache/spark/sql/comet/shims/ShimSparkErrorConverter.scala index 53cd3f671ef..fda2d760826 100644 --- a/spark/src/main/spark-4.x/org/apache/spark/sql/comet/shims/ShimSparkErrorConverter.scala +++ b/spark/src/main/spark-4.x/org/apache/spark/sql/comet/shims/ShimSparkErrorConverter.scala @@ -299,6 +299,13 @@ trait ShimSparkErrorConverter { Some(QueryExecutionErrors.castingCauseOverflowError(typedValue, fromType, toType)) + case "CastOverflowInTableInsert" => + Some( + QueryExecutionErrors.castingCauseOverflowErrorInTableInsert( + getDataType(params("fromType").toString), + getDataType(params("toType").toString), + params("columnName").toString)) + case "CannotParseDecimal" => Some(QueryExecutionErrors.cannotParseDecimalError()) diff --git a/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala b/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala index e44978c1d56..2bbfeddc789 100644 --- a/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala +++ b/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala @@ -78,6 +78,23 @@ class SparkErrorConverterSuite extends AnyFunSuite { error.getMessage) } + test("CastOverflowInTableInsert names the column as Spark does") { + val error = SparkErrorConverter + .convertErrorType( + "CastOverflowInTableInsert", + "CAST_OVERFLOW_IN_TABLE_INSERT", + Map("fromType" -> "DECIMAL(20,2)", "toType" -> "DECIMAL(10,2)", "columnName" -> "s.x"), + Array.empty, + "") + .getOrElse(fail("Expected CastOverflowInTableInsert to be converted")) + assert(error.isInstanceOf[ArithmeticException]) + assert(error.asInstanceOf[SparkThrowable].getErrorClass == "CAST_OVERFLOW_IN_TABLE_INSERT") + assert(error.asInstanceOf[SparkThrowable].getQueryContext.isEmpty) + assert(error.getMessage.contains("\"DECIMAL(20,2)\""), error.getMessage) + assert(error.getMessage.contains("\"DECIMAL(10,2)\""), error.getMessage) + assert(error.getMessage.contains("`s`.`x`"), error.getMessage) + } + test("ArithmeticOverflow without a suggested function remains compatible") { // Accept both older payloads without the key and unary errors with an empty function. for (params <- Seq( diff --git a/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala b/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala index 5f4efe8ce57..468157e1d3f 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/CometParquetWriterSuite.scala @@ -36,13 +36,14 @@ import org.apache.parquet.hadoop.ParquetFileReader import org.apache.parquet.hadoop.metadata.CompressionCodecName import org.apache.parquet.hadoop.util.HadoopInputFile import org.apache.parquet.schema.{MessageType, Type} -import org.apache.spark.SPARK_VERSION_SHORT +import org.apache.spark.{SPARK_VERSION_SHORT, SparkThrowable} import org.apache.spark.internal.io.{FileCommitProtocol, FileNameSpec} import org.apache.spark.internal.io.cloud.{BindingParquetOutputCommitter, PathOutputCommitProtocol} import org.apache.spark.memory.MemoryMode import org.apache.spark.sql.{AnalysisException, DataFrame, Row, SaveMode} import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow, TableIdentifier} -import org.apache.spark.sql.comet.{CometBatchScanExec, CometColumnarToRowExec, CometNativeScanExec, CometNativeWriteExec, CometScanExec, CometSparkToColumnarExec, CometWriteFilesExec} +import org.apache.spark.sql.catalyst.optimizer.ConstantFolding +import org.apache.spark.sql.comet.{CometBatchScanExec, CometColumnarToRowExec, CometNativeScanExec, CometNativeWriteExec, CometProjectExec, CometScanExec, CometSparkToColumnarExec, CometWriteFilesExec} import org.apache.spark.sql.execution.{ColumnarToRowTransition, FileSourceScanExec, SparkPlan, SQLExecution} import org.apache.spark.sql.execution.command.DataWritingCommandExec import org.apache.spark.sql.execution.datasources.{BasicWriteTaskStats, BasicWriteTaskStatsTracker, FileFormatWriter, SQLHadoopMapReduceCommitProtocol, WriteFilesExec, WriteJobStatsTracker, WriteTaskStats, WriteTaskStatsTracker} @@ -53,7 +54,7 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{ArrayType, IntegerType, LongType, MapType, Metadata, MetadataBuilder, StringType, StructField, StructType} import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} -import org.apache.comet.CometConf +import org.apache.comet.{CometConf, CometNativeException} import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus import org.apache.comet.hadoop.fs.RecordingFileSystem import org.apache.comet.rules.RevertNativeForTransitionHeavyStages @@ -1694,6 +1695,153 @@ class CometParquetWriterSuite extends CometParquetWriterTestBase { } } + private def withInsertWriter(nativeWriter: Boolean)(f: => Unit): Unit = + if (nativeWriter) withNativeWriter(f) else f + + private def assertCometInsert(plan: SparkPlan, nativeWriter: Boolean): Unit = + if (nativeWriter) { + assertHasCometNativeWriteExec(plan) + } else { + assert(plan.exists(_.isInstanceOf[CometProjectExec]), plan.treeString) + } + + private def withNarrowingTables(sourceType: String, targetType: String, values: Seq[String])( + f: => Unit): Unit = + withTable("comet_narrow_source", "comet_narrow_target") { + sql(s"CREATE TABLE comet_narrow_source(c $sourceType) USING parquet") + sql(s"CREATE TABLE comet_narrow_target(c $targetType) USING parquet") + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { + sql(s"INSERT INTO comet_narrow_source VALUES ${values.map(v => s"($v)").mkString(", ")}") + } + withClue(s"$sourceType -> $targetType: ")(f) + } + + /** Runs `select` into the target in Spark and in Comet, and returns Spark's error. */ + private def assertInsertFailsAsSpark( + select: String, + nativeWriter: Boolean): SparkThrowable with Throwable = { + val insert = s"INSERT INTO comet_narrow_target SELECT $select FROM comet_narrow_source" + def sparkError(e: Throwable): SparkThrowable with Throwable = + causeChain(e) + .collect { case t: SparkThrowable with Throwable => t } + .lastOption + .getOrElse(fail(s"$insert did not raise a SparkThrowable", e)) + var expected: SparkThrowable with Throwable = null + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { + expected = sparkError(intercept[Exception](sql(insert))) + } + withInsertWriter(nativeWriter) { + val (plan, e) = captureFailedWritePlan(sql(insert)) + assertCometInsert(plan, nativeWriter) + assert(!causeChain(e).exists(_.isInstanceOf[CometNativeException]), e) + val actual = sparkError(e) + assert(actual.getClass == expected.getClass) + assert(actual.getErrorClass == expected.getErrorClass) + assert(actual.getSqlState == expected.getSqlState) + assert(actual.getMessage == expected.getMessage) + } + expected + } + + for (nativeWriter <- Seq(false, true)) { + test(s"a narrowing INSERT stays in Comet (native writer: $nativeWriter)") { + val inserts = Seq( + ("bigint", "int", Seq("1", "NULL", "2147483647", "-2147483648")), + ("int", "tinyint", Seq("127", "-128")), + ("bigint", "smallint", Seq("32767", "-32768")), + ("double", "bigint", Seq("1.5", "-2.9", "NULL")), + ("decimal(20,2)", "decimal(10,2)", Seq("1.25", "-99999999.99", "NULL")), + ("decimal(20,2)", "int", Seq("1.99", "-7.5")), + ("struct", "struct", Seq("named_struct('x', 1)", "NULL"))) + inserts.foreach { case (sourceType, targetType, values) => + withNarrowingTables(sourceType, targetType, values) { + withTable("comet_narrow_expected") { + sql(s"CREATE TABLE comet_narrow_expected(c $targetType) USING parquet") + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { + sql("INSERT INTO comet_narrow_expected SELECT c FROM comet_narrow_source") + } + withInsertWriter(nativeWriter) { + val plan = captureWritePlan( + sql("INSERT INTO comet_narrow_target SELECT c FROM comet_narrow_source")) + assertCometInsert(plan, nativeWriter) + } + checkAnswer(spark.table("comet_narrow_target"), spark.table("comet_narrow_expected")) + } + } + } + } + + test( + s"a narrowing INSERT that overflows fails as Spark does (native writer: $nativeWriter)") { + val overflows = Seq( + ("bigint", "int", "3000000000"), + ("bigint", "int", "-2147483649"), + ("int", "tinyint", "128"), + ("bigint", "smallint", "-32769"), + ("double", "int", "double('NaN')"), + ("double", "bigint", "double('Infinity')"), + ("double", "bigint", "1e20"), + ("double", "decimal(10,2)", "1e12"), + ("decimal(20,2)", "decimal(10,2)", "123456789012.34"), + ("decimal(20,2)", "int", "3000000000.5"), + ("struct", "struct", "named_struct('x', 3000000000)")) + overflows.foreach { case (sourceType, targetType, value) => + withNarrowingTables(sourceType, targetType, Seq(value)) { + val error = assertInsertFailsAsSpark("c", nativeWriter) + assert(error.getErrorClass == "CAST_OVERFLOW_IN_TABLE_INSERT", error) + } + } + + // Spark reports any arithmetic error under the cast as an overflow of the column. + withNarrowingTables("bigint", "int", Seq("1")) { + withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { + val error = assertInsertFailsAsSpark("c div 0", nativeWriter) + assert(error.getErrorClass == "CAST_OVERFLOW_IN_TABLE_INSERT", error) + } + } + + // Other errors under the cast keep their class and query context. + withNarrowingTables("string", "int", Seq("'abc'")) { + withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { + val error = assertInsertFailsAsSpark("CAST(c AS BIGINT)", nativeWriter) + assert(error.getErrorClass == "CAST_INVALID_INPUT", error) + assert(error.getQueryContext.nonEmpty, error) + } + } + } + + test( + s"a narrowing INSERT with a rewritten cast keeps its error (native writer: $nativeWriter)") { + // SPARK-42286: Spark rethrows the child's error as it is when the optimizer has replaced the + // cast under CheckOverflowInTableInsert, here by pushing it into the CASE WHEN branches. + val inserts = Seq( + "tinyint" -> ("0 - (CASE WHEN c = 1.2345678901234567E19D THEN 1.2345678901234567E19D " + + "ELSE c END)"), + "decimal(9,0)" -> "0 - (CASE WHEN c = 1 THEN 1 ELSE c END)") + inserts.foreach { case (targetType, select) => + withNarrowingTables("double", targetType, Seq("1.2345678901234567E19D")) { + val error = assertInsertFailsAsSpark(select, nativeWriter) + assert(error.getErrorClass != "CAST_OVERFLOW_IN_TABLE_INSERT", error) + } + } + } + } + + test("a narrowing INSERT of an overflowing literal fails as Spark does") { + withNarrowingTables("bigint", "int", Seq("1")) { + withSQLConf(SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> ConstantFolding.ruleName) { + val error = assertInsertFailsAsSpark("3000000000L", nativeWriter = false) + assert(error.getErrorClass == "CAST_OVERFLOW_IN_TABLE_INSERT", error) + val (plan, _) = captureFailedWritePlan( + sql("INSERT INTO comet_narrow_target SELECT 3000000000L FROM comet_narrow_source")) + assertExpressionImpl( + plan, + native = Seq("checkoverflowintableinsert"), + dispatched = Seq("cast")) + } + } + } + test("an empty partition writes no file and still commits") { // The write task's `partitionId != 0 && !batches.hasNext` branch must skip newTaskTempFile // altogether and still commit the task, matching FileFormatWriter's EmptyDirectoryDataWriter. From 2604e1f22d8b8d10c66236e5b7ac59c652944ec4 Mon Sep 17 00:00:00 2001 From: FreakingMind <55959207+FreakingMind@users.noreply.github.com> Date: Tue, 17 Nov 2026 01:42:06 +0300 Subject: [PATCH 2/4] docs: move the CheckOverflowInTableInsert error difference to the compatibility guide --- docs/source/user-guide/latest/compatibility/index.md | 5 +++++ docs/source/user-guide/latest/expressions.md | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/source/user-guide/latest/compatibility/index.md b/docs/source/user-guide/latest/compatibility/index.md index dca663d9228..9e88e4dab7e 100644 --- a/docs/source/user-guide/latest/compatibility/index.md +++ b/docs/source/user-guide/latest/compatibility/index.md @@ -181,6 +181,11 @@ divergence: - Spark 4.2 introduced additional ANSI arithmetic overflow behavior differences that Comet does not yet track ([#4967](https://github.com/apache/datafusion-comet/issues/4967)). +- An `INSERT` that narrows a numeric column reports an overflow as `CAST_OVERFLOW_IN_TABLE_INSERT`, + whatever `spark.sql.ansi.enabled` is. When an expression under that cast occurs more than once in + the projection, Spark's subexpression elimination evaluates it outside the check and reports the + original error, such as `CAST_OVERFLOW`. Comet still reports `CAST_OVERFLOW_IN_TABLE_INSERT`, as + Spark does with `spark.sql.subexpressionElimination.enabled=false`. ## Known result-value divergences diff --git a/docs/source/user-guide/latest/expressions.md b/docs/source/user-guide/latest/expressions.md index 0831ced7a66..aa0396e6657 100644 --- a/docs/source/user-guide/latest/expressions.md +++ b/docs/source/user-guide/latest/expressions.md @@ -704,7 +704,7 @@ Comet also accelerates a number of Catalyst expressions that have no Spark SQL f - **Operator and optimizer-injected expressions:** runtime bloom-filter join probes (`BloomFilterMightContain`, `BloomFilterAggregate`), optimized `IN` sets (`InSet`), scalar subqueries (`ScalarSubquery`), and floating-point normalization (`KnownFloatingPointNormalized`). - **Accessor expressions (subscript and field access, not functions):** struct field access (`col.field`), array element access (`arr[i]`), and map value access (`map[key]`). - **Internal decimal arithmetic:** `CheckOverflow`, `MakeDecimal`, and `UnscaledValue`, which the analyzer inserts around decimal operations. -- **Table insert casts:** `CheckOverflowInTableInsert`, which Spark wraps around the cast an `INSERT` adds under the default `spark.sql.storeAssignmentPolicy=ANSI`, so that an arithmetic error from the cast is reported as `CAST_OVERFLOW_IN_TABLE_INSERT`. When an expression under the cast occurs more than once in the projection, Spark's subexpression elimination evaluates it outside the wrapper and Spark reports the original error, such as `CAST_OVERFLOW`. Comet reports `CAST_OVERFLOW_IN_TABLE_INSERT` there, as Spark does with `spark.sql.subexpressionElimination.enabled=false`. +- **Table insert casts:** `CheckOverflowInTableInsert`, which the analyzer wraps around the cast an `INSERT` adds to narrow a numeric column. - **Missing-data predicates:** `AtLeastNNonNulls`, used by `DataFrame.na.drop`, counts non-null and non-NaN values natively. - **User-defined functions:** Scala UDFs registered through the DataFrame or SQL API. - **DataSource V2 catalog functions:** Iceberg's system functions `bucket`, `truncate`, `years`, `months`, `days`, and `hours` (for example `system.bucket(16, id)`) run natively; see [Iceberg system functions](iceberg.md#iceberg-system-functions). Other DataSource V2 catalog functions run in Spark. From 22aadc647251f9dfbd10003be3308d17887dab11 Mon Sep 17 00:00:00 2001 From: FreakingMind <55959207+FreakingMind@users.noreply.github.com> Date: Tue, 17 Nov 2026 10:14:53 +0300 Subject: [PATCH 3/4] test: run Spark's numeric.sql without Comet until #6797 is fixed An INSERT in numeric.sql writes POWER(10, LN(val)) into a decimal column. It used to fall back to Spark, and now runs natively, where pow differs from Math.pow in the last digit (#6797). --- dev/diffs/3.5.9.diff | 14 ++++++++++++++ dev/diffs/4.0.4.diff | 14 ++++++++++++++ dev/diffs/4.1.3.diff | 14 ++++++++++++++ dev/diffs/4.2.0.diff | 14 ++++++++++++++ 4 files changed, 56 insertions(+) diff --git a/dev/diffs/3.5.9.diff b/dev/diffs/3.5.9.diff index c41291d25d5..7dc2484e9a8 100644 --- a/dev/diffs/3.5.9.diff +++ b/dev/diffs/3.5.9.diff @@ -256,6 +256,20 @@ index fac23b4a26f..2b73732c33f 100644 -- -- INT8 -- Test int8 64-bit integers. +diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql +index cbc9c2a26f6..e690156a325 100644 +--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql ++++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql +@@ -6,6 +6,9 @@ + -- https://github.com/postgres/postgres/blob/REL_12_BETA2/src/test/regress/sql/numeric.sql + -- + ++-- TODO: https://github.com/apache/datafusion-comet/issues/6797 ++--SET spark.comet.enabled = false ++ + -- [SPARK-28318] Decimal can only support precision up to 38. We rewrite numeric(210,10) to decimal(38,10). + CREATE TABLE num_data (id int, val decimal(38,10)) USING parquet; + CREATE TABLE num_exp_add (id1 int, id2 int, expected decimal(38,10)) USING parquet; diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql index 0efe0877e9b..423d3b3d76d 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql diff --git a/dev/diffs/4.0.4.diff b/dev/diffs/4.0.4.diff index 772ffbcdadb..bc79537db13 100644 --- a/dev/diffs/4.0.4.diff +++ b/dev/diffs/4.0.4.diff @@ -308,6 +308,20 @@ index fac23b4a26f..98b12ae5ccc 100644 CREATE TABLE INT8_TBL(q1 bigint, q2 bigint) USING parquet; -- PostgreSQL implicitly casts string literals to data with integral types, but +diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql +index cbc9c2a26f6..e690156a325 100644 +--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql ++++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql +@@ -6,6 +6,9 @@ + -- https://github.com/postgres/postgres/blob/REL_12_BETA2/src/test/regress/sql/numeric.sql + -- + ++-- TODO: https://github.com/apache/datafusion-comet/issues/6797 ++--SET spark.comet.enabled = false ++ + -- [SPARK-28318] Decimal can only support precision up to 38. We rewrite numeric(210,10) to decimal(38,10). + CREATE TABLE num_data (id int, val decimal(38,10)) USING parquet; + CREATE TABLE num_exp_add (id1 int, id2 int, expected decimal(38,10)) USING parquet; diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql index 0efe0877e9b..f9df0400c99 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql diff --git a/dev/diffs/4.1.3.diff b/dev/diffs/4.1.3.diff index daf53c6773c..4b4fece2c2a 100644 --- a/dev/diffs/4.1.3.diff +++ b/dev/diffs/4.1.3.diff @@ -310,6 +310,20 @@ index fac23b4a26f..98b12ae5ccc 100644 CREATE TABLE INT8_TBL(q1 bigint, q2 bigint) USING parquet; -- PostgreSQL implicitly casts string literals to data with integral types, but +diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql +index cbc9c2a26f6..e690156a325 100644 +--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql ++++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql +@@ -6,6 +6,9 @@ + -- https://github.com/postgres/postgres/blob/REL_12_BETA2/src/test/regress/sql/numeric.sql + -- + ++-- TODO: https://github.com/apache/datafusion-comet/issues/6797 ++--SET spark.comet.enabled = false ++ + -- [SPARK-28318] Decimal can only support precision up to 38. We rewrite numeric(210,10) to decimal(38,10). + CREATE TABLE num_data (id int, val decimal(38,10)) USING parquet; + CREATE TABLE num_exp_add (id1 int, id2 int, expected decimal(38,10)) USING parquet; diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql index 0efe0877e9b..f9df0400c99 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql diff --git a/dev/diffs/4.2.0.diff b/dev/diffs/4.2.0.diff index 40aad4b1d59..0bdb7579c9f 100644 --- a/dev/diffs/4.2.0.diff +++ b/dev/diffs/4.2.0.diff @@ -312,6 +312,20 @@ index fac23b4a26f..98b12ae5ccc 100644 CREATE TABLE INT8_TBL(q1 bigint, q2 bigint) USING parquet; -- PostgreSQL implicitly casts string literals to data with integral types, but +diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql +index cbc9c2a26f6..e690156a325 100644 +--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql ++++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql +@@ -6,6 +6,9 @@ + -- https://github.com/postgres/postgres/blob/REL_12_BETA2/src/test/regress/sql/numeric.sql + -- + ++-- TODO: https://github.com/apache/datafusion-comet/issues/6797 ++--SET spark.comet.enabled = false ++ + -- [SPARK-28318] Decimal can only support precision up to 38. We rewrite numeric(210,10) to decimal(38,10). + CREATE TABLE num_data (id int, val decimal(38,10)) USING parquet; + CREATE TABLE num_exp_add (id1 int, id2 int, expected decimal(38,10)) USING parquet; diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql index 0efe0877e9b..f9df0400c99 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql From 494a0a983681f1fbe8f6fb6844e505d04454e250 Mon Sep 17 00:00:00 2001 From: FreakingMind <55959207+FreakingMind@users.noreply.github.com> Date: Wed, 18 Nov 2026 04:55:23 +0300 Subject: [PATCH 4/4] fix: find an arithmetic error anywhere in the source chain in CheckOverflowInTableInsert Since #4653 the JNI layer takes the first Spark error from the source() chain, so a child error wrapped in Context or Shared still reaches Spark with its class. Look for it the same way, so a wrapped arithmetic error becomes CAST_OVERFLOW_IN_TABLE_INSERT too. --- .../check_overflow_in_table_insert.rs | 93 ++++++++++++------- 1 file changed, 61 insertions(+), 32 deletions(-) diff --git a/native/spark-expr/src/conversion_funcs/check_overflow_in_table_insert.rs b/native/spark-expr/src/conversion_funcs/check_overflow_in_table_insert.rs index 0994942814a..6a7d8661a7e 100644 --- a/native/spark-expr/src/conversion_funcs/check_overflow_in_table_insert.rs +++ b/native/spark-expr/src/conversion_funcs/check_overflow_in_table_insert.rs @@ -24,6 +24,7 @@ use arrow::record_batch::RecordBatch; use datafusion::common::{DataFusionError, Result}; use datafusion::logical_expr::ColumnarValue; use datafusion::physical_expr::PhysicalExpr; +use datafusion_comet_common::error_chain; use datafusion_comet_jni_bridge::errors::CometError; use crate::{SparkError, SparkErrorWithContext}; @@ -138,18 +139,20 @@ impl PhysicalExpr for CheckOverflowInTableInsert { /// Whether Spark raises this error as a `SparkArithmeticException`: a native `SparkError`, or an /// exception thrown by a child that runs Spark's own code through the JVM codegen dispatcher. -/// Only the top of a `DataFusionError::External` is checked, because that is where the JNI layer -/// reads a Spark error from. +/// Like the JNI layer, it takes the first Spark error in the `source()` chain. fn is_arithmetic_error(error: &DataFusionError) -> bool { - let DataFusionError::External(e) = error else { - return false; - }; - if let Some(CometError::JavaException { class, .. }) = e.downcast_ref::() { - return class == "org.apache.spark.SparkArithmeticException"; - } - e.downcast_ref::() - .or_else(|| e.downcast_ref::().map(|e| &e.error)) - .is_some_and(SparkError::is_arithmetic) + error_chain(error) + .find_map(|e| { + if let Some(CometError::JavaException { class, .. }) = e.downcast_ref::() { + Some(class == "org.apache.spark.SparkArithmeticException") + } else if let Some(e) = e.downcast_ref::() { + Some(e.error.is_arithmetic()) + } else { + e.downcast_ref::() + .map(SparkError::is_arithmetic) + } + }) + .unwrap_or(false) } #[cfg(test)] @@ -191,11 +194,13 @@ mod tests { } } - /// A child that returns column `c`, or fails. + /// A child that returns column `c`, or fails. A wrapped error comes inside + /// `DataFusionError::Context`, as DataFusion adds it on the way up. #[derive(Debug, PartialEq, Eq, Hash)] struct Child { failure: Option, with_context: bool, + wrapped: bool, } impl Display for Child { @@ -222,11 +227,19 @@ mod tests { return Column::new("c", 0).evaluate(batch); }; let error = failure.error(); - if self.with_context { + let error = if self.with_context { let context = QueryContext::new("SELECT c".to_string(), 7, 7, None, None, 1, 7); - Err(SparkErrorWithContext::with_context(error, Arc::new(context)).into()) + SparkErrorWithContext::with_context(error, Arc::new(context)).into() + } else { + error.into() + }; + if self.wrapped { + Err(DataFusionError::Context( + "child".to_string(), + Box::new(error), + )) } else { - Err(error.into()) + Err(error) } } @@ -242,7 +255,11 @@ mod tests { } } - fn evaluate(failure: Option, with_context: bool) -> Result { + fn evaluate( + failure: Option, + with_context: bool, + wrapped: bool, + ) -> Result { let schema = Arc::new(Schema::new(vec![Field::new("c", DataType::Int64, true)])); let batch = RecordBatch::try_new( schema, @@ -252,6 +269,7 @@ mod tests { let child = Child { failure, with_context, + wrapped, }; CheckOverflowInTableInsert::new( Arc::new(child), @@ -262,22 +280,27 @@ mod tests { .evaluate(&batch) } - /// The Spark error's message, and whether it carries a query context. + /// The first Spark error's message, and whether it carries a query context. fn spark_error(result: Result) -> (String, bool) { - let Err(DataFusionError::External(e)) = result else { + let Err(error) = result else { panic!("expected a SparkError, got {result:?}"); }; - if let Some(e) = e.downcast_ref::() { - (e.error.to_string(), e.context.is_some()) - } else { - let e = e.downcast_ref::().expect("a SparkError"); - (e.to_string(), false) - } + let found = error_chain(&error) + .find_map(|e| { + if let Some(e) = e.downcast_ref::() { + Some((e.error.to_string(), e.context.is_some())) + } else { + e.downcast_ref::() + .map(|e| (e.to_string(), false)) + } + }) + .expect("a SparkError"); + found } #[test] fn values_pass_through() { - let ColumnarValue::Array(array) = evaluate(None, false).unwrap() else { + let ColumnarValue::Array(array) = evaluate(None, false, false).unwrap() else { panic!("expected an array"); }; let array = array.as_any().downcast_ref::().unwrap(); @@ -292,25 +315,31 @@ mod tests { Failure::DivideByZero, ]; for failure in failures { - for with_context in [false, true] { - let (message, has_context) = spark_error(evaluate(Some(failure), with_context)); + for (with_context, wrapped) in + [(false, false), (true, false), (false, true), (true, true)] + { + let clue = format!("{failure:?} with context {with_context}, wrapped {wrapped}"); + let result = evaluate(Some(failure), with_context, wrapped); + let (message, has_context) = spark_error(result); assert_eq!( message, "[CAST_OVERFLOW_IN_TABLE_INSERT] Fail to assign a value of \"BIGINT\" type \ to the \"INT\" type column or variable `c` due to an overflow. Use \ `try_cast` on the input value to tolerate overflow and return NULL instead.", - "{failure:?} with context {with_context}" + "{clue}" ); - assert!(!has_context, "{failure:?} with context {with_context}"); + assert!(!has_context, "{clue}"); } } } #[test] fn other_errors_are_unchanged() { - for with_context in [false, true] { - let failure = Some(Failure::InvalidInput); - let (message, has_context) = spark_error(evaluate(failure, with_context)); + for (with_context, wrapped) in [(false, false), (true, false), (false, true), (true, true)] + { + let result = evaluate(Some(Failure::InvalidInput), with_context, wrapped); + assert_eq!(matches!(result, Err(DataFusionError::Context(..))), wrapped); + let (message, has_context) = spark_error(result); assert_eq!(message, Failure::InvalidInput.error().to_string()); assert_eq!(has_context, with_context); }