Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions dev/diffs/3.5.9.diff
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,20 @@ index fac23b4a26f..2b73732c33f 100644
--
-- INT8
-- Test int8 64-bit integers.
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql
index cbc9c2a26f6..e690156a325 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql
@@ -6,6 +6,9 @@
-- https://github.com/postgres/postgres/blob/REL_12_BETA2/src/test/regress/sql/numeric.sql
--

+-- TODO: https://github.com/apache/datafusion-comet/issues/6797
+--SET spark.comet.enabled = false
+
-- [SPARK-28318] Decimal can only support precision up to 38. We rewrite numeric(210,10) to decimal(38,10).
CREATE TABLE num_data (id int, val decimal(38,10)) USING parquet;
CREATE TABLE num_exp_add (id1 int, id2 int, expected decimal(38,10)) USING parquet;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql
index 0efe0877e9b..423d3b3d76d 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql
Expand Down
14 changes: 14 additions & 0 deletions dev/diffs/4.0.4.diff
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,20 @@ index fac23b4a26f..98b12ae5ccc 100644
CREATE TABLE INT8_TBL(q1 bigint, q2 bigint) USING parquet;

-- PostgreSQL implicitly casts string literals to data with integral types, but
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql
index cbc9c2a26f6..e690156a325 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql
@@ -6,6 +6,9 @@
-- https://github.com/postgres/postgres/blob/REL_12_BETA2/src/test/regress/sql/numeric.sql
--

+-- TODO: https://github.com/apache/datafusion-comet/issues/6797
+--SET spark.comet.enabled = false
+
-- [SPARK-28318] Decimal can only support precision up to 38. We rewrite numeric(210,10) to decimal(38,10).
CREATE TABLE num_data (id int, val decimal(38,10)) USING parquet;
CREATE TABLE num_exp_add (id1 int, id2 int, expected decimal(38,10)) USING parquet;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql
index 0efe0877e9b..f9df0400c99 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql
Expand Down
14 changes: 14 additions & 0 deletions dev/diffs/4.1.3.diff
Original file line number Diff line number Diff line change
Expand Up @@ -310,6 +310,20 @@ index fac23b4a26f..98b12ae5ccc 100644
CREATE TABLE INT8_TBL(q1 bigint, q2 bigint) USING parquet;

-- PostgreSQL implicitly casts string literals to data with integral types, but
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql
index cbc9c2a26f6..e690156a325 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql
@@ -6,6 +6,9 @@
-- https://github.com/postgres/postgres/blob/REL_12_BETA2/src/test/regress/sql/numeric.sql
--

+-- TODO: https://github.com/apache/datafusion-comet/issues/6797
+--SET spark.comet.enabled = false
+
-- [SPARK-28318] Decimal can only support precision up to 38. We rewrite numeric(210,10) to decimal(38,10).
CREATE TABLE num_data (id int, val decimal(38,10)) USING parquet;
CREATE TABLE num_exp_add (id1 int, id2 int, expected decimal(38,10)) USING parquet;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql
index 0efe0877e9b..f9df0400c99 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql
Expand Down
14 changes: 14 additions & 0 deletions dev/diffs/4.2.0.diff
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,20 @@ index fac23b4a26f..98b12ae5ccc 100644
CREATE TABLE INT8_TBL(q1 bigint, q2 bigint) USING parquet;

-- PostgreSQL implicitly casts string literals to data with integral types, but
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql
index cbc9c2a26f6..e690156a325 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/numeric.sql
@@ -6,6 +6,9 @@
-- https://github.com/postgres/postgres/blob/REL_12_BETA2/src/test/regress/sql/numeric.sql
--

+-- TODO: https://github.com/apache/datafusion-comet/issues/6797
+--SET spark.comet.enabled = false
+
-- [SPARK-28318] Decimal can only support precision up to 38. We rewrite numeric(210,10) to decimal(38,10).
CREATE TABLE num_data (id int, val decimal(38,10)) USING parquet;
CREATE TABLE num_exp_add (id1 int, id2 int, expected decimal(38,10)) USING parquet;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql
index 0efe0877e9b..f9df0400c99 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/select_having.sql
Expand Down
5 changes: 5 additions & 0 deletions docs/source/user-guide/latest/compatibility/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,11 @@ divergence:

- Spark 4.2 introduced additional ANSI arithmetic overflow behavior differences that Comet does
not yet track ([#4967](https://github.com/apache/datafusion-comet/issues/4967)).
- An `INSERT` that narrows a numeric column reports an overflow as `CAST_OVERFLOW_IN_TABLE_INSERT`,
whatever `spark.sql.ansi.enabled` is. When an expression under that cast occurs more than once in
the projection, Spark's subexpression elimination evaluates it outside the check and reports the
original error, such as `CAST_OVERFLOW`. Comet still reports `CAST_OVERFLOW_IN_TABLE_INSERT`, as
Spark does with `spark.sql.subexpressionElimination.enabled=false`.

## Known result-value divergences

Expand Down
1 change: 1 addition & 0 deletions docs/source/user-guide/latest/expressions.md
Original file line number Diff line number Diff line change
Expand Up @@ -704,6 +704,7 @@ Comet also accelerates a number of Catalyst expressions that have no Spark SQL f
- **Operator and optimizer-injected expressions:** runtime bloom-filter join probes (`BloomFilterMightContain`, `BloomFilterAggregate`), optimized `IN` sets (`InSet`), scalar subqueries (`ScalarSubquery`), and floating-point normalization (`KnownFloatingPointNormalized`).
- **Accessor expressions (subscript and field access, not functions):** struct field access (`col.field`), array element access (`arr[i]`), and map value access (`map[key]`).
- **Internal decimal arithmetic:** `CheckOverflow`, `MakeDecimal`, and `UnscaledValue`, which the analyzer inserts around decimal operations.
- **Table insert casts:** `CheckOverflowInTableInsert`, which the analyzer wraps around the cast an `INSERT` adds to narrow a numeric column.
- **Missing-data predicates:** `AtLeastNNonNulls`, used by `DataFrame.na.drop`, counts non-null and non-NaN values natively.
- **User-defined functions:** Scala UDFs registered through the DataFrame or SQL API.
- **DataSource V2 catalog functions:** Iceberg's system functions `bucket`, `truncate`, `years`, `months`, `days`, and `hours` (for example `system.bucket(16, id)`) run natively; see [Iceberg system functions](iceberg.md#iceberg-system-functions). Other DataSource V2 catalog functions run in Spark.
Expand Down
36 changes: 35 additions & 1 deletion native/common/src/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,15 @@ pub enum SparkError {
to_type: String,
},

#[error("[CAST_OVERFLOW_IN_TABLE_INSERT] Fail to assign a value of \"{from_type}\" type to the \
\"{to_type}\" type column or variable `{column_name}` due to an overflow. Use `try_cast` on \
the input value to tolerate overflow and return NULL instead.")]
CastOverflowInTableInsert {
from_type: String,
to_type: String,
column_name: String,
},

#[error("[CANNOT_PARSE_DECIMAL] Cannot parse decimal.")]
CannotParseDecimal,

Expand Down Expand Up @@ -358,6 +367,7 @@ impl SparkError {
SparkError::NumericValueOutOfRange { .. } => "NumericValueOutOfRange",
SparkError::NumericOutOfRange { .. } => "NumericOutOfRange",
SparkError::CastOverFlow { .. } => "CastOverFlow",
SparkError::CastOverflowInTableInsert { .. } => "CastOverflowInTableInsert",
SparkError::CannotParseDecimal => "CannotParseDecimal",
SparkError::ArithmeticOverflow { .. } => "ArithmeticOverflow",
SparkError::IntegralDivideOverflow => "IntegralDivideOverflow",
Expand Down Expand Up @@ -464,6 +474,17 @@ impl SparkError {
"toType": to_type,
})
}
SparkError::CastOverflowInTableInsert {
from_type,
to_type,
column_name,
} => {
serde_json::json!({
"fromType": from_type,
"toType": to_type,
"columnName": column_name,
})
}
SparkError::ArithmeticOverflow {
from_type,
function_name,
Expand Down Expand Up @@ -717,6 +738,15 @@ impl SparkError {
}
}

/// Whether Spark raises this error as a `SparkArithmeticException`.
pub fn is_arithmetic(&self) -> bool {
matches!(
self.exception_class(),
"org/apache/spark/SparkArithmeticException"
| "org/apache/spark/sql/comet/CastOverflowException"
)
}

/// Returns the appropriate Spark exception class for this error
pub fn exception_class(&self) -> &'static str {
match self {
Expand All @@ -732,7 +762,10 @@ impl SparkError {
| SparkError::BinaryArithmeticOverflow { .. }
| SparkError::IntervalArithmeticOverflowWithSuggestion { .. }
| SparkError::IntervalArithmeticOverflowWithoutSuggestion
| SparkError::DatetimeOverflow => "org/apache/spark/SparkArithmeticException",
| SparkError::DatetimeOverflow
| SparkError::CastOverflowInTableInsert { .. } => {
"org/apache/spark/SparkArithmeticException"
}

// CastOverflow gets special handling with CastOverflowException
SparkError::CastOverFlow { .. } => "org/apache/spark/sql/comet/CastOverflowException",
Expand Down Expand Up @@ -829,6 +862,7 @@ impl SparkError {
SparkError::CastInvalidValue { .. } => Some("CAST_INVALID_INPUT"),
SparkError::InvalidInputInCastToDatetime { .. } => Some("CAST_INVALID_INPUT"),
SparkError::CastOverFlow { .. } => Some("CAST_OVERFLOW"),
SparkError::CastOverflowInTableInsert { .. } => Some("CAST_OVERFLOW_IN_TABLE_INSERT"),
SparkError::NumericValueOutOfRange { .. } => {
Some("NUMERIC_VALUE_OUT_OF_RANGE.WITH_SUGGESTION")
}
Expand Down
17 changes: 13 additions & 4 deletions native/core/src/execution/planner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -121,10 +121,10 @@ use datafusion_comet_proto::{
use datafusion_comet_spark_expr::{
coerce_to_common_type, create_case_when, create_if_expr, decimal128_add_sub_width,
jvm_udf::JvmScalarUdfExpr, spark_in_list, ArrayInsert, AtLeastNNonNulls, Cast, CheckOverflow,
CreateNamedStruct, DecimalRescaleCheckOverflow, FloatOperands, GetArrayStructFields,
GetStructField, IfExpr, ListExtract, MakeDateExpr, NormalizeNaNAndZero, NormalizeNestedFloats,
NullShortCircuit, SparkCastOptions, ToJson, UnboundColumn, WideDecimalBinaryExpr,
WideDecimalOp,
CheckOverflowInTableInsert, CreateNamedStruct, DecimalRescaleCheckOverflow, FloatOperands,
GetArrayStructFields, GetStructField, IfExpr, ListExtract, MakeDateExpr, NormalizeNaNAndZero,
NormalizeNestedFloats, NullShortCircuit, SparkCastOptions, ToJson, UnboundColumn,
WideDecimalBinaryExpr, WideDecimalOp,
};
use itertools::Itertools;
use jni::objects::{Global, JObject};
Expand Down Expand Up @@ -932,6 +932,15 @@ impl PhysicalPlanner {
query_context,
)))
}
ExprStruct::CheckOverflowInTableInsert(expr) => {
let child = self.create_expr(expr.child.as_ref().unwrap(), input_schema)?;
Ok(Arc::new(CheckOverflowInTableInsert::new(
child,
expr.from_type.clone(),
expr.to_type.clone(),
expr.column_name.clone(),
)))
}
ExprStruct::ScalarFunc(expr) => {
let func = self.create_scalar_function_expr(expr, input_schema);
match expr.func.as_ref() {
Expand Down
9 changes: 9 additions & 0 deletions native/proto/src/proto/expr.proto
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ message Expr {
Uuid uuid = 74;
NativeScalarUdf native_scalar_udf = 75;
AtLeastNNonNulls at_least_n_non_nulls = 76;
CheckOverflowInTableInsert check_overflow_in_table_insert = 77;
}

reserved 20;
Expand Down Expand Up @@ -593,6 +594,14 @@ message CheckOverflow {
bool fail_on_error = 3;
}

// The types are Spark SQL type names, which the JVM parses back to build Spark's error.
message CheckOverflowInTableInsert {
Expr child = 1;
string from_type = 2;
string to_type = 3;
string column_name = 4;
}

message ScalarFunc {
string func = 1;
repeated Expr args = 2;
Expand Down
Loading