From 5280143c57c4f9d86c1f6429028ac7be40551b93 Mon Sep 17 00:00:00 2001 From: Ted Jenks <77486246+ted-jenks@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:58:42 -0500 Subject: [PATCH 1/5] fix: reject negative decimal scales in native cast kernels --- .../expressions/_category_template/cast.md | 3 +- .../src/conversion_funcs/numeric.rs | 58 +++++++++++++++++-- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md b/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md index 9bbf3da7f75..93c31d6e744 100644 --- a/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md +++ b/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md @@ -179,8 +179,7 @@ integer types to a negative-scale `DecimalType`. Comet reports these as unsuppor routes them through the JVM codegen dispatcher (Spark's own generated code) inside the Comet pipeline, so the results match Spark exactly. -The native kernels align scale by computing `10^|scale|` in fixed-point arithmetic, which -overflows when the scale is negative. See +The native kernels reject a negative scale with an error. See [#5013](https://github.com/apache/datafusion-comet/issues/5013). All other cast directions to and from negative-scale decimals run natively: `FloatType` and diff --git a/native/spark-expr/src/conversion_funcs/numeric.rs b/native/spark-expr/src/conversion_funcs/numeric.rs index 9a9a7bf915e..bbab9b3cca7 100644 --- a/native/spark-expr/src/conversion_funcs/numeric.rs +++ b/native/spark-expr/src/conversion_funcs/numeric.rs @@ -520,6 +520,16 @@ macro_rules! cast_float_to_int32_up { }}; } +/// 10^scale, or an error for a scale outside 0..=38. Negative scales need +/// spark.sql.legacy.allowNegativeScaleOfDecimal and can still arrive from the Parquet schema +/// adapter. +fn decimal_scale_factor(scale: i8, cast: &str) -> SparkResult { + u32::try_from(scale) + .ok() + .and_then(pow10_i128) + .ok_or_else(|| SparkError::Internal(format!("{cast}: unsupported decimal scale {scale}"))) +} + // When Spark casts to Byte/Short Types, it does not cast directly to Byte/Short. // It casts to Int first and then to Byte/Short. Because of potential overflows in the Int cast, // this can cause unexpected Short/Byte cast results. Replicate this behavior. @@ -538,7 +548,7 @@ macro_rules! cast_decimal_to_int16_down { .downcast_ref::() .expect("Expected a Decimal128ArrayType"); - let divisor = 10_i128.pow($scale as u32); + let divisor = decimal_scale_factor($scale, "cast_decimal_to_integral")?; // `try_unary` only visits valid slots, so a value under a null is neither divided nor, // in ANSI mode, checked for overflow. The legacy conversion cannot fail, but the division // is most of the cost for decimals, so it uses `try_unary` too, with a closure that @@ -593,7 +603,7 @@ macro_rules! cast_decimal_to_int32_up { .downcast_ref::() .expect("Expected a Decimal128ArrayType"); - let divisor = 10_i128.pow($scale as u32); + let divisor = decimal_scale_factor($scale, "cast_decimal_to_integral")?; // `try_unary` only visits valid slots, so a value under a null is neither divided nor, // in ANSI mode, checked for overflow. The legacy conversion cannot fail, but the division // is most of the cost for decimals, so it uses `try_unary` too, with a closure that @@ -771,7 +781,7 @@ where T: ArrowPrimitiveType, T::Native: Into, { - let multiplier = 10_i128.pow(scale as u32); + let multiplier = decimal_scale_factor(scale, "cast_int_to_decimal128")?; // Single spelling of the "does this value fit at the target precision after scaling" check, // shared between the vectorized pass and the ANSI rescan below. @@ -1349,7 +1359,7 @@ pub(crate) fn cast_decimal_to_timestamp( scale: i8, ) -> SparkResult { let arr = array_ref.as_primitive::(); - let scale_factor = 10_i128.pow(scale as u32); + let scale_factor = decimal_scale_factor(scale, "cast_decimal_to_timestamp")?; // `unary_opt` visits valid rows only. `unary` also divides null slots, and // that was slower on every measured shape, including a batch with no nulls. let micros = arr.unary_opt::<_, TimestampMicrosecondType>(|value| { @@ -2977,4 +2987,44 @@ mod tests { let out = cast_float_to_timestamp(&array, &tz, EvalMode::Ansi).unwrap(); assert_ts(&out, &[Some(1), Some(-1)], Some("UTC")); } + + #[test] + fn test_cast_kernels_reject_negative_scale() { + // Try routes decimal to integer through the DataFusion cast, not these kernels. + let decimals: ArrayRef = Arc::new( + Decimal128Array::from(vec![Some(12345), None]) + .with_precision_and_scale(10, -2) + .unwrap(), + ); + let ints: ArrayRef = Arc::new(Int32Array::from(vec![Some(7), None])); + let utc = Some(Arc::from("UTC")); + let cases = [ + (&decimals, DataType::Int8, "cast_decimal_to_integral"), + (&decimals, DataType::Int16, "cast_decimal_to_integral"), + (&decimals, DataType::Int32, "cast_decimal_to_integral"), + (&decimals, DataType::Int64, "cast_decimal_to_integral"), + ( + &ints, + DataType::Decimal128(10, -2), + "cast_int_to_decimal128", + ), + ( + &decimals, + DataType::Timestamp(TimeUnit::Microsecond, utc), + "cast_decimal_to_timestamp", + ), + ]; + for eval_mode in [EvalMode::Legacy, EvalMode::Ansi] { + let options = SparkCastOptions::new(eval_mode, "UTC"); + for (input, to_type, label) in &cases { + let err = cast_array(Arc::clone(input), to_type, &options) + .expect_err("a negative scale must be rejected"); + assert!( + err.to_string() + .contains(&format!("{label}: unsupported decimal scale -2")), + "{eval_mode:?} {to_type}: unexpected error: {err}" + ); + } + } + } } From ca2143a837ea4f88aff7804ba456804868a7ae3e Mon Sep 17 00:00:00 2001 From: Ted Jenks <77486246+ted-jenks@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:44:02 -0500 Subject: [PATCH 2/5] fall back for negative scale scans --- .../expressions/_category_template/cast.md | 4 ++- .../src/conversion_funcs/numeric.rs | 4 +-- .../apache/comet/rules/CometScanRule.scala | 6 ++++ .../comet/rules/CometScanRuleSuite.scala | 30 +++++++++++++++++++ 4 files changed, 41 insertions(+), 3 deletions(-) diff --git a/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md b/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md index 93c31d6e744..eb997b19d2b 100644 --- a/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md +++ b/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md @@ -180,7 +180,9 @@ routes them through the JVM codegen dispatcher (Spark's own generated code) insi pipeline, so the results match Spark exactly. The native kernels reject a negative scale with an error. See -[#5013](https://github.com/apache/datafusion-comet/issues/5013). +[#5013](https://github.com/apache/datafusion-comet/issues/5013). For the same reason, the native +Parquet scan falls back to Spark when the read schema has a negative-scale decimal, such as an +integer column read as `DECIMAL(12,-2)`. All other cast directions to and from negative-scale decimals run natively: `FloatType` and `DoubleType`, `StringType` in both directions (subject to the config gate described above), diff --git a/native/spark-expr/src/conversion_funcs/numeric.rs b/native/spark-expr/src/conversion_funcs/numeric.rs index bbab9b3cca7..10e6f8e5acb 100644 --- a/native/spark-expr/src/conversion_funcs/numeric.rs +++ b/native/spark-expr/src/conversion_funcs/numeric.rs @@ -521,8 +521,8 @@ macro_rules! cast_float_to_int32_up { } /// 10^scale, or an error for a scale outside 0..=38. Negative scales need -/// spark.sql.legacy.allowNegativeScaleOfDecimal and can still arrive from the Parquet schema -/// adapter. +/// spark.sql.legacy.allowNegativeScaleOfDecimal, and the serde and Parquet scan both leave those +/// to Spark. fn decimal_scale_factor(scale: i8, cast: &str) -> SparkResult { u32::try_from(scale) .ok() diff --git a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala index 955db40ce42..8a70bcb09ea 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala @@ -1243,6 +1243,12 @@ case class CometScanTypeChecker() extends DataTypeSupport with CometTypeShim { fallbackReasons += s"Unsupported $name of type $dt (native Parquet scan does not " + "support TIME)" false + case d: DecimalType if d.scale < 0 => + // Spark reads an integer column as a negative-scale decimal by rescaling each value, + // which the native cast kernels reject. + fallbackReasons += s"Unsupported $name of type $d (native Parquet scan does not " + + "support negative-scale decimals)" + false case dt if isStringCollationType(dt) => // we don't need specific support for collation in scans, but this // is a convenient place to force the whole query to fall back to Spark for now diff --git a/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala index 52c6159d11e..be0dd5c7bab 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala @@ -239,6 +239,36 @@ class CometScanRuleSuite extends CometTestBase { } } + test("CometScanTypeChecker declines negative-scale decimals") { + withSQLConf("spark.sql.legacy.allowNegativeScaleOfDecimal" -> "true") { + val negative = StructField("d", DecimalType(10, -2)) + val declined = Seq[(String, StructType)]( + "root" -> StructType(Array(negative)), + "nested struct" -> StructType(Array(StructField("s", StructType(Array(negative)))))) + for ((label, schema) <- declined) { + val reasons = ListBuffer.empty[String] + assert(!CometScanTypeChecker().isSchemaSupported(schema, reasons), s"$label: accepted") + assert(reasons.exists(_.contains("negative-scale decimals")), s"$label: $reasons") + } + val positive = StructType(Array(StructField("d", DecimalType(10, 2)))) + assert(CometScanTypeChecker().isSchemaSupported(positive, ListBuffer.empty)) + } + } + + test("an integer column read as a negative-scale decimal falls back to Spark") { + // Spark reads multiples of 100 as DECIMAL(12,-2) and keeps nulls. The native scan sent this + // read through the integer to decimal cast kernel, which cannot take a negative scale. + withSQLConf("spark.sql.legacy.allowNegativeScaleOfDecimal" -> "true") { + withTempPath { path => + Seq(Some(100), Some(200), None).toDF("a").repartition(1).write.parquet(path.toString) + val schema = StructType(Array(StructField("a", DecimalType(12, -2)))) + checkSparkAnswerAndFallbackReason( + spark.read.schema(schema).parquet(path.toString), + "native Parquet scan does not support negative-scale decimals") + } + } + } + test("DataTypeSupport finds sibling field names that collide case-insensitively") { // See https://github.com/apache/datafusion-comet/issues/6136. def fields(names: String*): StructType = From e4c4eb53158166955b00cd5e8da67c031986c8c2 Mon Sep 17 00:00:00 2001 From: Ted Jenks <77486246+ted-jenks@users.noreply.github.com> Date: Mon, 16 Nov 2026 11:22:38 -0500 Subject: [PATCH 3/5] nits --- .../src/conversion_funcs/numeric.rs | 37 +++++++------------ 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/native/spark-expr/src/conversion_funcs/numeric.rs b/native/spark-expr/src/conversion_funcs/numeric.rs index 10e6f8e5acb..18f75bd7cc4 100644 --- a/native/spark-expr/src/conversion_funcs/numeric.rs +++ b/native/spark-expr/src/conversion_funcs/numeric.rs @@ -523,11 +523,11 @@ macro_rules! cast_float_to_int32_up { /// 10^scale, or an error for a scale outside 0..=38. Negative scales need /// spark.sql.legacy.allowNegativeScaleOfDecimal, and the serde and Parquet scan both leave those /// to Spark. -fn decimal_scale_factor(scale: i8, cast: &str) -> SparkResult { +fn decimal_scale_factor(scale: i8) -> SparkResult { u32::try_from(scale) .ok() .and_then(pow10_i128) - .ok_or_else(|| SparkError::Internal(format!("{cast}: unsupported decimal scale {scale}"))) + .ok_or_else(|| SparkError::Internal(format!("unsupported decimal scale {scale}"))) } // When Spark casts to Byte/Short Types, it does not cast directly to Byte/Short. @@ -548,7 +548,7 @@ macro_rules! cast_decimal_to_int16_down { .downcast_ref::() .expect("Expected a Decimal128ArrayType"); - let divisor = decimal_scale_factor($scale, "cast_decimal_to_integral")?; + let divisor = decimal_scale_factor($scale)?; // `try_unary` only visits valid slots, so a value under a null is neither divided nor, // in ANSI mode, checked for overflow. The legacy conversion cannot fail, but the division // is most of the cost for decimals, so it uses `try_unary` too, with a closure that @@ -603,7 +603,7 @@ macro_rules! cast_decimal_to_int32_up { .downcast_ref::() .expect("Expected a Decimal128ArrayType"); - let divisor = decimal_scale_factor($scale, "cast_decimal_to_integral")?; + let divisor = decimal_scale_factor($scale)?; // `try_unary` only visits valid slots, so a value under a null is neither divided nor, // in ANSI mode, checked for overflow. The legacy conversion cannot fail, but the division // is most of the cost for decimals, so it uses `try_unary` too, with a closure that @@ -781,7 +781,7 @@ where T: ArrowPrimitiveType, T::Native: Into, { - let multiplier = decimal_scale_factor(scale, "cast_int_to_decimal128")?; + let multiplier = decimal_scale_factor(scale)?; // Single spelling of the "does this value fit at the target precision after scaling" check, // shared between the vectorized pass and the ANSI rescan below. @@ -1359,7 +1359,7 @@ pub(crate) fn cast_decimal_to_timestamp( scale: i8, ) -> SparkResult { let arr = array_ref.as_primitive::(); - let scale_factor = decimal_scale_factor(scale, "cast_decimal_to_timestamp")?; + let scale_factor = decimal_scale_factor(scale)?; // `unary_opt` visits valid rows only. `unary` also divides null slots, and // that was slower on every measured shape, including a batch with no nulls. let micros = arr.unary_opt::<_, TimestampMicrosecondType>(|value| { @@ -2999,29 +2999,20 @@ mod tests { let ints: ArrayRef = Arc::new(Int32Array::from(vec![Some(7), None])); let utc = Some(Arc::from("UTC")); let cases = [ - (&decimals, DataType::Int8, "cast_decimal_to_integral"), - (&decimals, DataType::Int16, "cast_decimal_to_integral"), - (&decimals, DataType::Int32, "cast_decimal_to_integral"), - (&decimals, DataType::Int64, "cast_decimal_to_integral"), - ( - &ints, - DataType::Decimal128(10, -2), - "cast_int_to_decimal128", - ), - ( - &decimals, - DataType::Timestamp(TimeUnit::Microsecond, utc), - "cast_decimal_to_timestamp", - ), + (&decimals, DataType::Int8), + (&decimals, DataType::Int16), + (&decimals, DataType::Int32), + (&decimals, DataType::Int64), + (&ints, DataType::Decimal128(10, -2)), + (&decimals, DataType::Timestamp(TimeUnit::Microsecond, utc)), ]; for eval_mode in [EvalMode::Legacy, EvalMode::Ansi] { let options = SparkCastOptions::new(eval_mode, "UTC"); - for (input, to_type, label) in &cases { + for (input, to_type) in &cases { let err = cast_array(Arc::clone(input), to_type, &options) .expect_err("a negative scale must be rejected"); assert!( - err.to_string() - .contains(&format!("{label}: unsupported decimal scale -2")), + err.to_string().contains("unsupported decimal scale -2"), "{eval_mode:?} {to_type}: unexpected error: {err}" ); } From a41bcd8bb6cafcb4c2ac9c4b3d39477b4acf5ad7 Mon Sep 17 00:00:00 2001 From: Ted Jenks <77486246+ted-jenks@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:03:06 -0500 Subject: [PATCH 4/5] fix test --- .../scala/org/apache/comet/rules/CometScanRuleSuite.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala index be0dd5c7bab..a694e6e0bef 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala @@ -260,7 +260,9 @@ class CometScanRuleSuite extends CometTestBase { // read through the integer to decimal cast kernel, which cannot take a negative scale. withSQLConf("spark.sql.legacy.allowNegativeScaleOfDecimal" -> "true") { withTempPath { path => - Seq(Some(100), Some(200), None).toDF("a").repartition(1).write.parquet(path.toString) + val ints = StructType(Array(StructField("a", IntegerType))) + val rows = spark.sparkContext.parallelize(Seq(Row(100), Row(200), Row(null)), 1) + spark.createDataFrame(rows, ints).write.parquet(path.toString) val schema = StructType(Array(StructField("a", DecimalType(12, -2)))) checkSparkAnswerAndFallbackReason( spark.read.schema(schema).parquet(path.toString), From 8a4e528c084d1181e9e0a3a4a246a482c14f36d0 Mon Sep 17 00:00:00 2001 From: Ted Jenks <77486246+ted-jenks@users.noreply.github.com> Date: Tue, 17 Nov 2026 13:07:55 -0500 Subject: [PATCH 5/5] fix test --- .../comet/rules/CometScanRuleSuite.scala | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala index a694e6e0bef..57be5d8d52a 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala @@ -29,7 +29,8 @@ import org.apache.spark.sql.execution.adaptive.QueryStageExec import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ -import org.apache.comet.{CometConf, DataTypeSupport} +import org.apache.comet.{CometConf, DataTypeSupport, ExtendedExplainInfo} +import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator} /** @@ -256,17 +257,28 @@ class CometScanRuleSuite extends CometTestBase { } test("an integer column read as a negative-scale decimal falls back to Spark") { - // Spark reads multiples of 100 as DECIMAL(12,-2) and keeps nulls. The native scan sent this - // read through the integer to decimal cast kernel, which cannot take a negative scale. + // The native scan sent this read through the integer to decimal cast kernel, which cannot + // take a negative scale. Spark 4.0+ reads multiples of 100 as DECIMAL(12,-2) and keeps nulls. withSQLConf("spark.sql.legacy.allowNegativeScaleOfDecimal" -> "true") { withTempPath { path => val ints = StructType(Array(StructField("a", IntegerType))) val rows = spark.sparkContext.parallelize(Seq(Row(100), Row(200), Row(null)), 1) spark.createDataFrame(rows, ints).write.parquet(path.toString) val schema = StructType(Array(StructField("a", DecimalType(12, -2)))) - checkSparkAnswerAndFallbackReason( - spark.read.schema(schema).parquet(path.toString), - "native Parquet scan does not support negative-scale decimals") + val reason = "native Parquet scan does not support negative-scale decimals" + val df = spark.read.schema(schema).parquet(path.toString) + if (isSpark40Plus) { + checkSparkAnswerAndFallbackReason(df, reason) + } else { + // Spark 3.5 cannot read an integer column as a decimal, so there is no answer to compare. + val reasons = + new ExtendedExplainInfo().getFallbackReasons(df.queryExecution.executedPlan) + assert(reasons.exists(_.contains(reason)), reasons) + checkSparkAnswerMaybeThrows(df) match { + case (Some(_), Some(_)) => + case result => fail(s"Expected both Spark and Comet to fail, got $result") + } + } } } }