diff --git a/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md b/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md index 323de0dcd30..cbeaa2d6a48 100644 --- a/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md +++ b/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md @@ -211,9 +211,10 @@ integer types to a negative-scale `DecimalType`. Comet reports these as unsuppor routes them through the JVM codegen dispatcher (Spark's own generated code) inside the Comet pipeline, so the results match Spark exactly. -The native kernels align scale by computing `10^|scale|` in fixed-point arithmetic, which -overflows when the scale is negative. See -[#5013](https://github.com/apache/datafusion-comet/issues/5013). +The native kernels reject a negative scale with an error. See +[#5013](https://github.com/apache/datafusion-comet/issues/5013). For the same reason, the native +Parquet scan falls back to Spark when the read schema has a negative-scale decimal, such as an +integer column read as `DECIMAL(12,-2)`. All other cast directions to and from negative-scale decimals run natively: `FloatType` and `DoubleType`, `StringType` in both directions (subject to the config gate described above), diff --git a/native/spark-expr/src/conversion_funcs/numeric.rs b/native/spark-expr/src/conversion_funcs/numeric.rs index f262004606e..f21c164bfa1 100644 --- a/native/spark-expr/src/conversion_funcs/numeric.rs +++ b/native/spark-expr/src/conversion_funcs/numeric.rs @@ -524,6 +524,16 @@ macro_rules! cast_float_to_int32_up { }}; } +/// 10^scale, or an error for a scale outside 0..=38. Negative scales need +/// spark.sql.legacy.allowNegativeScaleOfDecimal, and the serde and Parquet scan both leave those +/// to Spark. +fn decimal_scale_factor(scale: i8) -> SparkResult { + u32::try_from(scale) + .ok() + .and_then(pow10_i128) + .ok_or_else(|| SparkError::Internal(format!("unsupported decimal scale {scale}"))) +} + // When Spark casts to Byte/Short Types, it does not cast directly to Byte/Short. // It casts to Int first and then to Byte/Short. Because of potential overflows in the Int cast, // this can cause unexpected Short/Byte cast results. Replicate this behavior. @@ -542,7 +552,7 @@ macro_rules! cast_decimal_to_int16_down { .downcast_ref::() .expect("Expected a Decimal128ArrayType"); - let divisor = 10_i128.pow($scale as u32); + let divisor = decimal_scale_factor($scale)?; // `try_unary` only visits valid slots, so a value under a null is neither divided nor, // in ANSI mode, checked for overflow. The legacy conversion cannot fail, but the division // is most of the cost for decimals, so it uses `try_unary` too, with a closure that @@ -597,7 +607,7 @@ macro_rules! cast_decimal_to_int32_up { .downcast_ref::() .expect("Expected a Decimal128ArrayType"); - let divisor = 10_i128.pow($scale as u32); + let divisor = decimal_scale_factor($scale)?; // `try_unary` only visits valid slots, so a value under a null is neither divided nor, // in ANSI mode, checked for overflow. The legacy conversion cannot fail, but the division // is most of the cost for decimals, so it uses `try_unary` too, with a closure that @@ -775,7 +785,7 @@ where T: ArrowPrimitiveType, T::Native: Into + Bounded, { - let multiplier = 10_i128.pow(scale as u32); + let multiplier = decimal_scale_factor(scale)?; // Single spelling of the "does this value fit at the target precision after scaling" check, // shared between the vectorized pass and the ANSI rescan below. @@ -1371,7 +1381,7 @@ pub(crate) fn cast_decimal_to_timestamp( scale: i8, ) -> SparkResult { let arr = array_ref.as_primitive::(); - let scale_factor = 10_i128.pow(scale as u32); + let scale_factor = decimal_scale_factor(scale)?; // `unary_opt` visits valid rows only. `unary` also divides null slots, and // that was slower on every measured shape, including a batch with no nulls. let micros = arr.unary_opt::<_, TimestampMicrosecondType>(|value| { @@ -3105,6 +3115,37 @@ mod tests { assert_ts(&out, &[Some(1), Some(-1)], Some("UTC")); } + #[test] + fn test_cast_kernels_reject_negative_scale() { + // Try routes decimal to integer through the DataFusion cast, not these kernels. + let decimals: ArrayRef = Arc::new( + Decimal128Array::from(vec![Some(12345), None]) + .with_precision_and_scale(10, -2) + .unwrap(), + ); + let ints: ArrayRef = Arc::new(Int32Array::from(vec![Some(7), None])); + let utc = Some(Arc::from("UTC")); + let cases = [ + (&decimals, DataType::Int8), + (&decimals, DataType::Int16), + (&decimals, DataType::Int32), + (&decimals, DataType::Int64), + (&ints, DataType::Decimal128(10, -2)), + (&decimals, DataType::Timestamp(TimeUnit::Microsecond, utc)), + ]; + for eval_mode in [EvalMode::Legacy, EvalMode::Ansi] { + let options = SparkCastOptions::new(eval_mode, "UTC"); + for (input, to_type) in &cases { + let err = cast_array(Arc::clone(input), to_type, &options) + .expect_err("a negative scale must be rejected"); + assert!( + err.to_string().contains("unsupported decimal scale -2"), + "{eval_mode:?} {to_type}: unexpected error: {err}" + ); + } + } + } + #[test] fn test_cast_float_to_decimal_i64_path_matches_i128_path() { // Precision 18 rounds through `i64` and precision 19 still through `to_i128`; wherever the diff --git a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala index 73f19b2cfb2..2924a1587d9 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala @@ -1447,6 +1447,12 @@ case class CometScanTypeChecker() extends DataTypeSupport with CometTypeShim { fallbackReasons += s"Unsupported $name of type $dt (native Parquet scan does not " + "support TIME)" false + case d: DecimalType if d.scale < 0 => + // Spark reads an integer column as a negative-scale decimal by rescaling each value, + // which the native cast kernels reject. + fallbackReasons += s"Unsupported $name of type $d (native Parquet scan does not " + + "support negative-scale decimals)" + false case dt if isStringCollationType(dt) => // we don't need specific support for collation in scans, but this // is a convenient place to force the whole query to fall back to Spark for now diff --git a/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala index 52c6159d11e..57be5d8d52a 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala @@ -29,7 +29,8 @@ import org.apache.spark.sql.execution.adaptive.QueryStageExec import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ -import org.apache.comet.{CometConf, DataTypeSupport} +import org.apache.comet.{CometConf, DataTypeSupport, ExtendedExplainInfo} +import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator} /** @@ -239,6 +240,49 @@ class CometScanRuleSuite extends CometTestBase { } } + test("CometScanTypeChecker declines negative-scale decimals") { + withSQLConf("spark.sql.legacy.allowNegativeScaleOfDecimal" -> "true") { + val negative = StructField("d", DecimalType(10, -2)) + val declined = Seq[(String, StructType)]( + "root" -> StructType(Array(negative)), + "nested struct" -> StructType(Array(StructField("s", StructType(Array(negative)))))) + for ((label, schema) <- declined) { + val reasons = ListBuffer.empty[String] + assert(!CometScanTypeChecker().isSchemaSupported(schema, reasons), s"$label: accepted") + assert(reasons.exists(_.contains("negative-scale decimals")), s"$label: $reasons") + } + val positive = StructType(Array(StructField("d", DecimalType(10, 2)))) + assert(CometScanTypeChecker().isSchemaSupported(positive, ListBuffer.empty)) + } + } + + test("an integer column read as a negative-scale decimal falls back to Spark") { + // The native scan sent this read through the integer to decimal cast kernel, which cannot + // take a negative scale. Spark 4.0+ reads multiples of 100 as DECIMAL(12,-2) and keeps nulls. + withSQLConf("spark.sql.legacy.allowNegativeScaleOfDecimal" -> "true") { + withTempPath { path => + val ints = StructType(Array(StructField("a", IntegerType))) + val rows = spark.sparkContext.parallelize(Seq(Row(100), Row(200), Row(null)), 1) + spark.createDataFrame(rows, ints).write.parquet(path.toString) + val schema = StructType(Array(StructField("a", DecimalType(12, -2)))) + val reason = "native Parquet scan does not support negative-scale decimals" + val df = spark.read.schema(schema).parquet(path.toString) + if (isSpark40Plus) { + checkSparkAnswerAndFallbackReason(df, reason) + } else { + // Spark 3.5 cannot read an integer column as a decimal, so there is no answer to compare. + val reasons = + new ExtendedExplainInfo().getFallbackReasons(df.queryExecution.executedPlan) + assert(reasons.exists(_.contains(reason)), reasons) + checkSparkAnswerMaybeThrows(df) match { + case (Some(_), Some(_)) => + case result => fail(s"Expected both Spark and Comet to fail, got $result") + } + } + } + } + } + test("DataTypeSupport finds sibling field names that collide case-insensitively") { // See https://github.com/apache/datafusion-comet/issues/6136. def fields(names: String*): StructType =