Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -211,9 +211,10 @@ integer types to a negative-scale `DecimalType`. Comet reports these as unsuppor
routes them through the JVM codegen dispatcher (Spark's own generated code) inside the Comet
pipeline, so the results match Spark exactly.

The native kernels align scale by computing `10^|scale|` in fixed-point arithmetic, which
overflows when the scale is negative. See
[#5013](https://github.com/apache/datafusion-comet/issues/5013).
The native kernels reject a negative scale with an error. See
[#5013](https://github.com/apache/datafusion-comet/issues/5013). For the same reason, the native
Parquet scan falls back to Spark when the read schema has a negative-scale decimal, such as an
integer column read as `DECIMAL(12,-2)`.

All other cast directions to and from negative-scale decimals run natively: `FloatType` and
`DoubleType`, `StringType` in both directions (subject to the config gate described above),
Expand Down
49 changes: 45 additions & 4 deletions native/spark-expr/src/conversion_funcs/numeric.rs
Original file line number Diff line number Diff line change
Expand Up @@ -524,6 +524,16 @@ macro_rules! cast_float_to_int32_up {
}};
}

/// 10^scale, or an error for a scale outside 0..=38. Negative scales need
/// spark.sql.legacy.allowNegativeScaleOfDecimal, and the serde and Parquet scan both leave those
/// to Spark.
fn decimal_scale_factor(scale: i8) -> SparkResult<i128> {
u32::try_from(scale)
.ok()
.and_then(pow10_i128)
.ok_or_else(|| SparkError::Internal(format!("unsupported decimal scale {scale}")))
}

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit : given that this is seldom reached given the Scala guards, wonder if we could just print scale and not the cast expression itself ?


// When Spark casts to Byte/Short Types, it does not cast directly to Byte/Short.
// It casts to Int first and then to Byte/Short. Because of potential overflows in the Int cast,
// this can cause unexpected Short/Byte cast results. Replicate this behavior.
Expand All @@ -542,7 +552,7 @@ macro_rules! cast_decimal_to_int16_down {
.downcast_ref::<Decimal128Array>()
.expect("Expected a Decimal128ArrayType");

let divisor = 10_i128.pow($scale as u32);
let divisor = decimal_scale_factor($scale)?;
// `try_unary` only visits valid slots, so a value under a null is neither divided nor,
// in ANSI mode, checked for overflow. The legacy conversion cannot fail, but the division
// is most of the cost for decimals, so it uses `try_unary` too, with a closure that
Expand Down Expand Up @@ -597,7 +607,7 @@ macro_rules! cast_decimal_to_int32_up {
.downcast_ref::<Decimal128Array>()
.expect("Expected a Decimal128ArrayType");

let divisor = 10_i128.pow($scale as u32);
let divisor = decimal_scale_factor($scale)?;
// `try_unary` only visits valid slots, so a value under a null is neither divided nor,
// in ANSI mode, checked for overflow. The legacy conversion cannot fail, but the division
// is most of the cost for decimals, so it uses `try_unary` too, with a closure that
Expand Down Expand Up @@ -775,7 +785,7 @@ where
T: ArrowPrimitiveType,
T::Native: Into<i128> + Bounded,
{
let multiplier = 10_i128.pow(scale as u32);
let multiplier = decimal_scale_factor(scale)?;

// Single spelling of the "does this value fit at the target precision after scaling" check,
// shared between the vectorized pass and the ANSI rescan below.
Expand Down Expand Up @@ -1371,7 +1381,7 @@ pub(crate) fn cast_decimal_to_timestamp(
scale: i8,
) -> SparkResult<ArrayRef> {
let arr = array_ref.as_primitive::<Decimal128Type>();
let scale_factor = 10_i128.pow(scale as u32);
let scale_factor = decimal_scale_factor(scale)?;
// `unary_opt` visits valid rows only. `unary` also divides null slots, and
// that was slower on every measured shape, including a batch with no nulls.
let micros = arr.unary_opt::<_, TimestampMicrosecondType>(|value| {
Expand Down Expand Up @@ -3105,6 +3115,37 @@ mod tests {
assert_ts(&out, &[Some(1), Some(-1)], Some("UTC"));
}

#[test]
fn test_cast_kernels_reject_negative_scale() {
// Try routes decimal to integer through the DataFusion cast, not these kernels.
let decimals: ArrayRef = Arc::new(
Decimal128Array::from(vec![Some(12345), None])
.with_precision_and_scale(10, -2)
.unwrap(),
);
let ints: ArrayRef = Arc::new(Int32Array::from(vec![Some(7), None]));
let utc = Some(Arc::from("UTC"));
let cases = [
(&decimals, DataType::Int8),
(&decimals, DataType::Int16),
(&decimals, DataType::Int32),
(&decimals, DataType::Int64),
(&ints, DataType::Decimal128(10, -2)),
(&decimals, DataType::Timestamp(TimeUnit::Microsecond, utc)),
];
for eval_mode in [EvalMode::Legacy, EvalMode::Ansi] {
let options = SparkCastOptions::new(eval_mode, "UTC");
for (input, to_type) in &cases {
let err = cast_array(Arc::clone(input), to_type, &options)
.expect_err("a negative scale must be rejected");
assert!(
err.to_string().contains("unsupported decimal scale -2"),
"{eval_mode:?} {to_type}: unexpected error: {err}"
);
}
}
}

#[test]
fn test_cast_float_to_decimal_i64_path_matches_i128_path() {
// Precision 18 rounds through `i64` and precision 19 still through `to_i128`; wherever the
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1447,6 +1447,12 @@ case class CometScanTypeChecker() extends DataTypeSupport with CometTypeShim {
fallbackReasons += s"Unsupported $name of type $dt (native Parquet scan does not " +
"support TIME)"
false
case d: DecimalType if d.scale < 0 =>
// Spark reads an integer column as a negative-scale decimal by rescaling each value,
// which the native cast kernels reject.
fallbackReasons += s"Unsupported $name of type $d (native Parquet scan does not " +
"support negative-scale decimals)"
false
case dt if isStringCollationType(dt) =>
// we don't need specific support for collation in scans, but this
// is a convenient place to force the whole query to fall back to Spark for now
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@ import org.apache.spark.sql.execution.adaptive.QueryStageExec
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._

import org.apache.comet.{CometConf, DataTypeSupport}
import org.apache.comet.{CometConf, DataTypeSupport, ExtendedExplainInfo}
import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus
import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator}

/**
Expand Down Expand Up @@ -239,6 +240,49 @@ class CometScanRuleSuite extends CometTestBase {
}
}

test("CometScanTypeChecker declines negative-scale decimals") {
withSQLConf("spark.sql.legacy.allowNegativeScaleOfDecimal" -> "true") {
val negative = StructField("d", DecimalType(10, -2))
val declined = Seq[(String, StructType)](
"root" -> StructType(Array(negative)),
"nested struct" -> StructType(Array(StructField("s", StructType(Array(negative))))))
for ((label, schema) <- declined) {
val reasons = ListBuffer.empty[String]
assert(!CometScanTypeChecker().isSchemaSupported(schema, reasons), s"$label: accepted")
assert(reasons.exists(_.contains("negative-scale decimals")), s"$label: $reasons")
}
val positive = StructType(Array(StructField("d", DecimalType(10, 2))))
assert(CometScanTypeChecker().isSchemaSupported(positive, ListBuffer.empty))
}
}

test("an integer column read as a negative-scale decimal falls back to Spark") {
// The native scan sent this read through the integer to decimal cast kernel, which cannot
// take a negative scale. Spark 4.0+ reads multiples of 100 as DECIMAL(12,-2) and keeps nulls.
withSQLConf("spark.sql.legacy.allowNegativeScaleOfDecimal" -> "true") {
withTempPath { path =>
val ints = StructType(Array(StructField("a", IntegerType)))
val rows = spark.sparkContext.parallelize(Seq(Row(100), Row(200), Row(null)), 1)
spark.createDataFrame(rows, ints).write.parquet(path.toString)
val schema = StructType(Array(StructField("a", DecimalType(12, -2))))
val reason = "native Parquet scan does not support negative-scale decimals"
val df = spark.read.schema(schema).parquet(path.toString)
if (isSpark40Plus) {
checkSparkAnswerAndFallbackReason(df, reason)
} else {
// Spark 3.5 cannot read an integer column as a decimal, so there is no answer to compare.
val reasons =
new ExtendedExplainInfo().getFallbackReasons(df.queryExecution.executedPlan)
assert(reasons.exists(_.contains(reason)), reasons)
checkSparkAnswerMaybeThrows(df) match {
case (Some(_), Some(_)) =>
case result => fail(s"Expected both Spark and Comet to fail, got $result")
}
}
}
}
}

test("DataTypeSupport finds sibling field names that collide case-insensitively") {
// See https://github.com/apache/datafusion-comet/issues/6136.
def fields(names: String*): StructType =
Expand Down
Loading