From 5f71057399ddc33498109933aa3a0e00fdfd4ba3 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Thu, 19 Nov 2026 16:13:45 -0500 Subject: [PATCH 1/2] perf: divide once per row in decimal / and div --- native/spark-expr/benches/decimal_div.rs | 97 ++++++++++- native/spark-expr/src/math_funcs/div.rs | 211 ++++++++++++++++++++--- 2 files changed, 287 insertions(+), 21 deletions(-) diff --git a/native/spark-expr/benches/decimal_div.rs b/native/spark-expr/benches/decimal_div.rs index 89119987e85..4a8141a5da5 100644 --- a/native/spark-expr/benches/decimal_div.rs +++ b/native/spark-expr/benches/decimal_div.rs @@ -16,14 +16,36 @@ // under the License. use arrow::array::builder::Decimal128Builder; +use arrow::array::{ArrayRef, Decimal128Array}; use arrow::compute::cast; +use arrow::compute::kernels::numeric; use arrow::datatypes::DataType; use criterion::{criterion_group, criterion_main, Criterion}; use datafusion::physical_plan::ColumnarValue; use datafusion_comet_spark_expr::{spark_decimal_div, spark_decimal_integral_div, EvalMode}; +use rand::rngs::StdRng; +use rand::{RngExt, SeedableRng}; use std::hint::black_box; use std::sync::Arc; +const BATCH_SIZE: usize = 8192; + +/// A batch of nonzero values in `[-max, max]`. +fn decimal_array(rng: &mut StdRng, max: i128, precision: u8, scale: i8) -> ArrayRef { + let values = (0..BATCH_SIZE).map(|_| { + let v = rng.random_range(1..=max); + if rng.random_bool(0.5) { + v + } else { + -v + } + }); + Arc::new( + Decimal128Array::from_iter_values(values) + .with_data_type(DataType::Decimal128(precision, scale)), + ) +} + fn criterion_benchmark(c: &mut Criterion) { // create input data let mut c1 = Decimal128Builder::new(); @@ -40,7 +62,10 @@ fn criterion_benchmark(c: &mut Criterion) { let c2_type = DataType::Decimal128(10, 3); let c2 = cast(c2.as_ref(), &c2_type).unwrap(); - let args = [ColumnarValue::Array(c1), ColumnarValue::Array(c2)]; + let args = [ + ColumnarValue::Array(Arc::clone(&c1)), + ColumnarValue::Array(Arc::clone(&c2)), + ]; let mut group = c.benchmark_group("decimal div"); group.bench_function("decimal_div", |b| { @@ -63,6 +88,76 @@ fn criterion_benchmark(c: &mut Criterion) { )) }) }); + + // The cases below use Spark's result types and a full batch. + let mut rng = StdRng::seed_from_u64(42); + + // Decimal(10, 4) / Decimal(10, 3) is Decimal(24, 15). The dividend is scaled past 64 bits. + let args = [ + ColumnarValue::Array(decimal_array(&mut rng, 10_i128.pow(10) - 1, 10, 4)), + ColumnarValue::Array(decimal_array(&mut rng, 10_i128.pow(10) - 1, 10, 3)), + ]; + group.bench_function("decimal_div_10_4_by_10_3", |b| { + b.iter(|| { + black_box(spark_decimal_div( + black_box(&args), + &DataType::Decimal128(24, 15), + EvalMode::Legacy, + )) + }) + }); + // Decimal(10, 4) div Decimal(10, 3) is Decimal(9, 0). + group.bench_function("decimal_integral_div_10_4_by_10_3", |b| { + b.iter(|| { + black_box(spark_decimal_integral_div( + black_box(&args), + &DataType::Decimal128(9, 0), + EvalMode::Legacy, + false, + )) + }) + }); + + // Decimal(38, 10) / Decimal(38, 10) is Decimal(38, 6), which takes the BigInt branch. + let args = [ + ColumnarValue::Array(decimal_array(&mut rng, 10_i128.pow(20), 38, 10)), + ColumnarValue::Array(decimal_array(&mut rng, 10_i128.pow(15), 38, 10)), + ]; + group.bench_function("decimal_div_38_10_bigint", |b| { + b.iter(|| { + black_box(spark_decimal_div( + black_box(&args), + &DataType::Decimal128(38, 6), + EvalMode::Legacy, + )) + }) + }); + + // Integer `div` casts both operands to Decimal(19, 0). + for (name, max) in [ + ("integral_div_long_small", 1_000_000), + ("integral_div_long_random", i64::MAX as i128), + ] { + let args = [ + ColumnarValue::Array(decimal_array(&mut rng, max, 19, 0)), + ColumnarValue::Array(decimal_array(&mut rng, max, 19, 0)), + ]; + group.bench_function(name, |b| { + b.iter(|| { + black_box(spark_decimal_integral_div( + black_box(&args), + &DataType::Decimal128(19, 0), + EvalMode::Legacy, + false, + )) + }) + }); + } + + // Arrow's decimal division, which this kernel doesn't call. Its change is the run's noise. + group.bench_function("control_arrow_decimal_div", |b| { + b.iter(|| black_box(numeric::div(black_box(&c1), black_box(&c2)))) + }); } criterion_group!(benches, criterion_benchmark); diff --git a/native/spark-expr/src/math_funcs/div.rs b/native/spark-expr/src/math_funcs/div.rs index b31ff36bfd6..e6dcf57fd92 100644 --- a/native/spark-expr/src/math_funcs/div.rs +++ b/native/spark-expr/src/math_funcs/div.rs @@ -27,7 +27,7 @@ use arrow::{ }; use datafusion::common::DataFusionError; use datafusion::physical_plan::ColumnarValue; -use num::{BigInt, Signed, ToPrimitive, Zero}; +use num::{BigInt, Integer, Signed, ToPrimitive, Zero}; use std::sync::Arc; pub fn spark_decimal_div( @@ -49,8 +49,8 @@ pub fn spark_decimal_integral_div( // Let Decimal(p3, s3) as return type i.e. Decimal(p1, s1) / Decimal(p2, s2) = Decimal(p3, s3). // Conversely, Decimal(p1, s1) = Decimal(p2, s2) * Decimal(p3, s3). This means that, in order to -// get enough scale that matches with Spark behavior, it requires to widen s1 to s2 + s3 + 1. Since -// both s2 and s3 are 38 at max., s1 is 77 at max. DataFusion division cannot handle such scale > +// get enough scale that matches with Spark behavior, it requires to widen s1 to s2 + s3. Since +// both s2 and s3 are 38 at max., s1 is 76 at max. DataFusion division cannot handle such scale > // Decimal256Type::MAX_SCALE. Therefore, we need to implement this decimal division using BigInt. /// Convert a computed quotient to the `i128` stored in the result array, throwing /// ARITHMETIC_OVERFLOW when the integral divide overflow check applies and the quotient @@ -110,15 +110,15 @@ fn spark_decimal_div_internal( ))); } - let l_exp = ((s2 + s3 + 1) as u32).saturating_sub(s1 as u32); - let r_exp = (s1 as u32).saturating_sub((s2 + s3 + 1) as u32); + // Both branches divide once at scale s3 and round from the remainder. + let l_exp = ((s2 + s3) as u32).saturating_sub(s1 as u32); + let r_exp = (s1 as u32).saturating_sub((s2 + s3) as u32); let result = if p1 as u32 + l_exp > DECIMAL128_MAX_PRECISION as u32 || p2 as u32 + r_exp > DECIMAL128_MAX_PRECISION as u32 { let ten = BigInt::from(10); let l_mul = ten.pow(l_exp); let r_mul = ten.pow(r_exp); - let five = BigInt::from(5); let zero = BigInt::from(0); arrow::compute::kernels::arity::try_binary(left, right, |l, r| { let l = BigInt::from(l) * &l_mul; @@ -134,14 +134,20 @@ fn spark_decimal_div_internal( // `nullIfWhenPrimitive` wrapper applied in the Scala serde layer, so // `try_binary` will never invoke this closure for a zero `r` in legacy/try mode. // The fallback `zero.clone()` is therefore unreachable in practice. - let div = if r.eq(&zero) { zero.clone() } else { &l / &r }; - let res = if is_integral_div { - div - } else if div.is_negative() { - div - &five + if r.eq(&zero) { + return quotient_to_i128(&zero, check_divide_overflow); + } + let (q, rem) = l.div_rem(&r); + // Round half away from zero. + let res = if !is_integral_div && rem.abs() * 2 >= r.abs() { + if l.is_negative() == r.is_negative() { + q + 1 + } else { + q - 1 + } } else { - div + &five - } / &ten; + q + }; quotient_to_i128(&res, check_divide_overflow) }) } else { @@ -161,14 +167,22 @@ fn spark_decimal_div_internal( // `nullIfWhenPrimitive` wrapper applied in the Scala serde layer, so // `try_binary` will never invoke this closure for a zero `r` in legacy/try mode. // The fallback `0` is therefore unreachable in practice. - let div = if r == 0 { 0 } else { l / r }; - let res = if is_integral_div { - div - } else if div.is_negative() { - div - 5 + if r == 0 { + return quotient_to_i128(&0, check_divide_overflow); + } + let q = l / r; + // Round half away from zero. `|q * r| <= |l|`, so the product can't overflow, and + // `rem >= |r| - rem` is `2 * rem >= |r|` without the overflow of `2 * rem`. + let rem = (l - q * r).unsigned_abs(); + let res = if !is_integral_div && rem >= r.unsigned_abs() - rem { + if (l < 0) == (r < 0) { + q + 1 + } else { + q - 1 + } } else { - div + 5 - } / 10; + q + }; quotient_to_i128(&res, check_divide_overflow) }) }; @@ -181,6 +195,9 @@ fn spark_decimal_div_internal( mod tests { use super::*; use crate::SparkError; + use num::Integer; + use rand::rngs::StdRng; + use rand::{RngExt, SeedableRng}; fn decimal(value: i128, precision: u8, scale: i8) -> ColumnarValue { ColumnarValue::Array(Arc::new( @@ -256,6 +273,160 @@ mod tests { } } + /// The exact quotient `(l / 10^s1) / (r / 10^s2)` at scale `s3`, truncated for `div` and + /// rounded half away from zero for `/`, saturated the way the kernel saturates. Also returns + /// whether the exact quotient was halfway between two values at scale `s3`. + fn reference_div( + l: i128, + r: i128, + (s1, s2, s3): (i8, i8, i8), + is_integral_div: bool, + ) -> (i128, bool) { + if r == 0 { + return (0, false); + } + let ten = BigInt::from(10); + let num = BigInt::from(l) * ten.pow((s2 + s3) as u32); + let den = BigInt::from(r) * ten.pow(s1 as u32); + let (mut q, rem) = num.div_rem(&den); + let twice_rem = rem.abs() * 2; + let half = twice_rem == den.abs(); + if !is_integral_div && twice_rem >= den.abs() { + q += num.signum() * den.signum(); + } + (q.to_i128().unwrap_or(i128::MAX), half) + } + + /// Values at and around the bounds of precision `p`, values that make exact halves likely, + /// and random values, each with both signs. + fn test_values(p: u8, rng: &mut StdRng) -> Vec { + let max = 10_i128.pow(p as u32) - 1; + let mut v = vec![1, 2, 3, 5, 7, max, max - 1, max / 2, max / 3]; + v.push(10_i128.pow(p as u32 - 1)); + for j in [1, 5, 18, 37] { + if j < p as u32 { + v.push(2 * 10_i128.pow(j)); + v.push(3 * 10_i128.pow(j)); + } + } + v.extend((0..3).map(|_| rng.random_range(1..=max))); + v.extend(v.clone().into_iter().map(|x| -x)); + v.push(0); + v + } + + #[test] + fn test_decimal_div_matches_exact_quotient() { + let mut rng = StdRng::seed_from_u64(42); + let (mut halves, mut i128_branch, mut bigint_branch) = (0, 0, 0); + let precisions: Vec<(u8, Vec)> = [1u8, 10, 19, 38] + .into_iter() + .map(|p| (p, test_values(p, &mut rng))) + .collect(); + for (p1, ls) in &precisions { + for (p2, rs) in &precisions { + let (p1, p2) = (*p1, *p2); + // Every pair, plus a null on each side. + let (mut lv, mut rv): (Vec<_>, Vec<_>) = ls + .iter() + .flat_map(|&l| rs.iter().map(move |&r| (Some(l), Some(r)))) + .unzip(); + lv.extend([None, Some(1)]); + rv.extend([Some(1), None]); + let (left, right) = ( + Decimal128Array::from(lv.clone()), + Decimal128Array::from(rv.clone()), + ); + for s1 in [0, 1, p1 / 2, p1 - 1, p1] { + for s2 in [0, 1, p2 / 2, p2 - 1, p2] { + for s3 in [0u8, 1, 6, 18, 38] { + let scales = (s1 as i8, s2 as i8, s3 as i8); + // The branch choice before this kernel removed its extra digit. + let l_exp = (s2 + s3 + 1).saturating_sub(s1); + let r_exp = s1.saturating_sub(s2 + s3 + 1); + if p1 + l_exp > 38 || p2 + r_exp > 38 { + bigint_branch += 1; + } else { + i128_branch += 1; + } + let left = left + .clone() + .with_data_type(DataType::Decimal128(p1, s1 as i8)); + let right = right + .clone() + .with_data_type(DataType::Decimal128(p2, s2 as i8)); + let args = [ + ColumnarValue::Array(Arc::new(left)), + ColumnarValue::Array(Arc::new(right)), + ]; + let ret = DataType::Decimal128(38, s3 as i8); + for is_integral_div in [false, true] { + let result = if is_integral_div { + spark_decimal_integral_div(&args, &ret, EvalMode::Legacy, true) + } else { + spark_decimal_div(&args, &ret, EvalMode::Legacy) + }; + let ColumnarValue::Array(result) = result.unwrap() else { + unreachable!() + }; + let result = result.as_primitive::(); + assert_eq!(result.data_type(), &ret); + for (i, (l, r)) in lv.iter().zip(&rv).enumerate() { + let (Some(l), Some(r)) = (l, r) else { + assert!(result.is_null(i)); + continue; + }; + let (expected, half) = + reference_div(*l, *r, scales, is_integral_div); + halves += half as usize; + assert_eq!( + result.value(i), + expected, + "{l} / {r}, scales {scales:?}, div {is_integral_div}" + ); + } + } + } + } + } + } + } + assert!(halves > 1000, "only {halves} exact halves"); + assert!(i128_branch > 100 && bigint_branch > 100); + } + + #[test] + fn test_integral_divide_overflow_through_kernel() { + // (38, 0) div (38, 0) runs the i128 branch. (38, 0) div (38, 1) runs the BigInt branch. + for s2 in [0, 1] { + let divisor = 10_i128.pow(s2 as u32); + for (dividend, overflows) in [ + (i64::MAX as i128, false), + (i64::MIN as i128, false), + (i64::MAX as i128 + 1, true), + (i64::MIN as i128 - 1, true), + ] { + let result = spark_decimal_integral_div( + &[decimal(dividend, 38, 0), decimal(divisor, 38, s2)], + &DataType::Decimal128(38, 0), + EvalMode::Ansi, + true, + ); + if overflows { + assert!(matches!( + spark_error(result), + SparkError::IntegralDivideOverflow + )); + } else { + let ColumnarValue::Array(result) = result.unwrap() else { + unreachable!() + }; + assert_eq!(result.as_primitive::().value(0), dividend); + } + } + } + } + #[test] fn test_integral_divide_overflow_returns_spark_error() { let result = quotient_to_i128(&(i64::MAX as i128 + 1), true); From a0d8c4a646aea1466ff852f53a2ea430885e49b0 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Thu, 19 Nov 2026 16:41:35 -0500 Subject: [PATCH 2/2] perf: divide once per row in decimal / and div --- native/spark-expr/benches/decimal_div.rs | 1 + native/spark-expr/src/math_funcs/div.rs | 25 +++++++++++++++++------- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/native/spark-expr/benches/decimal_div.rs b/native/spark-expr/benches/decimal_div.rs index 4a8141a5da5..b17d7b91678 100644 --- a/native/spark-expr/benches/decimal_div.rs +++ b/native/spark-expr/benches/decimal_div.rs @@ -155,6 +155,7 @@ fn criterion_benchmark(c: &mut Criterion) { } // Arrow's decimal division, which this kernel doesn't call. Its change is the run's noise. + assert!(numeric::div(&c1, &c2).is_ok()); group.bench_function("control_arrow_decimal_div", |b| { b.iter(|| black_box(numeric::div(black_box(&c1), black_box(&c2)))) }); diff --git a/native/spark-expr/src/math_funcs/div.rs b/native/spark-expr/src/math_funcs/div.rs index e6dcf57fd92..a9090726c91 100644 --- a/native/spark-expr/src/math_funcs/div.rs +++ b/native/spark-expr/src/math_funcs/div.rs @@ -110,7 +110,7 @@ fn spark_decimal_div_internal( ))); } - // Both branches divide once at scale s3 and round from the remainder. + // Both branches divide once at scale s3. `/` rounds from the remainder. let l_exp = ((s2 + s3) as u32).saturating_sub(s1 as u32); let r_exp = (s1 as u32).saturating_sub((s2 + s3) as u32); let result = if p1 as u32 + l_exp > DECIMAL128_MAX_PRECISION as u32 @@ -172,9 +172,9 @@ fn spark_decimal_div_internal( } let q = l / r; // Round half away from zero. `|q * r| <= |l|`, so the product can't overflow, and - // `rem >= |r| - rem` is `2 * rem >= |r|` without the overflow of `2 * rem`. + // `rem < |r| <= 2^127`, so `2 * rem` fits in `u128`. let rem = (l - q * r).unsigned_abs(); - let res = if !is_integral_div && rem >= r.unsigned_abs() - rem { + let res = if !is_integral_div && 2 * rem >= r.unsigned_abs() { if (l < 0) == (r < 0) { q + 1 } else { @@ -319,6 +319,9 @@ mod tests { fn test_decimal_div_matches_exact_quotient() { let mut rng = StdRng::seed_from_u64(42); let (mut halves, mut i128_branch, mut bigint_branch) = (0, 0, 0); + // Scale combinations that moved to the i128 branch, and to the BigInt branch, when the + // kernel removed its extra digit. + let (mut to_i128, mut to_bigint) = (0, 0); let precisions: Vec<(u8, Vec)> = [1u8, 10, 19, 38] .into_iter() .map(|p| (p, test_values(p, &mut rng))) @@ -341,10 +344,17 @@ mod tests { for s2 in [0, 1, p2 / 2, p2 - 1, p2] { for s3 in [0u8, 1, 6, 18, 38] { let scales = (s1 as i8, s2 as i8, s3 as i8); - // The branch choice before this kernel removed its extra digit. - let l_exp = (s2 + s3 + 1).saturating_sub(s1); - let r_exp = s1.saturating_sub(s2 + s3 + 1); - if p1 + l_exp > 38 || p2 + r_exp > 38 { + let uses_bigint = |extra: u8| { + let l_exp = (s2 + s3 + extra).saturating_sub(s1); + let r_exp = s1.saturating_sub(s2 + s3 + extra); + p1 + l_exp > 38 || p2 + r_exp > 38 + }; + match (uses_bigint(1), uses_bigint(0)) { + (true, false) => to_i128 += 1, + (false, true) => to_bigint += 1, + _ => {} + } + if uses_bigint(0) { bigint_branch += 1; } else { i128_branch += 1; @@ -393,6 +403,7 @@ mod tests { } assert!(halves > 1000, "only {halves} exact halves"); assert!(i128_branch > 100 && bigint_branch > 100); + assert!(to_i128 > 0 && to_bigint > 0, "{to_i128} {to_bigint}"); } #[test]