diff --git a/docs/source/contributor-guide/expression-audits/agg_funcs.md b/docs/source/contributor-guide/expression-audits/agg_funcs.md index c3157c0ba6d..feaa5810d51 100644 --- a/docs/source/contributor-guide/expression-audits/agg_funcs.md +++ b/docs/source/contributor-guide/expression-audits/agg_funcs.md @@ -44,6 +44,7 @@ - Spark 3.4.3 (2026-05-26) - Spark 3.5.8 (2026-05-26): aggregate logic identical to 3.4.3 - Spark 4.0.1 (2026-05-26): aggregate logic identical to 3.5.8; only `QueryContext` import path differs. `YearMonthIntervalType` and `DayTimeIntervalType` inputs (supported by Spark) fall back to Spark in Comet. +- Performance (tuned 2026-11-17, PR [#7674](https://github.com/apache/datafusion-comet/pull/7674)): the grouped decimal accumulator loads the sum's precision bounds once and compares against them on every update and merge, instead of calling `Decimal128Type::is_valid_decimal_precision`, which release builds keep out of line. Benchmark: `benches/decimal_groups.rs`. ## bit_and @@ -118,4 +119,8 @@ - Spark 4.2.0 (audited 2026-10-12): the `TreeMap` is chosen for every pivot column type without proper equals (`!TypeUtils.typeWithProperEquals`), which adds binary and strings with a non-binary-equality collation, so binary pivot values now match by content. A `findPivotIndex` wrapper returns `-1` for a null key on the `TreeMap` path instead of throwing. - `CometPivotFirst` (in `spark/src/main/scala/org/apache/comet/serde/aggregates.scala`) forwards the aggregate to the native `SparkPivotFirst` UDAF (`native/spark-expr/src/agg_funcs/pivot_first.rs`) when the value type is in the supported set and the pivot column is a Boolean, Byte, Short, Int, Long, Float, Double, Decimal, UTF8_BINARY String, Date, Timestamp or TimestampNTZ. Those are the types Spark keys by value equality on every version, which the native lookup reproduces, including the float rules. Other pivot columns fall back, as does a list where `indexSize` is smaller than the list (duplicates, including `0.0` next to `-0.0`). State layout matches Spark's `aggBufferAttributes` (one scalar column per pivot slot) so the shuffle schema between Partial and Final stays consistent. The result reassembles the slots into a `ListArray` matching `PivotFirst.dataType = ArrayType(valueDataType)`. +## sum + +- Performance (tuned 2026-11-17, PR [#7674](https://github.com/apache/datafusion-comet/pull/7674)): the grouped decimal accumulator loads the result's precision bounds once and compares against them on every update, merge, and evaluate, instead of calling `Decimal128Type::is_valid_decimal_precision`, which release builds keep out of line. Benchmark: `benches/decimal_groups.rs`. + [Spark Expression Support]: ../../user-guide/latest/expressions.md diff --git a/native/spark-expr/Cargo.toml b/native/spark-expr/Cargo.toml index a99a6891a59..b7b8a7d0e19 100644 --- a/native/spark-expr/Cargo.toml +++ b/native/spark-expr/Cargo.toml @@ -518,3 +518,7 @@ harness = false [[bench]] name = "lazy_logical" harness = false + +[[bench]] +name = "decimal_groups" +harness = false diff --git a/native/spark-expr/benches/decimal_groups.rs b/native/spark-expr/benches/decimal_groups.rs new file mode 100644 index 00000000000..04ac77f8e63 --- /dev/null +++ b/native/spark-expr/benches/decimal_groups.rs @@ -0,0 +1,237 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Grouped decimal SUM and AVG accumulators, driven directly through +//! `GroupsAccumulator::update_batch` so the numbers cover the accumulators and not grouping or +//! the execution framework. + +use arrow::array::{ArrayRef, Decimal128Array}; +use arrow::datatypes::{DataType, Field, Schema}; +use criterion::{criterion_group, criterion_main, Criterion}; +use datafusion::functions_aggregate::sum::sum_udaf; +use datafusion::logical_expr::function::AccumulatorArgs; +use datafusion::logical_expr::{AggregateUDFImpl, EmitTo, GroupsAccumulator}; +use datafusion_comet_spark_expr::{create_query_context_map, AvgDecimal, EvalMode, SumDecimal}; +use std::hint::black_box; +use std::sync::Arc; + +const ROWS: usize = 8192; +const BATCHES: usize = 10; + +/// TPC-H Q1 grouping: 4 groups in random order, weighted 25%, 0.6%, 50% and 25%. +fn q1_groups() -> Vec { + let mut x: u64 = 0x9E37_79B9_7F4A_7C15; + (0..ROWS) + .map(|_| { + x = x + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + match (x >> 33) % 1000 { + 0..=249 => 0, + 250..=255 => 1, + 256..=755 => 2, + _ => 3, + } + }) + .collect() +} + +fn decimals(precision: u8, scale: i8, value: impl Fn(usize) -> Option) -> ArrayRef { + Arc::new( + (0..ROWS) + .map(value) + .collect::() + .with_precision_and_scale(precision, scale) + .unwrap(), + ) +} + +/// One input shape: values, group ids, and the (precision, scale) of the input and of the sum. +struct Shape<'a> { + name: &'a str, + values: &'a ArrayRef, + groups: &'a [usize], + num_groups: usize, + types: ((u8, i8), (u8, i8)), +} + +impl<'a> Shape<'a> { + fn new( + name: &'a str, + values: &'a ArrayRef, + groups: &'a [usize], + num_groups: usize, + types: ((u8, i8), (u8, i8)), + ) -> Self { + Self { + name, + values, + groups, + num_groups, + types, + } + } +} + +fn bench( + c: &mut Criterion, + name: &str, + make: &dyn Fn() -> Box, + shape: &Shape, +) { + let values = [Arc::clone(shape.values)]; + c.bench_function(name, |b| { + b.iter(|| { + let mut acc = make(); + for _ in 0..BATCHES { + acc.update_batch(&values, shape.groups, None, shape.num_groups) + .unwrap(); + } + black_box(acc.evaluate(EmitTo::All).unwrap()) + }) + }); +} + +fn criterion_benchmark(c: &mut Criterion) { + let q1 = q1_groups(); + let cyclic: Vec = (0..ROWS).map(|i| i % 1024).collect(); + // A bijection on 0..65536, so each batch touches 8192 scattered groups. + let scattered: Vec = (0..ROWS) + .map(|i| (i as u64 * 2654435761 % 65536) as usize) + .collect(); + + // Q1's `l_quantity`: DECIMAL(15,2) values 1.00..50.00 summed as DECIMAL(25,2). + let quantity = decimals(15, 2, |i| Some(((i % 50) as i128 + 1) * 100)); + let quantity_nulls = decimals(15, 2, |i| { + (i % 10 != 0).then(|| ((i % 50) as i128 + 1) * 100) + }); + // A DECIMAL(38,6) input keeps the sum at the capped precision 38. Values near 1e5 stay far + // from it. Values of 1e27 to 5e28 have absolute values that sum to 2.1e32 per batch, past + // the 1e32 limit, so a bound on a batch's total can't skip the per-row check. Each group's + // rows share one sign, and no running sum passes 3% of the limit, so no group latches. + let wide = decimals(38, 6, |i| Some(((i % 50) as i128 + 1) * 1_000_000_000)); + let near_limit = decimals(38, 6, |i| { + let v = ((i % 50) as i128 + 1) * 10_i128.pow(33); + Some(if i % 2 == 0 { v } else { -v }) + }); + + let narrow = ((15, 2), (25, 2)); + let capped = ((38, 6), (38, 6)); + let shapes = [ + Shape::new("(25,2), 4 groups", &quantity, &q1, 4, narrow), + Shape::new("(25,2), 1024 groups", &quantity, &cyclic, 1024, narrow), + Shape::new("(25,2), 64K groups", &quantity, &scattered, 65536, narrow), + Shape::new( + "(25,2), 1024 groups, 10% nulls", + &quantity_nulls, + &cyclic, + 1024, + narrow, + ), + Shape::new("(38,6), 1024 groups", &wide, &cyclic, 1024, capped), + Shape::new( + "(38,6), 1024 groups, near limit", + &near_limit, + &cyclic, + 1024, + capped, + ), + ]; + + for shape in &shapes { + let ((p, s), (sum_p, sum_s)) = shape.types; + let input_type = DataType::Decimal128(p, s); + let sum_type = DataType::Decimal128(sum_p, sum_s); + // Spark's AVG result type: DECIMAL(min(38, p + 4), min(38, s + 4)). + let avg_type = DataType::Decimal128((p + 4).min(38), (s + 4).min(38)); + let schema = Schema::new(vec![Field::new("c0", input_type.clone(), true)]); + let expr_fields = [Arc::new(Field::new("c0", input_type.clone(), true))]; + let args = |result_type: &DataType| AccumulatorArgs { + return_field: Arc::new(Field::new("r", result_type.clone(), true)), + schema: &schema, + ignore_nulls: false, + order_bys: &[], + name: "agg", + is_distinct: false, + is_reversed: false, + exprs: &[], + expr_fields: &expr_fields, + }; + + let sum = SumDecimal::try_new( + sum_type.clone(), + EvalMode::Legacy, + None, + create_query_context_map(), + ) + .unwrap(); + bench( + c, + &format!("sum {}", shape.name), + &|| sum.create_groups_accumulator(args(&sum_type)).unwrap(), + shape, + ); + + let avg = AvgDecimal::new( + avg_type.clone(), + sum_type.clone(), + EvalMode::Legacy, + None, + create_query_context_map(), + ); + bench( + c, + &format!("avg {}", shape.name), + &|| avg.create_groups_accumulator(args(&avg_type)).unwrap(), + shape, + ); + } + + // DataFusion's SUM checks no precision per row, so it marks the floor. + let shape = &shapes[1]; + let sum_type = DataType::Decimal128(25, 2); + let schema = Schema::new(vec![Field::new("c0", DataType::Decimal128(15, 2), true)]); + let expr_fields = [Arc::new(Field::new( + "c0", + DataType::Decimal128(15, 2), + true, + ))]; + let datafusion = sum_udaf(); + bench( + c, + "datafusion sum (25,2), 1024 groups", + &|| { + datafusion + .create_groups_accumulator(AccumulatorArgs { + return_field: Arc::new(Field::new("r", sum_type.clone(), true)), + schema: &schema, + ignore_nulls: false, + order_bys: &[], + name: "sum", + is_distinct: false, + is_reversed: false, + exprs: &[], + expr_fields: &expr_fields, + }) + .unwrap() + }, + shape, + ); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/native/spark-expr/src/agg_funcs/avg_decimal.rs b/native/spark-expr/src/agg_funcs/avg_decimal.rs index b71ccc4f1eb..4d7582eca58 100644 --- a/native/spark-expr/src/agg_funcs/avg_decimal.rs +++ b/native/spark-expr/src/agg_funcs/avg_decimal.rs @@ -30,7 +30,7 @@ use datafusion::logical_expr::{ use datafusion::physical_expr::expressions::format_state_name; use std::sync::Arc; -use crate::utils::{build_bool_state, unlikely}; +use crate::utils::{build_bool_state, unlikely, Decimal128Bounds}; use crate::{decimal_sum_overflow_error, EvalMode, SparkErrorWithContext}; use arrow::array::ArrowNativeTypeOp; use arrow::datatypes::{ @@ -367,8 +367,8 @@ struct AvgDecimalGroupsAccumulator { /// The type of the sum sum_data_type: DataType, - /// This is input_precision + 10 to be consistent with Spark - sum_precision: u8, + /// The values a sum holds. Its precision is input_precision + 10 to be consistent with Spark + sum_bounds: Decimal128Bounds, sum_scale: i8, /// Evaluation mode for error handling @@ -398,7 +398,7 @@ impl AvgDecimalGroupsAccumulator { target_precision, target_scale, sum_data_type: sum_data_type.clone(), - sum_precision, + sum_bounds: Decimal128Bounds::new(sum_precision), sum_scale, counts: vec![], sums: vec![], @@ -428,9 +428,7 @@ impl AvgDecimalGroupsAccumulator { self.counts[group_index] += 1; self.sums[group_index] = new_sum; - if unlikely( - is_overflow || !Decimal128Type::is_valid_decimal_precision(new_sum, self.sum_precision), - ) { + if unlikely(is_overflow || !self.sum_bounds.contains(new_sum)) { // Overflow: set to null. Error will be thrown during evaluate in ANSI mode. // This matches Spark's DecimalAddNoOverflowCheck behavior. self.is_not_null.set_bit(group_index, false); @@ -519,9 +517,7 @@ impl GroupsAccumulator for AvgDecimalGroupsAccumulator { let sum = self.sums[group_index]; let (new_sum, is_overflow) = sum.overflowing_add(new_value); - if is_overflow - || !Decimal128Type::is_valid_decimal_precision(new_sum, self.sum_precision) - { + if is_overflow || !self.sum_bounds.contains(new_sum) { // Preserve overflow in intermediate/spilled state without throwing. Like // Spark's decimal AVG, report ANSI overflow only when evaluate emits the // affected group; a prefix emission may not need this group yet. @@ -884,4 +880,39 @@ mod tests { } Ok(()) } + + #[test] + fn grouped_latches_at_the_sum_precision_below_38() -> Result<()> { + // A DECIMAL(15,2) input sums as DECIMAL(25,2) and averages as DECIMAL(19,6), so a latch + // that used the result precision would also null group 0. + let max = 10_i128.pow(25) - 1; + let new_acc = || { + AvgDecimalGroupsAccumulator::new( + &DataType::Decimal128(19, 6), + &DataType::Decimal128(25, 2), + 19, + 6, + 25, + 2, + EvalMode::Legacy, + None, + crate::create_query_context_map(), + ) + }; + let sums = || Decimal128Array::from(vec![max, max, 1]).with_precision_and_scale(25, 2); + let mut updated = new_acc(); + updated.update_batch(&[Arc::new(sums()?)], &[0, 1, 1], None, 2)?; + let mut merged = new_acc(); + merged.merge_batch( + &[Arc::new(sums()?), Arc::new(Int64Array::from(vec![1; 3]))], + &[0, 1, 1], + 2, + )?; + for mut acc in [updated, merged] { + let state = acc.state(EmitTo::All)?; + let sums = state[0].as_primitive::(); + assert_eq!(sums.iter().collect::>(), vec![Some(max), None]); + } + Ok(()) + } } diff --git a/native/spark-expr/src/agg_funcs/sum_decimal.rs b/native/spark-expr/src/agg_funcs/sum_decimal.rs index 1930001d857..ae05aeec366 100644 --- a/native/spark-expr/src/agg_funcs/sum_decimal.rs +++ b/native/spark-expr/src/agg_funcs/sum_decimal.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use crate::utils::Decimal128Bounds; use crate::{decimal_sum_overflow_error, EvalMode, SparkError, SparkErrorWithContext}; use arrow::array::{ cast::AsArray, @@ -366,7 +367,8 @@ struct SumDecimalGroupsAccumulator { sum: Vec>, is_empty: Vec, result_type: DataType, - precision: u8, + /// The values a sum of the result precision holds + bounds: Decimal128Bounds, eval_mode: EvalMode, expr_id: Option, registry: Arc, @@ -384,7 +386,7 @@ impl SumDecimalGroupsAccumulator { sum: Vec::new(), is_empty: Vec::new(), result_type, - precision, + bounds: Decimal128Bounds::new(precision), eval_mode, expr_id, registry, @@ -418,7 +420,7 @@ impl SumDecimalGroupsAccumulator { let running_sum = self.sum[group_index].unwrap_or(0); let (new_sum, is_overflow) = running_sum.overflowing_add(value); - if is_overflow || !Decimal128Type::is_valid_decimal_precision(new_sum, self.precision) { + if is_overflow || !self.bounds.contains(new_sum) { if self.eval_mode == EvalMode::Ansi { let error = decimal_sum_overflow_error("sum"); return Err(self.wrap_error_with_context(error)); @@ -478,14 +480,7 @@ impl GroupsAccumulator for SumDecimalGroupsAccumulator { None } else { match sum { - Some(v) - if Decimal128Type::is_valid_decimal_precision( - v, - self.precision, - ) => - { - Some(v) - } + Some(v) if self.bounds.contains(v) => Some(v), _ => None, } } @@ -507,14 +502,7 @@ impl GroupsAccumulator for SumDecimalGroupsAccumulator { None } else { match sum { - Some(v) - if Decimal128Type::is_valid_decimal_precision( - v, - self.precision, - ) => - { - Some(v) - } + Some(v) if self.bounds.contains(v) => Some(v), _ => None, } } @@ -591,7 +579,7 @@ impl GroupsAccumulator for SumDecimalGroupsAccumulator { let right = that_sum_val.unwrap(); let (new_sum, is_overflow) = left.overflowing_add(right); - if is_overflow || !Decimal128Type::is_valid_decimal_precision(new_sum, self.precision) { + if is_overflow || !self.bounds.contains(new_sum) { if self.eval_mode == EvalMode::Ansi { let error = decimal_sum_overflow_error("sum"); return Err(self.wrap_error_with_context(error)); @@ -778,6 +766,47 @@ mod tests { assert_eq!(result.value(0), 40); // 10 + 30 = 40 } + #[test] + fn grouped_latches_at_the_result_precision_below_38() { + use datafusion::logical_expr::{EmitTo, GroupsAccumulator}; + + // Inputs within their declared precision need more than 10^10 rows to reach a sum + // precision below 38, so only a unit test can check that the latch uses the result + // precision there. + let data_type = DataType::Decimal128(25, 2); + let max = 10_i128.pow(25) - 1; + let decimals = |values: Vec| -> ArrayRef { + Arc::new(Decimal128Array::from(values).with_data_type(data_type.clone())) + }; + for eval_mode in [EvalMode::Legacy, EvalMode::Ansi] { + let new_acc = || { + SumDecimalGroupsAccumulator::new( + data_type.clone(), + 25, + eval_mode, + None, + crate::create_query_context_map(), + ) + }; + // Group 0 reaches the largest value that fits and group 1 goes one past it. + let mut updated = new_acc(); + let update = updated.update_batch(&[decimals(vec![max, max, 1])], &[0, 1, 1], None, 2); + let mut merged = new_acc(); + let is_empty: ArrayRef = Arc::new(BooleanArray::from(vec![false; 3])); + let merge = merged.merge_batch(&[decimals(vec![max, max, 1]), is_empty], &[0, 1, 1], 2); + for (result, mut acc) in [(update, updated), (merge, merged)] { + if eval_mode == EvalMode::Ansi { + assert!(result.is_err()); + } else { + result.unwrap(); + let sums = acc.evaluate(EmitTo::All).unwrap(); + let sums = sums.as_primitive::(); + assert_eq!(sums.iter().collect::>(), vec![Some(max), None]); + } + } + } + } + /// `merge_batch` must consume every row — previously it asserted /// `states[*].len() == 1` and silently dropped the rest. #[test] diff --git a/native/spark-expr/src/utils.rs b/native/spark-expr/src/utils.rs index e393de172cf..adad7363796 100644 --- a/native/spark-expr/src/utils.rs +++ b/native/spark-expr/src/utils.rs @@ -15,7 +15,9 @@ // specific language governing permissions and limitations // under the License. -use arrow::datatypes::{DataType, TimeUnit}; +use arrow::datatypes::{ + DataType, TimeUnit, MAX_DECIMAL128_FOR_EACH_PRECISION, MIN_DECIMAL128_FOR_EACH_PRECISION, +}; use arrow::{ array::{ cast::as_primitive_array, @@ -322,6 +324,34 @@ fn pre_timestamp_cast(array: ArrayRef, timezone: String) -> Result. +#[derive(Debug, Clone, Copy)] +pub(crate) struct Decimal128Bounds { + min: i128, + max: i128, +} + +impl Decimal128Bounds { + pub(crate) fn new(precision: u8) -> Self { + match ( + MIN_DECIMAL128_FOR_EACH_PRECISION.get(precision as usize), + MAX_DECIMAL128_FOR_EACH_PRECISION.get(precision as usize), + ) { + (Some(&min), Some(&max)) => Self { min, max }, + // No value fits a precision past Decimal128's maximum. + _ => Self { min: 1, max: 0 }, + } + } + + #[inline] + pub(crate) fn contains(self, value: i128) -> bool { + self.min <= value && value <= self.max + } +} + /// Build a boolean buffer from the state and reset the state, based on the emit_to /// strategy. pub fn build_bool_state(state: &mut BooleanBufferBuilder, emit_to: &EmitTo) -> BooleanBuffer { @@ -488,4 +518,24 @@ mod tests { .timestamp_micros(); assert_eq!(out.value(0), expected); } + + #[test] + fn decimal128_bounds_match_arrow() { + use arrow::datatypes::{Decimal128Type, DecimalType}; + for precision in [0_u8, 1, 18, 25, 38, 39] { + let bounds = Decimal128Bounds::new(precision); + let mut values = vec![0, 1, -1, i128::MAX, i128::MIN]; + // 10^39 does not fit an i128, so precision 39 only has the values above. + if let Some(edge) = 10_i128.checked_pow(precision as u32) { + values.extend([edge - 1, edge, -(edge - 1), -edge]); + } + for value in values { + assert_eq!( + bounds.contains(value), + Decimal128Type::is_valid_decimal_precision(value, precision), + "precision {precision}, value {value}" + ); + } + } + } }