From 83514229e195880569cc5d9d07f51e1d096bf594 Mon Sep 17 00:00:00 2001 From: comphead Date: Mon, 16 Nov 2026 13:41:31 -0800 Subject: [PATCH 1/2] perf: scan the values buffer in `CheckOverflow` when there are no nulls (1.4x faster) The no-overflow check iterated `Option`s with `.iter().flatten().all()`, reading the null bitmap for every slot and branching per value. Without nulls every slot holds a value, so scan the values buffer directly, with one unsigned comparison per value against the precision bounds shifted by the minimum, and no branch inside each 64-value chunk. Arrays with nulls keep the existing scan. Add dense-null and ANSI-with-nulls shapes to the benchmark, and a test for an overflow past the first chunk and a sliced array. --- .../expression-audits/math_funcs.md | 1 + native/spark-expr/benches/check_overflow.rs | 13 ++++ .../src/math_funcs/internal/checkoverflow.rs | 69 ++++++++++++++++--- 3 files changed, 73 insertions(+), 10 deletions(-) diff --git a/docs/source/contributor-guide/expression-audits/math_funcs.md b/docs/source/contributor-guide/expression-audits/math_funcs.md index bba19f5cbe9..674e853e067 100644 --- a/docs/source/contributor-guide/expression-audits/math_funcs.md +++ b/docs/source/contributor-guide/expression-audits/math_funcs.md @@ -46,6 +46,7 @@ Internal decimal wrapper emitted around every decimal `+ - * /`, `sum`, and `avg` result to null out (non-ANSI) or raise on (ANSI) values that exceed the declared precision. Native impl: `math_funcs/internal/checkoverflow.rs`. - Performance (tuned 2026-07-15, PR [#4937](https://github.com/apache/datafusion-comet/pull/4937)): both the ANSI and non-ANSI paths share a no-overflow fast path built on `is_valid_decimal_precision`, a small inlined bounds check scanned with `all` (short-circuits at the first overflow). When nothing overflows (the common shape) the input buffers are reused via `to_data()` (cheap Arc metadata clone) instead of allocating through `null_if_overflow_precision` (non-ANSI) or running the heavier per-value `validate_decimal_precision` (ANSI). The ANSI path only falls back to `validate_decimal_precision` when an overflow is present, to build the precise Spark error. ~10% faster on the no-overflow shape, ~17% with nulls, and ~69% faster for ANSI no-overflow (down to parity with non-ANSI); overflow shapes unchanged. Benchmark: `benches/check_overflow.rs`. +- Performance (tuned 2026-11-16): without nulls the no-overflow scan reads the values buffer directly, with one unsigned comparison per value against the shifted precision bounds and no branch inside each 64-value chunk, instead of iterating `Option`s through the null bitmap. ~30% faster on the no-null, no-overflow shape in both ANSI and non-ANSI. Shapes with nulls or overflows unchanged. Benchmark: `benches/check_overflow.rs`, which gains dense-null and ANSI-with-nulls shapes. ## abs diff --git a/native/spark-expr/benches/check_overflow.rs b/native/spark-expr/benches/check_overflow.rs index 1bdda3cc46e..9f0f72a6243 100644 --- a/native/spark-expr/benches/check_overflow.rs +++ b/native/spark-expr/benches/check_overflow.rs @@ -67,6 +67,7 @@ fn criterion_benchmark(c: &mut Criterion) { // The common TPC-DS shape: decimal arithmetic result that fits the declared precision. let no_overflow = build(size, 0, 0); let no_overflow_nulls = build(size, 17, 0); + let no_overflow_dense_nulls = build(size, 2, 0); let sparse_overflow = build(size, 0, 17); let dense_overflow = build(size, 0, 2); @@ -76,6 +77,15 @@ fn criterion_benchmark(c: &mut Criterion) { c.bench_function("check_overflow: no overflow, with nulls", |b| { b.iter(|| black_box(legacy.evaluate(black_box(&no_overflow_nulls)).unwrap())) }); + c.bench_function("check_overflow: no overflow, dense nulls", |b| { + b.iter(|| { + black_box( + legacy + .evaluate(black_box(&no_overflow_dense_nulls)) + .unwrap(), + ) + }) + }); c.bench_function("check_overflow: sparse overflow", |b| { b.iter(|| black_box(legacy.evaluate(black_box(&sparse_overflow)).unwrap())) }); @@ -85,6 +95,9 @@ fn criterion_benchmark(c: &mut Criterion) { c.bench_function("check_overflow: ansi no overflow", |b| { b.iter(|| black_box(ansi.evaluate(black_box(&no_overflow)).unwrap())) }); + c.bench_function("check_overflow: ansi no overflow, with nulls", |b| { + b.iter(|| black_box(ansi.evaluate(black_box(&no_overflow_nulls)).unwrap())) + }); } criterion_group!(benches, criterion_benchmark); diff --git a/native/spark-expr/src/math_funcs/internal/checkoverflow.rs b/native/spark-expr/src/math_funcs/internal/checkoverflow.rs index 60e44c5f219..124319cac68 100644 --- a/native/spark-expr/src/math_funcs/internal/checkoverflow.rs +++ b/native/spark-expr/src/math_funcs/internal/checkoverflow.rs @@ -18,7 +18,10 @@ use arrow::datatypes::{DataType, Schema}; use arrow::{ array::{as_primitive_array, Array, ArrayRef, Decimal128Array}, - datatypes::{Decimal128Type, DecimalType}, + datatypes::{ + Decimal128Type, DecimalType, MAX_DECIMAL128_FOR_EACH_PRECISION, + MIN_DECIMAL128_FOR_EACH_PRECISION, + }, record_batch::RecordBatch, }; use datafusion::common::{DataFusionError, ScalarValue}; @@ -119,16 +122,12 @@ impl PhysicalExpr for CheckOverflow { let decimal_array = as_primitive_array::(&array); - // Fast path shared by both ANSI and non-ANSI: `is_valid_decimal_precision` is a - // small, inlined bounds check and `all` short-circuits at the first overflow. When - // nothing overflows (the common shape for decimal arithmetic in TPC-DS) we reuse the - // input buffers via `to_data()`, which only clones cheap Arc metadata. This avoids - // the heavier per-value `validate_decimal_precision` scan (ANSI) or the allocating + // Fast path shared by both ANSI and non-ANSI. When nothing overflows (the common + // shape for decimal arithmetic in TPC-DS) we reuse the input buffers via + // `to_data()`, which only clones cheap Arc metadata. This avoids the heavier + // per-value `validate_decimal_precision` scan (ANSI) or the allocating // `null_if_overflow_precision` (non-ANSI) below. - let no_overflow = decimal_array - .iter() - .flatten() - .all(|v| Decimal128Type::is_valid_decimal_precision(v, *precision)); + let no_overflow = fits_precision(decimal_array, *precision); let casted_array = if no_overflow { Decimal128Array::from(decimal_array.to_data()) @@ -249,6 +248,38 @@ impl PhysicalExpr for CheckOverflow { } } +/// Values per chunk in the no-null scan of [`fits_precision`]. +const FITS_PRECISION_CHUNK: usize = 64; + +/// Whether every non-null value of `array` fits `precision`, the check that +/// `Decimal128Type::is_valid_decimal_precision` makes for one value. +#[inline] +fn fits_precision(array: &Decimal128Array, precision: u8) -> bool { + let bounds = ( + MIN_DECIMAL128_FOR_EACH_PRECISION.get(precision as usize), + MAX_DECIMAL128_FOR_EACH_PRECISION.get(precision as usize), + ); + match bounds { + (Some(&min), Some(&max)) if array.null_count() == 0 => { + // Without nulls every slot holds a value, so scan the values buffer directly instead + // of reading the null bitmap per slot. Shifting by `min` turns the two bounds checks + // into one unsigned comparison. The check is branchless within a chunk, which lets + // the compiler unroll it, and the scan still stops at the first chunk that holds an + // overflow. + let range = max.wrapping_sub(min) as u128; + array.values().chunks(FITS_PRECISION_CHUNK).all(|chunk| { + chunk.iter().fold(true, |fits, &v| { + fits & (v.wrapping_sub(min) as u128 <= range) + }) + }) + } + _ => array + .iter() + .flatten() + .all(|v| Decimal128Type::is_valid_decimal_precision(v, precision)), + } +} + #[cfg(test)] mod tests { use super::*; @@ -509,4 +540,22 @@ mod tests { let out = eval_array(&array_check_overflow(3, 0, false), &batch); assert_eq!(out.iter().collect::>(), vec![Some(999), None]); } + + #[test] + fn test_array_no_nulls_overflow_past_first_chunk_and_sliced() { + // The no-null scan reads the values buffer in chunks, so an overflow in a later chunk + // must still be found, and a sliced array must only be checked within its slice. + let mut values = vec![Some(5); 200]; + values[130] = Some(1000); + let batch = array_batch(values, 38, 0); + let out = eval_array(&array_check_overflow(3, 0, false), &batch); + assert_eq!(out.null_count(), 1); + assert!(out.is_null(130)); + + // Starts right after the overflowing value, so the slice has a non-zero offset. + let sliced = batch.slice(131, 69); + let out = eval_array(&array_check_overflow(3, 0, true), &sliced); + assert_eq!(out.len(), 69); + assert_eq!(out.null_count(), 0); + } } From 5f164b99ba86d5a9685ed9e9219e4995605a36b4 Mon Sep 17 00:00:00 2001 From: comphead Date: Wed, 18 Nov 2026 11:57:33 -0800 Subject: [PATCH 2/2] perf: scan null slots too in `CheckOverflow`, and read a validity bit only where needed The scan now covers every slot of the values buffer, null or not, so arrays with nulls take it as well. A null slot can hold any value, so a value that doesn't fit is looked up in the null bitmap, and the scan resumes after it if its slot is null. Re-running the scan that skips nulls instead made a null slot that doesn't fit cost both passes. A plain loop replaces the chunked fold. On aarch64 the fold compiled to a vectorized reduction with stack spills, and the plain loop was as fast or faster on every shape. The bench gains a last-row overflow and a null slot that doesn't fit. The tests move a value that doesn't fit through every row, cover precision 38's bounds where the shift wraps, and cover null slots that don't fit, including in a slice. --- .../expression-audits/math_funcs.md | 2 +- native/spark-expr/benches/check_overflow.rs | 34 +++++ .../src/math_funcs/internal/checkoverflow.rs | 140 ++++++++++++++---- 3 files changed, 146 insertions(+), 30 deletions(-) diff --git a/docs/source/contributor-guide/expression-audits/math_funcs.md b/docs/source/contributor-guide/expression-audits/math_funcs.md index 674e853e067..71cb88b2e4f 100644 --- a/docs/source/contributor-guide/expression-audits/math_funcs.md +++ b/docs/source/contributor-guide/expression-audits/math_funcs.md @@ -46,7 +46,7 @@ Internal decimal wrapper emitted around every decimal `+ - * /`, `sum`, and `avg` result to null out (non-ANSI) or raise on (ANSI) values that exceed the declared precision. Native impl: `math_funcs/internal/checkoverflow.rs`. - Performance (tuned 2026-07-15, PR [#4937](https://github.com/apache/datafusion-comet/pull/4937)): both the ANSI and non-ANSI paths share a no-overflow fast path built on `is_valid_decimal_precision`, a small inlined bounds check scanned with `all` (short-circuits at the first overflow). When nothing overflows (the common shape) the input buffers are reused via `to_data()` (cheap Arc metadata clone) instead of allocating through `null_if_overflow_precision` (non-ANSI) or running the heavier per-value `validate_decimal_precision` (ANSI). The ANSI path only falls back to `validate_decimal_precision` when an overflow is present, to build the precise Spark error. ~10% faster on the no-overflow shape, ~17% with nulls, and ~69% faster for ANSI no-overflow (down to parity with non-ANSI); overflow shapes unchanged. Benchmark: `benches/check_overflow.rs`. -- Performance (tuned 2026-11-16): without nulls the no-overflow scan reads the values buffer directly, with one unsigned comparison per value against the shifted precision bounds and no branch inside each 64-value chunk, instead of iterating `Option`s through the null bitmap. ~30% faster on the no-null, no-overflow shape in both ANSI and non-ANSI. Shapes with nulls or overflows unchanged. Benchmark: `benches/check_overflow.rs`, which gains dense-null and ANSI-with-nulls shapes. +- Performance (tuned 2026-11-16, PR [#7658](https://github.com/apache/datafusion-comet/pull/7658)): the no-overflow scan reads every slot of the values buffer, null or not, with one unsigned comparison per value against the shifted precision bounds, instead of iterating `Option`s through the null bitmap. A null slot can hold any value, so the scan reads a slot's validity bit only at a value that doesn't fit and resumes after a null one. Benchmark: `benches/check_overflow.rs`, which gains dense-null, ANSI-with-nulls, last-row-overflow and out-of-range-null-slot shapes. ## abs diff --git a/native/spark-expr/benches/check_overflow.rs b/native/spark-expr/benches/check_overflow.rs index 9f0f72a6243..7738f1d1596 100644 --- a/native/spark-expr/benches/check_overflow.rs +++ b/native/spark-expr/benches/check_overflow.rs @@ -16,6 +16,8 @@ // under the License. use arrow::array::{Array, Decimal128Array}; +use arrow::buffer::NullBuffer; +use arrow::compute::concat_batches; use arrow::datatypes::{DataType, Field, Schema}; use arrow::record_batch::RecordBatch; use criterion::{criterion_group, criterion_main, Criterion}; @@ -98,6 +100,38 @@ fn criterion_benchmark(c: &mut Criterion) { c.bench_function("check_overflow: ansi no overflow, with nulls", |b| { b.iter(|| black_box(ansi.evaluate(black_box(&no_overflow_nulls)).unwrap())) }); + + // One overflow in the last row, so the scan reads the whole batch before the slow path runs. + // `sparse overflow` and `dense overflow` overflow at row 0 and stop in the first chunk. + let head = build(size - 1, 0, 0); + let last_row_overflow = concat_batches(&head.schema(), [&head, &build(1, 0, 1)]).unwrap(); + c.bench_function("check_overflow: last row overflow", |b| { + b.iter(|| black_box(legacy.evaluate(black_box(&last_row_overflow)).unwrap())) + }); + + // The `with nulls` shape, except that its last row is a null slot holding a value that + // doesn't fit. A scan over every slot reaches it at the end of the batch and has to find + // that the slot is null, though nothing overflows. + let null_slot_overflow = { + let nulls = build(size, 17, 0); + let nulls = nulls + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let mut values = nulls.values().to_vec(); + values[size - 1] = OVERFLOW_VALUE; + let mut validity: Vec = (0..size).map(|i| nulls.is_valid(i)).collect(); + validity[size - 1] = false; + let array = Decimal128Array::new(values.into(), Some(NullBuffer::from(validity))) + .with_precision_and_scale(IN_PRECISION, SCALE) + .unwrap(); + let schema = Schema::new(vec![Field::new("d", array.data_type().clone(), true)]); + RecordBatch::try_new(Arc::new(schema), vec![Arc::new(array)]).unwrap() + }; + c.bench_function("check_overflow: no overflow, null slot out of range", |b| { + b.iter(|| black_box(legacy.evaluate(black_box(&null_slot_overflow)).unwrap())) + }); } criterion_group!(benches, criterion_benchmark); diff --git a/native/spark-expr/src/math_funcs/internal/checkoverflow.rs b/native/spark-expr/src/math_funcs/internal/checkoverflow.rs index 124319cac68..cfc7c61c3b5 100644 --- a/native/spark-expr/src/math_funcs/internal/checkoverflow.rs +++ b/native/spark-expr/src/math_funcs/internal/checkoverflow.rs @@ -248,9 +248,6 @@ impl PhysicalExpr for CheckOverflow { } } -/// Values per chunk in the no-null scan of [`fits_precision`]. -const FITS_PRECISION_CHUNK: usize = 64; - /// Whether every non-null value of `array` fits `precision`, the check that /// `Decimal128Type::is_valid_decimal_precision` makes for one value. #[inline] @@ -259,30 +256,38 @@ fn fits_precision(array: &Decimal128Array, precision: u8) -> bool { MIN_DECIMAL128_FOR_EACH_PRECISION.get(precision as usize), MAX_DECIMAL128_FOR_EACH_PRECISION.get(precision as usize), ); - match bounds { - (Some(&min), Some(&max)) if array.null_count() == 0 => { - // Without nulls every slot holds a value, so scan the values buffer directly instead - // of reading the null bitmap per slot. Shifting by `min` turns the two bounds checks - // into one unsigned comparison. The check is branchless within a chunk, which lets - // the compiler unroll it, and the scan still stops at the first chunk that holds an - // overflow. - let range = max.wrapping_sub(min) as u128; - array.values().chunks(FITS_PRECISION_CHUNK).all(|chunk| { - chunk.iter().fold(true, |fits, &v| { - fits & (v.wrapping_sub(min) as u128 <= range) - }) - }) - } - _ => array + let (Some(&min), Some(&max)) = bounds else { + return array .iter() .flatten() - .all(|v| Decimal128Type::is_valid_decimal_precision(v, precision)), + .all(|v| Decimal128Type::is_valid_decimal_precision(v, precision)); + }; + // Scan every slot of the values buffer, null or not, instead of reading the null bitmap per + // slot. Shifting by `min` turns the two bounds checks into one unsigned comparison, so the + // loop is one compare per value and a branch that is only taken at a value that doesn't fit. + let range = max.wrapping_sub(min) as u128; + let values = array.values(); + let nulls = array.nulls(); + let mut start = 0; + while let Some(offset) = values[start..] + .iter() + .position(|&v| v.wrapping_sub(min) as u128 > range) + { + // A null slot can hold any value, so a value that doesn't fit only counts in a valid + // slot. The null bitmap is read only at such a value, and the scan resumes after it. + let index = start + offset; + if !nulls.is_some_and(|nulls| nulls.is_null(index)) { + return false; + } + start = index + 1; } + true } #[cfg(test)] mod tests { use super::*; + use arrow::buffer::NullBuffer; use arrow::datatypes::{Field, Schema}; use arrow::record_batch::RecordBatch; use std::fmt::{Display, Formatter}; @@ -539,23 +544,100 @@ mod tests { let batch = array_batch(vec![Some(999), Some(9999)], 38, 0); let out = eval_array(&array_check_overflow(3, 0, false), &batch); assert_eq!(out.iter().collect::>(), vec![Some(999), None]); + + // The scan shifts by the precision's minimum and compares unsigned, so also check the + // values where that shift wraps, at the widest precision. + let max = MAX_DECIMAL128_FOR_EACH_PRECISION[38]; + let min = MIN_DECIMAL128_FOR_EACH_PRECISION[38]; + let fits = array_batch(vec![Some(max), Some(min), Some(0)], 38, 0); + let out = eval_array(&array_check_overflow(38, 0, true), &fits); + assert_eq!(out.null_count(), 0); + for v in [max + 1, min - 1, i128::MAX, i128::MIN] { + let batch = array_batch(vec![Some(0), Some(v)], 38, 0); + let out = eval_array(&array_check_overflow(38, 0, false), &batch); + assert!(out.is_null(1), "{v}"); + let ansi = array_check_overflow(38, 0, true).evaluate(&batch); + assert!(ansi.is_err(), "{v}"); + } } #[test] - fn test_array_no_nulls_overflow_past_first_chunk_and_sliced() { - // The no-null scan reads the values buffer in chunks, so an overflow in a later chunk - // must still be found, and a sliced array must only be checked within its slice. + fn test_array_overflow_at_every_position_and_sliced() { + // The scan stops at the first value that doesn't fit, and a compiled loop can split the + // batch into an unrolled part and a remainder, so move one value that doesn't fit + // through every row. 203 rows are not a multiple of any unroll factor. + for overflow in [1000, -1000] { + for i in 0..203 { + let mut values = vec![Some(5); 203]; + values[i] = Some(overflow); + let batch = array_batch(values, 38, 0); + let out = eval_array(&array_check_overflow(3, 0, false), &batch); + assert_eq!(out.null_count(), 1, "{overflow} at {i}"); + assert!(out.is_null(i), "{overflow} at {i}"); + let ansi = array_check_overflow(3, 0, true).evaluate(&batch); + assert!(ansi.is_err(), "{overflow} at {i}"); + } + } + + // A sliced array must only be checked within its slice. This one starts right after + // the overflowing value, so the slice has a non-zero offset. let mut values = vec![Some(5); 200]; values[130] = Some(1000); - let batch = array_batch(values, 38, 0); - let out = eval_array(&array_check_overflow(3, 0, false), &batch); - assert_eq!(out.null_count(), 1); - assert!(out.is_null(130)); - - // Starts right after the overflowing value, so the slice has a non-zero offset. - let sliced = batch.slice(131, 69); + let sliced = array_batch(values, 38, 0).slice(131, 69); let out = eval_array(&array_check_overflow(3, 0, true), &sliced); assert_eq!(out.len(), 69); assert_eq!(out.null_count(), 0); } + + /// A Decimal128(38, 0) batch whose null slots keep the values given for them. + fn array_batch_with_null_slots(values: Vec, valid: Vec) -> RecordBatch { + let arr = Decimal128Array::new(values.into(), Some(NullBuffer::from(valid))) + .with_precision_and_scale(38, 0) + .unwrap(); + let schema = Schema::new(vec![Field::new("d", arr.data_type().clone(), true)]); + RecordBatch::try_new(Arc::new(schema), vec![Arc::new(arr)]).unwrap() + } + + #[test] + fn test_array_null_slot_out_of_range_is_ignored() { + // A null slot can hold any value. One that doesn't fit the precision must not null out + // valid values or raise an ANSI error, and the scan must still find a value that doesn't + // fit in a valid slot after it, next to it or at the end. + let n = 70; + for i in 0..n - 1 { + let mut values = vec![5_i128; n]; + let mut valid = vec![true; n]; + values[i] = 1000; + valid[i] = false; + let mut expected = vec![Some(5_i128); n]; + expected[i] = None; + let batch = array_batch_with_null_slots(values.clone(), valid.clone()); + for fail_on_error in [false, true] { + let out = eval_array(&array_check_overflow(3, 0, fail_on_error), &batch); + assert_eq!(out.iter().collect::>(), expected, "null slot {i}"); + } + for j in [i + 1, n - 1] { + let mut values = values.clone(); + values[j] = -1000; + let batch = array_batch_with_null_slots(values, valid.clone()); + let out = eval_array(&array_check_overflow(3, 0, false), &batch); + let mut expected = expected.clone(); + expected[j] = None; + assert_eq!(out.iter().collect::>(), expected, "{i}, {j}"); + let ansi = array_check_overflow(3, 0, true).evaluate(&batch); + assert!(ansi.is_err(), "null slot {i}, overflow at {j}"); + } + } + + // In a slice, the values and the null bitmap both count from the slice's first row. + let mut values = vec![5_i128; 100]; + let mut valid = vec![true; 100]; + values[40] = 1000; + values[60] = 1000; + valid[60] = false; + let sliced = array_batch_with_null_slots(values, valid).slice(41, 59); + let out = eval_array(&array_check_overflow(3, 0, true), &sliced); + assert_eq!(out.null_count(), 1); + assert!(out.is_null(19)); + } }