diff --git a/native/shuffle/Cargo.toml b/native/shuffle/Cargo.toml index 03ad0a91ff2..00a8d79eb82 100644 --- a/native/shuffle/Cargo.toml +++ b/native/shuffle/Cargo.toml @@ -83,3 +83,7 @@ harness = false [[bench]] name = "shuffle_reader" harness = false + +[[bench]] +name = "pmod" +harness = false diff --git a/native/shuffle/benches/pmod.rs b/native/shuffle/benches/pmod.rs new file mode 100644 index 00000000000..563b1229393 --- /dev/null +++ b/native/shuffle/benches/pmod.rs @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Benchmarks the mapping from row hash to shuffle partition. + +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; +use datafusion_comet_shuffle::bench_support::hash_partition_ids; +use std::hint::black_box; + +const NUM_ROWS: usize = 8192; + +/// Pseudo-random hashes from a fixed-seed xorshift, so runs are comparable. +fn random_hashes() -> Vec { + let mut x: u32 = 0x9e37_79b9; + (0..NUM_ROWS) + .map(|_| { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + x + }) + .collect() +} + +fn criterion_benchmark(c: &mut Criterion) { + let random = random_hashes(); + // Clearing the sign bit makes every hash non-negative as an i32, so the sign is predictable. + let non_negative: Vec = random.iter().map(|h| h & 0x7fff_ffff).collect(); + let mut partition_ids = vec![0u32; NUM_ROWS]; + + let mut group = c.benchmark_group("pmod"); + for num_partitions in [16, 200] { + for (shape, hashes) in [("random", &random), ("non_negative", &non_negative)] { + group.bench_with_input( + BenchmarkId::new(shape, num_partitions), + hashes, + |b, hashes| { + b.iter(|| { + hash_partition_ids( + black_box(hashes), + black_box(num_partitions), + &mut partition_ids, + ); + black_box(&partition_ids); + }) + }, + ); + } + } + group.finish(); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/native/shuffle/src/bench_support.rs b/native/shuffle/src/bench_support.rs index a04493adc05..ccd51f7b2a3 100644 --- a/native/shuffle/src/bench_support.rs +++ b/native/shuffle/src/bench_support.rs @@ -26,6 +26,7 @@ //! //! Not an API, and not used outside `benches/`. +pub use crate::comet_partitioning::hash_partition_ids; use crate::metrics::ShufflePartitionerMetrics; use crate::partitioners::{MultiPartitionShuffleRepartitioner, ShufflePartitioner}; use crate::writers::PartitionWriter; diff --git a/native/shuffle/src/comet_partitioning.rs b/native/shuffle/src/comet_partitioning.rs index 5b15f2e49d6..521c102a0a2 100644 --- a/native/shuffle/src/comet_partitioning.rs +++ b/native/shuffle/src/comet_partitioning.rs @@ -121,11 +121,14 @@ impl CometPartitioning { } pub(crate) fn pmod(hash: u32, n: usize) -> usize { - let hash = hash as i32; - let n = n as i32; - let r = hash % n; - let result = if r < 0 { (r + n) % n } else { r }; - result as usize + (hash as i32).rem_euclid(n as i32) as usize +} + +/// Maps each hash to its partition the way hash partitioning does. +pub fn hash_partition_ids(hashes: &[u32], num_partitions: usize, partition_ids: &mut [u32]) { + for (id, &hash) in partition_ids.iter_mut().zip(hashes) { + *id = pmod(hash, num_partitions) as u32; + } } #[cfg(test)] @@ -142,6 +145,28 @@ mod tests { assert_eq!(result, expected); } + /// Spark's `pmod` on the hash as an `i32`, which for a positive `n` is the Euclidean remainder. + #[test] + fn test_pmod_edge_cases() { + let hashes = [ + 0, + 1, + 199, + 200, + 201, + i32::MAX as u32, + 0x8000_0000, + 0x8000_0001, + u32::MAX, + ]; + for n in [1, 2, 13, 16, 200, i32::MAX as usize] { + for hash in hashes { + let expected = (hash as i32 as i64).rem_euclid(n as i64) as usize; + assert_eq!(pmod(hash, n), expected, "hash {hash:#x}, n {n}"); + } + } + } + /// A group that a previous batch left part-way through is finished by the next batch, rather /// than restarting at a group boundary. #[test] diff --git a/native/shuffle/src/partitioners/multi_partition.rs b/native/shuffle/src/partitioners/multi_partition.rs index f217165d38f..a7d51e8f029 100644 --- a/native/shuffle/src/partitioners/multi_partition.rs +++ b/native/shuffle/src/partitioners/multi_partition.rs @@ -15,14 +15,14 @@ // specific language governing permissions and limitations // under the License. -use crate::comet_partitioning::positional_runs; +use crate::comet_partitioning::{hash_partition_ids, positional_runs}; use crate::metrics::ShufflePartitionerMetrics; use crate::partitioners::partitioned_batch_iterator::{ BufferedRun, PartitionIndices, PartitionedBatchesProducer, }; use crate::partitioners::ShufflePartitioner; use crate::writers::PartitionWriter; -use crate::{comet_partitioning, CometPartitioning, RoundRobinStrategy}; +use crate::{CometPartitioning, RoundRobinStrategy}; use arrow::array::{Array, ArrayData, RecordBatch}; use datafusion::common::utils::proxy::VecAllocExt; use datafusion::common::{DataFusionError, HashSet}; @@ -299,13 +299,8 @@ impl MultiPartitionShuffleRepartitioner { { // Hash arrays and compute partition ids based on number of partitions. let partition_ids = &mut scratch.partition_ids[..num_rows]; - create_murmur3_hashes(&arrays, hashes_buf)? - .iter() - .enumerate() - .for_each(|(idx, hash)| { - partition_ids[idx] = - comet_partitioning::pmod(*hash, *num_output_partitions) as u32; - }); + let hashes = create_murmur3_hashes(&arrays, hashes_buf)?; + hash_partition_ids(hashes, *num_output_partitions, partition_ids); } // We now have partition ids for every input row, map that to partition starts @@ -435,10 +430,7 @@ impl MultiPartitionShuffleRepartitioner { // Assign partition IDs based on hash (same as hash partitioning) let partition_ids = &mut scratch.partition_ids[..num_rows]; - hashes_buf.iter().enumerate().for_each(|(idx, hash)| { - partition_ids[idx] = - comet_partitioning::pmod(*hash, *num_output_partitions) as u32; - }); + hash_partition_ids(hashes_buf, *num_output_partitions, partition_ids); // We now have partition ids for every input row, map that to partition // starts and partition indices to eventually write these rows to diff --git a/native/spark-expr/benches/wide_decimal.rs b/native/spark-expr/benches/wide_decimal.rs index 6de9fb5e83b..13808658a13 100644 --- a/native/spark-expr/benches/wide_decimal.rs +++ b/native/spark-expr/benches/wide_decimal.rs @@ -35,11 +35,22 @@ const BATCH_SIZE: usize = 8192; /// Build a RecordBatch with two Decimal128 columns. fn make_decimal_batch(p1: u8, s1: i8, p2: u8, s2: i8) -> RecordBatch { + make_decimal_batch_from(p1, s1, p2, s2, 123456789012345, 987654321098765) +} + +fn make_decimal_batch_from( + p1: u8, + s1: i8, + p2: u8, + s2: i8, + left_start: i128, + right_start: i128, +) -> RecordBatch { let mut left = Decimal128Builder::new(); let mut right = Decimal128Builder::new(); for i in 0..BATCH_SIZE as i128 { - left.append_value(123456789012345_i128 + i * 1000); - right.append_value(987654321098765_i128 - i * 1000); + left.append_value(left_start + i * 1000); + right.append_value(right_start - i * 1000); } let left = left.finish().with_data_type(DataType::Decimal128(p1, s1)); let right = right.finish().with_data_type(DataType::Decimal128(p2, s2)); @@ -169,6 +180,28 @@ fn criterion_benchmark(c: &mut Criterion) { bench_case(&mut group, "multiply", &batch, &old, &new); } + // Case 3b: Multiply with 48-digit products, which make the i256 division longer + { + let batch = make_decimal_batch_from( + 38, + 10, + 38, + 10, + 123456789012345678901234, + 987654321098765432109876, + ); + let old = build_old_expr( + 38, + 10, + 38, + 10, + Operator::Multiply, + DataType::Decimal128(38, 6), + ); + let new = build_new_expr(WideDecimalOp::Multiply, 38, 6, EvalMode::Legacy); + bench_case(&mut group, "multiply_48_digit_products", &batch, &old, &new); + } + // Case 4: Subtract with same scale - Decimal128(38,18) - Decimal128(38,18) -> Decimal128(38,18) { let batch = make_decimal_batch(38, 18, 38, 18); diff --git a/native/spark-expr/src/math_funcs/wide_decimal_binary_expr.rs b/native/spark-expr/src/math_funcs/wide_decimal_binary_expr.rs index a501a012784..6a709e4cf6d 100644 --- a/native/spark-expr/src/math_funcs/wide_decimal_binary_expr.rs +++ b/native/spark-expr/src/math_funcs/wide_decimal_binary_expr.rs @@ -121,7 +121,9 @@ impl Display for WideDecimalBinaryExpr { /// Compute `value / divisor` with HALF_UP rounding. #[inline] fn div_round_half_up(value: i256, divisor: i256) -> i256 { - let (quot, rem) = (value / divisor, value % divisor); + let quot = value / divisor; + // `quot` truncates toward zero, so `|quot * divisor| <= |value|` and neither step overflows. + let rem = value.wrapping_sub(quot.wrapping_mul(divisor)); // HALF_UP: if |remainder| * 2 >= |divisor|, round away from zero let abs_rem_x2 = if rem < i256::ZERO { rem.wrapping_neg() @@ -492,6 +494,55 @@ mod tests { } } + /// HALF_UP division computed from `/` and `%` separately. + fn div_round_half_up_reference(value: i256, divisor: i256) -> i256 { + let (quot, rem) = (value / divisor, value % divisor); + let twice_rem = rem.wrapping_abs().wrapping_mul(i256::from_i128(2)); + if twice_rem >= divisor.wrapping_abs() { + if (value < i256::ZERO) != (divisor < i256::ZERO) { + quot.wrapping_sub(i256::ONE) + } else { + quot.wrapping_add(i256::ONE) + } + } else { + quot + } + } + + #[test] + fn test_div_round_half_up_matches_reference() { + // Callers divide by 10^k for k in 1..=76. + for exp in [1, 2, 6, 18, 19, 38, 39, 76] { + let divisor = i256_pow10(exp); + let half = divisor / i256::from_i128(2); + let values = [ + i256::ZERO, + i256::ONE, + half - i256::ONE, + half, + half + i256::ONE, + divisor - i256::ONE, + divisor, + divisor + half, + i256::from_i128(i128::MAX), + i256::from_i128(i128::MIN), + i256::MAX, + i256::MIN, + i256::MAX - half, + i256::MIN + half, + ]; + for value in values { + for value in [value, value.wrapping_neg()] { + assert_eq!( + div_round_half_up(value, divisor), + div_round_half_up_reference(value, divisor), + "value {value}, divisor 10^{exp}" + ); + } + } + } + } + #[test] fn test_add_same_scale() { // Decimal128(38, 10) + Decimal128(38, 10) -> Decimal128(38, 10)