From 07765c20b50ec090ed256d3f5d6b0ae3ddbac707 Mon Sep 17 00:00:00 2001 From: comphead Date: Sun, 15 Nov 2026 11:26:52 -0800 Subject: [PATCH 1/2] perf: slice contiguous rows in native shuffle instead of gathering them one at a time Hash, range, and hash round robin partitioning record one (batch, row) pair per row and build each output chunk with `interleave_record_batch`, which copies primitive and string columns a row at a time even when the rows sit together. A constant hash key, which an Iceberg write gets when Spark rebalances on a partition column holding one value per run, sends every row of a batch to one partition, so every chunk is one long run. `RowIterator` now splits each chunk into runs of consecutive rows of one buffered batch. When they average at least 128 rows it slices them out and concatenates them, and a chunk that is a single run reaches the writer as a slice. Scattered chunks and schemas with view types are gathered as before. The native shuffle guide no longer says positional placement is the only strategy that hands the IPC writer slices. --- .../contributor-guide/native_shuffle.md | 11 +- native/shuffle/benches/shuffle_writer.rs | 43 +++- .../partitioned_batch_iterator.rs | 220 +++++++++++++++--- native/shuffle/src/shuffle_writer.rs | 11 +- 4 files changed, 244 insertions(+), 41 deletions(-) diff --git a/docs/source/contributor-guide/native_shuffle.md b/docs/source/contributor-guide/native_shuffle.md index 30e24b950f6..6d2138087d2 100644 --- a/docs/source/contributor-guide/native_shuffle.md +++ b/docs/source/contributor-guide/native_shuffle.md @@ -446,11 +446,12 @@ one `(batch, row)` pair per row, so the index list charged against the spill res smaller, and `RunIterator` builds each output chunk by slicing and concatenating runs. A run that covers an entire `batch_size` buffered batch is passed through without copying. -One schema-level restriction is applied in `create_repartitioner`: positional placement is the only -strategy that hands a sliced array to the IPC writer, and while the writer truncates a slice's -buffers for every other type, for `Utf8View` and `BinaryView` it serializes every shared data -buffer in full. A schema containing a view type anywhere therefore falls back to `HashAll`, with -the same `maxHashColumns`. +One schema-level restriction is applied in `create_repartitioner`. The IPC writer truncates a +slice's buffers for every type except `Utf8View` and `BinaryView`, where it serializes every shared +data buffer in full. Positional placement always hands the writer slices, so a schema containing a +view type anywhere falls back to `HashAll`, with the same `maxHashColumns`. The other strategies +also slice when a chunk of a partition's rows sits in long contiguous runs, and on a schema with a +view type they gather those rows instead. #### Retry safety under `RowGroups` diff --git a/native/shuffle/benches/shuffle_writer.rs b/native/shuffle/benches/shuffle_writer.rs index b9c9444e679..9cbab2ea444 100644 --- a/native/shuffle/benches/shuffle_writer.rs +++ b/native/shuffle/benches/shuffle_writer.rs @@ -25,7 +25,7 @@ use arrow::row::{RowConverter, SortField}; use criterion::{criterion_group, criterion_main, BatchSize, Bencher, Criterion}; use datafusion::datasource::memory::MemorySourceConfig; use datafusion::datasource::source::DataSourceExec; -use datafusion::physical_expr::expressions::{col, Column}; +use datafusion::physical_expr::expressions::{col, lit, Column}; use datafusion::physical_expr::{LexOrdering, PhysicalSortExpr}; use datafusion::physical_plan::metrics::Time; use datafusion::{ @@ -467,6 +467,45 @@ fn partitioning_benchmark(c: &mut Criterion) { group.finish(); } +/// Hash placement when every row of a task goes to one output partition, as when a V2 write +/// rebalances on a partition column holding one value per run, such as an hourly or daily `ds`. +/// The literal key places rows as such a column would, so each partition's rows are contiguous +/// in the buffered batches. A per-row key over the same rows is the scattered control. +/// `rows_per_batch=8192` makes every output chunk one whole buffered batch, `rows_per_batch=1000` +/// makes a chunk span several. The writer discards its batches, as in `partitioning_benchmark`. +fn contiguous_hash_benchmark(c: &mut Criterion) { + const BATCH_SIZE: usize = 8192; + const NUM_PARTITIONS: usize = 200; + + let mut group = c.benchmark_group("shuffle_hash_contiguous"); + for (rows_per_batch, num_batches) in [(BATCH_SIZE, 8), (1000, 66)] { + let batches: Vec<_> = (0..num_batches) + .map(|b| create_batch_from(b * rows_per_batch, rows_per_batch, true)) + .collect(); + for (key, expr) in [ + ("constant", lit(20261103i32)), + ("per_row", col("c0", batches[0].schema().as_ref()).unwrap()), + ] { + let partitioning = CometPartitioning::Hash(vec![expr], NUM_PARTITIONS); + group.bench_function( + format!("place+gather (key={key}, rows_per_batch={rows_per_batch})"), + |b| { + b.iter_batched( + || BenchRepartitioner::try_new(partitioning.clone(), BATCH_SIZE).unwrap(), + |mut repartitioner| { + repartitioner.place(&batches).unwrap(); + repartitioner.gather().unwrap(); + repartitioner + }, + BatchSize::LargeInput, + ); + }, + ); + } + } + group.finish(); +} + /// Benchmarks the per-block IPC encoding cost (schema + record batch) in isolation, using the /// `None` codec so that compression does not obscure the schema-encoding cost. Covers a wide flat /// schema and a deeply nested schema, where the schema flatbuffer is largest. @@ -684,6 +723,6 @@ fn config() -> Criterion { criterion_group! { name = benches; config = config(); - targets = criterion_benchmark, boolean_state_benchmark, partitioning_benchmark, schema_encoding_benchmark, ipc_context_reuse_benchmark + targets = criterion_benchmark, boolean_state_benchmark, partitioning_benchmark, contiguous_hash_benchmark, schema_encoding_benchmark, ipc_context_reuse_benchmark } criterion_main!(benches); diff --git a/native/shuffle/src/partitioners/partitioned_batch_iterator.rs b/native/shuffle/src/partitioners/partitioned_batch_iterator.rs index 7740c8037f2..fb5b876f389 100644 --- a/native/shuffle/src/partitioners/partitioned_batch_iterator.rs +++ b/native/shuffle/src/partitioners/partitioned_batch_iterator.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use crate::shuffle_writer::contains_view_type; use arrow::array::{cast::AsArray, Array, ArrayRef, BooleanArray, RecordBatch}; use arrow::buffer::{BooleanBuffer, NullBuffer}; use arrow::compute::{concat_batches, interleave, interleave_record_batch}; @@ -42,7 +43,8 @@ pub(crate) struct BufferedRun { /// space at a 64-row group, which matters because this list is charged against the spill /// reservation. The payoff is at flush: a run is copied with one bulk copy per buffer, where a /// row list has to be gathered a row at a time through `interleave_record_batch`, re-walking -/// every column and every nested child. +/// every column and every nested child, unless its rows happen to be contiguous (see +/// [`RowIterator`]). pub(crate) enum PartitionIndices { /// One `(batch, row)` pair per row, in the order the rows should be written. Rows(Vec>), @@ -97,6 +99,9 @@ pub(super) struct PartitionedBatchesProducer { batch_size: usize, // Every partition borrows the same immutable schema. Check it once per flush. has_boolean_columns: bool, + /// Slicing a view array keeps every data buffer it shares, so rows of a schema with a view + /// type are always gathered (see `contains_view_type`). + has_view_types: bool, } impl PartitionedBatchesProducer { @@ -105,18 +110,23 @@ impl PartitionedBatchesProducer { indices: PartitionIndices, batch_size: usize, ) -> Self { - let has_boolean_columns = buffered_batches.first().is_some_and(|batch| { - batch - .schema_ref() - .fields() - .iter() - .any(|field| field.data_type() == &DataType::Boolean) - }); + let schema_has = |matches: fn(&DataType) -> bool| { + buffered_batches.first().is_some_and(|batch| { + batch + .schema_ref() + .fields() + .iter() + .any(|field| matches(field.data_type())) + }) + }; + let has_boolean_columns = schema_has(|data_type| data_type == &DataType::Boolean); + let has_view_types = schema_has(contains_view_type); Self { partition_indices: indices, buffered_batches, batch_size, has_boolean_columns, + has_view_types, } } @@ -128,8 +138,8 @@ impl PartitionedBatchesProducer { } /// `interleave_time` is the shuffle writer's `interleave_time` metric, which times the gather - /// out of the partition index for either shape: `interleave_record_batch` over rows, or the - /// slicing and concatenation of runs. + /// out of the partition index for either shape: `interleave_record_batch` over scattered + /// rows, or the slicing and concatenation of runs. pub(super) fn produce<'a>( &'a self, refs: &'a [&'a RecordBatch], @@ -150,6 +160,7 @@ impl PartitionedBatchesProducer { self.batch_size, interleave_time, self.has_boolean_columns, + self.has_view_types, )), PartitionIndices::Runs(runs) => PartitionedBatchIterator::Runs(RunIterator::new( &runs[partition_id], @@ -182,8 +193,12 @@ impl Iterator for PartitionedBatchIterator<'_> { } /// Produces a partition's output by gathering individually named rows. +/// +/// Rows that already sit together, such as a batch whose rows all hash to one partition, are +/// sliced out instead of copied a row at a time. See [`has_long_runs`]. pub(crate) struct RowIterator<'a> { has_boolean_columns: bool, + has_view_types: bool, record_batches: &'a [&'a RecordBatch], batch_size: usize, indices: &'a [(u32, u32)], @@ -191,6 +206,9 @@ pub(crate) struct RowIterator<'a> { /// expects. Reused across chunks so each partition costs one small allocation /// (capacity at most `batch_size`) rather than re-materializing its whole index list. chunk_scratch: Vec<(usize, usize)>, + /// Scratch for the slices cut from the current chunk's runs. Grown only by chunks that + /// take the slice path, so scattered partitions never allocate it. + slice_scratch: Vec, pos: usize, interleave_time: &'a Time, } @@ -202,25 +220,30 @@ impl<'a> RowIterator<'a> { batch_size: usize, interleave_time: &'a Time, has_boolean_columns: bool, + has_view_types: bool, ) -> Self { if indices.is_empty() { // Avoid unnecessary allocations when the partition is empty return Self { has_boolean_columns, + has_view_types, record_batches: &[], batch_size, indices: &[], chunk_scratch: vec![], + slice_scratch: vec![], pos: 0, interleave_time, }; } Self { has_boolean_columns, + has_view_types, record_batches, batch_size, indices, chunk_scratch: Vec::with_capacity(batch_size.min(indices.len())), + slice_scratch: vec![], pos: 0, interleave_time, } @@ -236,17 +259,27 @@ impl Iterator for RowIterator<'_> { } let indices_end = std::cmp::min(self.pos + self.batch_size, self.indices.len()); - self.chunk_scratch.clear(); - self.chunk_scratch.extend( - self.indices[self.pos..indices_end] - .iter() - .map(|(i_batch, i_row)| (*i_batch as usize, *i_row as usize)), - ); + let chunk = &self.indices[self.pos..indices_end]; let mut timer = self.interleave_time.timer(); - let result = if self.has_boolean_columns { - interleave_shuffle_batches(self.record_batches, &self.chunk_scratch) + let result = if !self.has_view_types && has_long_runs(chunk) { + self.slice_scratch.clear(); + self.slice_scratch.extend(contiguous_runs(chunk).map(|run| { + let (batch, start) = run[0]; + self.record_batches[batch as usize].slice(start as usize, run.len()) + })); + concat_slices(&mut self.slice_scratch) } else { - interleave_record_batch(self.record_batches, &self.chunk_scratch) + self.chunk_scratch.clear(); + self.chunk_scratch.extend( + chunk + .iter() + .map(|(i_batch, i_row)| (*i_batch as usize, *i_row as usize)), + ); + if self.has_boolean_columns { + interleave_shuffle_batches(self.record_batches, &self.chunk_scratch) + } else { + interleave_record_batch(self.record_batches, &self.chunk_scratch) + } }; timer.stop(); match result { @@ -262,6 +295,40 @@ impl Iterator for RowIterator<'_> { } } +/// The shortest average run that [`RowIterator`] slices out of a chunk rather than gathering row +/// by row. Slicing costs a fixed amount per run and array, gathering a fixed amount per row and +/// array. Slicing starts to win at runs of about 32 rows on flat schemas and about 64 on nested +/// ones, so 128 keeps it ahead on both. +const MIN_AVG_RUN_ROWS: usize = 128; + +/// The maximal runs of consecutive rows of one buffered batch in `chunk`, in order. +fn contiguous_runs(chunk: &[(u32, u32)]) -> impl Iterator { + chunk.chunk_by(|&(batch, row), &(next_batch, next_row)| { + next_batch == batch && next_row == row + 1 + }) +} + +/// Whether the runs of `chunk` average at least [`MIN_AVG_RUN_ROWS`] rows. Counting stops one run +/// past that budget and before anything is sliced, so scattered rows cost only a short scan. +fn has_long_runs(chunk: &[(u32, u32)]) -> bool { + let max_runs = chunk.len() / MIN_AVG_RUN_ROWS; + max_runs > 0 && contiguous_runs(chunk).take(max_runs + 1).count() <= max_runs +} + +/// One output batch from the slices making up a chunk. +/// +/// `concat_batches` over a single slice returns the slice rather than compacting it (arrow's +/// `concat` short-circuits at one input), so skip the call and let the slice through: the IPC +/// writer truncates a sliced array's buffers per type, and the one family it does not, the view +/// types, cannot reach here (see `contains_view_type`). +fn concat_slices(slices: &mut Vec) -> Result { + if slices.len() == 1 { + Ok(slices.pop().expect("one slice")) + } else { + concat_batches(&slices[0].schema(), slices.iter()) + } +} + /// Gather scattered Boolean state without rebuilding Arrow's MutableArrayData /// descriptors for every reducer. Clustered selections retain Arrow's range copies. /// Upstream tracking: https://github.com/apache/arrow-rs/issues/11379 @@ -394,15 +461,7 @@ impl Iterator for RunIterator<'_> { } } - // `concat_batches` over a single slice returns the slice rather than compacting it - // (arrow's `concat` short-circuits at one input), so skip the call and let the slice - // through: the IPC writer truncates a sliced array's buffers per type, and the one - // family it does not, the view types, cannot reach here (see `create_repartitioner`). - let result = if self.chunk_scratch.len() == 1 { - Ok(self.chunk_scratch.pop().expect("one slice")) - } else { - concat_batches(&self.chunk_scratch[0].schema(), self.chunk_scratch.iter()) - }; + let result = concat_slices(&mut self.chunk_scratch); timer.stop(); match result { Ok(batch) => Some(Ok(batch)), @@ -417,7 +476,7 @@ impl Iterator for RunIterator<'_> { #[cfg(test)] mod tests { use super::*; - use arrow::array::{Decimal128Array, Int32Array, Int64Array}; + use arrow::array::{Decimal128Array, Int32Array, Int64Array, StringArray, StringViewArray}; use arrow::datatypes::{DataType, Field, Schema}; use std::sync::Arc; @@ -754,4 +813,107 @@ mod tests { let (values, _) = run_values(vec![], 4); assert!(values.is_empty()); } + + /// Rows that sit in long runs are sliced out rather than gathered, and come out the same as + /// the gather. The chunks of 300 rows are one whole batch, two runs, and part of a batch, over + /// sources that are themselves slices. The whole-batch chunk shares its source buffers, which + /// shows that the slice path was taken. + #[test] + fn contiguous_rows_are_sliced_not_gathered() { + let schema = Arc::new(Schema::new(vec![ + Field::new("v", DataType::Int64, false), + Field::new("s", DataType::Utf8, true), + Field::new("b", DataType::Boolean, true), + ])); + let buffered: Vec = (0..3) + .map(|b| { + let rows = b * 1000..b * 1000 + 305; + RecordBatch::try_new( + Arc::clone(&schema), + vec![ + Arc::new(Int64Array::from_iter_values(rows.clone())), + Arc::new(StringArray::from_iter( + rows.clone() + .map(|r| (r % 7 != 0).then(|| format!("row {r}"))), + )), + Arc::new(BooleanArray::from_iter( + rows.map(|r| (r % 5 != 0).then_some(r % 3 == 0)), + )), + ], + ) + .unwrap() + .slice(5, 300) + }) + .collect(); + let indices: Vec<(u32, u32)> = (0..300) + .map(|r| (0, r)) + .chain((20..220).map(|r| (1, r))) + .chain((0..300).map(|r| (2, r))) + .collect(); + let batch_size = 300; + let producer = PartitionedBatchesProducer::new( + buffered.clone(), + PartitionIndices::Rows(vec![indices.clone()]), + batch_size, + ); + let refs = producer.batch_refs(); + let time = Time::default(); + let produced: Vec = producer + .produce(&refs, 0, &time) + .collect::>() + .unwrap(); + + let expected: Vec = indices + .chunks(batch_size) + .map(|chunk| { + let chunk: Vec<_> = chunk + .iter() + .map(|&(b, r)| (b as usize, r as usize)) + .collect(); + interleave_record_batch(&refs, &chunk).unwrap() + }) + .collect(); + assert_eq!(produced, expected); + assert_eq!( + produced[0].column(0).to_data().buffers()[0] + .data_ptr() + .as_ptr() as usize, + buffered[0].column(0).to_data().buffers()[0] + .data_ptr() + .as_ptr() as usize, + "expected the source allocation, not a copy" + ); + } + + /// A sliced view array keeps every data buffer it shares, so view columns are gathered even + /// when their rows are contiguous. + #[test] + fn view_columns_are_gathered_even_when_contiguous() { + let values = StringViewArray::from_iter_values( + (0..300).map(|r| format!("a view value longer than twelve bytes {r}")), + ); + let batch = RecordBatch::try_from_iter([("s", Arc::new(values) as ArrayRef)]).unwrap(); + let views = batch.column(0).to_data().buffers()[0].clone(); + // Long enough to take the slice path if the schema allowed it. + let producer = PartitionedBatchesProducer::new( + vec![batch.clone()], + PartitionIndices::Rows(vec![(0..300).map(|r| (0, r)).collect()]), + 300, + ); + let refs = producer.batch_refs(); + let time = Time::default(); + let produced: Vec = producer + .produce(&refs, 0, &time) + .collect::>() + .unwrap(); + + assert_eq!(produced, vec![batch]); + assert_ne!( + produced[0].column(0).to_data().buffers()[0] + .data_ptr() + .as_ptr() as usize, + views.data_ptr().as_ptr() as usize, + "expected the views to be gathered, not sliced" + ); + } } diff --git a/native/shuffle/src/shuffle_writer.rs b/native/shuffle/src/shuffle_writer.rs index fe6088d3d4c..aa0057c2b9f 100644 --- a/native/shuffle/src/shuffle_writer.rs +++ b/native/shuffle/src/shuffle_writer.rs @@ -415,11 +415,12 @@ async fn external_shuffle( /// the views buffer but serializes every shared data buffer in full, since proving that no /// surviving view references a buffer is not cheap. /// -/// [`RoundRobinStrategy::RowGroups`] is the only placement that can hand the writer a sliced -/// array; everything else materializes a fresh batch through `interleave_record_batch`. So a view -/// column anywhere in the schema would let a short run drag a whole batch's data buffers into the -/// shuffle output, which is the opposite of what the strategy is for. -fn contains_view_type(data_type: &DataType) -> bool { +/// [`RoundRobinStrategy::RowGroups`], which records runs, hands the writer sliced arrays, and so +/// does a row-indexed gather whose rows already sit in long runs. So a view column anywhere in the +/// schema would let a run drag a whole batch's data buffers into the shuffle output. Positional +/// placement falls back to hashing for such a schema (see `partitioning_for_schema`), and the row +/// gather interleaves rather than slices (see `PartitionedBatchesProducer`). +pub(crate) fn contains_view_type(data_type: &DataType) -> bool { match data_type { DataType::Utf8View | DataType::BinaryView => true, DataType::List(field) From 2ad22a305b471f91be8e09e03fad5df4f2219309 Mon Sep 17 00:00:00 2001 From: comphead Date: Mon, 16 Nov 2026 11:47:40 -0800 Subject: [PATCH 2/2] perf: rule out short-run chunks cheaply and slice from 64-row runs The exact run count reads about L/T of a chunk whose runs of L rows fall under the threshold T before the chunk is gathered anyway, which made place+gather about 2.4% slower for clustered runs just under the threshold. A probe every T rows now gives a lower bound on the run count that rules those chunks out first without changing any decision, and the threshold drops back to 64 rows, where slicing breaks even on nested schemas. Every mapped type now also round-trips through the block writer as a slice that starts past row 0, and the shuffle guide describes how hash and range partitions are written. --- .../contributor-guide/native_shuffle.md | 15 ++++- native/shuffle/benches/shuffle_writer.rs | 28 +++++++--- .../partitioned_batch_iterator.rs | 55 +++++++++++++++++-- .../src/writers/shuffle_block_writer.rs | 31 ++++++----- 4 files changed, 100 insertions(+), 29 deletions(-) diff --git a/docs/source/contributor-guide/native_shuffle.md b/docs/source/contributor-guide/native_shuffle.md index 6d2138087d2..2eff1d80c06 100644 --- a/docs/source/contributor-guide/native_shuffle.md +++ b/docs/source/contributor-guide/native_shuffle.md @@ -337,6 +337,13 @@ Native shuffle implements Spark-compatible hash partitioning: - Computes hash of partition key columns - Applies modulo by partition count: `partition_id = hash % num_partitions` +The partitioner writes each partition's rows out of the buffered batches in chunks of up to +`batch_size` rows. When a chunk's rows sit in contiguous runs that average at least 64 rows +(`MIN_AVG_RUN_ROWS`), as when every row of a batch hashes to one partition, it slices the runs out +and concatenates them, and it hands a chunk that is a single run to the IPC writer as a slice of +the buffered batch. It gathers any other chunk a row at a time with `interleave_record_batch`, as it +does every chunk when the schema has a view type. + ### Range Partitioning For range partitioning: @@ -346,6 +353,10 @@ For range partitioning: 3. Native code converts sort key columns to comparable row format. 4. Binary search (`partition_point`) determines which partition each row belongs to. +The partitioner then writes each partition's rows as described in +[Hash Partitioning](#hash-partitioning), so input that is already clustered on the sort key is +sliced rather than gathered. + ### Single Partition The simplest case: all rows go to partition 0. Uses `SinglePartitionShufflePartitioner`, which @@ -450,8 +461,8 @@ One schema-level restriction is applied in `create_repartitioner`. The IPC write slice's buffers for every type except `Utf8View` and `BinaryView`, where it serializes every shared data buffer in full. Positional placement always hands the writer slices, so a schema containing a view type anywhere falls back to `HashAll`, with the same `maxHashColumns`. The other strategies -also slice when a chunk of a partition's rows sits in long contiguous runs, and on a schema with a -view type they gather those rows instead. +also slice chunks that sit in long contiguous runs (see [Hash Partitioning](#hash-partitioning)), +and on a schema with a view type they gather those rows instead. #### Retry safety under `RowGroups` diff --git a/native/shuffle/benches/shuffle_writer.rs b/native/shuffle/benches/shuffle_writer.rs index 9cbab2ea444..94b757dca1f 100644 --- a/native/shuffle/benches/shuffle_writer.rs +++ b/native/shuffle/benches/shuffle_writer.rs @@ -25,7 +25,8 @@ use arrow::row::{RowConverter, SortField}; use criterion::{criterion_group, criterion_main, BatchSize, Bencher, Criterion}; use datafusion::datasource::memory::MemorySourceConfig; use datafusion::datasource::source::DataSourceExec; -use datafusion::physical_expr::expressions::{col, lit, Column}; +use datafusion::logical_expr::Operator; +use datafusion::physical_expr::expressions::{binary, col, lit, Column}; use datafusion::physical_expr::{LexOrdering, PhysicalSortExpr}; use datafusion::physical_plan::metrics::Time; use datafusion::{ @@ -470,9 +471,11 @@ fn partitioning_benchmark(c: &mut Criterion) { /// Hash placement when every row of a task goes to one output partition, as when a V2 write /// rebalances on a partition column holding one value per run, such as an hourly or daily `ds`. /// The literal key places rows as such a column would, so each partition's rows are contiguous -/// in the buffered batches. A per-row key over the same rows is the scattered control. -/// `rows_per_batch=8192` makes every output chunk one whole buffered batch, `rows_per_batch=1000` -/// makes a chunk span several. The writer discards its batches, as in `partitioning_benchmark`. +/// in the buffered batches. The clustered keys repeat over runs of 48 and 96 rows, just under and +/// over the shortest average run that is sliced, and a per-row key over the same rows is the +/// scattered control. `rows_per_batch=8192` makes every output chunk one whole buffered batch, +/// `rows_per_batch=1000` makes a chunk span several. The writer discards its batches, as in +/// `partitioning_benchmark`. fn contiguous_hash_benchmark(c: &mut Criterion) { const BATCH_SIZE: usize = 8192; const NUM_PARTITIONS: usize = 200; @@ -482,10 +485,19 @@ fn contiguous_hash_benchmark(c: &mut Criterion) { let batches: Vec<_> = (0..num_batches) .map(|b| create_batch_from(b * rows_per_batch, rows_per_batch, true)) .collect(); - for (key, expr) in [ - ("constant", lit(20261103i32)), - ("per_row", col("c0", batches[0].schema().as_ref()).unwrap()), - ] { + let schema = batches[0].schema(); + let row = col("c0", &schema).unwrap(); + let mut keys = vec![ + ("constant".to_string(), lit(20261103i32)), + ("per_row".to_string(), Arc::clone(&row)), + ]; + if rows_per_batch == BATCH_SIZE { + for run in [48, 96] { + let key = binary(Arc::clone(&row), Operator::Divide, lit(run), &schema).unwrap(); + keys.push((format!("clustered_{run}"), key)); + } + } + for (key, expr) in keys { let partitioning = CometPartitioning::Hash(vec![expr], NUM_PARTITIONS); group.bench_function( format!("place+gather (key={key}, rows_per_batch={rows_per_batch})"), diff --git a/native/shuffle/src/partitioners/partitioned_batch_iterator.rs b/native/shuffle/src/partitioners/partitioned_batch_iterator.rs index fb5b876f389..0e8c43d9d24 100644 --- a/native/shuffle/src/partitioners/partitioned_batch_iterator.rs +++ b/native/shuffle/src/partitioners/partitioned_batch_iterator.rs @@ -297,9 +297,9 @@ impl Iterator for RowIterator<'_> { /// The shortest average run that [`RowIterator`] slices out of a chunk rather than gathering row /// by row. Slicing costs a fixed amount per run and array, gathering a fixed amount per row and -/// array. Slicing starts to win at runs of about 32 rows on flat schemas and about 64 on nested -/// ones, so 128 keeps it ahead on both. -const MIN_AVG_RUN_ROWS: usize = 128; +/// array. Slicing wins from runs of about 32 rows on flat schemas and breaks even at about 64 on +/// nested ones. +const MIN_AVG_RUN_ROWS: usize = 64; /// The maximal runs of consecutive rows of one buffered batch in `chunk`, in order. fn contiguous_runs(chunk: &[(u32, u32)]) -> impl Iterator { @@ -308,11 +308,28 @@ fn contiguous_runs(chunk: &[(u32, u32)]) -> impl Iterator }) } -/// Whether the runs of `chunk` average at least [`MIN_AVG_RUN_ROWS`] rows. Counting stops one run -/// past that budget and before anything is sliced, so scattered rows cost only a short scan. +/// Whether the runs of `chunk` average at least [`MIN_AVG_RUN_ROWS`] rows. +/// +/// The exact count stops one run past the budget, which for runs just under the minimum is most +/// of the chunk, all of it wasted before the gather. So a lower bound comes first. A stretch of +/// `MIN_AVG_RUN_ROWS` steps whose ends are not that many rows apart in one batch holds a run +/// boundary, and the stretches are disjoint, so one probe per stretch rules out any chunk whose +/// runs are all shorter than the minimum. fn has_long_runs(chunk: &[(u32, u32)]) -> bool { let max_runs = chunk.len() / MIN_AVG_RUN_ROWS; - max_runs > 0 && contiguous_runs(chunk).take(max_runs + 1).count() <= max_runs + if max_runs == 0 { + return false; + } + let last = chunk.len() - 1; + let broken = (0..last) + .step_by(MIN_AVG_RUN_ROWS) + .filter(|&start| { + let end = (start + MIN_AVG_RUN_ROWS).min(last); + let (batch, row) = chunk[start]; + chunk[end] != (batch, row + (end - start) as u32) + }) + .count(); + broken < max_runs && contiguous_runs(chunk).take(max_runs + 1).count() <= max_runs } /// One output batch from the slices making up a chunk. @@ -916,4 +933,30 @@ mod tests { "expected the views to be gathered, not sliced" ); } + + /// The probe-per-stretch lower bound in `has_long_runs` never changes the exact count's + /// decision, on random mixes of short and long runs that need not follow each other in row + /// order. + #[test] + fn long_run_bound_matches_exact_count() { + let mut state = 0x9E37_79B9_7F4A_7C15_u64; + let mut next = move || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }; + for case in 0..500 { + let len = 1 + (next() % 9_000) as usize; + let mut chunk = Vec::with_capacity(len); + while chunk.len() < len { + let run = 1 + (next() % [2, 40, 70, 200, 9_000][case % 5]) as usize; + let (batch, row) = ((next() % 16) as u32, (next() % 4_096) as u32); + chunk.extend((0..run.min(len - chunk.len())).map(|k| (batch, row + k as u32))); + } + let max_runs = chunk.len() / MIN_AVG_RUN_ROWS; + let exact = max_runs > 0 && contiguous_runs(&chunk).count() <= max_runs; + assert_eq!(has_long_runs(&chunk), exact, "case {case}"); + } + } } diff --git a/native/shuffle/src/writers/shuffle_block_writer.rs b/native/shuffle/src/writers/shuffle_block_writer.rs index dc1bbf40dc9..33532f0a5fc 100644 --- a/native/shuffle/src/writers/shuffle_block_writer.rs +++ b/native/shuffle/src/writers/shuffle_block_writer.rs @@ -918,12 +918,14 @@ mod tests { /// Every mapped type round-trips under every codec: without lengths, with lengths as the /// public constructor allows them (zstd only), and with lengths forced on. The forced case /// shows the reader rebuilds offsets whatever the codec, so allowing another codec later is a - /// writer-only change. + /// writer-only change. Each also round-trips as a slice that starts past row 0, the shape + /// positional placement and the row gather's contiguous runs hand the writer. #[test] #[cfg_attr(miri, ignore)] // miri can't call foreign function `ZSTD_createCCtx` fn offsets_as_lengths_round_trip_every_mapped_type() { let batch = offsets_batch(); let schema = batch.schema(); + let inputs = [batch.slice(5, 40), batch]; for codec in CODECS { let zstd = matches!(codec, CompressionCodec::Zstd(_)); for (writer, flagged) in [ @@ -941,18 +943,21 @@ mod tests { true, ), ] { - let block = write_block(&writer.unwrap(), &batch); - assert_eq!(lengths_flagged(&block), flagged, "{codec:?}"); - assert_eq!( - read_ipc_compressed(&block[16..]).unwrap(), - batch, - "{codec:?}" - ); - assert_eq!( - read_ipc_compressed_validated(&block[16..]).unwrap(), - batch, - "{codec:?} validated" - ); + let writer = writer.unwrap(); + for input in &inputs { + let block = write_block(&writer, input); + assert_eq!(lengths_flagged(&block), flagged, "{codec:?}"); + assert_eq!( + read_ipc_compressed(&block[16..]).unwrap(), + *input, + "{codec:?}" + ); + assert_eq!( + read_ipc_compressed_validated(&block[16..]).unwrap(), + *input, + "{codec:?} validated" + ); + } } } }