From d1819fd7fe1024fdc0fe7ae8ff7113e9c00e0587 Mon Sep 17 00:00:00 2001 From: Parth Chandra Date: Tue, 17 Nov 2026 11:24:22 -0800 Subject: [PATCH] perf: combine small input batches before a native sort that merges them A shuffle from many map tasks gives each reduce task one tiny batch per block. DataFusion's sort merges one stream per buffered batch when it spills, so such sorts become very slow. Add SortInputCoalesceExec in front of a full native sort over unordered input. It passes batches through until the input reaches the sort's in-place threshold, then combines small batches into batches of at most spark.comet.batchSize rows and half that threshold in bytes. It reserves exactly what it holds and holds nothing while the sort takes a batch. Controlled by spark.comet.exec.sort.coalesceInput.enabled (default true). Closes #5686. --- .../contributor-guide/memory_management.md | 24 +- docs/source/user-guide/latest/metrics.md | 14 + .../user-guide/latest/migration-guide.md | 11 + .../src/execution/memory_pools/fair_pool.rs | 91 ++ native/core/src/execution/planner.rs | 38 +- native/core/src/execution/planner/join.rs | 11 +- .../planner/tests/sort_input_coalesce.rs | 151 +++ native/operators/Cargo.toml | 4 + .../operators/benches/sort_small_batches.rs | 171 +++ native/operators/src/lib.rs | 2 + native/operators/src/sort_input_coalesce.rs | 1160 +++++++++++++++++ native/proto/src/proto/operator.proto | 3 + .../scala/org/apache/comet/CometConf.scala | 20 + .../apache/spark/sql/comet/operators.scala | 9 +- .../operators/sort_coalesce_input.sql | 124 ++ .../sql/comet/CometTaskMetricsSuite.scala | 44 + 16 files changed, 1862 insertions(+), 15 deletions(-) create mode 100644 native/core/src/execution/planner/tests/sort_input_coalesce.rs create mode 100644 native/operators/benches/sort_small_batches.rs create mode 100644 native/operators/src/sort_input_coalesce.rs create mode 100644 spark/src/test/resources/sql-tests/operators/sort_coalesce_input.sql diff --git a/docs/source/contributor-guide/memory_management.md b/docs/source/contributor-guide/memory_management.md index c90b5b1e853..dcbeda394ae 100644 --- a/docs/source/contributor-guide/memory_management.md +++ b/docs/source/contributor-guide/memory_management.md @@ -413,7 +413,7 @@ partially sorted, but it is likewise checked only against the pool and Spark's g `PartialMerge` over input sorted on every grouping key stays on the hash aggregate, since the final aggregate has no spill path there and would only halve the limit of a sort below it. -Three details matter for tuning: +Four details matter for tuning: - The pool is task-shared (see below), so the consumers that divide the task's share include those of every native plan in the task, not just the plan making the request. Registering a consumer @@ -433,7 +433,21 @@ Three details matter for tuning: the task's whole share, because the limit is then zero. Spark lets such a consumer hold more than `off_heap_size / running_tasks` while other tasks have not yet asked for memory. A larger `spark.memory.offHeap.size`, a smaller `spark.comet.batchSize` or `spark.comet.exec.memoryPool=greedy_unified` - avoids it, and the migration guide says so too. + avoids it, and the migration guide says so too. For a sort over a shuffle with many small + blocks, `spark.comet.exec.sort.coalesceInput.enabled=false` also avoids it, as described next. +- `CometSort` can put a consumer that cannot spill in front of the sort, which combines small input + batches, such as shuffle blocks from many map tasks, before the sort buffers them. It reserves + twice the bytes of the batches it holds, enough for them and the combined copy, and releases that + before the sort reserves the combined batch, so it holds nothing while the sort takes a batch. A + combined batch is larger than the blocks it replaces, so a sort whose share is very close to the + least it can run with may still fail with the error above where the blocks fit, as it can with a + full-size batch from any other input. A combined batch holds at most half the sort's in-place + threshold, 512 KiB by default, so the sort reserves at most about that threshold for it. That cap + is kept between 64 KiB and 4 MiB, so with a threshold below 128 KiB or above 8 MiB it is not half. + While it holds a group, its reservation, like any that cannot spill, lowers the share of the + task's consumers that can spill, such as another plan's sort in the same task. When the pool + refuses it a reservation, it passes the next 64 small batches through, then asks again. + `spark.comet.exec.sort.coalesceInput.enabled=false` turns it off. ### Task-shared pools and their lifetime @@ -562,8 +576,10 @@ unbounded, so nobody charged for them at allocation time. Whether they are charg on who holds them. DataFusion's `ExternalSorter` reserves `get_reserved_bytes_for_record_batch` for every batch it retains, and the hash join build side reserves `get_record_batch_memory_size` for each incoming batch; both read buffer sizes off the `ArrayData` and apply equally to imported -buffers. So an imported batch that a sort or join has buffered _is_ reserved in Comet's pool, and -through a unified pool is charged to Spark. The same batch held by an operator that does not reserve +buffers. The coalescer that can sit in front of a native sort also reserves the batches it holds, +at twice their size to cover the combined copy. So an imported batch that a sort, a join or that +coalescer has buffered _is_ reserved in Comet's pool, and through a unified pool is charged to +Spark. The same batch held by an operator that does not reserve (a projection or filter, or `ScanExec` itself between polls) is charged to nothing. **Native → JVM (`CometExecIterator`).** DataFusion produces the batch in Rust, and whether it is diff --git a/docs/source/user-guide/latest/metrics.md b/docs/source/user-guide/latest/metrics.md index 0fcacd0ba8f..e438f59c19f 100644 --- a/docs/source/user-guide/latest/metrics.md +++ b/docs/source/user-guide/latest/metrics.md @@ -87,6 +87,20 @@ Native aggregates with grouping keys report these additional metrics: Spill bytes from native sorts, aggregates, and sort-merge joins are also added to Spark's task-level `diskBytesSpilled` metric in every stage, not only in shuffle stages. +### CometSort + +Native sorts report these additional metrics: + +| Metric | Description | +| ------------------------------------------------------------- | --------------------------------------------------------------------------------------------------- | +| `number of spills` | Number of times the sort spilled to disk. | +| `total spilled bytes` | Bytes written to sort spill files. | +| `number of small input batches combined into larger ones` | Small input batches, such as shuffle blocks from many map tasks, combined before the sort. | +| `number of batches produced by combining small input batches` | Batches the sort received from that combining. Zero means the sort got its input batches unchanged. | + +The combining is controlled by `spark.comet.exec.sort.coalesceInput.enabled`. Its time is included in +the sort's `total time`. + ### Hash Joins With `spark.comet.exec.join.dynamicFilter.enabled=true`, native broadcast and shuffled hash joins diff --git a/docs/source/user-guide/latest/migration-guide.md b/docs/source/user-guide/latest/migration-guide.md index 9308e00b4dd..a8a86655a03 100644 --- a/docs/source/user-guide/latest/migration-guide.md +++ b/docs/source/user-guide/latest/migration-guide.md @@ -80,6 +80,17 @@ Kubernetes can kill an executor that runs out of it. The plugin warns at startup but check it before you upgrade; see [Configuring Comet Memory](tuning/memory.md#configuring-comet-memory). +### Sorts Combine Small Input Batches + +`CometSort` now combines small input batches before sorting them, once a task's sort input is too +large to sort in place. A shuffle written by many map tasks gives each reduce task many tiny +batches, and DataFusion's sort merges one stream per batch, at least when it spills or sorts on more +than one key, which made such sorts very slow. The results do not change. A combined batch is +larger than the blocks it replaces, so a sort with a very small memory share can fail with "Not +enough memory to continue external sort" where it used to run. If that happens, set +`spark.comet.exec.sort.coalesceInput.enabled=false`, or use the remedies in +[Memory Pool Limits](#memory-pool-limits). + ### Deprecated and Removed Settings `spark.comet.exec.columnarToRow.native.enabled` is deprecated and will be removed in a future major diff --git a/native/core/src/execution/memory_pools/fair_pool.rs b/native/core/src/execution/memory_pools/fair_pool.rs index 93174ba10e9..50985ea54fa 100644 --- a/native/core/src/execution/memory_pools/fair_pool.rs +++ b/native/core/src/execution/memory_pools/fair_pool.rs @@ -335,6 +335,97 @@ mod tests { (0..n).map(|_| tasks.start()).collect() } + /// Sorts 3000 batches of 60 wide rows in `pool`, optionally combining them first. Returns + /// how many rows came out, how many times the sort spilled, and how many input batches were + /// combined. + async fn sort_small_batches( + pool: &Arc, + coalesce: bool, + ) -> datafusion::common::Result<(usize, usize, usize)> { + use arrow::array::{ArrayRef, Int64Array, RecordBatch, StringArray}; + use datafusion::datasource::memory::MemorySourceConfig; + use datafusion::execution::runtime_env::RuntimeEnvBuilder; + use datafusion::physical_expr::expressions::col; + use datafusion::physical_expr::{LexOrdering, PhysicalSortExpr}; + use datafusion::physical_plan::sorts::sort::SortExec; + use datafusion::physical_plan::{collect, ExecutionPlan}; + use datafusion::prelude::{SessionConfig, SessionContext}; + use datafusion_comet_operators::SortInputCoalesceExec; + + let input: Vec = (0..3000) + .map(|b| { + let ids: Vec = (0..60).map(|i| (b * 60 + i) * 7919 % 180_000).collect(); + let text = ids.iter().map(|i| format!("{i:0>210}")); + RecordBatch::try_from_iter([ + ("id", Arc::new(Int64Array::from(ids.clone())) as ArrayRef), + ( + "s", + Arc::new(StringArray::from_iter_values(text)) as ArrayRef, + ), + ]) + .unwrap() + }) + .collect(); + let schema = input[0].schema(); + let runtime = RuntimeEnvBuilder::new() + .with_memory_pool(Arc::clone(pool)) + .build_arc()?; + let config = SessionConfig::new().with_sort_spill_reservation_bytes(64 * 1024); + let gate = config.options().execution.sort_in_place_threshold_bytes; + let ctx = SessionContext::new_with_config_rt(config, runtime); + let source: Arc = + MemorySourceConfig::try_new_exec(&[input], Arc::clone(&schema), None)?; + let coalescer = Arc::new(SortInputCoalesceExec::new(Arc::clone(&source), 8192, gate)); + let sort_input: Arc = if coalesce { + Arc::clone(&coalescer) as _ + } else { + source + }; + let ordering = LexOrdering::new(vec![PhysicalSortExpr::new( + col("id", &schema)?, + Default::default(), + )]) + .unwrap(); + let sort = Arc::new(SortExec::new(ordering, sort_input)); + let output = collect(Arc::clone(&sort) as _, ctx.task_ctx()).await?; + let spills = sort + .metrics() + .and_then(|metrics| metrics.spill_count()) + .unwrap_or(0); + let combined = coalescer + .metrics() + .and_then(|metrics| metrics.sum_by_name("coalesced_input_batches")) + .map_or(0, |value| value.as_usize()); + Ok(( + output.iter().map(|batch| batch.num_rows()).sum(), + spills, + combined, + )) + } + + /// The coalescer in front of a sort cannot spill, so this pool limits it only by the pool + /// size. A combined batch must still be small enough for the sort's fair share, or the sort + /// fails after a spill where the uncombined batches fit. + #[tokio::test(flavor = "multi_thread")] + async fn a_sort_fits_its_share_with_combined_input_where_it_fits_without() { + let fake = FakeSpark::with(usize::MAX); + let pool = pool(&fake, 3 * 1024 * 1024); + // Another consumer that can spill, such as a second sort in the task, halves the share. + let _other_sort = spillable("ExternalSorter[1]", &pool); + + let (rows, spills, _) = sort_small_batches(&pool, false).await.unwrap(); + assert_eq!(rows, 180_000); + assert!( + spills > 0, + "the plain sort must spill for this test to mean anything" + ); + let (rows, spills, combined) = sort_small_batches(&pool, true).await.unwrap(); + assert_eq!(rows, 180_000); + assert!(spills > 0, "the sort over combined input must spill too"); + assert!(combined > 0, "no input batches were combined"); + assert_eq!(pool.reserved(), 0); + } + #[test] fn consumers_that_can_spill_share_their_tasks_share_of_the_pool() { // Eight running tasks share a 1040-byte pool, and Spark grants this one its 130 bytes. diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index 73870b75fc8..a26a0319d1a 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -79,7 +79,9 @@ use datafusion::{ }, prelude::SessionContext, }; -use datafusion_comet_operators::{range_exec, range_sort_expr, ExpandExec, SampleExec}; +use datafusion_comet_operators::{ + range_exec, range_sort_expr, ExpandExec, SampleExec, SortInputCoalesceExec, +}; use datafusion_comet_spark_expr::{ create_comet_physical_fun, create_comet_physical_fun_with_eval_mode, BinaryOutputStyle, BloomFilterMightContain, CheckedBinaryExpr, CsvWriteOptions, EvalMode, SparkArraysZipFunc, @@ -1860,12 +1862,32 @@ impl PhysicalPlanner { .collect(); let fetch = sort.fetch.map(|num| num as usize); + let ordering = LexOrdering::new(exprs?).unwrap(); + + // Only a full sort over unordered input merges: TopK never does, and a sort passes + // ordered input through. + let (sort_input, coalescer): (Arc, Vec<_>) = + if sort.coalesce_input + && fetch.is_none() + && !child + .native_plan + .properties() + .equivalence_properties() + .ordering_satisfy(ordering.clone())? + { + let config = self.session_ctx.copied_config(); + let coalescer: Arc = + Arc::new(SortInputCoalesceExec::new( + Arc::clone(&child.native_plan), + config.batch_size(), + config.options().execution.sort_in_place_threshold_bytes, + )); + (Arc::clone(&coalescer), vec![coalescer]) + } else { + (Arc::clone(&child.native_plan), vec![]) + }; - let sort_plan = SortExec::new( - LexOrdering::new(exprs?).unwrap(), - Arc::clone(&child.native_plan), - ) - .with_fetch(fetch); + let sort_plan = SortExec::new(ordering, sort_input).with_fetch(fetch); let mut sort_exec: Arc = if sort.dynamic_filter_enabled { match TopKReaderFilterExec::try_new( &sort_plan, @@ -1885,10 +1907,11 @@ impl PhysicalPlanner { Ok(( scans, shuffle_scans, - Arc::new(SparkPlan::new( + Arc::new(SparkPlan::new_with_additional( spark_plan.plan_id, sort_exec, vec![Arc::clone(&child)], + coalescer, )), )) } @@ -3183,6 +3206,7 @@ mod tests { mod empty_native_scan; mod multi_store_scan; mod semi_join_reduction; + mod sort_input_coalesce; use futures::{poll, StreamExt}; use std::{ diff --git a/native/core/src/execution/planner/join.rs b/native/core/src/execution/planner/join.rs index f135c6afa70..43d041c0ec0 100644 --- a/native/core/src/execution/planner/join.rs +++ b/native/core/src/execution/planner/join.rs @@ -47,7 +47,7 @@ use datafusion::{ }, }; use datafusion_comet_operators::{ - CometFilterExec, CometProjectionExec, MembershipKeyReductionExec, + CometFilterExec, CometProjectionExec, MembershipKeyReductionExec, SortInputCoalesceExec, }; use datafusion_comet_proto::{ spark_expression::Expr, @@ -220,7 +220,8 @@ fn reduce_before_sort( let Some(sort) = params.right.native_plan.downcast_ref::() else { return Ok(None); }; - let input = sort.input(); + // The reducer packs its own output, so it replaces any input coalescer. + let input = SortInputCoalesceExec::unwrap_input(sort.input()); let ([(_, key)], [options]) = (params.join_on.as_slice(), sort_options) else { return Ok(None); }; @@ -268,7 +269,11 @@ fn reduce_before_sort( vec![Arc::clone(&reducer)], ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), )?; - Arc::make_mut(&mut params.right).native_plan = sort; + let right = Arc::make_mut(&mut params.right); + right.native_plan = sort; + right + .additional_native_plans + .retain(|plan| !plan.is::()); // The Spark sort collects from its replacement. The join owns the reducer's named metrics. Ok(Some(reducer)) } diff --git a/native/core/src/execution/planner/tests/sort_input_coalesce.rs b/native/core/src/execution/planner/tests/sort_input_coalesce.rs new file mode 100644 index 00000000000..0360d771a79 --- /dev/null +++ b/native/core/src/execution/planner/tests/sort_input_coalesce.rs @@ -0,0 +1,151 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::{ + create_bound_reference, create_proto_datatype, create_scan, create_sort_order, PhysicalPlanner, +}; +use datafusion_comet_operators::{MembershipKeyReductionExec, SortInputCoalesceExec}; +use datafusion_comet_proto::spark_operator::{self as op, operator::OpStruct, JoinType, Operator}; + +fn sort(coalesce_input: bool, fetch: Option) -> Operator { + sort_with_skip(coalesce_input, fetch, None) +} + +fn sort_with_skip(coalesce_input: bool, fetch: Option, skip: Option) -> Operator { + Operator { + children: vec![create_scan()], + op_struct: Some(OpStruct::Sort(op::Sort { + sort_orders: vec![create_sort_order( + 0, + create_proto_datatype().type_id, + false, + true, + )], + fetch, + skip, + coalesce_input, + ..Default::default() + })), + ..Default::default() + } +} + +#[test] +fn wraps_the_input_of_a_full_sort_only() { + let plan = |op: Operator| { + PhysicalPlanner::default() + .create_plan(&op, &mut vec![], 1) + .unwrap() + .2 + }; + + let full = plan(sort(true, None)); + let input = &full.native_plan.children()[0]; + assert!(input.is::()); + // The session's batch size and the sort's in-place threshold reach the coalescer. + let defaults = datafusion::prelude::SessionConfig::new(); + assert_eq!( + datafusion::physical_plan::displayable(input.as_ref()) + .one_line() + .to_string() + .trim(), + format!( + "CometSortInputCoalesceExec: target_rows={}, gate_bytes={}, max_bytes={}", + defaults.batch_size(), + defaults.options().execution.sort_in_place_threshold_bytes, + defaults.options().execution.sort_in_place_threshold_bytes / 2 + ) + ); + // Its metrics are reported under the sort. + assert_eq!(full.additional_native_plans.len(), 1); + assert!(full.additional_native_plans[0].is::()); + + for op in [sort(false, None), sort(true, Some(10))] { + let planned = plan(op); + assert!(!planned.native_plan.children()[0].is::()); + assert!(planned.additional_native_plans.is_empty()); + } +} + +#[test] +fn semi_join_key_reduction_replaces_the_coalescer() { + let key = create_bound_reference(0); + let sorted = sort(true, None); + let join = Operator { + children: vec![sorted.clone(), sorted], + op_struct: Some(OpStruct::SortMergeJoin(op::SortMergeJoin { + left_join_keys: vec![key.clone()], + right_join_keys: vec![key], + join_type: JoinType::LeftSemi as i32, + sort_options: vec![create_sort_order( + 0, + create_proto_datatype().type_id, + false, + true, + )], + right_key_reduction_enabled: true, + ..Default::default() + })), + ..Default::default() + }; + let (_, _, plan) = PhysicalPlanner::default() + .create_plan(&join, &mut vec![], 1) + .unwrap(); + let right = &plan.children[1]; + assert!(right.native_plan.children()[0].is::()); + assert!(right.additional_native_plans.is_empty()); + // The left sort keeps its coalescer. + assert!(plan.children[0].native_plan.children()[0].is::()); +} + +#[test] +fn a_sort_with_only_a_skip_still_gets_its_input_combined() { + let (_, _, plan) = PhysicalPlanner::default() + .create_plan(&sort_with_skip(true, None, Some(1)), &mut vec![], 1) + .unwrap(); + // The skip wraps the sort in a limit; the sort itself still has no fetch. + let sort = &plan.native_plan.children()[0]; + assert!(sort.children()[0].is::()); + assert_eq!(plan.additional_native_plans.len(), 1); +} + +#[test] +fn a_sort_over_input_already_in_its_order_is_not_wrapped() { + // The inner sort's output is already in the outer sort's order, so the outer sort passes it + // through and combining it would only copy. + let outer = Operator { + children: vec![sort(true, None)], + op_struct: Some(OpStruct::Sort(op::Sort { + sort_orders: vec![create_sort_order( + 0, + create_proto_datatype().type_id, + false, + true, + )], + coalesce_input: true, + ..Default::default() + })), + ..Default::default() + }; + let (_, _, plan) = PhysicalPlanner::default() + .create_plan(&outer, &mut vec![], 1) + .unwrap(); + assert!(!plan.native_plan.children()[0].is::()); + assert!(plan.additional_native_plans.is_empty()); + // The inner sort, over an unordered scan, still is. + assert!(plan.children[0].native_plan.children()[0].is::()); +} diff --git a/native/operators/Cargo.toml b/native/operators/Cargo.toml index 07c7f61e0b9..15240456359 100644 --- a/native/operators/Cargo.toml +++ b/native/operators/Cargo.toml @@ -51,3 +51,7 @@ harness = false [[bench]] name = "window" harness = false + +[[bench]] +name = "sort_small_batches" +harness = false diff --git a/native/operators/benches/sort_small_batches.rs b/native/operators/benches/sort_small_batches.rs new file mode 100644 index 00000000000..a3812c04de5 --- /dev/null +++ b/native/operators/benches/sort_small_batches.rs @@ -0,0 +1,171 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Micro-benchmarks for `SortInputCoalesceExec` in front of DataFusion's `SortExec`. +//! +//! A reduce task that reads a shuffle from many map tasks gets one small batch per block. The +//! sort merges one stream per buffered batch, so its cost grows with the number of batches. +//! This compares the sort with and without combining its input, over a range of input batch +//! sizes, with enough memory to sort in memory and with a memory limit that forces spills, and +//! with one sort key and with two. DataFusion combines runs itself only for one key in memory. +//! Keys arrive either scattered, so every batch spans the whole key range, or already in order, +//! the most predictable merge. Keys have a fixed length and no value is null. + +use std::sync::Arc; + +use arrow::array::{ArrayRef, Int64Array, StringArray, StructArray}; +use arrow::compute::SortOptions; +use arrow::datatypes::{DataType, Field, Fields}; +use arrow::record_batch::RecordBatch; +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; +use datafusion::datasource::memory::MemorySourceConfig; +use datafusion::execution::runtime_env::RuntimeEnvBuilder; +use datafusion::physical_expr::expressions::col; +use datafusion::physical_expr::{LexOrdering, PhysicalSortExpr}; +use datafusion::physical_plan::sorts::sort::SortExec; +use datafusion::physical_plan::{common::collect, ExecutionPlan}; +use datafusion::prelude::{SessionConfig, SessionContext}; +use datafusion_comet_operators::SortInputCoalesceExec; +use tokio::runtime::Runtime; + +const ROWS: usize = 200_000; +/// The session batch size, which is also the combined batch's row target. +const BATCH_SIZE: usize = 8192; +/// About a fifth of the input, so the sort spills several times. +const SPILL_MEMORY_LIMIT: usize = 24 * 1024 * 1024; + +/// A string key, a low-cardinality group key, and wide rows: eight longs and a struct, like an +/// aggregate's state. +fn batches(rows_per_batch: usize, scattered: bool) -> Vec { + let fields = Fields::from(vec![ + Field::new("a", DataType::Int64, true), + Field::new("b", DataType::Utf8, true), + ]); + (0..ROWS) + .step_by(rows_per_batch) + .map(|start| { + let rows = rows_per_batch.min(ROWS - start); + let ids: Vec = (start..start + rows) + .map(|i| { + if scattered { + i.wrapping_mul(2_654_435_761) % ROWS + } else { + i + } + }) + .collect(); + let mut columns: Vec<(String, ArrayRef)> = vec![( + "key".into(), + Arc::new(StringArray::from_iter_values( + ids.iter().map(|i| format!("https://example.com/{i:08}")), + )), + )]; + columns.push(( + "g".into(), + Arc::new(Int64Array::from_iter_values( + ids.iter().map(|&i| (i % 64) as i64), + )), + )); + for c in 0..8 { + columns.push(( + format!("v{c}"), + Arc::new(Int64Array::from_iter_values( + ids.iter().map(|&i| (i * (c + 1)) as i64), + )), + )); + } + columns.push(( + "s".into(), + Arc::new(StructArray::new( + fields.clone(), + vec![ + Arc::new(Int64Array::from_iter_values(ids.iter().map(|&i| i as i64))), + Arc::new(StringArray::from_iter_values( + ids.iter().map(|i| format!("value-{}", i % 1000)), + )), + ], + None, + )), + )); + RecordBatch::try_from_iter(columns).unwrap() + }) + .collect() +} + +fn sort( + rt: &Runtime, + input: &[RecordBatch], + keys: &[&str], + coalesce: bool, + memory_limit: Option, +) { + let schema = input[0].schema(); + let mut runtime = RuntimeEnvBuilder::new(); + if let Some(limit) = memory_limit { + runtime = runtime.with_memory_limit(limit, 1.0); + } + let config = SessionConfig::new() + .with_batch_size(BATCH_SIZE) + .with_sort_spill_reservation_bytes(1024 * 1024); + let ctx = SessionContext::new_with_config_rt(config.clone(), runtime.build_arc().unwrap()); + let source = + MemorySourceConfig::try_new_exec(&[input.to_vec()], Arc::clone(&schema), None).unwrap(); + let sort_input: Arc = if coalesce { + let gate = config.options().execution.sort_in_place_threshold_bytes; + Arc::new(SortInputCoalesceExec::new(source, BATCH_SIZE, gate)) + } else { + source + }; + let ordering = + LexOrdering::new(keys.iter().map(|name| { + PhysicalSortExpr::new(col(name, &schema).unwrap(), SortOptions::default()) + })) + .unwrap(); + let sort = Arc::new(SortExec::new(ordering, sort_input)); + let output = rt + .block_on(collect(sort.execute(0, ctx.task_ctx()).unwrap())) + .unwrap(); + assert_eq!(output.iter().map(|b| b.num_rows()).sum::(), ROWS); +} + +fn bench(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + let key_sets: [(&str, &[&str]); 2] = [("one_key", &["key"]), ("two_keys", &["g", "key"])]; + for (name, memory_limit) in [("in_memory", None), ("spill", Some(SPILL_MEMORY_LIMIT))] { + for (keys_name, keys) in key_sets { + for (shape, scattered) in [("scattered", true), ("in_order", false)] { + let mut group = + c.benchmark_group(format!("sort_small_batches/{name}/{keys_name}/{shape}")); + group.sample_size(10); + for rows_per_batch in [16, 64, 256, 384, 512, 1024, 8192] { + let input = batches(rows_per_batch, scattered); + for (label, coalesce) in [("raw", false), ("coalesced", true)] { + group.bench_with_input( + BenchmarkId::new(label, rows_per_batch), + &input, + |b, input| b.iter(|| sort(&rt, input, keys, coalesce, memory_limit)), + ); + } + } + group.finish(); + } + } + } +} + +criterion_group!(benches, bench); +criterion_main!(benches); diff --git a/native/operators/src/lib.rs b/native/operators/src/lib.rs index 05a0324567c..72a973c5c7f 100644 --- a/native/operators/src/lib.rs +++ b/native/operators/src/lib.rs @@ -34,6 +34,7 @@ mod projection; mod range; mod rank_limit; mod sample; +mod sort_input_coalesce; pub use expand::ExpandExec; pub use explode::ExplodeExec; @@ -43,3 +44,4 @@ pub use projection::CometProjectionExec; pub use range::{range_exec, range_sort_expr}; pub use rank_limit::{PartitionedRankLimitExec, WindowFnKind}; pub use sample::SampleExec; +pub use sort_input_coalesce::SortInputCoalesceExec; diff --git a/native/operators/src/sort_input_coalesce.rs b/native/operators/src/sort_input_coalesce.rs new file mode 100644 index 00000000000..544992e37d2 --- /dev/null +++ b/native/operators/src/sort_input_coalesce.rs @@ -0,0 +1,1160 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Combines small input batches before a native sort. +//! +//! On its spill path DataFusion's sort merges one stream per buffered batch, so thousands of +//! tiny shuffle blocks make it very slow. Below the in-place threshold the sort concatenates its +//! input itself, so batches pass through until a partition's input reaches that threshold. The +//! gate never closes again, so a sort whose share is below the threshold gets a copy it does not +//! need. +//! +//! This sits in front of the sort rather than in the shuffle reader because only the sort's merge +//! cost grows with the number of batches. Arrow's `BatchCoalescer` has no byte limit and copies +//! as batches arrive, and `DecodedBatchCoalescer` takes only flat types, so neither lets the held +//! memory be reserved exactly for any input. + +use std::fmt::Formatter; +use std::sync::Arc; + +use arrow::compute::concat_batches; +use arrow::datatypes::DataType; +use arrow::record_batch::RecordBatch; +use datafusion::common::tree_node::TreeNodeRecursion; +use datafusion::common::utils::memory::get_record_batch_memory_size; +use datafusion::common::{DataFusionError, Result}; +use datafusion::execution::memory_pool::{MemoryConsumer, MemoryReservation}; +use datafusion::execution::{async_try_stream, TaskContext}; +use datafusion::physical_expr::PhysicalExpr; +use datafusion::physical_plan::execution_plan::CardinalityEffect; +use datafusion::physical_plan::metrics::{ + BaselineMetrics, Count, ExecutionPlanMetricsSet, MetricBuilder, MetricsSet, +}; +use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +use datafusion::physical_plan::{ + DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, SendableRecordBatchStream, +}; +use futures::TryStreamExt; + +/// The cap is half the in-place threshold because the sort reserves about twice a batch's size. +/// The bounds also keep string and binary offsets far below the i32 limit. +const MIN_GROUP_BYTES: usize = 64 * 1024; +const MAX_GROUP_BYTES: usize = 4 * 1024 * 1024; + +/// In `sort_small_batches`, combining 512-row batches made a spilling single-key sort slower, +/// while 384 rows or fewer was never slower. +const MAX_SMALL_BATCH_ROWS: usize = 512; + +/// Small batches passed through after a refusal, so a pool under pressure is not asked again for +/// each one. +const REFUSED_BACKOFF_BATCHES: usize = 64; + +#[derive(Debug)] +pub struct SortInputCoalesceExec { + input: Arc, + target_rows: usize, + gate_bytes: usize, + max_bytes: usize, + metrics: ExecutionPlanMetricsSet, +} + +impl SortInputCoalesceExec { + /// `gate_bytes` is the sort's in-place threshold. + pub fn new(input: Arc, target_rows: usize, gate_bytes: usize) -> Self { + Self { + input, + target_rows: target_rows.max(1), + gate_bytes, + max_bytes: (gate_bytes / 2).clamp(MIN_GROUP_BYTES, MAX_GROUP_BYTES), + metrics: ExecutionPlanMetricsSet::new(), + } + } + + #[cfg(test)] + fn with_max_bytes(mut self, max_bytes: usize) -> Self { + self.max_bytes = max_bytes; + self + } + + pub fn input(&self) -> &Arc { + &self.input + } + + /// The plan below a coalescer, or `plan` itself when it is not one. + pub fn unwrap_input(plan: &Arc) -> &Arc { + plan.downcast_ref::().map_or(plan, Self::input) + } +} + +impl DisplayAs for SortInputCoalesceExec { + fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { + write!( + f, + "CometSortInputCoalesceExec: target_rows={}, gate_bytes={}, max_bytes={}", + self.target_rows, self.gate_bytes, self.max_bytes + ) + } +} + +impl ExecutionPlan for SortInputCoalesceExec { + fn name(&self) -> &str { + "CometSortInputCoalesceExec" + } + + fn properties(&self) -> &Arc { + self.input.properties() + } + + fn maintains_input_order(&self) -> Vec { + vec![true] + } + + fn children(&self) -> Vec<&Arc> { + vec![&self.input] + } + + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> Result, + ) -> Result { + Ok(TreeNodeRecursion::Continue) + } + + fn cardinality_effect(&self) -> CardinalityEffect { + CardinalityEffect::Equal + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + let [input]: [_; 1] = children.try_into().map_err(|_| { + DataFusionError::Internal("Sort input coalescing requires one child".into()) + })?; + Ok(Arc::new(Self { + input, + target_rows: self.target_rows, + gate_bytes: self.gate_bytes, + max_bytes: self.max_bytes, + // Shared, so the copy reports through the node the Spark plan registered for its + // metrics if a rewrite rebuilds the sort's input. + metrics: self.metrics.clone(), + })) + } + + fn execute( + &self, + partition: usize, + context: Arc, + ) -> Result { + let mut input = self.input.execute(partition, Arc::clone(&context))?; + let reservation = MemoryConsumer::new(format!("SortInputCoalesceExec[{partition}]")) + .register(context.memory_pool()); + let baseline = BaselineMetrics::new(&self.metrics, partition); + let mut grouper = Grouper::new(self, reservation, partition); + let stream = async_try_stream(|mut out| async move { + while let Some(batch) = input.try_next().await? { + // A group that has to go out first is emitted before the new batch is held, so + // the sort reserves it while this operator holds nothing. + let (step, flushed) = { + let _timer = baseline.elapsed_compute().timer(); + grouper.prepare(&batch)? + }; + if let Some(batch) = flushed { + baseline.record_output(batch.num_rows()); + out.emit(batch).await; + } + let ready = { + let _timer = baseline.elapsed_compute().timer(); + grouper.add(batch, step)? + }; + for batch in ready.into_iter().flatten() { + baseline.record_output(batch.num_rows()); + out.emit(batch).await; + } + } + let last = { + let _timer = baseline.elapsed_compute().timer(); + grouper.flush()? + }; + if let Some(batch) = last { + baseline.record_output(batch.num_rows()); + out.emit(batch).await; + } + Ok(()) + }); + Ok(Box::pin(RecordBatchStreamAdapter::new( + self.schema(), + stream, + ))) + } + + fn metrics(&self) -> Option { + Some(self.metrics.clone_inner()) + } +} + +/// Concatenating dictionaries merges their values, which can overflow a small key type where +/// the batches on their own fit. Dictionaries nested in structs, lists and maps count too. +fn has_small_dictionary_keys(batch: &RecordBatch) -> bool { + fn small(data_type: &DataType) -> bool { + match data_type { + DataType::Dictionary(key, value) => { + key.primitive_width().is_some_and(|width| width < 4) || small(value) + } + DataType::Struct(fields) => fields.iter().any(|f| small(f.data_type())), + DataType::List(f) + | DataType::LargeList(f) + | DataType::ListView(f) + | DataType::LargeListView(f) + | DataType::FixedSizeList(f, _) + | DataType::Map(f, _) => small(f.data_type()), + // Run ends count rows, so a small run-end type can overflow too. + DataType::RunEndEncoded(run_ends, values) => { + run_ends + .data_type() + .primitive_width() + .is_some_and(|width| width < 4) + || small(values.data_type()) + } + _ => false, + } + } + batch.schema().fields().iter().any(|f| small(f.data_type())) +} + +/// The bytes the batch's rows take, without unused buffer capacity, as DataFusion's sort counts +/// it. +fn sliced_size(batch: &RecordBatch) -> Result { + batch.columns().iter().try_fold(0usize, |total, column| { + Ok(total + column.to_data().get_slice_memory_size()?) + }) +} + +enum Step { + Skip, + PassThrough, + /// Size in bytes. + Hold(usize), +} + +/// Groups whole input batches in arrival order and concatenates each group. Inputs are never +/// split, so the reservation always covers the batches being held. +struct Grouper { + target_rows: usize, + small_rows: usize, + gate_bytes: usize, + max_bytes: usize, + seen_bytes: usize, + past_gate: bool, + pending: Vec, + rows: usize, + bytes: usize, + reservation: MemoryReservation, + backoff: usize, + input_batches: Count, + output_batches: Count, +} + +impl Grouper { + fn new(exec: &SortInputCoalesceExec, reservation: MemoryReservation, partition: usize) -> Self { + let metrics = MetricBuilder::new(&exec.metrics); + Self { + target_rows: exec.target_rows, + small_rows: exec.target_rows.div_ceil(2).min(MAX_SMALL_BATCH_ROWS), + gate_bytes: exec.gate_bytes, + max_bytes: exec.max_bytes, + seen_bytes: 0, + past_gate: exec.gate_bytes == 0, + pending: Vec::new(), + rows: 0, + bytes: 0, + reservation, + backoff: 0, + input_batches: metrics + .clone() + .counter("coalesced_input_batches", partition), + output_batches: metrics.counter("coalesced_output_batches", partition), + } + } + + /// Decides what `add` does with `batch`. Returns the held group too when it has to go out + /// first: before a batch that passes through, or before one that does not fit in the group. + fn prepare(&mut self, batch: &RecordBatch) -> Result<(Step, Option)> { + let rows = batch.num_rows(); + if rows == 0 { + return Ok((Step::Skip, None)); + } + // A held batch is sized by every buffer it keeps alive, so a small slice of a large batch + // counts as large and passes through. + let bytes = if !self.past_gate { + // Counted as DataFusion's `get_reserved_bytes_for_record_batch` counts it, which the + // sort compares with the threshold. + self.seen_bytes = self + .seen_bytes + .saturating_add(get_record_batch_memory_size(batch) + sliced_size(batch)?); + self.past_gate = self.seen_bytes >= self.gate_bytes; + None + } else if rows >= self.small_rows || has_small_dictionary_keys(batch) { + None + } else if self.backoff > 0 { + self.backoff -= 1; + None + } else { + Some(get_record_batch_memory_size(batch)) + }; + match bytes.filter(|&bytes| bytes <= self.max_bytes) { + Some(bytes) => { + // A batch with another schema starts a new group, since the concat needs one. + let fits = rows <= self.target_rows - self.rows + && bytes <= self.max_bytes - self.bytes + && self + .pending + .first() + .is_none_or(|held| held.schema() == batch.schema()); + let flushed = if fits { None } else { self.flush()? }; + Ok((Step::Hold(bytes), flushed)) + } + None => Ok((Step::PassThrough, self.flush()?)), + } + } + + /// Returns what is ready, in input order: a flushed group, then `batch` when it is not held. + fn add(&mut self, batch: RecordBatch, step: Step) -> Result<[Option; 2]> { + let bytes = match step { + Step::Skip => return Ok([None, None]), + Step::PassThrough => return Ok([None, Some(batch)]), + Step::Hold(bytes) => bytes, + }; + let rows = batch.num_rows(); + if !self.reserve(bytes)? { + // The sort can spill and this cannot, so hand it everything. + self.backoff = REFUSED_BACKOFF_BATCHES; + return Ok([self.flush()?, Some(batch)]); + } + self.pending.push(batch); + self.rows += rows; + self.bytes += bytes; + if self.rows == self.target_rows || self.bytes == self.max_bytes { + return Ok([self.flush()?, None]); + } + Ok([None, None]) + } + + #[cfg(test)] + fn push(&mut self, batch: RecordBatch) -> Result> { + let (step, flushed) = self.prepare(&batch)?; + let mut ready: Vec<_> = flushed.into_iter().collect(); + ready.extend(self.add(batch, step)?.into_iter().flatten()); + Ok(ready) + } + + /// Reserves the held bytes plus `bytes`, twice over for the combined copy, and no more: a + /// consumer that cannot spill is checked only against the whole pool, so asking ahead could + /// make Spark spill another consumer for memory never used. Returns false when refused. + fn reserve(&mut self, bytes: usize) -> Result { + let needed = 2 * (self.bytes + bytes); + let size = self.reservation.size(); + if needed <= size { + return Ok(true); + } + match self.reservation.try_grow(needed - size) { + Ok(()) => Ok(true), + Err(DataFusionError::ResourcesExhausted(_)) => Ok(false), + Err(error) => Err(error), + } + } + + fn flush(&mut self) -> Result> { + let out = match self.pending.len() { + 0 => None, + 1 => self.pending.pop(), + n => { + // The inputs' own schema, which a batch that passes through also keeps. + let combined = concat_batches(&self.pending[0].schema(), &self.pending)?; + self.pending.clear(); + self.input_batches.add(n); + self.output_batches.add(1); + Some(combined) + } + }; + // The sort reserves what it buffers, so hold nothing between groups. + self.reservation.free(); + self.rows = 0; + self.bytes = 0; + Ok(out) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{ + Array, ArrayRef, AsArray, Date32Array, Decimal128Array, Int32Array, Int64Array, ListArray, + MapArray, StringArray, StringDictionaryBuilder, StructArray, TimestampMicrosecondArray, + }; + use arrow::datatypes::{DataType, Field, Fields, Int32Type, Int64Type, Schema, SchemaRef}; + use datafusion::datasource::memory::MemorySourceConfig; + use datafusion::execution::memory_pool::{GreedyMemoryPool, MemoryPool}; + use datafusion::execution::runtime_env::RuntimeEnvBuilder; + use datafusion::physical_plan::common::collect; + use datafusion::prelude::{SessionConfig, SessionContext}; + + fn ints(values: std::ops::Range) -> RecordBatch { + RecordBatch::try_from_iter([( + "k", + Arc::new(Int32Array::from_iter_values(values)) as ArrayRef, + )]) + .unwrap() + } + + fn ten_row_batches(n: i32) -> Vec { + (0..n).map(|i| ints(i * 10..i * 10 + 10)).collect() + } + + fn run_with( + input: Vec, + schema: SchemaRef, + exec: impl FnOnce(Arc) -> SortInputCoalesceExec, + ctx: SessionContext, + ) -> (Vec, MetricsSet) { + let source = MemorySourceConfig::try_new_exec(&[input], schema, None).unwrap(); + let exec = exec(source); + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + let output = rt + .block_on(collect(exec.execute(0, ctx.task_ctx()).unwrap())) + .unwrap(); + (output, exec.metrics().unwrap()) + } + + fn run( + input: Vec, + target_rows: usize, + gate_bytes: usize, + max_bytes: usize, + ) -> (Vec, MetricsSet) { + let schema = input[0].schema(); + run_with( + input, + schema, + |source| { + SortInputCoalesceExec::new(source, target_rows, gate_bytes) + .with_max_bytes(max_bytes) + }, + SessionContext::new(), + ) + } + + fn row_counts(batches: &[RecordBatch]) -> Vec { + batches.iter().map(|b| b.num_rows()).collect() + } + + fn same_rows(output: &[RecordBatch], input: &[RecordBatch]) { + let schema = input[0].schema(); + assert_eq!( + concat_batches(&schema, output).unwrap(), + concat_batches(&schema, input).unwrap() + ); + } + + fn metric(metrics: &MetricsSet, name: &str) -> usize { + metrics.sum_by_name(name).map(|v| v.as_usize()).unwrap_or(0) + } + + #[test] + fn small_partitions_pass_through_without_a_copy() { + let input = ten_row_batches(10); + let (output, _) = run(input.clone(), 1024, usize::MAX, MAX_GROUP_BYTES); + assert_eq!(output, input); + for (out, inp) in output.iter().zip(&input) { + assert!(Arc::ptr_eq(out.column(0), inp.column(0))); + } + } + + #[test] + fn combines_small_batches_after_the_gate_in_input_order() { + let input = ten_row_batches(10); + // A zero gate starts combining with the first batch. + let (output, metrics) = run(input.clone(), 40, 0, MAX_GROUP_BYTES); + assert_eq!(row_counts(&output), [40, 40, 20]); + same_rows(&output, &input); + assert_eq!(metric(&metrics, "coalesced_input_batches"), 10); + assert_eq!(metric(&metrics, "coalesced_output_batches"), 3); + assert_eq!(metrics.output_rows(), Some(100)); + } + + #[test] + fn a_rebuilt_coalescer_reports_through_the_original() { + let input = ten_row_batches(10); + let schema = input[0].schema(); + let source = MemorySourceConfig::try_new_exec(&[input], schema, None).unwrap(); + let original = Arc::new(SortInputCoalesceExec::new( + Arc::clone(&source) as Arc, + 40, + 0, + )); + let rebuilt = (Arc::clone(&original) as Arc) + .replace_children( + vec![source], + datafusion::physical_plan::ReplaceChildrenOptions::new( + datafusion::physical_plan::ChildrenPropertiesMode::Recompute, + ), + ) + .unwrap(); + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + rt.block_on(collect( + rebuilt + .execute(0, SessionContext::new().task_ctx()) + .unwrap(), + )) + .unwrap(); + let metrics = original.metrics().unwrap(); + assert_eq!(metric(&metrics, "coalesced_input_batches"), 10); + } + + #[test] + fn the_gate_counts_bytes_before_combining() { + let input = ten_row_batches(10); + // The gate counts each batch as the sort does: memory size plus sliced size. + let one = get_record_batch_memory_size(&input[0]) + sliced_size(&input[0]).unwrap(); + // The first three batches pass through; the third one crosses the gate. + let (output, _) = run(input.clone(), 40, one * 2 + 1, MAX_GROUP_BYTES); + assert_eq!(row_counts(&output), [10, 10, 10, 40, 30]); + same_rows(&output, &input); + } + + #[test] + fn byte_cap_flushes_before_the_row_target() { + let input = ten_row_batches(6); + let one = get_record_batch_memory_size(&input[0]); + let (output, _) = run(input.clone(), 1000, 0, one * 2); + assert_eq!(row_counts(&output), [20, 20, 20]); + same_rows(&output, &input); + } + + #[test] + fn large_batches_pass_through_after_flushing_the_group() { + let small = ints(0..10); + let large = ints(100..150); + let input = vec![small.clone(), small.clone(), large.clone(), small.clone()]; + let (output, _) = run(input.clone(), 100, 0, MAX_GROUP_BYTES); + assert_eq!(row_counts(&output), [20, 50, 10]); + assert!(Arc::ptr_eq(output[1].column(0), large.column(0))); + same_rows(&output, &input); + } + + #[test] + fn batches_of_512_rows_or_more_pass_through_at_the_default_target() { + let mid = ints(0..MAX_SMALL_BATCH_ROWS as i32); + let small = ints(0..(MAX_SMALL_BATCH_ROWS - 1) as i32); + let input = vec![mid.clone(), mid.clone(), small.clone(), small.clone()]; + let (output, _) = run(input.clone(), 8192, 0, MAX_GROUP_BYTES); + assert_eq!(row_counts(&output), [512, 512, 1022]); + assert!(Arc::ptr_eq(output[0].column(0), mid.column(0))); + same_rows(&output, &input); + } + + #[test] + fn empty_input_and_empty_batches() { + let schema = ints(0..1).schema(); + let (output, _) = run_with( + vec![], + Arc::clone(&schema), + |source| SortInputCoalesceExec::new(source, 8, 0), + SessionContext::new(), + ); + assert!(output.is_empty()); + + let input = vec![ints(0..0), ints(0..3), ints(0..0), ints(3..5)]; + let (output, _) = run(input.clone(), 8, 0, MAX_GROUP_BYTES); + assert_eq!(row_counts(&output), [5]); + same_rows(&output, &input); + } + + #[test] + fn passes_through_when_memory_is_short() { + let input = ten_row_batches(4); + let schema = input[0].schema(); + let pool: Arc = Arc::new(GreedyMemoryPool::new(1)); + let runtime = RuntimeEnvBuilder::new() + .with_memory_pool(Arc::clone(&pool)) + .build_arc() + .unwrap(); + let ctx = SessionContext::new_with_config_rt(SessionConfig::new(), runtime); + let (output, _) = run_with( + input.clone(), + schema, + |source| SortInputCoalesceExec::new(source, 40, 0), + ctx, + ); + assert_eq!(output, input); + assert_eq!(pool.reserved(), 0); + } + + #[test] + fn the_group_cap_is_half_the_in_place_threshold() { + let cap = |gate| { + let source = + MemorySourceConfig::try_new_exec(&[vec![]], ints(0..1).schema(), None).unwrap(); + SortInputCoalesceExec::new(source, 8192, gate).max_bytes + }; + assert_eq!(cap(1024 * 1024), 512 * 1024); + assert_eq!(cap(0), MIN_GROUP_BYTES); + assert_eq!(cap(usize::MAX), MAX_GROUP_BYTES); + } + + #[test] + fn reservation_covers_a_group_and_is_released_after_it() { + let input = ten_row_batches(3); + let pool: Arc = Arc::new(GreedyMemoryPool::new(1 << 30)); + let source = MemorySourceConfig::try_new_exec(&[vec![]], input[0].schema(), None).unwrap(); + let exec = SortInputCoalesceExec::new(source, 1000, 0); + let mut grouper = Grouper::new(&exec, MemoryConsumer::new("test").register(&pool), 0); + // The reservation covers exactly the held batches and their copy. + let mut held = 0; + for batch in &input { + held += get_record_batch_memory_size(batch); + assert!(grouper.push(batch.clone()).unwrap().is_empty()); + assert_eq!(pool.reserved(), 2 * held); + } + let combined = grouper.flush().unwrap().unwrap(); + assert_eq!( + combined, + concat_batches(&input[0].schema(), &input).unwrap() + ); + // The sort reserves the combined batch itself. + assert_eq!(pool.reserved(), 0); + assert!(grouper.flush().unwrap().is_none()); + } + + fn run_in_pool( + input: Vec, + target_rows: usize, + pool_bytes: usize, + ) -> Vec { + let schema = input[0].schema(); + let pool: Arc = Arc::new(GreedyMemoryPool::new(pool_bytes)); + let runtime = RuntimeEnvBuilder::new() + .with_memory_pool(Arc::clone(&pool)) + .build_arc() + .unwrap(); + let ctx = SessionContext::new_with_config_rt(SessionConfig::new(), runtime); + let (output, _) = run_with( + input, + schema, + |source| SortInputCoalesceExec::new(source, target_rows, 0), + ctx, + ); + assert_eq!(pool.reserved(), 0); + output + } + + #[test] + fn combines_when_the_pool_holds_the_group_but_not_a_full_one() { + let input = ten_row_batches(3); + let one = get_record_batch_memory_size(&input[0]); + // Room for these three batches and their copy, far less than a full group. + let output = run_in_pool(input.clone(), 30, 2 * 3 * one); + assert_eq!(row_counts(&output), [30]); + same_rows(&output, &input); + } + + #[test] + fn a_refused_reservation_flushes_the_held_group_first() { + let input = ten_row_batches(4); + let one = get_record_batch_memory_size(&input[0]); + // Room for two batches and their copy. The third is refused, so the first two go out + // combined and the third passes through. The fourth starts a new group. + let output = run_in_pool(input.clone(), 1000, 2 * 2 * one); + assert_eq!(row_counts(&output), [20, 10, 10]); + same_rows(&output, &input); + } + + #[tokio::test] + async fn holds_no_memory_while_the_sort_takes_a_batch() { + let input = ten_row_batches(10); + let schema = input[0].schema(); + let pool: Arc = Arc::new(GreedyMemoryPool::new(1 << 30)); + let runtime = RuntimeEnvBuilder::new() + .with_memory_pool(Arc::clone(&pool)) + .build_arc() + .unwrap(); + let ctx = SessionContext::new_with_config_rt(SessionConfig::new(), runtime); + let source = MemorySourceConfig::try_new_exec(&[input], schema, None).unwrap(); + // A 25-row target makes every third batch overflow the group. + let exec = SortInputCoalesceExec::new(source, 25, 0); + let mut stream = exec.execute(0, ctx.task_ctx()).unwrap(); + let mut rows = vec![]; + while let Some(batch) = stream.try_next().await.unwrap() { + assert_eq!(pool.reserved(), 0, "held memory while handing on a batch"); + rows.push(batch.num_rows()); + } + assert_eq!(rows, [20, 20, 20, 20, 20]); + } + + #[test] + fn after_a_refusal_the_next_batches_pass_through() { + let input = ten_row_batches(6); + let one = get_record_batch_memory_size(&input[0]); + // Room for two batches and their copy, so the third is refused. The ones after it pass + // through without asking the pool again. + let output = run_in_pool(input.clone(), 1000, 2 * 2 * one); + assert_eq!(row_counts(&output), [20, 10, 10, 10, 10]); + same_rows(&output, &input); + } + + #[test] + fn combining_resumes_after_the_backoff() { + let input = ten_row_batches(70); + let one = get_record_batch_memory_size(&input[0]); + // Room for two batches and their copy. The third is refused, it and the next 64 pass + // through, and then two more are combined before the next refusal. + let output = run_in_pool(input.clone(), 1000, 2 * 2 * one); + let mut expected = vec![20]; + expected.extend([10; 1 + REFUSED_BACKOFF_BATCHES]); + expected.extend([20, 10]); + assert_eq!(row_counts(&output), expected); + same_rows(&output, &input); + } + + #[test] + fn a_small_slice_of_a_large_batch_passes_through() { + // The size counts the whole buffer the slice shares, which is over the cap. + let slice = ints(0..100_000).slice(0, 10); + let input = vec![slice.clone(), slice.clone()]; + let output = run_in_pool(input.clone(), 1000, 1 << 30); + assert_eq!(row_counts(&output), [10, 10]); + assert!(Arc::ptr_eq(output[0].column(0), slice.column(0))); + } + + #[test] + fn a_schema_change_starts_a_new_group() { + let nullable = |values: std::ops::Range| { + let schema = Arc::new(Schema::new(vec![Field::new("k", DataType::Int32, true)])); + RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from_iter_values(values))]) + .unwrap() + }; + let input = vec![ + ints(0..10), + ints(10..20), + nullable(20..30), + nullable(30..40), + ]; + let (output, _) = run(input.clone(), 1000, 0, MAX_GROUP_BYTES); + assert_eq!(row_counts(&output), [20, 20]); + assert_eq!(output[0].schema(), input[0].schema()); + assert_eq!(output[1].schema(), input[2].schema()); + } + + #[test] + fn batches_with_small_dictionary_keys_pass_through() { + use arrow::datatypes::Int8Type; + let batch = |start: usize| { + let mut dict = StringDictionaryBuilder::::new(); + for i in start..start + 10 { + dict.append_value(format!("v{i}")); + } + RecordBatch::try_from_iter([("d", Arc::new(dict.finish()) as ArrayRef)]).unwrap() + }; + let input: Vec<_> = (0..20).map(|i| batch(i * 10)).collect(); + let (output, _) = run(input.clone(), 1000, 0, MAX_GROUP_BYTES); + assert_eq!(output, input); + } + + #[test] + fn batches_with_small_dictionary_keys_in_a_struct_pass_through() { + use arrow::datatypes::Int8Type; + let batch = |start: usize| { + let mut dict = StringDictionaryBuilder::::new(); + for i in start..start + 10 { + dict.append_value(format!("v{i}")); + } + let dict: ArrayRef = Arc::new(dict.finish()); + let field = Field::new("d", dict.data_type().clone(), false); + let column = StructArray::new(Fields::from(vec![field]), vec![dict], None); + RecordBatch::try_from_iter([("s", Arc::new(column) as ArrayRef)]).unwrap() + }; + // Combined, the 200 distinct values would overflow the Int8 keys. + let input: Vec<_> = (0..20).map(|i| batch(i * 10)).collect(); + let (output, _) = run(input.clone(), 1000, 0, MAX_GROUP_BYTES); + assert_eq!(output, input); + } + + #[test] + fn batches_with_small_dictionary_keys_in_a_list_pass_through() { + use arrow::array::ListBuilder; + use arrow::datatypes::Int8Type; + let batch = |start: usize| { + let mut list = ListBuilder::new(StringDictionaryBuilder::::new()); + for i in start..start + 10 { + list.values().append_value(format!("v{i}")); + list.append(true); + } + RecordBatch::try_from_iter([("l", Arc::new(list.finish()) as ArrayRef)]).unwrap() + }; + let input: Vec<_> = (0..20).map(|i| batch(i * 10)).collect(); + let (output, _) = run(input.clone(), 1000, 0, MAX_GROUP_BYTES); + assert_eq!(output, input); + } + + #[tokio::test] + async fn an_input_error_reaches_the_sort_and_releases_memory() { + use datafusion::physical_plan::streaming::{PartitionStream, StreamingTableExec}; + + #[derive(Debug)] + struct FailsAfterTwo(SchemaRef); + impl PartitionStream for FailsAfterTwo { + fn schema(&self) -> &SchemaRef { + &self.0 + } + fn execute(&self, _ctx: Arc) -> SendableRecordBatchStream { + let items = vec![ + Ok(ints(0..10)), + Ok(ints(10..20)), + Err(DataFusionError::Execution("input failed".into())), + ]; + Box::pin(RecordBatchStreamAdapter::new( + Arc::clone(&self.0), + futures::stream::iter(items), + )) + } + } + + let schema = ints(0..1).schema(); + let source = Arc::new( + StreamingTableExec::try_new( + Arc::clone(&schema), + vec![Arc::new(FailsAfterTwo(Arc::clone(&schema)))], + None, + vec![], + false, + None, + ) + .unwrap(), + ); + let pool: Arc = Arc::new(GreedyMemoryPool::new(1 << 30)); + let runtime = RuntimeEnvBuilder::new() + .with_memory_pool(Arc::clone(&pool)) + .build_arc() + .unwrap(); + let ctx = SessionContext::new_with_config_rt(SessionConfig::new(), runtime); + let exec = SortInputCoalesceExec::new(source, 1000, 0); + let error = collect(exec.execute(0, ctx.task_ctx()).unwrap()) + .await + .unwrap_err(); + assert!(error.to_string().contains("input failed"), "{error}"); + assert_eq!(pool.reserved(), 0); + } + + #[tokio::test] + async fn dropping_the_stream_early_releases_the_reservation() { + let input = ten_row_batches(10); + let schema = input[0].schema(); + let pool: Arc = Arc::new(GreedyMemoryPool::new(1 << 30)); + let runtime = RuntimeEnvBuilder::new() + .with_memory_pool(Arc::clone(&pool)) + .build_arc() + .unwrap(); + let ctx = SessionContext::new_with_config_rt(SessionConfig::new(), runtime); + let source = MemorySourceConfig::try_new_exec(&[input], schema, None).unwrap(); + let exec = SortInputCoalesceExec::new(source, 40, 0); + let mut stream = exec.execute(0, ctx.task_ctx()).unwrap(); + // The first group is out and the second is being held. + assert_eq!(stream.try_next().await.unwrap().unwrap().num_rows(), 40); + drop(stream); + assert_eq!(pool.reserved(), 0); + } + + /// Sorts `input` by `keys` (column name, descending) under a memory limit that forces + /// spills. + async fn spilling_sort( + input: Vec, + keys: &[(&str, bool)], + coalesce: bool, + ) -> (Vec, MetricsSet, Option) { + use datafusion::physical_expr::expressions::col; + use datafusion::physical_expr::{LexOrdering, PhysicalSortExpr}; + use datafusion::physical_plan::sorts::sort::SortExec; + + let schema = input[0].schema(); + let runtime = RuntimeEnvBuilder::new() + .with_memory_limit(2 * 1024 * 1024, 1.0) + .build_arc() + .unwrap(); + let config = SessionConfig::new() + .with_batch_size(256) + .with_sort_spill_reservation_bytes(64 * 1024) + .with_sort_in_place_threshold_bytes(64 * 1024); + let ctx = SessionContext::new_with_config_rt(config, runtime); + let source = MemorySourceConfig::try_new_exec(&[input], Arc::clone(&schema), None).unwrap(); + let (sort_input, coalescer): (Arc, _) = if coalesce { + let coalescer = Arc::new(SortInputCoalesceExec::new(source, 256, 64 * 1024)); + (Arc::clone(&coalescer) as _, Some(coalescer)) + } else { + (source, None) + }; + let ordering = LexOrdering::new(keys.iter().map(|&(name, descending)| { + PhysicalSortExpr::new( + col(name, &schema).unwrap(), + arrow::compute::SortOptions { + descending, + nulls_first: false, + }, + ) + })) + .unwrap(); + let sort = Arc::new(SortExec::new(ordering, sort_input)); + let output = collect(sort.execute(0, ctx.task_ctx()).unwrap()) + .await + .unwrap(); + ( + output, + sort.metrics().unwrap(), + coalescer.map(|c| c.metrics().unwrap()), + ) + } + + async fn check_spilling_sort(keys: &[(&str, bool)]) { + // 24,000 wide rows in 8-row batches, shuffled so the sort has real work to do. + let mut input: Vec<_> = (0..3000).map(|i| wide((i * 389) % 3000 * 8, 8)).collect(); + input.rotate_left(17); + + let (expected, raw_sort, _) = spilling_sort(input.clone(), keys, false).await; + let (actual, sort, coalescer) = spilling_sort(input, keys, true).await; + + assert!( + raw_sort.spill_count().unwrap_or(0) > 0, + "the plain sort must spill" + ); + assert!( + sort.spill_count().unwrap_or(0) > 0, + "the sort must still spill" + ); + let schema = expected[0].schema(); + assert_eq!( + concat_batches(&schema, &actual).unwrap(), + concat_batches(&schema, &expected).unwrap() + ); + let coalescer = coalescer.unwrap(); + // A little input passes through first. After that 8-row inputs are grouped up to + // 256 rows or the 64 KiB cap that the 64 KiB threshold gives, whichever comes first. + let combined_inputs = metric(&coalescer, "coalesced_input_batches"); + let combined_outputs = metric(&coalescer, "coalesced_output_batches"); + assert!(combined_inputs > 2500, "combined {combined_inputs} inputs"); + assert!( + combined_outputs * 2 < combined_inputs, + "{combined_inputs} inputs gave {combined_outputs} outputs" + ); + } + + #[tokio::test] + async fn a_spilling_sort_gives_the_same_result_from_combined_input() { + check_spilling_sort(&[("id", true)]).await; + } + + #[tokio::test] + async fn a_spilling_multi_key_sort_gives_the_same_result_from_combined_input() { + // A two-value date first, then a unique id, so both keys decide the order. + check_spilling_sort(&[("d", false), ("id", true)]).await; + } + + /// A sort in a fair pool so tight that it spills about every 50 KiB. It failed with combined + /// input when the coalescer still held memory while the sort reserved a combined batch. + async fn sort_in_tight_fair_pool( + input: Vec, + coalesce: bool, + ) -> Result { + use datafusion::execution::memory_pool::FairSpillPool; + use datafusion::physical_expr::expressions::col; + use datafusion::physical_expr::{LexOrdering, PhysicalSortExpr}; + use datafusion::physical_plan::sorts::sort::SortExec; + + let schema = input[0].schema(); + let runtime = RuntimeEnvBuilder::new() + .with_memory_pool(Arc::new(FairSpillPool::new(2560 * 1024))) + .build_arc() + .unwrap(); + let config = SessionConfig::new().with_sort_spill_reservation_bytes(64 * 1024); + let gate = config.options().execution.sort_in_place_threshold_bytes; + let ctx = SessionContext::new_with_config_rt(config, runtime); + let source = MemorySourceConfig::try_new_exec(&[input], Arc::clone(&schema), None).unwrap(); + let coalescer = Arc::new(SortInputCoalesceExec::new( + Arc::clone(&source) as _, + 8192, + gate, + )); + let sort_input: Arc = if coalesce { + Arc::clone(&coalescer) as _ + } else { + source + }; + let ordering = LexOrdering::new(vec![PhysicalSortExpr::new( + col("id", &schema).unwrap(), + Default::default(), + )]) + .unwrap(); + let sort = Arc::new(SortExec::new(ordering, sort_input)); + collect(sort.execute(0, ctx.task_ctx())?).await?; + Ok(coalescer.metrics().unwrap()) + } + + #[tokio::test] + async fn combined_input_does_not_fail_a_sort_in_a_tight_fair_pool() { + let input: Vec<_> = (0..4000) + .map(|b| { + let ids: Vec = (0..50) + .map(|i| ((b * 50 + i) * 7919 % 200_000) as i64) + .collect(); + let text = ids.iter().map(|i| format!("value-{i:010}-padding-padding")); + RecordBatch::try_from_iter([ + ("id", Arc::new(Int64Array::from(ids.clone())) as ArrayRef), + ( + "s", + Arc::new(StringArray::from_iter_values(text)) as ArrayRef, + ), + ]) + .unwrap() + }) + .collect(); + sort_in_tight_fair_pool(input.clone(), false) + .await + .expect("the plain sort fits"); + let metrics = sort_in_tight_fair_pool(input, true) + .await + .expect("the sort over combined batches fits too"); + assert!(metric(&metrics, "coalesced_input_batches") > 0); + } + + /// A batch of `rows` rows covering nested, decimal, temporal, dictionary and non-ASCII data. + fn wide(start: usize, rows: usize) -> RecordBatch { + let ids: Vec = (start..start + rows).map(|i| i as i64).collect(); + let point_fields = Fields::from(vec![ + Field::new("x", DataType::Int32, false), + Field::new("label", DataType::Utf8, true), + ]); + let points = StructArray::new( + point_fields.clone(), + vec![ + Arc::new(Int32Array::from_iter_values((0..rows).map(|i| i as i32))) as ArrayRef, + Arc::new(StringArray::from_iter( + (0..rows).map(|i| (i % 3 != 0).then(|| format!("点{i}"))), + )), + ], + None, + ); + let list = ListArray::from_iter_primitive::( + (0..rows).map(|i| (i % 4 != 0).then(|| (0..(i % 3) as i64).map(Some))), + ); + let entries = StructArray::from(vec![ + ( + Arc::new(Field::new("key", DataType::Utf8, false)), + Arc::new(StringArray::from_iter_values( + (0..rows * 2).map(|i| format!("键{i}")), + )) as ArrayRef, + ), + ( + Arc::new(Field::new("value", DataType::Int32, true)), + Arc::new(Int32Array::from_iter( + (0..rows * 2).map(|i| (i % 5 != 0).then_some(i as i32)), + )) as ArrayRef, + ), + ]); + let offsets: Vec = (0..=rows).map(|i| (i * 2) as i32).collect(); + let map = MapArray::try_new( + Arc::new(Field::new("entries", entries.data_type().clone(), false)), + arrow::buffer::OffsetBuffer::new(offsets.into()), + entries, + None, + false, + ) + .unwrap(); + let nested = StructArray::new( + Fields::from(vec![ + Field::new("m", map.data_type().clone(), true), + Field::new("l", list.data_type().clone(), true), + ]), + vec![Arc::new(map) as ArrayRef, Arc::new(list) as ArrayRef], + Some((0..rows).map(|i| i % 7 != 0).collect()), + ); + let decimals = Decimal128Array::from_iter((0..rows).map(|i| match i % 3 { + 0 => Some(99_999_999_999_999_999_999_999_999_999_999_999_999i128), + 1 => Some(-99_999_999_999_999_999_999_999_999_999_999_999_999i128), + _ => None, + })) + .with_precision_and_scale(38, 10) + .unwrap(); + let timestamps = TimestampMicrosecondArray::from_iter_values((0..rows).map(|i| { + if i % 2 == 0 { + -62_135_596_800_000_000 + } else { + 253_402_300_799_999_999 + } + })) + .with_timezone("America/Los_Angeles"); + let dates = Date32Array::from_iter_values((0..rows).map(|i| { + if i % 2 == 0 { + -719_162 + } else { + 2_932_896 + } + })); + let mut dict = StringDictionaryBuilder::::new(); + for i in 0..rows { + dict.append_value(["東京", "Zürich", "москва"][(start + i) % 3]); + } + RecordBatch::try_from_iter([ + ("id", Arc::new(Int64Array::from(ids)) as ArrayRef), + ("point", Arc::new(points) as ArrayRef), + ("nested", Arc::new(nested) as ArrayRef), + ("dec", Arc::new(decimals) as ArrayRef), + ("ts", Arc::new(timestamps) as ArrayRef), + ("d", Arc::new(dates) as ArrayRef), + ("dict", Arc::new(dict.finish()) as ArrayRef), + ]) + .unwrap() + } + + #[test] + fn combines_nested_and_typed_columns() { + let input: Vec<_> = (0..12).map(|i| wide(i * 7, 7)).collect(); + let (output, _) = run(input.clone(), 30, 0, MAX_GROUP_BYTES); + assert_eq!(row_counts(&output), [28, 28, 28]); + same_rows(&output, &input); + let dict = output[0].column(6).as_dictionary::(); + assert_eq!(dict.len(), 28); + assert_eq!(output[0].schema(), input[0].schema()); + } + + #[test] + fn zero_column_batches_keep_their_row_counts() { + let schema = Arc::new(Schema::empty()); + let batch = |rows| { + RecordBatch::try_new_with_options( + Arc::clone(&schema), + vec![], + &arrow::record_batch::RecordBatchOptions::new().with_row_count(Some(rows)), + ) + .unwrap() + }; + let (output, _) = run(vec![batch(3), batch(4)], 100, 0, MAX_GROUP_BYTES); + assert_eq!(row_counts(&output), [7]); + } +} diff --git a/native/proto/src/proto/operator.proto b/native/proto/src/proto/operator.proto index 1b64d22923e..041691ab019 100644 --- a/native/proto/src/proto/operator.proto +++ b/native/proto/src/proto/operator.proto @@ -938,6 +938,9 @@ message Sort { optional int32 skip = 4; // Attach this TopK execution's live threshold to an eligible native reader. bool dynamic_filter_enabled = 5; + // Combine small input batches before a sort without a fetch, once the input is past the + // sort's in-place threshold. + bool coalesce_input = 6; } message HashAggregate { diff --git a/spark/src/main/scala/org/apache/comet/CometConf.scala b/spark/src/main/scala/org/apache/comet/CometConf.scala index a4e54555246..323635cfadc 100644 --- a/spark/src/main/scala/org/apache/comet/CometConf.scala +++ b/spark/src/main/scala/org/apache/comet/CometConf.scala @@ -399,6 +399,26 @@ object CometConf extends ShimCometConf { createExecEnabledConfig("filter", defaultValue = true) val COMET_EXEC_SORT_ENABLED: ConfigEntry[Boolean] = createExecEnabledConfig("sort", defaultValue = true) + val COMET_EXEC_SORT_COALESCE_INPUT_ENABLED: ConfigEntry[Boolean] = + conf(s"$COMET_EXEC_CONFIG_PREFIX.sort.coalesceInput.enabled") + .category(CATEGORY_EXEC) + .doc( + "Whether `CometSort` combines small input batches before sorting them. A shuffle " + + "with many map tasks and many partitions gives each reduce task many tiny Arrow " + + "batches. DataFusion's sort then merges one stream per batch, at least when it " + + "spills or sorts on more than one key, which is slow. Batches pass through " + + "unchanged until a task's sort input is large enough that the sort would merge " + + "instead of sorting in place, so small sorts are not affected. After that, batches " + + "with fewer rows than half of `spark.comet.batchSize`, and fewer than 512, are " + + "combined into batches of at most `spark.comet.batchSize` rows and half of " + + "DataFusion's in-place sort threshold in bytes, 512 KiB by default, kept between 64 " + + "KiB and 4 MiB. The threshold is " + + "`spark.comet.datafusion.execution.sort_in_place_threshold_bytes`, which takes " + + "effect with `spark.comet.exec.respectDataFusionConfigs=true`. A sort with a " + + "limit is not affected. If a sort with a very small memory share fails with " + + "\"Not enough memory to continue external sort\", turning this off may avoid it.") + .booleanConf + .createWithDefault(true) val COMET_EXEC_LOCAL_LIMIT_ENABLED: ConfigEntry[Boolean] = createExecEnabledConfig("localLimit", defaultValue = true) val COMET_EXEC_GLOBAL_LIMIT_ENABLED: ConfigEntry[Boolean] = diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala b/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala index 5e637c6858d..2ea715bc1a7 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala @@ -1601,6 +1601,7 @@ object CometSortExec extends CometOperatorSerde[SortExec] { val sortBuilder = OperatorOuterClass.Sort .newBuilder() .addAllSortOrders(sortOrders.map(_.get).asJava) + .setCoalesceInput(CometConf.COMET_EXEC_SORT_COALESCE_INPUT_ENABLED.get(op.conf)) Some(builder.setSort(sortBuilder).build()) } else { withFallbackReason(op, "sort order not supported") @@ -1655,7 +1656,13 @@ case class CometSortExec( CometMetricNode.baselineMetrics(sparkContext) ++ Map( "spill_count" -> SQLMetrics.createMetric(sparkContext, "number of spills"), - "spilled_bytes" -> SQLMetrics.createSizeMetric(sparkContext, "total spilled bytes")) + "spilled_bytes" -> SQLMetrics.createSizeMetric(sparkContext, "total spilled bytes"), + "coalesced_input_batches" -> SQLMetrics.createMetric( + sparkContext, + "number of small input batches combined into larger ones"), + "coalesced_output_batches" -> SQLMetrics.createMetric( + sparkContext, + "number of batches produced by combining small input batches")) } object CometLocalLimitExec extends CometOperatorSerde[LocalLimitExec] { diff --git a/spark/src/test/resources/sql-tests/operators/sort_coalesce_input.sql b/spark/src/test/resources/sql-tests/operators/sort_coalesce_input.sql new file mode 100644 index 00000000000..530a1032559 --- /dev/null +++ b/spark/src/test/resources/sql-tests/operators/sort_coalesce_input.sql @@ -0,0 +1,124 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- CometSort combines small input batches before sorting them once the input passes the +-- in-place sort threshold. These queries give each sort task many tiny shuffle blocks and +-- check that the result matches Spark. + +-- Many small scan partitions, so each sort task reads small blocks from many map tasks. +-- Config: spark.sql.files.maxPartitionBytes=1024 +-- Config: spark.sql.files.openCostInBytes=1 +-- Config: spark.sql.shuffle.partitions=4 +-- Config: spark.sql.adaptive.coalescePartitions.enabled=false +-- A combined batch holds at most this many rows. +-- Config: spark.comet.batchSize=64 +-- Config: spark.comet.exec.respectDataFusionConfigs=true +-- 0 starts combining with the first batch and makes the sort merge instead of sorting in +-- place. The default, 1 MiB, lets the first part of each sort task's input pass through. +-- ConfigMatrix: spark.comet.datafusion.execution.sort_in_place_threshold_bytes=0,1048576 +-- Write year 1 and year 9999 values without the legacy calendar rebase. +-- Config: spark.sql.parquet.datetimeRebaseModeInWrite=CORRECTED +-- Config: spark.sql.parquet.int96RebaseModeInWrite=CORRECTED +-- Config: spark.comet.exec.sort.coalesceInput.enabled=true +-- The second run gives the sort a tiny memory pool and merge reservation so that it spills. +-- Config: spark.comet.datafusion.execution.sort_spill_reservation_bytes=65536 +-- ConfigMatrix: spark.comet.exec.memoryPool.fraction=1.0,0.002 +-- ConfigMatrix: spark.sql.session.timeZone=UTC,America/Los_Angeles + +statement +CREATE TABLE sort_coalesce( + id INT, + k STRING, + d DECIMAL(38, 10), + ts TIMESTAMP, + tsn TIMESTAMP_NTZ, + dt DATE, + s STRUCT, l: ARRAY>>, + m MAP>, + arr ARRAY) USING parquet + +-- 200 write tasks give 200 small files, so each sort task reads blocks of about 12 rows, well +-- under the size that gets combined. +statement +INSERT INTO sort_coalesce +SELECT + CAST(id AS INT), + CASE id % 7 + WHEN 0 THEN NULL + WHEN 1 THEN concat('東京', id % 50) + WHEN 2 THEN concat('Zürich-', id % 50) + WHEN 3 THEN concat('москва', id % 50) + WHEN 4 THEN concat('😀', id % 50) + WHEN 5 THEN '' + ELSE concat('k', id % 50) + END, + CASE id % 5 + WHEN 0 THEN CAST('999999999999999999999999999.9999999999' AS DECIMAL(38, 10)) + WHEN 1 THEN CAST('-999999999999999999999999999.9999999999' AS DECIMAL(38, 10)) + WHEN 2 THEN NULL + ELSE CAST(id AS DECIMAL(38, 10)) / 7 + END, + CASE id % 4 + WHEN 0 THEN TIMESTAMP '0001-01-01 00:00:00' + WHEN 1 THEN TIMESTAMP '9999-12-31 23:59:59.999999' + WHEN 2 THEN NULL + ELSE timestamp_micros(id * 3600000000) + END, + CASE id % 4 + WHEN 0 THEN TIMESTAMP_NTZ '0001-01-01 00:00:00' + WHEN 1 THEN TIMESTAMP_NTZ '9999-12-31 23:59:59.999999' + WHEN 2 THEN NULL + ELSE TIMESTAMP_NTZ '2024-03-10 02:30:00' + make_interval(0, 0, 0, 0, CAST(id AS INT), 0, 0) + END, + CASE id % 3 + WHEN 0 THEN DATE '0001-01-01' + WHEN 1 THEN DATE '9999-12-31' + ELSE date_add(DATE '1970-01-01', CAST(id AS INT)) + END, + IF(id % 11 = 0, NULL, named_struct( + 'a', CAST(id % 97 AS INT), + 'm', map(concat('键', id % 3), CAST(id AS INT)), + 'l', array( + named_struct('x', id, 'y', concat('值', id % 9)), + named_struct('x', -id, 'y', CAST(NULL AS STRING))))), + IF(id % 13 = 0, NULL, map('a', array(CAST(id AS INT), NULL), 'b', array())), + IF(id % 17 = 0, NULL, array(concat('é', id % 5), NULL, 'z')) +FROM range(0, 10000, 1, 200) + +-- A single string key with non-ASCII values, an empty string and nulls. +query +SELECT * FROM sort_coalesce ORDER BY k NULLS FIRST, id + +query +SELECT * FROM sort_coalesce ORDER BY k DESC NULLS LAST, id DESC + +-- Several keys of different types and directions. +query +SELECT id, dt, d, ts, tsn FROM sort_coalesce +ORDER BY dt DESC NULLS LAST, d ASC NULLS FIRST, tsn, id + +-- Timestamp key, which depends on the session time zone only through the input values. +query +SELECT id, ts, k FROM sort_coalesce ORDER BY ts NULLS LAST, id + +-- An expression over a struct field as the key, with the nested columns carried through. +query +SELECT id, s, m, arr FROM sort_coalesce ORDER BY s.a DESC NULLS FIRST, id + +-- ORDER BY with LIMIT plans as a TopK, which is never combined. It shares the input above. +query +SELECT id, k, s FROM sort_coalesce ORDER BY k NULLS LAST, id LIMIT 25 diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala index d9452017130..ff00ea03b4f 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometTaskMetricsSuite.scala @@ -341,6 +341,50 @@ class CometTaskMetricsSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } + test("native sort combines small shuffle blocks before it spills") { + val value = "sort-coalesce-input-" * 8 + withTempPath { path => + // 80 files, read as 80 scan partitions below, so the shuffle has 80 map tasks. A + // repartition in the query would not do: Spark collapses it into the one above it. + spark + .createDataFrame((0 until 40000).map(index => (index, value))) + .repartition(80) + .write + .parquet(path.getAbsolutePath) + + withParquetTable(path.getAbsolutePath, "tbl") { + Seq(true, false).foreach { enabled => + withSQLConf( + SQLConf.FILES_MAX_PARTITION_BYTES.key -> "1024", + SQLConf.FILES_OPEN_COST_IN_BYTES.key -> "1", + SQLConf.COALESCE_PARTITIONS_ENABLED.key -> "false", + CometConf.COMET_BATCH_SIZE.key -> "1024", + CometConf.COMET_OFFHEAP_MEMORY_POOL_FRACTION.key -> "0.002", + CometConf.COMET_RESPECT_DATAFUSION_CONFIGS.key -> "true", + "spark.comet.datafusion.execution.sort_spill_reservation_bytes" -> "65536", + CometConf.COMET_EXEC_SORT_COALESCE_INPUT_ENABLED.key -> enabled.toString) { + // 80 map tasks write to 2 partitions, so each sort task reads 80 blocks of about + // 250 rows, more than the input that passes through before combining starts. + val sorted = sql("SELECT * FROM tbl") + .repartition(2, $"_1") + .sortWithinPartitions($"_1".desc) + val (_, plan) = checkSparkAnswerAndOperator(sorted) + val sorts = collect(plan) { case sort: CometSortExec => sort } + assert(sorts.nonEmpty, s"Expected a native sort:\n$plan") + val combined = sorts.map(_.metrics("coalesced_input_batches").value).sum + val spilled = sorts.map(_.metrics("spill_count").value).sum + assert(spilled > 0L, s"The sort did not spill (enabled=$enabled):\n$plan") + if (enabled) { + assert(combined > 0L, s"The sort did not combine any input batches:\n$plan") + } else { + assert(combined == 0L, s"The sort combined input batches while disabled:\n$plan") + } + } + } + } + } + } + test("failed non-shuffle native stage attempts preserve disk spill metrics") { val failureRow = 8192 val compressibleValue = "non-shuffle-failed-spill-metrics-" * 8